Refactor filter code into its own class

apps/build_disk_index.cpp

@@ -1,15 +1,15 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license.
 
-#include <omp.h>
 #include <boost/program_options.hpp>
+#include <omp.h>
 
-#include "utils.h"
 #include "disk_utils.h"
-#include "math_utils.h"
 #include "index.h"
+#include "math_utils.h"
 #include "partition.h"
 #include "program_options_utils.hpp"
+#include "utils.h"
 
 namespace po = boost::program_options;
 

apps/build_memory_index.cpp

@@ -1,13 +1,13 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license.
 
-#include <omp.h>
-#include <cstring>
 #include <boost/program_options.hpp>
+#include <cstring>
+#include <omp.h>
 
 #include "index.h"
-#include "utils.h"
 #include "program_options_utils.hpp"
+#include "utils.h"
 
 #ifndef _WINDOWS
 #include <sys/mman.h>
@@ -16,9 +16,9 @@
 #include <Windows.h>
 #endif
 
-#include "memory_mapper.h"
 #include "ann_exception.h"
 #include "index_factory.h"
+#include "memory_mapper.h"
 
 namespace po = boost::program_options;
 

apps/build_stitched_index.cpp

@@ -1,24 +1,24 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license.
 
+#include "filter_utils.h"
 #include <boost/program_options.hpp>
 #include <chrono>
 #include <cstdio>
 #include <cstring>
+#include <omp.h>
 #include <random>
 #include <string>
 #include <tuple>
-#include "filter_utils.h"
-#include <omp.h>
 #ifndef _WINDOWS
 #include <sys/uio.h>
 #endif
 
 #include "index.h"
 #include "memory_mapper.h"
 #include "parameters.h"
-#include "utils.h"
 #include "program_options_utils.hpp"
+#include "utils.h"
 
 namespace po = boost::program_options;
 typedef std::tuple<std::vector<std::vector<uint32_t>>, uint64_t> stitch_indices_return_values;

apps/range_search_disk_index.cpp

@@ -2,26 +2,26 @@
 // Licensed under the MIT license.
 
 #include <atomic>
+#include <boost/program_options.hpp>
 #include <cstring>
 #include <iomanip>
 #include <omp.h>
 #include <set>
-#include <boost/program_options.hpp>
 
-#include "index.h"
 #include "disk_utils.h"
+#include "index.h"
 #include "math_utils.h"
 #include "memory_mapper.h"
-#include "pq_flash_index.h"
 #include "partition.h"
-#include "timer.h"
+#include "pq_flash_index.h"
 #include "program_options_utils.hpp"
+#include "timer.h"
 
 #ifndef _WINDOWS
+#include "linux_aligned_file_reader.h"
 #include <sys/mman.h>
 #include <sys/stat.h>
 #include <unistd.h>
-#include "linux_aligned_file_reader.h"
 #else
 #ifdef USE_BING_INFRA
 #include "bing_aligned_file_reader.h"

apps/search_disk_index.cpp

@@ -4,21 +4,21 @@
 #include "common_includes.h"
 #include <boost/program_options.hpp>
 
-#include "index.h"
 #include "disk_utils.h"
+#include "index.h"
 #include "math_utils.h"
 #include "memory_mapper.h"
 #include "partition.h"
-#include "pq_flash_index.h"
-#include "timer.h"
 #include "percentile_stats.h"
+#include "pq_flash_index.h"
 #include "program_options_utils.hpp"
+#include "timer.h"
 
 #ifndef _WINDOWS
+#include "linux_aligned_file_reader.h"
 #include <sys/mman.h>
 #include <sys/stat.h>
 #include <unistd.h>
-#include "linux_aligned_file_reader.h"
 #else
 #ifdef USE_BING_INFRA
 #include "bing_aligned_file_reader.h"
@@ -123,8 +123,8 @@ int search_disk_index(diskann::Metric &metric, const std::string &index_path_pre
     diskann::cout << "Caching " << num_nodes_to_cache << " nodes around medoid(s)" << std::endl;
     _pFlashIndex->cache_bfs_levels(num_nodes_to_cache, node_list);
     // if (num_nodes_to_cache > 0)
-    //     _pFlashIndex->generate_cache_list_from_sample_queries(warmup_query_file, 15, 6, num_nodes_to_cache,
-    //     num_threads, node_list);
+    //     _pFlashIndex->generate_cache_list_from_sample_queries(warmup_query_file,
+    //     15, 6, num_nodes_to_cache, num_threads, node_list);
     _pFlashIndex->load_cache_list(node_list);
     node_list.clear();
     node_list.shrink_to_fit();

apps/search_memory_index.cpp

@@ -1,14 +1,14 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license.
 
+#include <algorithm>
+#include <boost/program_options.hpp>
 #include <cstring>
 #include <iomanip>
-#include <algorithm>
 #include <numeric>
 #include <omp.h>
 #include <set>
 #include <string.h>
-#include <boost/program_options.hpp>
 
 #ifndef _WINDOWS
 #include <sys/mman.h>
@@ -18,10 +18,10 @@
 #endif
 
 #include "index.h"
+#include "index_factory.h"
 #include "memory_mapper.h"
-#include "utils.h"
 #include "program_options_utils.hpp"
-#include "index_factory.h"
+#include "utils.h"
 
 namespace po = boost::program_options;
 
@@ -323,9 +323,9 @@ int main(int argc, char **argv)
         optional_configs.add_options()("num_threads,T",
                                        po::value<uint32_t>(&num_threads)->default_value(omp_get_num_procs()),
                                        program_options_utils::NUMBER_THREADS_DESCRIPTION);
-        optional_configs.add_options()(
-            "dynamic", po::value<bool>(&dynamic)->default_value(false),
-            "Whether the index is dynamic. Dynamic indices must have associated tags.  Default false.");
+        optional_configs.add_options()("dynamic", po::value<bool>(&dynamic)->default_value(false),
+                                       "Whether the index is dynamic. Dynamic indices must have associated "
+                                       "tags.  Default false.");
         optional_configs.add_options()("tags", po::value<bool>(&tags)->default_value(false),
                                        "Whether to search with external identifiers (tags). Default false.");
         optional_configs.add_options()("fail_if_recall_below",

apps/test_insert_deletes_consolidate.cpp

@@ -1,19 +1,19 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license.
 
+#include <boost/program_options.hpp>
+#include <future>
 #include <index.h>
 #include <numeric>
 #include <omp.h>
 #include <string.h>
 #include <time.h>
 #include <timer.h>
-#include <boost/program_options.hpp>
-#include <future>
 
-#include "utils.h"
 #include "filter_utils.h"
-#include "program_options_utils.hpp"
 #include "index_factory.h"
+#include "program_options_utils.hpp"
+#include "utils.h"
 
 #ifndef _WINDOWS
 #include <sys/mman.h>

apps/test_streaming_scenario.cpp

@@ -1,20 +1,20 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license.
 
+#include <abstract_index.h>
+#include <boost/program_options.hpp>
+#include <future>
 #include <index.h>
+#include <index_factory.h>
 #include <numeric>
 #include <omp.h>
 #include <string.h>
 #include <time.h>
 #include <timer.h>
-#include <boost/program_options.hpp>
-#include <future>
-#include <abstract_index.h>
-#include <index_factory.h>
 
-#include "utils.h"
 #include "filter_utils.h"
 #include "program_options_utils.hpp"
+#include "utils.h"
 
 #ifndef _WINDOWS
 #include <sys/mman.h>

apps/utils/bin_to_fvecs.cpp

@@ -1,8 +1,8 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license.
 
-#include <iostream>
 #include "util.h"
+#include <iostream>
 
 void block_convert(std::ifstream &writr, std::ofstream &readr, float *read_buf, float *write_buf, uint64_t npts,
                    uint64_t ndims)

apps/utils/bin_to_tsv.cpp

@@ -1,8 +1,8 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license.
 
-#include <iostream>
 #include "utils.h"
+#include <iostream>
 
 template <class T>
 void block_convert(std::ofstream &writer, std::ifstream &reader, T *read_buf, size_t npts, size_t ndims)

apps/utils/calculate_recall.cpp

@@ -9,8 +9,8 @@
 #include <string>
 #include <vector>
 
-#include "utils.h"
 #include "disk_utils.h"
+#include "utils.h"
 
 int main(int argc, char **argv)
 {

apps/utils/compute_groundtruth.cpp

@@ -1,25 +1,25 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license.
 
-#include <string>
-#include <iostream>
-#include <fstream>
 #include <cassert>
+#include <fstream>
+#include <iostream>
+#include <string>
 
-#include <vector>
 #include <algorithm>
+#include <boost/program_options.hpp>
 #include <cassert>
 #include <cstddef>
-#include <random>
-#include <limits>
 #include <cstring>
-#include <queue>
-#include <omp.h>
+#include <limits>
 #include <mkl.h>
-#include <boost/program_options.hpp>
-#include <unordered_map>
+#include <omp.h>
+#include <queue>
+#include <random>
 #include <tsl/robin_map.h>
 #include <tsl/robin_set.h>
+#include <unordered_map>
+#include <vector>
 
 #ifdef _WINDOWS
 #include <malloc.h>

apps/utils/compute_groundtruth_for_filters.cpp

@@ -1,25 +1,25 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license.
 
-#include <string>
-#include <iostream>
-#include <fstream>
 #include <cassert>
+#include <fstream>
+#include <iostream>
+#include <string>
 
-#include <vector>
 #include <algorithm>
+#include <boost/program_options.hpp>
 #include <cassert>
 #include <cstddef>
-#include <random>
-#include <limits>
 #include <cstring>
-#include <queue>
-#include <omp.h>
+#include <limits>
 #include <mkl.h>
-#include <boost/program_options.hpp>
-#include <unordered_map>
+#include <omp.h>
+#include <queue>
+#include <random>
 #include <tsl/robin_map.h>
 #include <tsl/robin_set.h>
+#include <unordered_map>
+#include <vector>
 
 #ifdef _WINDOWS
 #include <malloc.h>

apps/utils/count_bfs_levels.cpp

@@ -1,14 +1,14 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license.
 
+#include <algorithm>
+#include <boost/program_options.hpp>
 #include <cstring>
 #include <iomanip>
-#include <algorithm>
 #include <numeric>
 #include <omp.h>
 #include <set>
 #include <string.h>
-#include <boost/program_options.hpp>
 
 #ifndef _WINDOWS
 #include <sys/mman.h>
@@ -17,9 +17,9 @@
 #include <unistd.h>
 #endif
 
-#include "utils.h"
 #include "index.h"
 #include "memory_mapper.h"
+#include "utils.h"
 
 namespace po = boost::program_options;
 

apps/utils/create_disk_layout.cpp

@@ -8,9 +8,9 @@
 #include <limits>
 #include <vector>
 
-#include "utils.h"
-#include "disk_utils.h"
 #include "cached_io.h"
+#include "disk_utils.h"
+#include "utils.h"
 
 template <typename T> int create_disk_layout(char **argv)
 {

apps/utils/float_bin_to_int8.cpp

@@ -1,8 +1,8 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license.
 
-#include <iostream>
 #include "utils.h"
+#include <iostream>
 
 void block_convert(std::ofstream &writer, int8_t *write_buf, std::ifstream &reader, float *read_buf, size_t npts,
                    size_t ndims, float bias, float scale)

apps/utils/fvecs_to_bin.cpp

@@ -1,8 +1,8 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license.
 
-#include <iostream>
 #include "utils.h"
+#include <iostream>
 
 // Convert float types
 void block_convert_float(std::ifstream &reader, std::ofstream &writer, float *read_buf, float *write_buf, size_t npts,