IVF-HNSW
 All Classes Functions Variables Typedefs Pages
utils.h
1 #ifndef IVF_HNSW_LIB_UTILS_H
2 #define IVF_HNSW_LIB_UTILS_H
3 
4 #include <queue>
5 #include <limits>
6 #include <cmath>
7 #include <chrono>
8 #include <fstream>
9 #include <iostream>
10 #include <sys/time.h>
11 
12 #include <faiss/utils.h>
13 
14 #ifdef _MSC_VER
15 #include <intrin.h>
16 #include <stdexcept>
17 
18 #define __builtin_popcount(t) __popcnt(t)
19 #else
20 #include <x86intrin.h>
21 #endif
22 
23 #define USE_AVX
24 #if defined(__GNUC__)
25 #define PORTABLE_ALIGN32 __attribute__((aligned(32)))
26 #else
27 #define PORTABLE_ALIGN32 __declspec(align(32))
28 #endif
29 
30 #define EPS 0.00001
31 
32 namespace ivfhnsw {
34  class StopW {
35  std::chrono::steady_clock::time_point time_begin;
36  public:
37  StopW() {
38  time_begin = std::chrono::steady_clock::now();
39  }
40 
41  float getElapsedTimeMicro() {
42  std::chrono::steady_clock::time_point time_end = std::chrono::steady_clock::now();
43  return (std::chrono::duration_cast<std::chrono::microseconds>(time_end - time_begin).count());
44  }
45 
46  void reset() {
47  time_begin = std::chrono::steady_clock::now();
48  }
49  };
50 
52  template<typename T>
53  void read_variable(std::istream &in, T &podRef) {
54  in.read((char *) &podRef, sizeof(T));
55  }
56 
58  template<typename T>
59  void read_vector(std::istream &in, std::vector<T> &vec)
60  {
61  uint32_t size;
62  in.read((char *) &size, sizeof(uint32_t));
63  vec.resize(size);
64  in.read((char *) vec.data(), size * sizeof(T));
65  }
66 
68  template<typename T>
69  void write_variable(std::ostream &out, const T &val) {
70  out.write((char *) &val, sizeof(T));
71  }
72 
74  template<typename T>
75  void write_vector(std::ostream &out, std::vector<T> &vec)
76  {
77  uint32_t size = vec.size();
78  out.write((char *) &size, sizeof(uint32_t));
79  out.write((char *) vec.data(), size * sizeof(T));
80  }
81 
82 
84  template<typename T>
85  void readXvec(std::ifstream &in, T *data, const size_t d, const size_t n = 1)
86  {
87  uint32_t dim = d;
88  for (size_t i = 0; i < n; i++) {
89  in.read((char *) &dim, sizeof(uint32_t));
90  if (dim != d) {
91  std::cout << "file error\n";
92  exit(1);
93  }
94  in.read((char *) (data + i * dim), dim * sizeof(T));
95  }
96  }
97 
99  template<typename T>
100  void writeXvec(std::ofstream &out, T *data, const size_t d, const size_t n = 1)
101  {
102  uint32_t dim = d;
103  for (size_t i = 0; i < n; i++) {
104  out.write((char *) &dim, sizeof(uint32_t));
105  out.write((char *) (data + i * dim), dim * sizeof(T));
106  }
107  }
108 
110  template<typename T>
111  void readXvecFvec(std::ifstream &in, float *data, const size_t d, const size_t n = 1)
112  {
113  uint32_t dim = d;
114  T mass[d];
115 
116  for (size_t i = 0; i < n; i++) {
117  in.read((char *) &dim, sizeof(uint32_t));
118  if (dim != d) {
119  std::cout << "file error\n";
120  exit(1);
121  }
122  in.read((char *) mass, dim * sizeof(T));
123  for (size_t j = 0; j < d; j++)
124  data[i * dim + j] = (1.0) * mass[j];
125  }
126  }
127 
129  inline bool exists(const char *path) {
130  std::ifstream f(path);
131  return f.good();
132  }
133 
135  void random_subset(const float *x, float *x_out, size_t d, size_t nx, size_t sub_nx);
136 
138  float fvec_L2sqr(const float *x, const float *y, size_t d);
139 }
140 #endif //IVF_HNSW_LIB_UTILS_H
Clock class.
Definition: utils.h:34