All files HistoricalRequests.js

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
1x
1x
 
1x
13x
1x
 
 
 
 
 
4x
4x
4x
 
 
 
 
3x
 
 
4x
4x
 
 
3x
 
 
3x
 
3x
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3x
3x
 
 
 
1x
 
var Api = require('./Api')
var lodash = require('lodash')
 
const customizer = (a, b) => {
  if (lodash.isArray(a)) {
    return a.concat(b);
  }
}
 
class HistoricalRequests {
  constructor(opts, config, logger) {
    this.opts = opts
    this.logger = logger
    this.api = new Api(config)
  }
  getRangeForFile(url) {
    // The URL format is:
    // https://<bucket>.s3.amazonaws.com/<timestamp>/requests_split_historical/<account>/requests/range/
    return url.split('/')[7]
  }
  run(cb) {
    this.api.getSync((err, response) => {
      if (err) return cb(err)
      //
      // Get all the not-partial files and figure out their range
      const files = response.files
        .filter((file) => file.table == 'requests')
        .filter((file) => !file.partial);
      const results = lodash.merge({}, ...files
        .map((file) => {
          return {
            [this.getRangeForFile(file.url)]: [file]
          }
        }), customizer)
 
      // const results = files
      // const results = lodash.merge({}, ...files
      //   .filter((dump) => !dump.partial)
      //   .map((dump) => {
      //     const maps = dump.files.map((file) => {
      //       return {
      //         dumpId: dump.dumpId,
      //         ranges: {
      //           [this.getRangeForFile(file.url)]: [file]
      //         }
      //       }
      //     })
      //     return lodash.merge({}, ...maps, customizer)
      //   }), customizer)
      this.logger.info(JSON.stringify(results, null, 2))
      cb(null, results)
    })
  }
}
module.exports = HistoricalRequests