| 1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146 | 1x
1x
1x
1x
1x
1x
1x
1x
1x
9x
9x
9x
9x
9x
2x
2x
2x
2x
2x
2x
2x
2x
2x
1x
1x
1x
1x
1x
1x
1x
2x
2x
2x
2x
2x
2x
2x
2x
3x
3x
3x
1x
1x
1x
2x
1x
1x
1x
1x
1x
1x
1x
1x
22x
3x
18x
3x
3x
3x
3x
2x
1x
1x
2x
2x
2x
1x
2x
2x
2x
2x
2x
2x
2x
2x
1x
1x
1x
1x
1x
4x
4x
1x
1x
1x
1x
8x
1x
1x
1x
| var path = require('path')
var fs = require('fs')
var Api = require('./Api')
var FileDownloader = require('./FileDownloader')
var async = require('async')
var mkdirp = require('mkdirp')
var glob = require('glob')
const DEFAULT_LIMIT = 50
const CONCURRENCY_LIMIT = 5
class Sync {
constructor(opts, config, logger) {
this.opts = opts
this.logger = logger
this.api = new Api(config)
this.fileDownloader = new FileDownloader(logger)
this.saveLocation = path.resolve(process.cwd(), config.saveLocation)
}
run(cb) {
this.getSync((err, toSync) => {
Iif (err) return cb(err)
this.downloadSchema(toSync.schemaVersion, (err) => {
Iif (err) return cb(err)
async.mapLimit(toSync.files, CONCURRENCY_LIMIT, this.processFile.bind(this), (err, results) => {
Iif (err) return cb(err)
var splitResults = this.splitResults(results)
this.logResults(splitResults)
if (splitResults.erroredFiles.length) {
this.logger.warn(`${splitResults.erroredFiles.length} files failed to download, please try running the sync again, if this error persists, open a ticket. No files will be cleaned up`)
this.logger.warn(splitResults.erroredFiles)
return cb(new Error('failed to download some files, try running sync again'))
}
this.cleanupFiles(results, (err) => {
Iif (err) return cb(err)
this.logger.info('finished cleanup, done!')
cb()
})
})
})
})
}
splitResults(results) {
var erroredFiles = results.filter((res) => res.error)
var newDownloaded = results.filter((res) => res.didDownload).map((res) => res.filename)
var cached = results.filter((res) => !res.error && !res.didDownload).map((res) => res.filename)
return {erroredFiles, newDownloaded, cached, results}
}
logResults(splitResults) {
Iif (splitResults.cached.length === splitResults.results.length) {
this.logger.info('no new files were downloaded')
} else {
this.logger.info(`downloaded ${splitResults.newDownloaded.length} new files out of ${splitResults.results.length} total files`)
this.logger.debug('new files downloaded', splitResults.newDownloaded)
this.logger.debug('cached files', splitResults.cached)
}
}
getSync(cb) {
this.logger.info('fetching current list of files from API...')
this.api.getSync((err, toSync) => {
if (err && err.errorCode === 404) {
this.logger.error('no files exist for account, cannot sync')
err.silence = true
return cb(err)
}
if (err) return cb(err)
Iif (toSync.incomplete) this.logger.warn(`Could not retrieve a full list of files! Some incremental data will be missing!`)
this.logger.info(`total number of files: ${toSync.files.length} files`)
cb(null, toSync)
})
}
downloadSchema(schemaVersion, cb) {
mkdirp(this.saveLocation, (err) => {
Iif (err) return cb(err)
this.api.getSchemaVersion(schemaVersion, (err, schema) => {
Iif (err) return cb(err)
fs.writeFile(path.join(this.saveLocation, 'schema.json'), JSON.stringify(schema, 0, 2), cb)
})
})
}
buildDir(fileInfo) {
return path.join(this.saveLocation, fileInfo.table)
}
buildTempPath(fileInfo) {
return this.buildRealPath(fileInfo) + '.tmp'
}
buildRealPath(fileInfo) {
return path.join(this.buildDir(fileInfo), fileInfo.filename)
}
processFile(fileInfo, cb) {
var filename = this.buildRealPath(fileInfo)
this.logger.info(`checking for existence of ${fileInfo.filename}`)
this.fileExists(filename, (err, exists) => {
if (err) return cb(err)
if (!exists) return this.downloadFile(fileInfo, cb)
this.logger.info(`already have ${fileInfo.filename}, no need to redownload`)
return cb(null, {error: null, table: fileInfo.table, filename: fileInfo.filename, savedTo: filename, didDownload: false})
})
}
fileExists(filename, cb) {
fs.stat(filename, (err, stat) => {
Iif (err && err.code !== 'ENOENT') return cb(err)
if (err && err.code === 'ENOENT') return cb(null, false)
cb(null, true)
})
}
downloadFile(fileInfo, cb) {
var filename = this.buildRealPath(fileInfo)
var tmpFilename = this.buildTempPath(fileInfo)
this.logger.info(`${filename} does not exist, downloading`)
mkdirp(this.buildDir(fileInfo), (err) => {
Iif (err) return cb(err)
this.fileDownloader.downloadToFile(fileInfo, {tableName: fileInfo.table}, tmpFilename, (err) => {
this.logger.info(`${filename} finished`)
if (err) return cb(null, {error: err, table: fileInfo.table, filename: fileInfo.filename})
this.logger.debug(`rename ${tmpFilename} to ${filename}`)
fs.rename(tmpFilename, filename, (err) => {
Iif (err) return cb(err)
cb(null, {error: null, table: fileInfo.table, filename: fileInfo.filename, savedTo: filename, didDownload: true})
})
})
})
}
cleanupFiles(downloadedFiles, cb) {
var byFilename = {}
for (var file of downloadedFiles) {
byFilename[path.relative(this.saveLocation, file.savedTo)] = true
}
this.logger.info('searching for old files to remove')
glob('**/*', {cwd: this.saveLocation, nodir: true}, (err, files) => {
Iif (err) return cb(err)
// rewrite paths because glob returns inproper path seperators on windows (/ instead of \)
var toRemove = files.map((f) => f.split('/').join(path.sep)).filter((f) => {
return f !== 'schema.json' && !byFilename[f]
})
this.logger.debug('will remove files', toRemove)
async.map(toRemove.map((name) => path.join(this.saveLocation, name)), fs.unlink, cb)
})
}
}
module.exports = Sync
|