// @ts-nocheck import { s3Lib as ownS3 } from '@ownzones/lib'; import * as commander from 'commander'; import * as _ from 'lodash'; import * as fs from 'fs'; import * as iam from '@ownzones/iam-service'; async function clearDirectory(bucket, prefix, suffix, orgSlug, orgId) { console.log(`Scanning... Bucket: ${bucket} Key: ${prefix}`); let response: any = {}; try { await s3.headBucket({ Bucket: bucket }).promise(); } catch (err) { console.log(`Skipping bucket: ${bucket}`); console.log(err); return; } do { const query: any = { Bucket: bucket, Prefix: prefix, Delimiter: '/', // MaxKeys: 1000, }; if (response.IsTruncated) { query.ContinuationToken = response.NextContinuationToken; } response = await s3.listObjectsV2(query).promise(); // console.log(`Query ... MaxKeys: ${query.MaxKeys}, ContinuationToken: ${query.ContinuationToken}, Returned: ${response.KeyCount}`); // Process files await Promise.all(response.Contents.filter( f => f.Key.endsWith(suffix), ).map(async (f) => { const regex = /([0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})_[0-9]*_[0-9]*\.raw$/i; const matched = f.Key.match(regex); if (!matched || !matched[1]) { console.log(`Invalid file: ${f.Key} ... skipping...`); } else { console.log(`Removing... ${f.Key}`); removedFiles.push(matched[1]); fs.writeFileSync(csvPath, `${matched[1]},${orgSlug},${orgId},\n`, { flag: 'a' }); if (!commander.dryRun) { await s3.deleteObject({ Bucket: bucket, Key: f.Key }).promise(); } } })); // Scan directories for (const dir of response.CommonPrefixes) { await clearDirectory(bucket, dir.Prefix, suffix, orgSlug, orgId); } } while (response.IsTruncated); } const s3 = ownS3.s3; let removedFiles; const csvPath = './clear-extract-channels-s3-cache-output.csv'; const csvUniquePath = './clear-extract-channels-s3-cache-output-unique.csv'; fs.writeFileSync(csvPath, `id,orgSlug,orgId,\n`, { flag: 'w+' }); fs.writeFileSync(csvUniquePath, `id,orgSlug,orgId,\n`, { flag: 'w+' }); (async () => { commander .option('-o, --org-slug ', 'Org slug') .option('--no-dry-run', 'Don\'t run in dry run mode') .parse(process.argv); console.log(`Running in '${commander.dryRun ? 'dry run' : 'no dry run'}' mode`); const orgArr = (commander.orgSlug ? [await iam.getOrganizationBySlug(commander.orgSlug, iam.setContext())] : await iam.getAllOrganizations(iam.setContext()) ).filter(org => org); for (const org of orgArr) { removedFiles = []; // Clear directory await clearDirectory( org.mediaViewCacheFileLocator.bucket, org.mediaViewCacheFileLocator.key.replace(new RegExp('\/$'), ''), '.raw', org.slug, org.id, ); // Output const uniqueFiles = _.uniq(removedFiles); console.log('Cache removed for files:'); console.log(JSON.stringify(uniqueFiles, null, 2)); for (const file of uniqueFiles) { fs.writeFileSync(csvUniquePath, `${file},${org.slug},${org.id},\n`, { flag: 'a' }); } } // Exit process.exit(0); })();