import * as s from '@eddiewang/sia.js' import { Client } from 'siajs-lib' import * as agent from 'superagent' import cache from './cache' const connect = s.connect const path = require('path') const fs = require('fs').promises const readDirRecursive = require('fs-readdir-recursive') let Sia = null let SiaAddress = '' const hasSignedIn = () => true const initClient = async options => { const { siaClientConfig } = options try { Sia = new Client(siaClientConfig) return { apiInitialized: true, apiSignedIn: hasSignedIn() } } catch (e) { console.log("Can't init Sia Client", e) return { apiInitialized: false, apiSignedIn: false } } } const init = async options => { return await initClient(options) } // measurePerformance takes the method name and t0 t1 items to generate a log // output. It will filter times lower than the default set time in ms. const measurePerformance = (name, t0, t1) => { // const time = t1 - t0 // if (time > 250) { // console.log(`Call for ${name} took ${time} milliseconds.`) // } } // const FILE_CACHE_KEY = 'FILE_CACHE' // const FILE_CACHE_EXPIRE = 1000 const getFiles = async () => { // if (cache.get(FILE_CACHE_KEY)) { // return cache.get(FILE_CACHE_KEY) // } const t0 = performance.now() const { files } = await Sia.call('/renter/files') const filesAppendRoot = files.map((f: SiaResource) => { return { ...f, siapath: path.posix.join('root', f.siapath) } }) // cache.set(FILE_CACHE_KEY, filesAppendRoot, FILE_CACHE_EXPIRE) const t1 = performance.now() measurePerformance('getFiles', t0, t1) return filesAppendRoot as SiaResource[] } // const DIR_KEY = 'DIRECTORY_CACHE_' // const DIR_CACHE_EXPIRE = 5000 const getDirectory = async (p): Promise => { console.log('getDirectory for path', p) // if (cache.get(DIR_KEY + p)) { // return cache.get(DIR_KEY + p) // } const t0 = performance.now() const dir: DirResponse = await Sia.call(`/renter/dir/${p}`) // cache.set(DIR_KEY + p, dir, DIR_CACHE_EXPIRE) const t1 = performance.now() measurePerformance('getDirectory', t0, t1) return dir } // Returns 'root' id const getRootId = async () => { return 'root' } // The resource object returned by Sia interface SiaResource { siapath: string localpath: string filesize: number available: boolean renewing: boolean redundancy: number health: number | string uploadedbytes: number uploadprogress: number expiration: number recoverable: boolean accesstime: string changetime: string createtime: string modtime: string } const NORMALIZE_RESOURCE_CACHE_KEY = 'NORMALIZE_RESOURCE_CACHE_KEY_' const NORMALIZE_RESOURCE_EXPIRE = 5000 const normalizedResource = async (resource: SiaResource, filetype = 'file') => { const t0 = performance.now() const cacheKey = NORMALIZE_RESOURCE_CACHE_KEY + resource.siapath if (cache.get(cacheKey)) { return cache.get(cacheKey) } const res = { createdDate: Date.parse(resource.createtime), modifiedDate: Date.parse(resource.changetime), id: resource.siapath, title: path.posix.basename(resource.siapath), type: filetype, redundancy: resource.redundancy, health: filetype === 'file' ? `${resource.health}%` : `${resource.health}%`, mimeType: 'application/octet-stream', downloadUrl: SiaAddress + '/renter/stream/' + resource.siapath .split(path.posix.sep) .slice(1) .join(path.posix.sep), size: resource.filesize, parents: [], capabilities: { canDelete: true, canRename: false, canCopy: false, canEdit: false, canDownload: resource.available }, siaResource: resource } cache.set(cacheKey, res, NORMALIZE_RESOURCE_EXPIRE) const t1 = performance.now() measurePerformance('normalizedResource', t0, t1) return res } const removeResources = async (apiOptions, selectedResources = []) => { const t0 = performance.now() for (const x of selectedResources) { const siaPath = cleanRootFromPath(x.id) if (x.type === 'file') { await deleteFileById(siaPath) } if (x.type === 'dir') { await deleteFolderById(siaPath) } } const t1 = performance.now() measurePerformance('removeResources', t0, t1) return true } const moveResources = async (dropRowResource, items = []) => { try { for (const i of items) { const currPath = i.siaResource.siapath const currName = path.posix.basename(i.siaResource.siapath) let newName = '' if (dropRowResource.siaResource.siapath === '..') { const newDir = path.posix.dirname(path.posix.dirname(i.siaResource.siapath)) if (newDir !== '.') { newName = path.posix.join(newDir, currName) } else { newName = currName } } else { newName = path.posix.join(dropRowResource.siaResource.siapath, currName) } if (i.type === 'dir') { await Sia.call({ url: `/renter/dir/${currPath}`, method: 'POST', qs: { action: 'rename', newsiapath: newName } }) } else { await Sia.call({ url: `/renter/rename/${currPath}`, method: 'POST', qs: { newsiapath: newName } }) } } } catch (e) { console.log('error moving resource', e) } } const cleanRootFromPath = (p: string) => { const t0 = performance.now() const res = p .split(path.posix.sep) .slice(1) .join(path.posix.sep) const t1 = performance.now() measurePerformance('cleanRootFromPath', t0, t1) return res } const getSiaResourceById = async id => { console.log('gettingSia Resource', id) const t0 = performance.now() const cleanPath = cleanRootFromPath(id) // const parentPath = path.posix.dirname(cleanPath) console.log('cleanPath', cleanPath) const dir = await getDirectory(cleanPath) const t1 = performance.now() measurePerformance('getSiaByResourceId', t0, t1) return (dir.files || []).find(s => s.siapath === cleanPath) } interface SiaDir { aggregatenumfiles: number aggregatenumstuckchunks: number aggregatesize: number aggregatehealth: number aggregatemaxhealthpercentage: number health: number lasthealthchecktime: string maxhealth: number aggregateminredundancy: number minredundancy: number mostrecentmodtime: string stuckhealth: number numfiles: number numsubdirs: number siapath: string } interface DirResponse { directories: SiaDir[] files: SiaResource[] } const getSiaDirById = async id => { const t0 = performance.now() const p = id .split(path.posix.sep) .slice(1) .join(path.posix.sep) const dir: DirResponse = await getDirectory(p) const t1 = performance.now() measurePerformance('getSiaDirById', t0, t1) return dir } const getCapabilitiesForResource = (options, resource) => { return resource.capabilities || [] } const createFolder = async (apiOptions, resourceId, folderName) => { const t0 = performance.now() console.log('createFolder', resourceId, folderName) const pathToCreate = path.posix.join(cleanRootFromPath(resourceId), folderName) console.log('pathToCreate', pathToCreate) try { const res = await Sia.call({ url: `/renter/dir/${pathToCreate}`, method: 'POST', qs: { action: 'create' } }) const t1 = performance.now() measurePerformance('createFolder', t0, t1) console.log('results are', res) } catch (e) { console.log('error creating sia folder', e) } } const deleteFileById = async siapath => { const t0 = performance.now() try { console.log('deleting', siapath) const result = await Sia.call({ url: '/renter/delete/' + encodeURI(siapath), timeout: 20000, method: 'POST' }) const t1 = performance.now() measurePerformance('deleteFileById', t0, t1) return result } catch (e) { console.log('error deleting file', e) } } const deleteFolderById = async siapath => { const t0 = performance.now() try { console.log('deleting folder', siapath) const result = await Sia.call({ url: '/renter/dir/' + encodeURI(siapath), timeout: 20000, method: 'POST', qs: { action: 'delete' } }) const t1 = performance.now() measurePerformance('deleteFolderById', t0, t1) return result } catch (e) { console.log('error deleting file', e) } } const uploadFileToId = (siapath, source) => { return Sia.call({ url: '/renter/upload/' + encodeURI(siapath), timeout: 20000, method: 'POST', qs: { source } }) } const sumFileSize = (files: SiaResource[]) => { const t0 = performance.now() const res = (files || []).reduce((x, y) => x + y.filesize, 0) const t1 = performance.now() measurePerformance('sumFileSize', t0, t1) return res } const sumUploadBytes = (files: SiaResource[]) => { const t0 = performance.now() const res = (files || []).reduce((x, y) => x + y.uploadedbytes, 0) const t1 = performance.now() measurePerformance('sumUploadBytes', t0, t1) return res } const topOfList = (n: number[]) => { const t0 = performance.now() const res = Math.max(...n) const t1 = performance.now() measurePerformance('topOfList', t0, t1) return res } const normalizedDirResource = async id => { const t0 = performance.now() const dirPath = cleanRootFromPath(id) const resource = await getSiaDirById(id) const selectedDir = resource.directories.filter(i => i.siapath === dirPath)[0] const dirObj: SiaResource = { siapath: id, localpath: id, filesize: selectedDir.aggregatesize, available: false, renewing: true, expiration: 1000, recoverable: true, health: selectedDir.aggregatemaxhealthpercentage, redundancy: selectedDir.aggregateminredundancy, uploadedbytes: sumUploadBytes(resource.files), uploadprogress: 100, accesstime: new Date( topOfList((resource.directories || []).map(n => Date.parse(n.mostrecentmodtime))) ).toISOString(), changetime: new Date( topOfList((resource.directories || []).map(n => Date.parse(n.mostrecentmodtime))) ).toISOString(), createtime: new Date( topOfList((resource.directories || []).map(n => Date.parse(n.mostrecentmodtime))) ).toISOString(), modtime: new Date( topOfList((resource.directories || []).map(n => Date.parse(n.mostrecentmodtime))) ).toISOString() } const res = await normalizedResource(dirObj, 'dir') const t1 = performance.now() measurePerformance('normalizedDirResource', t0, t1) // cache.set(cacheKey, res, NORMALIZE_DIR_CACHE_EXPIRE) return res } const createFileResource = async (resource, id) => { return { createdDate: Date.parse(resource.createtime), modifiedDate: Date.parse(resource.changetime), id, title: path.posix.basename(resource.siapath), type: 'file', redundancy: resource.redundancy, health: `${resource.maxhealthpercent}%`, mimeType: 'application/octet-stream', downloadUrl: SiaAddress + '/renter/stream/' + resource.siapath .split(path.posix.sep) .slice(1) .join(path.posix.sep), size: resource.filesize, parents: [], capabilities: { canDelete: true, canRename: false, canCopy: false, canEdit: false, canDownload: resource.available }, siaResource: resource } } const createDirResource = async (resource, id) => { const t0 = performance.now() const time = new Date(resource.mostrecentmodtime).toISOString() const dirObj: SiaResource = { siapath: resource.siapath, localpath: id, filesize: resource.aggregatesize, available: false, renewing: true, expiration: 1000, recoverable: true, redundancy: resource.aggregateminredundancy, health: `${resource.aggregatemaxhealthpercentage}%`, uploadedbytes: 0, uploadprogress: 100, accesstime: time, changetime: time, createtime: time, modtime: time } // console.log('dirObj', dirObj) // const parents = await createParentResources(id) const res = { createdDate: Date.parse(dirObj.createtime), modifiedDate: Date.parse(dirObj.changetime), id, title: dirObj.siapath === '' ? 'root' : path.posix.basename(dirObj.siapath), type: 'dir', redundancy: dirObj.redundancy, health: dirObj.health, mimeType: 'application/octet-stream', downloadUrl: SiaAddress + '/renter/stream/' + resource.siapath .split(path.posix.sep) .slice(1) .join(path.posix.sep), size: dirObj.filesize, parents: [], capabilities: { canDelete: true, canRename: false, canCopy: false, canEdit: false, canDownload: resource.available }, siaResource: resource } // console.log('create FR', res) const t1 = performance.now() measurePerformance('createDirResource', t0, t1) return res } const createRootResource = async () => { const date = Date.now().toString() const rootObj: SiaResource = { siapath: 'root', localpath: 'root', filesize: null, available: false, renewing: true, expiration: 1000, recoverable: true, health: 0, redundancy: 1, uploadedbytes: 0, uploadprogress: 100, accesstime: date, changetime: date, createtime: date, modtime: date } const res = await normalizedResource(rootObj, 'dir') return res } // Returns the normalized resource given an id (or path, in this case) const getResourceById = async (options, id) => { // console.log('getting resource by id', id) const t0 = performance.now() if (id === 'root') { const date = Date.now().toString() const rootObj: SiaResource = { siapath: 'root', localpath: 'root', filesize: null, available: false, renewing: true, expiration: 1000, recoverable: true, redundancy: 1, health: 0, uploadedbytes: 0, uploadprogress: 100, accesstime: date, changetime: date, createtime: date, modtime: date } const res = await normalizedResource(rootObj, 'dir') return res } const file = await getSiaResourceById(id) if (file) { const resource = await normalizedResource(file) const t1 = performance.now() measurePerformance('getResourceById', t0, t1) return resource } else { const resource = await normalizedDirResource(id) const t1 = performance.now() measurePerformance('getResourceById', t0, t1) return resource } } const normalizeDirResource = async id => {} // My assumption is that getParentsForId is currently only used to build a path // to resolve at the bottom of the FM. Therefore we should be able to create a fake parent free. const createFakeParent = async parentPath => { const siaPath = cleanRootFromPath(parentPath) const date = Date.now().toString() const dirObj: SiaResource = { siapath: siaPath, localpath: parentPath, filesize: 0, available: false, renewing: true, expiration: 1000, recoverable: true, redundancy: 0, health: 0, uploadedbytes: 0, uploadprogress: 100, accesstime: date, changetime: date, createtime: date, modtime: date } // console.log('dirObj', dirObj) // const parents = await createParentResources(id) const res = { createdDate: Date.parse(dirObj.createtime), modifiedDate: Date.parse(dirObj.changetime), id: parentPath, title: dirObj.siapath === '' ? 'root' : path.posix.basename(dirObj.siapath), type: 'dir', redundancy: dirObj.redundancy, health: dirObj.health, mimeType: 'application/octet-stream', downloadUrl: SiaAddress + '/renter/stream/' + parentPath .split(path.posix.sep) .slice(1) .join(path.posix.sep), size: dirObj.filesize, parents: [], capabilities: { canDelete: true, canRename: false, canCopy: false, canEdit: false, canDownload: false }, siaResource: dirObj } return res } const getParentsForId = async (options, id) => { let currentId = id let res = [] let mem = {} while (currentId !== 'root') { const parentPath = path.posix.dirname(currentId) currentId = parentPath const parentResource = await createFakeParent(parentPath) res = [parentResource].concat(res) // const res = await createParentResources(allFiles, parentPath, [parentResource].concat(result)) } return res } // const getParentsForId = async (options, id) => { // console.log('getting parents for id ', id) // let currentId = id // let res = [] // let mem = {} // while (currentId !== 'root') { // const parentPath = path.posix.dirname(currentId) // currentId = parentPath // const dirPath = cleanRootFromPath(parentPath) // console.log('dirPath is', dirPath) // // try to get from mem // let parentSiaResource = null // if (mem[dirPath]) { // parentSiaResource = mem[dirPath] // } else { // const dir: DirResponse = await getDirectory(dirPath) // parentSiaResource = dir.directories.filter(i => i.siapath === dirPath)[0] // console.log('got resource', parentSiaResource) // mem[dirPath] = parentSiaResource // } // const parentResource = await createDirResource(parentSiaResource, parentPath) // res = [parentResource].concat(res) // // const res = await createParentResources(allFiles, parentPath, [parentResource].concat(result)) // } // console.log('getParentsForId', res) // return res // } // Returns a list of the parent resources going back to the root // /media/movies/avengers.mp4 -> [Resource, Resource] // const getParentsForIdLegacy = async (options, id, result = []) => { // if (id === 'root') { // return result // } // const parentPath = path.posix.dirname(id) // const parent = await normalizedDirResource(parentPath) // const res = await getParentsForIdLegacy(options, parentPath, [parent].concat(result)) // const t1 = performance.now() // return res // } // naturalSorter sorts two strings alphanumerically export const naturalSorter = (as, bs) => { var a, b, a1, b1, i = 0, n, L, rx = /(\.\d+)|(\d+(\.\d+)?)|([^\d.]+)|(\.\D+)|(\.$)/g if (as === bs) return 0 a = as.toLowerCase().match(rx) b = bs.toLowerCase().match(rx) L = a.length while (i < L) { if (!b[i]) return 1 ;(a1 = a[i]), (b1 = b[i++]) if (a1 !== b1) { n = a1 - b1 if (!isNaN(n)) return n return a1 > b1 ? 1 : -1 } } return b[i] ? -1 : 0 } // getChildrenForId is only called on dirs const getChildrenForId = async (options, { id, sortBy = 'title', sortDirection = 'ASC' }) => { const t0 = performance.now() let results = {} const dirPath = cleanRootFromPath(id) const allFiles = await getDirectory(dirPath) for (const x of allFiles.files || []) { const finalPath = path.posix.join('root', x.siapath) const resource = await createFileResource(x, finalPath) if (!(finalPath in results)) { results[finalPath] = resource } } for (const y of allFiles.directories || []) { const finalPath = path.posix.join('root', y.siapath) const resource = await createDirResource(y, finalPath) if (!(finalPath in results) && y.siapath !== dirPath) { results[finalPath] = resource } } const finalResourceList = Object.keys(results).map(x => results[x]) const t1 = performance.now() measurePerformance('getChildrenForId', t0, t1) // we have to modify the final res list with the following rules: // 1) folders are displayed first // 2) alphanumeric sorting let dirList = [] let fileList = [] let res = [] // first we will push them into seperate stacks for (const singleResource of finalResourceList) { if (singleResource.type === 'dir') { dirList.push(singleResource) } else { fileList.push(singleResource) } } // then apply any user defined transformations switch (sortDirection.toUpperCase()) { case 'ASC': dirList = dirList.sort((a, b) => naturalSorter(a[sortBy], b[sortBy])) fileList = fileList.sort((a, b) => naturalSorter(a[sortBy], b[sortBy])) break case 'DESC': dirList = dirList.sort((a, b) => naturalSorter(a[sortBy], b[sortBy])).reverse() fileList = fileList.sort((a, b) => naturalSorter(a[sortBy], b[sortBy])).reverse() break default: } res = [...dirList, ...fileList] return res } const getResourceName = (options, resource) => { return resource.title } // Get the nearest parent id for a specified resource const getParentIdForResource = async (options, resource) => { if (resource.parents.length) { return 'root' } return resource.parents[0].id } const downloadResource = async ({ resource, params, onProgress, i = 0, l = 1 }) => { const { downloadUrl, direct, mimeType, fileName } = params let res try { res = await agent .get('http://' + downloadUrl) .responseType('blob') .on('progress', event => { onProgress((i * 100 + event.percent) / l) }) } catch (err) { throw new Error(`failed to download resource: ${err}`) } return { downloadUrl, direct, file: res.body, mimeType } } interface SiaDownloadObject { destination: string destinationtype: string filesize: number length: number offset: number siapath: string completed: boolean endtime: string error: string received: number starttime: string starttimeunix: number totaldatatransferred: number } interface SiaUploadObject { accesstime: string available: boolean changetime: string ciphertype: string createtime: string expiration: number filesize: number health: number localpath: string maxhealth: number maxhealthpercent: number modtime: string numstuckchunks: number ondisk: boolean recoverable: boolean redundancy: number renewing: boolean siapath: string stuck: boolean stuckhealth: number uploadedbytes: number uploadprogress: number } const allDownloads = async () => { const t0 = performance.now() const { downloads } = await Sia.call('/renter/downloads') const filteredDownloads = (downloads || []).filter( (d: SiaDownloadObject) => d.destinationtype === 'file' ) const map = {} // We run this through a loop to remove streaming objects bug from siad. filteredDownloads.forEach((d: SiaDownloadObject) => { if (map.hasOwnProperty(d.siapath)) { if (map[d.siapath].starttime < d.starttime) { map[d.siapath] = d } } else { map[d.siapath] = d } }) const results = Object.values(map) const t1 = performance.now() measurePerformance('allDownloads', t0, t1) return results as SiaDownloadObject[] } const allUploads = async () => { const t0 = performance.now() const { files } = await Sia.call('/renter/files') const t1 = performance.now() measurePerformance('allUploads', t0, t1) return (files || []) as SiaUploadObject[] } const queueDownload = async (siapath: string, downloadpath: string) => { const t0 = performance.now() const data = await Sia.call({ url: `/renter/download/${siapath}`, qs: { destination: downloadpath, async: true } }) const t1 = performance.now() measurePerformance('queueDownload', t0, t1) return data } // const uploadAction = async (paths) => { // const onFail = err => emitter.emit('notification', err) // // this function will go through each path and upload it to Sia // for (let eachPath of paths) { // // first, let's first out if it's a dir or file // const fileStatus = await fs.stat(eachPath) // // Handler for directories // if (fileStatus.isDirectory()) { // // first we'll recursively get the subpaths of the dir // const files = readDirRecursive(eachPath) // // loop over each subpath and create the siapath to be uploaded // for (let subPath of files) { // const folderName = path.basename(eachPath) // const uploadPath = path.join(eachPath, subPath) // const combinedPath = path.posix.join(resource.id, folderName, subPath).split(path.posix.sep) // const siaPath = combinedPath.slice(1).join(path.posix.sep) // try { // await uploadFileToId(siaPath, uploadPath) // } catch (err) { // console.log('error uploading file', err) // } // } // // onFail('directories are not support just yet') // } else { // // default handler for files // const fileName = path.basename(eachPath) // const combinedPath = path.posix.join(resource.id, fileName).split(path.posix.sep) // const siaPath = combinedPath.slice(1).join(path.posix.sep) // try { // await uploadFileToId(siaPath, eachPath) // } catch (err) { // console.log('error uploading file', err) // onFail(err) // } // } // } // } export default { init, hasSignedIn, getResourceById, getChildrenForId, getRootId, getParentsForId, getParentIdForResource, getCapabilitiesForResource, getResourceName, downloadResource, allDownloads, allUploads, uploadFileToId, cleanRootFromPath, queueDownload, createFolder, removeResources, moveResources }