// The file-server product: its two events and ASSETS_FOR_TEMPLATE // (file-server v8.15.9, scripts/platform-preset/seeds/file-server@8.15.9/ // scripts: file-server-eventhandler.kts below as EH, file-server-reqrep.kts as // RR), and the HTTP endpoints its router script serves (file-server-web- // handler.kts in file-server-app, which uploads, replaces, downloads and // deletes files). A file's bytes are kept in memory, by FILE_STORAGE_ID. import { randomUUID } from 'node:crypto'; import type { ServerResponse } from 'node:http'; import { checkUnique } from '../handlers/crudEvents.ts'; import { NackError } from '../protocol/errors.ts'; import type { EventHandler, EventHandlerCtx, EventSchemaOverride, HttpRouteDef, RequestReplyDef, Row, } from '../types.ts'; import { BodyTooLarge, boundaryOf, declaredTooLarge, type FormPart, parseMultipart, readBody, } from './multipart.ts'; import { dto } from './notify.ts'; import type { FileContent } from './reconciliation.ts'; const SCHEMAS: Record = { EVENT_FILE_STORAGE_DELETE: { table: 'FILE_STORAGE', pkOnly: true, className: 'global.genesis.gen.dao.FileStorage.ById', }, EVENT_LINK_TEMPLATE_ASSETS: { className: 'global.genesis.file.messages.EventLinkTemplateAssets', fields: [ dto('TEMPLATE_ID', 'STRING'), dto('ASSET_IDS', 'ARRAY', { items: { type: 'STRING' } }), ], }, }; // A FILE_STORAGE_ID as the uuid strategy builds one: a UUID, the sequence // (FF), the location (LO) and 1. export const newFileStorageId = () => `${randomUUID()}FFLO1`; // The sysdef's ALLOWED_FILE_EXTENSIONS default. const ALLOWED_EXTENSIONS = [ 'html', 'pdf', 'csv', 'txt', 'jpeg', 'jpg', 'xlsx', 'png', 'svg', 'css', 'ftl', 'xml', ]; const MAX_FILE_NAME = 128; // The largest upload body taken. GSF's MultiPartConfig.maxFileSize is 1 GiB // (HttpConfig.kt:88 @v8.15.29), which it spools to disk; the mock keeps // files in memory. const BYTES_PER_MIB = 1_048_576; const MAX_UPLOAD_MIB = 100; export const MAX_UPLOAD_BYTES = MAX_UPLOAD_MIB * BYTES_PER_MIB; const FILE_TYPES = ['USER_DOCUMENT', 'DOCUMENT_TEMPLATE', 'TEMPLATE_ASSET']; // MimetypesFileTypeMap's types for the allowed extensions; the rest are // application/octet-stream. const CONTENT_TYPES: Record = { html: 'text/html', txt: 'text/plain', jpeg: 'image/jpeg', jpg: 'image/jpeg', png: 'image/png', }; const STATUS_TEXT: Record = { 400: 'Bad Request', 401: 'Unauthorized', 404: 'Not Found', 406: 'Not Acceptable', 422: 'Unprocessable Entity', 500: 'Internal Server Error', }; const OK = 200; const BAD_REQUEST = 400; const UNAUTHORIZED = 401; const NOT_FOUND = 404; const NOT_ACCEPTABLE = 406; const UNPROCESSABLE = 422; const SERVER_ERROR = 500; // The router's error reply (HttpHandler): an IllegalArgumentException's // message as 422, anything else 500 "Error while processing request", in a // MSG_NACK. function sendNack(res: ServerResponse, status: number, text: string) { const STATUS_CODE = `${status} ${STATUS_TEXT[status]}`; sendJson(res, status, { MESSAGE_TYPE: 'MSG_NACK', // What the router mints for a request with none. SOURCE_REF: `AUTO_${randomUUID()}`, ERROR: [{ CODE: status, TEXT: text, STATUS_CODE }], DETAILS: { ERROR: text, STATUS_CODE }, }); } function sendJson(res: ServerResponse, status: number, body: unknown) { res.writeHead(status, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(body)); } // An endpoint's own exceptionHandler: its text as it is. function sendText(res: ServerResponse, status: number, text: string) { res.writeHead(status, { 'Content-Type': 'application/json' }); res.end(text); } // The file name after its last /, as File.separator splits it. const baseName = (name: string) => name.slice(name.lastIndexOf('/') + 1); const extensionOf = (name: string) => name.slice(name.lastIndexOf('.') + 1); // FileStorageContentValidator.performValidationChecks: empty content, then // the extension, the length and the characters. Its content-type sniffing // (Tika) isn't modelled. The empty file is an IOException: a 500. function contentError(part: FormPart, name: string): [number, string] | undefined { if (part.data.length <= 0) return [SERVER_ERROR, 'Error while processing request']; const extension = extensionOf(name); if (!ALLOWED_EXTENSIONS.includes(extension)) { return [UNPROCESSABLE, `File extension ${extension} is not on the list of allowed file types`]; } if (name.length > MAX_FILE_NAME) return [UNPROCESSABLE, 'File name must be 128 characters or fewer']; if (/[/\\:*?"<>|\0\n\r]|\.\./.test(name)) { return [UNPROCESSABLE, 'File name must not contain special characters']; } return undefined; } export interface FileServerBacking { eventHandlers: Record; eventSchemas: Record; requestReplies: Record; httpRoutes: HttpRouteDef[]; // The bytes the mock keeps for a file, if it has any. contents: Map; // FileStorageClient.getFileContents, for reconciliation's FILE datasources: // the file's name, or the fetch's failure. fileContent: FileContent; } const IDENTITY_HASH = 0xffffffff; const HEX = 16; // FILE_MANAGER has no file of that id: what DataSourceFactory reports. Its // FileContentRequest has no toString(), so it prints as Object does. const fileContent: FileContent = (store, id) => { const file = store.getRow('FILE_STORAGE', id); if (file) return { fileName: String(file.FILE_NAME) }; const hash = Math.floor(Math.random() * IDENTITY_HASH).toString(HEX); return { error: `Exception caught while attempting to request file contents from FILE_MANAGER with request global.genesis.file.message.request.FileContentRequest@${hash}`, }; }; export function fileServerBacking(): FileServerBacking { const contents = new Map(); const eventHandlers: Record = { // EH :5-38: the template's links replaced by those sent. EVENT_LINK_TEMPLATE_ASSETS: (details, ctx) => { const template = ctx.store.getRow('FILE_STORAGE', details.TEMPLATE_ID); if (!template) { throw new NackError( `Template for ID ${details.TEMPLATE_ID} does not exist in FILE_STORAGE!`, ); } if (template.FILE_TYPE !== 'DOCUMENT_TEMPLATE') { throw new NackError(`File for ID ${details.TEMPLATE_ID} has not been marked as template!`); } for (const assetId of details.ASSET_IDS as string[]) { if (!ctx.store.getRow('FILE_STORAGE', assetId)) { throw new NackError(`Asset for ID ${assetId} does not exist in FILE_STORAGE!`); } } if (ctx.validate) return {}; for (const link of linksOf(ctx, details.TEMPLATE_ID)) { const key = ctx.store.rowKey('TEMPLATE_ASSET', link); ctx.store.deleteRow('TEMPLATE_ASSET', key); ctx.broadcastTableChange('TEMPLATE_ASSET', 'DELETE', key); } for (const ASSET_ID of details.ASSET_IDS as string[]) { const row = { TEMPLATE_ID: details.TEMPLATE_ID, ASSET_ID }; checkUnique('TEMPLATE_ASSET', ctx.config.tables!.TEMPLATE_ASSET, row, ctx.store); ctx.store.insertRow('TEMPLATE_ASSET', row); ctx.broadcastTableChange( 'TEMPLATE_ASSET', 'INSERT', ctx.store.rowKey('TEMPLATE_ASSET', row), ); } return {}; }, // EH :41-57: verify { hasEntry }, then the file manager's delete. EVENT_FILE_STORAGE_DELETE: (details, ctx) => { const id = details.FILE_STORAGE_ID; if (!ctx.store.getRow('FILE_STORAGE', id)) { throw new NackError(`FILE_STORAGE ById(fileStorageId=${id}) not found in database`); } if (ctx.validate) return {}; deleteFile(ctx, id); return {}; }, }; // AbstractFileStorageManager.deleteFile: the row and the bytes; the // template links to it are left. function deleteFile(ctx: Pick, id: string) { if (ctx.store.deleteRow('FILE_STORAGE', id)) ctx.broadcastTableChange('FILE_STORAGE', 'DELETE', id); contents.delete(id); } const linksOf = (ctx: { store: EventHandlerCtx['store'] }, templateId: unknown) => ctx.store.getAllRows('TEMPLATE_ASSET').filter((link) => link.TEMPLATE_ID === templateId); return { contents, fileContent, eventHandlers, eventSchemas: SCHEMAS, requestReplies: { // RR :69-80: the template's assets that still exist. ASSETS_FOR_TEMPLATE: { source: 'FILE_STORAGE', requestFields: ['FILE_STORAGE_ID'], resolver: ({ request, store }) => linksOf({ store }, request.FILE_STORAGE_ID).flatMap((link) => { const file = store.getRow('FILE_STORAGE', link.ASSET_ID); return file ? [file as Row] : []; }), }, }, httpRoutes: httpRoutes(contents, deleteFile), }; } // file-server-web-handler.kts (file-server-app): the router's file-server/ // endpoints, each for a logged-in user only. function httpRoutes( contents: Map, deleteFile: (ctx: Pick, id: string) => void, ): HttpRouteDef[] { const endpoint = (name: string) => new RegExp(`/file-server/${name}/?$`); const queryOf = (url: string | undefined) => new URL(url ?? '', 'http://mock').searchParams; const loggedOut = (res: ServerResponse) => sendNack( res, UNAUTHORIZED, 'This request needs the user to be logged in, but the current authentication status is logged out', ); // A required query parameter missing: no endpoint matches (406), before // any auth. const missingId = (res: ServerResponse, name: string) => sendNack( res, NOT_ACCEPTABLE, `Resource file-server/${name} does not accept your request, acceptable requests are:\n- Required query parameter "fileStorageId"`, ); // A body over MAX_UPLOAD_BYTES: the router's refusal, before the endpoint // runs; the connection closes rather than read the rest. const tooLarge = (res: ServerResponse) => { res.setHeader('Connection', 'close'); sendNack(res, BAD_REQUEST, new BodyTooLarge(MAX_UPLOAD_BYTES).message); }; // The multipart body's files and fields; undefined once a body that grew // past MAX_UPLOAD_BYTES has been refused. const formOf = async (req: Parameters[0], res: ServerResponse) => { const boundary = boundaryOf(req.headers['content-type']); let parts: FormPart[] = []; try { parts = boundary ? parseMultipart(await readBody(req, MAX_UPLOAD_BYTES), boundary) : []; } catch (error) { if (!(error instanceof BodyTooLarge)) throw error; tooLarge(res); return undefined; } const fields = new Map( parts .filter((part) => part.fileName === undefined) .map((part) => [part.name, part.data.toString('utf8')]), ); return { files: parts.filter((part) => part.fileName !== undefined), fields }; }; // Where LocalFileStorageManager keeps a file: SUB_DIR/name, relative to // its root, with (1), (2)... before the extension when that's taken; it // then counts as taken. const locationOf = (taken: Set, subDir: string | undefined, name: string) => { const path = (subDir ? `${subDir}/${name}` : name).replace(/\/+/g, '/').replace(/^\//, ''); const dot = path.lastIndexOf('.'); const [stem, extension] = dot > path.lastIndexOf('/') ? [path.slice(0, dot), path.slice(dot)] : [path, '']; let location = path; for (let attempt = 1; taken.has(location); attempt += 1) location = `${stem}(${attempt})${extension}`; taken.add(location); return location; }; // The bytes of files whose rows are gone some other way than the two // deletes (the store, setTable). const prune = (store: EventHandlerCtx['store']) => { for (const id of contents.keys()) if (!store.getRow('FILE_STORAGE', id)) contents.delete(id); }; return [ { method: 'POST', path: endpoint('upload'), handler: async (req, res, ctx) => { if (declaredTooLarge(req, MAX_UPLOAD_BYTES)) return tooLarge(res); if (!ctx.userName) return loggedOut(res); const form = await formOf(req, res); if (!form) return undefined; const { files, fields } = form; if (!files.length) return sendNack(res, UNPROCESSABLE, 'No files uploaded'); const fileType = fields.get('FILE_TYPE') ?? 'USER_DOCUMENT'; if (!FILE_TYPES.includes(fileType)) { return sendNack( res, UNPROCESSABLE, `No enum constant global.genesis.gen.dao.enums.FileType.${fileType}`, ); } prune(ctx.store); const subDir = fields.get('SUB_DIR')?.trim() ? fields.get('SUB_DIR') : undefined; const taken = new Set( ctx.store.getAllRows('FILE_STORAGE').map((row) => row.LOCATION_DETAILS), ); const now = Date.now(); const ids: Row[] = []; // One file after another, each checked as it is saved // (HttpFileServerExtension saveFile): the files before a refused one // are kept. for (const part of files) { const name = baseName(part.fileName!); const error = contentError(part, name); if (error) return sendNack(res, ...error); const row = ctx.store.insertRow('FILE_STORAGE', { FILE_STORAGE_ID: newFileStorageId(), STORAGE_MANAGER: 'LOCAL_STORAGE', FILE_NAME: name, FILE_SIZE: part.data.length, MODIFIED_AT: now, MODIFIED_BY: ctx.userName, CREATED_BY: ctx.userName, CREATED_AT: now, LOCATION_DETAILS: locationOf(taken, subDir, name), FILE_TYPE: fileType, }); contents.set(row.FILE_STORAGE_ID, part.data); ctx.broadcastTableChange('FILE_STORAGE', 'INSERT', row.FILE_STORAGE_ID); ids.push({ FILE_STORAGE_ID: row.FILE_STORAGE_ID }); } return sendJson(res, OK, ids); }, }, { method: 'POST', path: endpoint('replace'), handler: async (req, res, ctx) => { if (declaredTooLarge(req, MAX_UPLOAD_BYTES)) return tooLarge(res); if (!ctx.userName) return loggedOut(res); const form = await formOf(req, res); if (!form) return undefined; const { files, fields } = form; if (!files.length) return sendNack(res, UNPROCESSABLE, 'No files uploaded'); if (files.length > 1) return sendNack(res, UNPROCESSABLE, 'Only one file can be replaced at a time'); const id = fields.get('FILE_STORAGE_ID'); if (!id) return sendNack( res, UNPROCESSABLE, 'FILE_STORAGE_ID of file to replace content must be provided', ); // modifyFile checks the content before it reads the row. const [file] = files; if (!file.data.length) { return sendText( res, BAD_REQUEST, 'Invalid file content. Please check that the uploaded file has content and is valid.', ); } if (!ctx.store.getRow('FILE_STORAGE', id)) { return sendText( res, NOT_FOUND, 'File not found. Please check the FILE_STORAGE_ID parameter provided in the request body.', ); } // modifyFile: the content, its size and who changed it, under the // same name and location. ctx.store.updateRow('FILE_STORAGE', id, { MODIFIED_AT: Date.now(), MODIFIED_BY: ctx.userName, FILE_SIZE: file.data.length, }); contents.set(id, file.data); ctx.broadcastTableChange('FILE_STORAGE', 'MODIFY', id); return sendJson(res, OK, { FILE_STORAGE_ID: id }); }, }, { method: 'GET', path: endpoint('download'), handler: (req, res, ctx) => { const id = queryOf(req.url).get('fileStorageId'); if (id === null) return missingId(res, 'download'); if (!ctx.userName) return loggedOut(res); const file = ctx.store.getRow('FILE_STORAGE', id); const data = contents.get(id); if (!file || !data) { return sendText( res, NOT_FOUND, 'File not found. Please check correct FILE_STORAGE_ID parameter provided in the request body.', ); } res.writeHead(OK, { 'Content-Type': CONTENT_TYPES[extensionOf(file.FILE_NAME)] ?? 'application/octet-stream', // Unencoded, as FileResponseComposer.kt:90 writes it; Netty puts // each UTF-16 unit above U+00FF on the wire as '?' // (AsciiString.c2b), and Node writes the rest as Latin-1. 'Content-disposition': `attachment; filename=${String(file.FILE_NAME).replace(/[^\u0000-\u00ff]/g, '?')}`, 'Access-Control-Expose-Headers': 'Content-disposition, Content-Type, Accept, X-Requested-With', 'Content-Length': data.length, }); res.end(data); return undefined; }, }, { method: 'GET', path: endpoint('all-files'), handler: (_req, res, ctx) => { if (!ctx.userName) return loggedOut(res); return sendJson( res, OK, ctx.store.getAllRows('FILE_STORAGE').map(({ RECORD_ID, TIMESTAMP, ...row }) => row), ); }, }, { method: 'DELETE', path: endpoint('delete'), handler: (req, res, ctx) => { const id = queryOf(req.url).get('fileStorageId'); if (id === null) return missingId(res, 'delete'); if (!ctx.userName) return loggedOut(res); if (!id.trim()) return sendNack( res, UNPROCESSABLE, "Mandatory query parameter is missing: 'fileStorageId'", ); // deleteFile: nothing to do for an unknown id. deleteFile(ctx, id); return sendJson(res, OK, { FILE_STORAGE_ID: id }); }, }, ]; }