process.env.NODE_ENV = "production"; import os from "os"; import path from "path"; import fsp from "fs/promises"; import { SocketFunction } from "socket-function/SocketFunction"; import { getExternalIP } from "socket-function/src/networking"; import { formatDateTimeDetailed } from "socket-function/src/formatting/format"; import { RequireController } from "socket-function/require/RequireController"; import { hostServer } from "../../misc/https/hostServer"; import { RemoteStorageController } from "./storageController"; import { setStorageServerConfig, setWritesRejectedReason, addExtraListenPort, removeExtraListenPort } from "./serverConfig"; import { detectDeployTakeover, setAltPort, getAltPortListenEnd, getMainPortAcquireDelay } from "./deployTakeover"; import { parseStorageUrl } from "./ArchivesRemote"; // Import browser code, so it is allowed to be required by the client import "./accessPage"; const DEFAULT_LOW_SPACE_THRESHOLD_BYTES = 25 * 1024 ** 3; const DISK_SPACE_CHECK_INTERVAL_MS = 15 * 60 * 1000; // Below this fraction of the warn threshold, we start rejecting writes so the server itself doesn't tip the machine into instability. Reads/deletes still work so users can free space. const HARD_REJECT_FRACTION = 0.1; // This machine's LAN (internal) IPv4 - the address other machines on the same network reach us at. Used with --internal, where the ip-domain is based on the internal address instead of our public one. function getInternalIP(): string { let candidates: string[] = []; for (let addrs of Object.values(os.networkInterfaces())) { for (let addr of addrs || []) { if (addr.family !== "IPv4" || addr.internal) continue; candidates.push(addr.address); } } if (!candidates.length) { throw new Error(`--internal was set, but no non-loopback IPv4 network interface was found to use as the internal address`); } // Prefer a private-range address (the actual LAN ip) over any other non-loopback interface return candidates.find(ip => /^(10\.|192\.168\.|172\.(1[6-9]|2\d|3[01])\.)/.test(ip)) || candidates[0]; } // The remote storage server, as a library function: consumers call hostStorageServer() from their own process to start hosting (or use the storageserver bin, see storageServerCli.ts). Who may talk to it comes from the signed authorized_keys repo, read through security/machines - there is nothing to grant here. export type HostStorageServerConfig = { // Full URL of this storage server, e.g. "https://1-2-3-4.example.com:4444". The subdomain must be an ip domain - this machine's external IP with dashes, or 127-0-0-1 for local testing (see the validation in hostStorageServer). The domain and port are extracted from it (bucket routing URLs clients use look like // https://1-2-3-4.example.com:4444/file///storage/storagerouting.json). url: string; folder: string; // When free space on the folder's drive drops below this many bytes, the server console.errors every 15 minutes. Below 10% of it, the server also rejects write operations (creating files, large uploads, new buckets) — reads, findInfo, and deletes still work so the user can free space. Default 25 GiB. lowSpaceThresholdBytes?: number; // LAN-only mode: do NOT forward ports (no UPnP/NAT mapping), and the ip-domain must be based on this machine's INTERNAL ipv4 (its LAN address) instead of its external ip. For a server other machines only reach over the local network. internal?: boolean; // Serve a self-signed TLS cert signed by this machine's CA, instead of obtaining a real (ACME/Cloudflare) certificate. For a domain we don't control DNS for. Clients trust it by trusting our CA, or by visiting the server once and accepting the certificate in the browser. selfSigned?: boolean; }; function formatBytes(bytes: number): string { return `${(bytes / 1024 ** 3).toFixed(2)} GiB`; } async function checkDiskSpace(config: { folder: string; threshold: number }): Promise { let { folder, threshold } = config; let stats = await fsp.statfs(folder); let free = Number(stats.bavail) * Number(stats.bsize); let hardLimit = threshold * HARD_REJECT_FRACTION; if (free >= threshold) { setWritesRejectedReason(undefined); return; } let under = threshold - free; let rejecting = free < hardLimit; console.error( `Storage folder ${folder} is low on disk: ${formatBytes(free)} free` + ` (warn threshold ${formatBytes(threshold)}, ${formatBytes(under)} under;` + ` hard-reject threshold ${formatBytes(hardLimit)}${rejecting ? ", REACHED — write ops now rejected" : ""}).` ); if (rejecting) { setWritesRejectedReason( `Storage server is out of disk space: only ${formatBytes(free)} free on ${folder}` + ` (hard-reject threshold ${formatBytes(hardLimit)}, warn threshold ${formatBytes(threshold)}).` + ` Write operations (create/append/new bucket) are rejected; reads, findInfo, and deletes still work — please free space.` ); } else { setWritesRejectedReason(undefined); } } export async function hostStorageServer(config: HostStorageServerConfig): Promise { let { url, folder } = config; let { address: domain, port } = parseStorageUrl(url); let rootDomain = domain.split(".").slice(-2).join("."); // With --internal the ip-domain is based on our LAN address; otherwise on our external ip let ip = config.internal ? getInternalIP() : (await getExternalIP()).trim(); // The subdomain must be an ip domain: the domain's A record points at exactly one machine, so a dynamic name would let the same script run on two servers and silently fight over it. Encoding the IP makes that mistake fail loudly - the wrong machine's domain just doesn't match. let allowedDomains = [`127-0-0-1.${rootDomain}`, `${ip.replaceAll(".", "-")}.${rootDomain}`]; if (!allowedDomains.includes(domain)) { throw new Error(`The storage server domain is based on the machine's ${config.internal ? "INTERNAL (LAN)" : "external"} IP (the subdomain is the IP with dots replaced by dashes). Expected ${allowedDomains.join(" or ")}, was ${domain}. This machine's ${config.internal ? "internal" : "external"} IP is ${ip}.`); } await fsp.mkdir(folder, { recursive: true }); let lowSpaceThreshold = config.lowSpaceThresholdBytes ?? DEFAULT_LOW_SPACE_THRESHOLD_BYTES; setStorageServerConfig({ domain, port, rootDomain, folder: path.resolve(folder), }); RequireController.allowAllNodeModules(); SocketFunction.expose(RequireController); SocketFunction.expose(RemoteStorageController); // Every HTTP path goes through httpEntry: /file///... serves public bucket files, everything else serves the access page (the path is the account name, see accessPage.tsx). A full URL, so the page resolves modules from the origin root even when served at /accountName (a relative require would resolve inside the account path). // The module path is resolved server-side against its working directory, which is the HOST application's folder - so it is computed from where this file actually is, instead of assuming the storage server runs from inside sliftutils (in a host app it lives under node_modules/sliftutils). let accessPagePath = "./" + path.relative(process.cwd(), path.join(__dirname, "accessPage.tsx")).replaceAll("\\", "/"); SocketFunction.setDefaultHTTPCall(RemoteStorageController, "httpEntry", { requireCalls: [`https://${domain}:${port}/${accessPagePath}`], }); // Initial check so a server starting under-limit immediately rejects writes; then keep checking every 15 minutes so recovery (freed disk space) is picked up automatically. await checkDiskSpace({ folder, threshold: lowSpaceThreshold }); let interval = setInterval(() => { void checkDiskSpace({ folder, threshold: lowSpaceThreshold }) .catch(e => console.error(`Disk space check failed for ${folder}:`, e)); }, DISK_SPACE_CHECK_INTERVAL_MS); (interval as { unref?: () => void }).unref?.(); await hostServer({ domain, port, setDNSRecord: true, publicIp: ip, internal: config.internal, selfSigned: config.selfSigned, portFallback: { getAcquireDelay: getMainPortAcquireDelay, onPortInUse: async () => { await detectDeployTakeover(); }, onListening: (listeningPort, isMainPort) => { if (isMainPort) return; addExtraListenPort(listeningPort); setAltPort(listeningPort); let listenEnd = getAltPortListenEnd(); let timer = setTimeout(() => { console.log(`Alternate port ${listeningPort} reached its end of life at ${formatDateTimeDetailed(listenEnd)}; it no longer counts as one of our addresses (the socket itself stays bound, as the main-port relay forwards into it)`); removeExtraListenPort(listeningPort); }, Math.max(0, listenEnd - Date.now())); (timer as { unref?: () => void }).unref?.(); }, }, }); }