import _ from 'lodash'; import path from 'path'; import process from 'process'; import fse from 'fs-extra'; import os from 'os'; import { v4 as uuid } from 'uuid'; import { Editor, ContentStoreAdapter, Worker, getRemoteContentSources, installRemoteContentSources, checkAndLogContentSourceVersions } from '@stackbit/dev-common'; import type { Extension } from '@stackbit/dev-common'; import { findStackbitConfigFile, loadConfig, LoadConfigResult, LoadConfigResultWithReloadDestroy, ConfigLoadError, StackbitConfigNotFoundError, SSGRunOptions, DEFAULT_SSG_OPTIONS } from '@stackbit/sdk'; import type { FSWatcher } from 'chokidar'; import { watchDir } from '../services/file-watcher'; import logger from '../services/logger'; import consts from '../consts'; import { logAnnotationErrors } from '../services/annotation-errors'; import { APIMethodInterface, APIMethodCreateDocument, APIMethodCreatePreset, APIMethodDeleteDocument, APIMethodDeletePreset, APIMethodDuplicateDocument, APIMethodGetAssets, APIMethodUpdateDocument, APIMethodUploadAssets, APIMethodValidateDocuments, APIMethodPublishDocuments, APIMethodCreateAndLinkDocument, APIMethodUploadAndLinkAsset, APIMethodUpdateAsset, APIMethodSearchDocuments, APIMethodHasAccess, APIMethodOnWebhook, APIMethodCreateAndCrossLinkDocument, APIMethodCreateScheduledAction, APIMethodGetScheduledActions, APIMethodCancelScheduledAction, APIMethodUpdateScheduledAction, APIMethodGetCustomActions, APIMethodGetRunningCustomActions, APIMethodRunCustomAction, APIMethodGetDocumentVersions, APIMethodGetDocumentForVersion, APIMethodGetCollections, APIMethodGetSiteMap, APIMethodGetTreeViews, APIMethodGetSchema, APIMethodGetObjects, APIMethodGetObjectsWithAnnotations, APIMethodGetDocuments, APIMethodGetCSIDocuments, APIMethodUnpublishDocuments, APIMethodArchiveDocument, APIMethodGetStagedChanges, APIMethodUnarchiveDocument } from '../types/api-methods'; import * as StackbitTypes from '@stackbit/types'; import { getCommandRunner } from '@stackbit/cms-core'; export interface RunnerOptions { rootDir: string; noProxy?: boolean; runnableDir?: string; cmsType?: string; csiEnabled?: boolean; csiWebhookUrl?: string; contentfulAccessToken?: string; contentfulSpaceIds?: [string]; contentfulPreviewTokens?: [string]; sanityToken?: string; sanityStudioPath?: string; sanityDataset?: string; sanityProjectId?: string; sanityStudioUrl?: string; logger?: StackbitTypes.Logger; isLocalDev: boolean; isDevServer: boolean; teamId?: string; siteId?: string; netlifyAccessToken?: string; remoteContentSourceBaseURL?: string; apiSecret?: string; projectId?: string; repoUrl?: string; repoBranch?: string; repoPublishBranch?: string; deployKey?: string; } export default class Runner implements APIMethodInterface { private readonly contentStoreAdapter: ContentStoreAdapter; private readonly workers: { gitApp: Worker }; private readonly rootDir: string; private readonly appDir: string; private readonly logger: StackbitTypes.Logger; private readonly isLocalDev: boolean; private readonly isDevServer: boolean; private readonly noProxy: boolean; private readonly teamId?: string; private readonly siteId?: string; private readonly netlifyAccessToken?: string; private readonly apiSecret?: string; private readonly projectId?: string; private readonly deployKey?: string; private readonly repoUrl?: string; private readonly remoteContentSourceBaseURL?: string; private stackbitConfigResult!: LoadConfigResultWithReloadDestroy; private editor!: Editor; private watcher: FSWatcher | undefined; private ssg!: SSGRunOptions; constructor(options: RunnerOptions) { this.rootDir = options.rootDir; this.repoUrl = options.repoUrl; this.appDir = options.runnableDir ? path.join(this.rootDir, options.runnableDir) : this.rootDir; this.logger = options.logger ?? logger; this.noProxy = !!options.noProxy; this.teamId = options.teamId; this.siteId = options.siteId; this.netlifyAccessToken = options.netlifyAccessToken; this.apiSecret = options.apiSecret; this.projectId = options.projectId; this.remoteContentSourceBaseURL = options.remoteContentSourceBaseURL; this.isLocalDev = options.isLocalDev; this.isDevServer = options.isDevServer; this.deployKey = options.deployKey; this.workers = { gitApp: new Worker() }; // todo add support for: // commitAndPushChanges, devAppRestartNeeded, userCommandSpawner (does that needed?) this.contentStoreAdapter = new ContentStoreAdapter({ cmsType: options.cmsType!, csiEnabled: options.csiEnabled, webhookUrl: options.csiWebhookUrl, repoUrl: options.repoUrl, repoBranch: options.repoBranch, repoPublishBranch: options.repoPublishBranch, workers: this.workers, rootDir: this.rootDir, projectDir: this.rootDir, appDir: this.appDir, logger: this.logger, userLogger: this.logger, isLocalDev: this.isLocalDev, isDevServer: this.isDevServer, contentfulAccessToken: options.contentfulAccessToken, runEnv: this.isLocalDev ? undefined : this.getRunEnv(), contentfulSpaces: _.zip(options.contentfulSpaceIds, options.contentfulPreviewTokens) .filter((pair): pair is [string, string] => !!pair[0] && !!pair[1]) .map(([spaceId, previewToken]) => ({ spaceId, previewToken })), sanityProject: { projectId: options.sanityProjectId!, token: options.sanityToken!, studioPath: options.sanityStudioPath!, dataset: options.sanityDataset!, projectUrl: options.sanityStudioUrl! }, assetIdPrefix: consts.ASSET_ID_PREFIX, staticAssetsFilePath: consts.STATIC_ASSETS_FILE_PATH, staticAssetsPublicPath: consts.STATIC_ASSETS_PUBLIC_PATH, staticThemeAssetsFilePath: consts.STATIC_THEME_ASSETS_FILE_PATH, staticThemeAssetsPublicPath: consts.STATIC_THEME_ASSETS_PUBLIC_PATH, defaultContentTypeExtensions: consts.DEFAULT_CONTENT_TYPE_EXTENSIONS }); } async install() { this.logger.debug('Initializing runner...'); await this.ensureRepoConfigured(); const { config, errors } = await this.loadStackbitConfigWithRemoteContentSources(); for (const error of errors) { if (error instanceof StackbitConfigNotFoundError) { // If stackbit config was not found, log the error message. this.logger.warn(error.message); } else if (error instanceof ConfigLoadError) { // if stackbit config was found, but it had a fatal error, exit the process // the error will be logged by the catch block of the dev command throw error; } else { // stackbit config was loaded with some validation errors, log them and continue running this.logger.warn(`error in stackbit config: ` + error.message); } } this.editor = new Editor({ contentStoreAdapter: this.contentStoreAdapter, stackbitConfig: config, logger: this.logger, userLogger: this.logger, stackbitUrlPathField: consts.STACKBIT_URL_PATH_FIELD, staticAssetsPublicPath: consts.STATIC_ASSETS_PUBLIC_PATH }); await this.contentStoreAdapter.init({ stackbitConfig: config, runEnv: this.getRunEnv() }); this.watcher = watchDir(this.rootDir, (filePaths) => { this.logger.debug(`File change detected: ${filePaths}`); this.handleFileChanges(filePaths); }); this.ssg = _.mergeWith({}, DEFAULT_SSG_OPTIONS, config?.internalStackbitRunnerOptions, (value, srcValue) => { if (Array.isArray(srcValue)) { return srcValue; } }); const connectURL = new URL('http://localhost:8000/__graphql'); const { port, host } = config?.contentEngine ?? {}; if (port) { connectURL.port = port.toString(); } // We only use this in local dev, because it makes no sense (and is a security risk) to allow host changes in production if (host) { connectURL.hostname = host; } this.ssg.directRoutes!['/.netlify/connect'] = connectURL.toString(); this.ssg.directPaths!.push('/.netlify/**'); if (!this.noProxy) { if (await fse.pathExists(path.join(this.appDir, 'sourcebit.js'))) { this.ssg.directPaths!.push('/socket.io/**'); this.ssg.directRoutes!['/socket.io'] = `http://localhost:${consts.NEXTJS_SOURCEBIT_LIVE_UPDATE_PORT}`; } if (config?.ssgName === 'nextjs') { this.ssg.directPaths!.push('/_next/**', '/nextjs-live-updates/**'); } if (config?.ssgName === 'gatsby') { this.ssg.directChangeSocketOrigin = true; this.ssg.directPaths!.push('/socket.io/**'); } } } async loadStackbitConfig() { const stackbitConfigFilePath = await findStackbitConfigFile([this.appDir, this.rootDir]); if (!stackbitConfigFilePath) { return { config: null, errors: [new StackbitConfigNotFoundError()] }; } else { this.logger.debug(`Found Stackbit configuration file at: ${stackbitConfigFilePath}`); this.stackbitConfigResult = await loadConfig({ dirPath: path.dirname(stackbitConfigFilePath), watchCallback: this.onStackbitConfigUpdate.bind(this), logger: this.logger }); if (this.stackbitConfigResult?.config?.contentSources) { await checkAndLogContentSourceVersions({ contentSources: this.stackbitConfigResult.config.contentSources, logger: this.logger, isLocalDev: true }); } this.logger.debug('loaded stackbit config'); return { config: this.stackbitConfigResult.config, errors: this.stackbitConfigResult.errors }; } } async loadStackbitConfigWithRemoteContentSources() { let remoteContentSources = []; let fetchedContentSources: Extension[] = []; if (this.siteId && this.teamId && this.remoteContentSourceBaseURL) { // remote content sources has to be fetched and installed first to prevent colliding with config loader and watching node_modules changes this.logger.debug('Fetching remote content sources...'); fetchedContentSources = await getRemoteContentSources({ siteId: this.siteId, teamId: this.teamId, netlifyAccessToken: this.netlifyAccessToken, apiSecret: this.apiSecret, projectId: this.projectId, baseURL: this.remoteContentSourceBaseURL }); if (fetchedContentSources.length) { this.logger.debug('Installing remote content sources...'); const installedInfo = await installRemoteContentSources({ extensions: fetchedContentSources, appDir: this.appDir, logger: this.logger }); if (installedInfo.length) { remoteContentSources = await Promise.all( installedInfo.map(async (installedInfo) => { const ContentSource = (await import(`file://${installedInfo.path}?c=${Math.random()}`)).default; return new ContentSource({ options: installedInfo.config }); }) ); } } else { this.logger.debug('No remote content sources found'); } } const { config, errors } = await this.loadStackbitConfig(); if (config && fetchedContentSources.length) { this.logger.debug('Extending config content sources with remote content sources...'); config.contentSources = [...(config.contentSources ?? []), ...remoteContentSources]; } return { config, errors }; } async onStackbitConfigUpdate(result: LoadConfigResult) { this.logger.debug('onStackbitConfigUpdate'); const { config, errors } = result; for (const error of errors) { this.logger.warn(error.message); } // if user broke the config file, don't reload anything until the file is fixed if (!config) { return; } this.editor.onStackbitConfigChange({ stackbitConfig: config }); await this.contentStoreAdapter.onStackbitConfigChange({ stackbitConfig: config }); } async handleFileChanges(filePaths: string[]): Promise { try { this.logger.debug('handleFileChanges', { filePaths }); await this.contentStoreAdapter.handleFileChanges(filePaths); } catch (err) { this.logger.debug('Handle file change error', { err, filePaths }); this.logger.error('Error handling file changes: ' + filePaths.join(', ')); } } getDirectPaths() { return this.ssg.directPaths ?? []; } /** * Function retuning a map or a function that rewrites direct proxy requests. * https://github.com/chimurai/http-proxy-middleware#http-proxy-middleware-options * @return {Function|Object} */ getDirectRoutes(): { [hostOrPath: string]: string } { return this.ssg.directRoutes!; } shouldProxyWebsockets(): boolean { return this.ssg.proxyWebsockets ?? false; } getDirectChangeOrigin() { return this.ssg.directChangeOrigin; } getDirectChangeSocketOrigin() { return this.ssg.directChangeSocketOrigin; } async getObjects({ objectIds, locale, user }: APIMethodGetObjects['data']): Promise { return this.editor.getObjects({ objectIds, locale, user }); } async getDocuments({ documentSpecs, user }: APIMethodGetDocuments['data']): Promise { return this.contentStoreAdapter.getApiDocuments({ documentSpecs, user }); } async getCSIDocuments(data: APIMethodGetCSIDocuments['data'] = {}): Promise { return this.contentStoreAdapter.getCSIDocuments(data); } async getStagedChanges(data: APIMethodGetStagedChanges['data']): Promise { const changes = await this.contentStoreAdapter.getStagedChanges(data); return { changes }; } async getObjectsWithAnnotations({ annotationTree = null, clientAnnotationErrors = [], resolveAllReferences, sourcemaps, locale, user }: APIMethodGetObjectsWithAnnotations['data']): Promise { const result = await this.editor.getObjectsWithAnnotations({ annotationTree, clientAnnotationErrors, resolveAllReferences, locale, user, reportAllErrors: true }); const errors = [...(result.errors ?? []), ...clientAnnotationErrors]; logAnnotationErrors(errors, this.logger, this.rootDir, sourcemaps); return result; } async getCollections({ locale, user }: APIMethodGetCollections['data']): Promise { return this.editor.getCollections({ locale, user }); } async getSiteMap({ locale, user }: APIMethodGetSiteMap['data']): Promise { return this.editor.getSiteMap({ locale, user }); } async getTreeViews({ user }: APIMethodGetTreeViews['data']): Promise { return this.contentStoreAdapter.getTreeViews({ user }); } async getSchema({ locale, user }: APIMethodGetSchema['data']): Promise { return this.editor.getSchema({ locale, user }); } async getUrl(srcDocumentId: string, srcProjectId: string, srcType: string, locale?: string): Promise { return this.editor.getUrl(srcDocumentId, srcProjectId, srcType, locale); } async getObject(objectId: string, projectId: string): Promise { return this.contentStoreAdapter.getObject_deprecated({ objectId, projectId }); } async listAssets(filterParams: any): Promise { return this.contentStoreAdapter.listAssets_deprecated(filterParams); } async uploadAsset(url: string, filename: string, user: any): Promise { return this.contentStoreAdapter.uploadAsset_deprecated({ url, fileName: filename, user }); } async hasAccess(data: APIMethodHasAccess['data']): Promise { return this.contentStoreAdapter.hasAccess(data); } async createDocument(data: APIMethodCreateDocument['data']): Promise { return this.contentStoreAdapter.createDocument(data); } async createAndLinkDocument(data: APIMethodCreateAndLinkDocument['data']): Promise { return this.contentStoreAdapter.createAndLinkDocument(data); } async createAndCrossLinkDocument(data: APIMethodCreateAndCrossLinkDocument['data']): Promise { return this.contentStoreAdapter.createAndLinkDocument(data); } async uploadAndLinkAsset(data: APIMethodUploadAndLinkAsset['data']): Promise { return this.contentStoreAdapter.uploadAndLinkAsset(data); } async duplicateDocument(data: APIMethodDuplicateDocument['data']): Promise { return this.contentStoreAdapter.duplicateDocument(data); } async updateDocument(data: APIMethodUpdateDocument['data']): Promise { return this.contentStoreAdapter.updateDocument(data); } async deleteDocument(data: APIMethodDeleteDocument['data']): Promise { return this.contentStoreAdapter.deleteDocument(data); } async archiveDocument(data: APIMethodArchiveDocument['data']): Promise { return this.contentStoreAdapter.archiveDocument(data); } async unarchiveDocument(data: APIMethodUnarchiveDocument['data']): Promise { return this.contentStoreAdapter.unarchiveDocument(data); } async getScheduledActions(): Promise { return this.contentStoreAdapter.getScheduledActions(); } async createScheduledAction(data: APIMethodCreateScheduledAction['data']): Promise { return this.contentStoreAdapter.createScheduledAction(data); } async cancelScheduledAction(data: APIMethodCancelScheduledAction['data']): Promise { return this.contentStoreAdapter.cancelScheduledAction(data); } async updateScheduledAction(data: APIMethodUpdateScheduledAction['data']): Promise { return this.contentStoreAdapter.updateScheduledAction(data); } async searchDocuments(data: APIMethodSearchDocuments['data']): Promise { return this.editor.searchDocuments(data); } async onWebhook(data: APIMethodOnWebhook['data']): Promise { return this.contentStoreAdapter.onWebhook(data); } async validateDocuments(data: APIMethodValidateDocuments['data']): Promise { return this.contentStoreAdapter.validateDocuments(data); } async publishDocuments(data: APIMethodPublishDocuments['data']): Promise { return this.contentStoreAdapter.publishDocuments(data); } async unpublishDocuments(data: APIMethodUnpublishDocuments['data']): Promise { return this.contentStoreAdapter.unpublishDocuments(data); } async getAssets(data: APIMethodGetAssets['data']): Promise { return this.contentStoreAdapter.getApiAssets(data); } async uploadAssets(data: APIMethodUploadAssets['data']): Promise { return this.contentStoreAdapter.uploadAssets(data); } async updateAsset(data: APIMethodUpdateAsset['data']): Promise { return this.contentStoreAdapter.updateAsset(data); } async createPreset(data: APIMethodCreatePreset['data']): Promise { return this.editor.createPreset(data.fieldDataPath, _.omit(data, ['fieldDataPath', 'user', 'dryRun']) as any, data.dryRun, data.user); } async deletePreset(data: APIMethodDeletePreset['data']): Promise { return this.editor.deletePreset(data.presetId, data.user); } async getCustomActions(data: APIMethodGetCustomActions['data']): Promise { return this.contentStoreAdapter.getCustomActions(data); } async getRunningCustomActions(data: APIMethodGetRunningCustomActions['data']): Promise { return this.contentStoreAdapter.getRunningCustomActions(data); } async runCustomAction(data: APIMethodRunCustomAction['data']): Promise { return this.contentStoreAdapter.runCustomAction(data); } async getDocumentVersions(data: APIMethodGetDocumentVersions['data']): Promise { return this.editor.getDocumentVersions(data); } async getDocumentForVersion(data: APIMethodGetDocumentForVersion['data']): Promise { return this.editor.getDocumentForVersion(data); } async makeAction(action: string, data: any): Promise { switch (action) { case 'createPage': return this.contentStoreAdapter.createObject_deprecated(data); case 'duplicatePage': return this.contentStoreAdapter.duplicateObject_deprecated(data); case 'updatePage': return this.contentStoreAdapter.updateObject_deprecated(data); case 'deleteObject': return this.contentStoreAdapter.deleteObject_deprecated(data); case 'getAssets': return this.contentStoreAdapter.getAssets_deprecated(data); case 'uploadAssets': return this.contentStoreAdapter.uploadAssets(data); case 'createPreset': return this.editor.createPreset(data.fieldDataPath, _.omit(data, ['fieldDataPath', 'user', 'dryRun']) as any, data.user, data.dryRun); case 'deletePreset': return this.editor.deletePreset(data.presetId, data.user); case 'hasAccess': // for old non csi git based projects return { hasConnection: true, hasPermissions: true }; default: throw new Error('Unsupported Operation: ' + data.action); } } keepAlive() { this.contentStoreAdapter.keepAlive(); } isCsiEnabled() { return this.contentStoreAdapter.isContentStoreEnabled(); } getAssetFilePath(url: string) { const assetFilePath = this.contentStoreAdapter.getAssetFilePath(url); if (assetFilePath) { return path.join(this.rootDir, assetFilePath); } return assetFilePath; } pullContent() { return this.contentStoreAdapter.pullContent(); } private getRunEnv() { return { PATH: process.env.PATH, GIT_SSH: process.env.GIT_SSH, HOME: process.env.HOME, STACKBIT_PREVIEW: 'true' }; } async ensureRepoConfigured() { if (!this.isDevServer) { return; // runs only on dev server } this.logger.debug('Ensuring repo configuration'); const commandRunner = getCommandRunner({ env: process.env }); if (this.repoUrl) { this.logger.debug(`Changing repo origin url to ${this.repoUrl}`); await commandRunner('git', ['remote', 'set-url', 'origin', this.repoUrl], { cwd: this.rootDir }); } if (process.env.GIT_SSH) { this.logger.debug('Ssh authorization was already set, skipping'); return; // skip if git_ssh is set meaning devbot has deploy key already } if (!this.deployKey) { this.logger.debug('Could not use deploy key since its not provided'); return; } this.logger.debug('Setting deploy key'); const tmpDir = path.join(os.tmpdir(), uuid()); await fse.mkdir(tmpDir); const keyPath = path.join(tmpDir, 'deploykey'); await fse.writeFile(keyPath, this.deployKey, { encoding: 'utf-8', mode: 0o600 }); const wrapper = `#!/bin/bash unset SSH_AUTH_SOCK ssh -o CheckHostIP=no -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o PasswordAuthentication=no -o IdentityFile=${keyPath} $*`; const wrapperPath = path.join(tmpDir, 'sshwrapper'); await fse.writeFile(wrapperPath, wrapper, { encoding: 'utf-8', mode: 0o755 }); process.env.GIT_SSH = wrapperPath; } async destroy() { this.watcher?.close(); await this.stackbitConfigResult?.destroy?.(); await this.contentStoreAdapter?.destroy(); } }