import _ from 'lodash'; import path from 'path'; import gitUrlParse from 'git-url-parse'; import type { Asset, ContentSourceInterface, Document, DocumentStatus, User, GitAuthor, GitServiceInterface, InitOptions, Model, UpdateOperation, UpdateOperationField, ValidationError, Version, Logger, Schema, Cache, ContentChanges } from '@stackbit/types'; import * as stackbitUtils from '@stackbit/types'; import { FileSystemContentSource, FileSystemContentSourceOptions } from './fs-content-source'; import { DocumentContext, AssetContext, getFieldNameFromVirtualSlug, isVirtualSlug } from './content-converter'; export type GitContentSourceOptions = FileSystemContentSourceOptions & { /** * When working in local development mode, the GitContentSource will not push * changes to the remote git repository to prevent accidental content changes * in the production data. * * Setting the `localDevSync` object, will push content changes made in the * local visual editor to the remote git repository. * * The `localDevSync` object needs to include three properties: `repoUrl`, * `repoBranch`, and `repoPublishBranch`. */ localDevSync?: LocalDevSyncOptions; }; export type LocalDevSyncOptions = { /** * Set `repoUrl` to the remote git URL you wish to the updated content * to be pushed to. */ repoUrl: string; /** * Set `repoBranch` to the remote git branch you wish the updated * content to be pushed to. */ repoWorkingBranch: string; /** * Set `repoPublishBranch` to the remote git branch you wish to publish * the content to. When clicking the "publish" button in the visual * editor, the changes from `repoWorkingBranch` will be merged into * `repoPublishBranch`. */ repoPublishBranch: string; }; type GitChange = { updatedBy: string[]; updatedAt: string; status: DocumentStatus; fromFilePath?: string; }; const DEFAULT_AUTHOR_EMAIL = 'projects@stackbit.com'; export class GitContentSource implements ContentSourceInterface { private readonly fileSystemContentSource: FileSystemContentSource; private readonly rootPath: string; private projectUrl!: string; private environment!: string; private git!: GitServiceInterface; private cache!: Cache; private localDev!: boolean; private logger!: Logger; private localDevSync?: LocalDevSyncOptions; private syncRemote!: boolean; // internal flag which shows that current class (and the one which inherits it) is aimed to work with files within the repo protected isGitCms = true; constructor(options: GitContentSourceOptions) { this.fileSystemContentSource = new FileSystemContentSource(options); this.rootPath = options.rootPath; this.projectUrl = ''; this.localDevSync = options.localDevSync; } async getVersion(): Promise { return stackbitUtils.getVersion({ packageJsonPath: path.join(__dirname, '../package.json') }); } async init(options: InitOptions): Promise { this.git = options.git; this.localDev = options.localDev; this.logger = options.logger.createLogger({ label: 'cms-git' }); this.cache = options.cache; this.syncRemote = !this.localDev; if (this.localDev && this.localDevSync && this.localDevSync.repoUrl && this.localDevSync.repoWorkingBranch && this.localDevSync.repoPublishBranch) { this.git.setRepoUrl(this.localDevSync.repoUrl); this.git.setRepoBranch(this.localDevSync.repoWorkingBranch); this.git.setRepoPublishBranch(this.localDevSync.repoPublishBranch); this.syncRemote = true; } let repoUrl = this.git.getRepoUrl(); const repoBranch = this.git.getRepoBranch(); if (repoUrl) { // remove user/pass from url before passing to gitUrlParse repoUrl = repoUrl.replace(/\/\/.*?@/, '//'); try { const gitUrlResult = gitUrlParse(repoUrl); let gitUrlResource = gitUrlResult.resource; let gitUrlFullName = gitUrlResult.full_name; let path = ''; if (gitUrlResource === 'github.com') { path = `/blob/${repoBranch}/`; } else if (gitUrlResource === 'gitlab.com') { path = `/-/blob/${repoBranch}/`; } else if (gitUrlResource === 'bitbucket.org') { path = `/src/${repoBranch}/`; } else if (gitUrlResource === 'dev.azure.com') { gitUrlResource = 'dev.azure.com'; path = '?path='; } else if (gitUrlResource === 'ssh.dev.azure.com') { gitUrlResource = 'dev.azure.com'; gitUrlFullName = `${gitUrlResult.organization}/${gitUrlResult.owner}/_git/${gitUrlResult.name}`; path = '?path='; } this.projectUrl = 'https://' + gitUrlResource + '/' + gitUrlFullName + path; } catch (err) { this.logger.warn('Failed to parse repo url', { repoUrl, err }); } } this.environment = 'preview'; if (repoBranch) { this.environment = repoBranch; } this.onGitPull = this.onGitPull.bind(this); await this.fileSystemContentSource.init({ ...options, cache: { ...options.cache, updateContent: async (contentChanges: ContentChanges): Promise => { const changesMap = await this.getChangesMap(); return options.cache.updateContent({ ...contentChanges, documents: (contentChanges.documents ?? []).map((document) => ({ ...document, ...changesMap?.[document.context.filePath] })), assets: (contentChanges.assets ?? []).map((asset) => { const filePath = this.getAssetFilePath(asset); return { ...asset, ...(filePath ? changesMap?.[filePath] : {}) }; }) }); } } }); } async destroy(): Promise { await this.fileSystemContentSource.destroy(); } async reset(): Promise { await this.fileSystemContentSource.reset(); } private getCommitAuthor(userContext?: User): GitAuthor { return { email: userContext?.email ?? DEFAULT_AUTHOR_EMAIL, name: userContext?.name ?? 'Stackbit' }; } private getCommitMessage(updateOperations: UpdateOperation[]): string { const diff = (updateOperation: UpdateOperation) => { let result = 'updated'; const path = _.filter(updateOperation.fieldPath, (part) => _.isString(part)); // skip array indexes const [field = null, parent = null] = _.take(_.reverse(path), 2); if (field) { if (_.isString(field) && isVirtualSlug(field)) { result += ' ' + getFieldNameFromVirtualSlug(field); } else { result += ' ' + field; } } if (parent) { result += ' in ' + parent; } return result; }; if (updateOperations.length > 1) { return 'Multiple updates:\n' + _.map(updateOperations, (updateOperation) => '* ' + diff(updateOperation)).join('\n'); } else if (updateOperations.length) { return diff(updateOperations[0]!); } return 'updated'; } getContentSourceType(): string { return 'git'; } getProjectId(): string { // user underlying content source's project id because git-service my not be available yet return this.fileSystemContentSource.getProjectId(); } getProjectEnvironment(): string { return this.environment; } getProjectManageUrl(): string { return this.projectUrl; } onWebhook(data: { data: unknown; headers: Record }): void {} async onFilesChange?({ updatedFiles }: { updatedFiles: string[]; }): Promise<{ invalidateSchema?: boolean; contentChanges?: ContentChanges }> { this.logger.debug('onFilesChange', { updatedFiles }); const result = await this.fileSystemContentSource.onFilesChange!({ updatedFiles }); if (!this.syncRemote || !result.contentChanges) { return result; } const changesMap = await this.getChangesMap(); return { ...result, contentChanges: { ...result.contentChanges, documents: (result.contentChanges.documents ?? []).map((document) => ({ ...document, ...changesMap?.[document.context.filePath] })), assets: (result.contentChanges.assets ?? []).map((asset) => { const filePath = this.getAssetFilePath(asset); return { ...asset, ...(filePath ? changesMap?.[filePath] : {}) }; }) } }; } private async onGitPull({ branch, updatedFiles }: { branch: string; updatedFiles: string[] }) { this.logger.debug('onGitPull', { branch, updatedFiles }); if (branch !== this.git.getRepoPublishBranch()) { return; } const result = await this.fileSystemContentSource.onFilesChange!({ updatedFiles }); if (!result.contentChanges) { return; } const changesMap = await this.getChangesMap(); await this.cache.updateContent({ ...result.contentChanges, documents: (result.contentChanges.documents ?? []).map((document) => ({ ...document, ...changesMap?.[document.context.filePath] })), assets: (result.contentChanges.assets ?? []).map((asset) => { const filePath = this.getAssetFilePath(asset); return { ...asset, ...(filePath ? changesMap?.[filePath] : {}) }; }) }); } startWatchingContentUpdates(): void { this.fileSystemContentSource.startWatchingContentUpdates(); this.git.addPullListener(this.onGitPull); } stopWatchingContentUpdates(): void { this.fileSystemContentSource.stopWatchingContentUpdates(); this.git.removePullListener(this.onGitPull); } getSchema(): Promise { return this.fileSystemContentSource.getSchema(); } async getDocuments(): Promise[]> { const changesMap = await this.getChangesMap(); const documents = await this.fileSystemContentSource.getDocuments(); return documents.map((document) => { return { ...document, manageUrl: `${this.projectUrl}${document.context.filePath}`, ...changesMap?.[document.context.filePath] }; }); } async getAssets(): Promise[]> { const changesMap = await this.getChangesMap(); const assets = await this.fileSystemContentSource.getAssets(); return assets.map((asset) => { const filePath = this.getAssetFilePath(asset); return { ...asset, manageUrl: `${this.projectUrl}${filePath}`, ...(filePath ? changesMap?.[filePath] : {}) }; }); } getAssetFilePath(asset: Asset): string | undefined { const assetField = asset.fields.file; const relFilePath = assetField?.localized === true ? assetField.locales[Object.keys(assetField.locales)[0] ?? '']?.url : assetField?.url; if (!relFilePath) { return relFilePath; } return path.relative(this.git.getRepoDir(), path.join(this.rootPath, relFilePath)); } getDocumentFilePath(document: Document): string { return path.relative(this.git.getRepoDir(), path.join(this.rootPath, document.context.filePath)); } async hasAccess(options: { userContext?: User }): Promise<{ hasConnection: boolean; hasPermissions: boolean }> { // we're assuming that we always have access to the repository return { hasConnection: true, hasPermissions: true }; } async createDocument(options: { updateOperationFields: Record; model: Model; locale?: string | undefined; defaultLocaleDocumentId?: string | undefined; userContext?: User; }): Promise<{ documentId: string }> { const document = await this.fileSystemContentSource.createDocument(options); if (this.syncRemote) { await this.git.commitAndPush(this.getCommitAuthor(options.userContext), [ { filePath: this.getDocumentFilePath(document), description: 'added' } ]); const updatedDocument = { ...document, status: 'added' as const, ...this.getGitChange(document, 'added', options.userContext) }; this.cache.updateContent({ documents: [updatedDocument], assets: [], deletedDocumentIds: [], deletedAssetIds: [] }); } return { documentId: document.id }; } async updateDocument(options: { document: Document; operations: UpdateOperation[]; userContext?: User }): Promise { const { document } = options; let updatedDocument = (await this.fileSystemContentSource.updateDocument(options)) as Document; if (this.syncRemote) { const files = [ { filePath: this.getDocumentFilePath(updatedDocument), description: this.getCommitMessage(options.operations) } ]; const deletedDocumentIds: string[] = []; if (document.context.filePath !== updatedDocument.context.filePath) { files.push({ filePath: this.getDocumentFilePath(document), description: `moved to ${updatedDocument.context.filePath}` }); // Delete the previous document only if its id is different. // When using file-ids, the id remains the same when copying the file. if (document.id !== updatedDocument.id) { deletedDocumentIds.push(document.id); } } await this.git.commitAndPush(this.getCommitAuthor(options.userContext), files); const existingStatus = this.cache.getDocumentById(options.document.id)?.status; updatedDocument = { ...updatedDocument, ...this.getGitChange(updatedDocument, existingStatus === 'added' ? 'added' : 'modified', options.userContext) }; this.cache.updateContent({ documents: [updatedDocument], assets: [], deletedDocumentIds, deletedAssetIds: [] }); } } async deleteDocument(options: { document: Document; userContext?: User }): Promise { const { document } = options; const result = await this.fileSystemContentSource.deleteDocument(options); if (this.syncRemote) { await this.git.commitAndPush(this.getCommitAuthor(options.userContext), [ { filePath: this.getDocumentFilePath(document), description: 'removed' } ]); this.cache.updateContent({ documents: [], assets: [], deletedDocumentIds: [document.id], deletedAssetIds: [] }); } return result; } async uploadAsset(options: { url?: string | undefined; base64?: string | undefined; fileName: string; mimeType: string; locale?: string | undefined; userContext?: User; }): Promise> { let asset = await this.fileSystemContentSource.uploadAsset(options); if (this.syncRemote) { const filePath = this.getAssetFilePath(asset); if (filePath) { await this.git.commitAndPush(this.getCommitAuthor(options.userContext), [ { filePath, description: 'uploaded asset' } ]); asset = { ...asset, status: 'added', ...this.getGitChange(asset, 'added', options.userContext) }; this.cache.updateContent({ documents: [], assets: [asset], deletedDocumentIds: [], deletedAssetIds: [] }); } else { this.logger.error('Error comitting uploaded asset'); } } return asset; } async validateDocuments(options: { documents: Document[]; assets: Asset[]; locale?: string | undefined; userContext?: User; }): Promise<{ errors: ValidationError[] }> { return this.fileSystemContentSource.validateDocuments(options); } async publishDocuments(options: { documents: Document[]; assets: Asset[]; userContext?: User }): Promise { const { documents, assets } = options; await this.fileSystemContentSource.publishDocuments(options); const changesMap = await this.getChangesMap(); if (this.syncRemote) { const filePathsToPublish = [ ...documents.map((document) => this.getDocumentFilePath(document)), ...assets.map((asset) => this.getAssetFilePath(asset) ?? '').filter(Boolean) ]; let hasChanges = true; let filePaths: string[] | undefined; if (changesMap) { const changedFiles = Object.keys(_.pickBy(changesMap, (change) => change.status !== 'published')).reverse(); hasChanges = !_.isEmpty(changedFiles); // if a file was moved, we need to publish both the old and new file for (const changedFile of changedFiles) { // we're adding renamed files as we go through the changed files list in reverse order (most recent first) if (filePathsToPublish.includes(changedFile) && changesMap[changedFile]!.fromFilePath) { filePathsToPublish.push(changesMap[changedFile]!.fromFilePath!); } } const isFullSitePublish = _.isEmpty(_.xor(filePathsToPublish, changedFiles)); // prefer full-site publish mechanism if we're publish all files anyways if (!isFullSitePublish) { filePaths = filePathsToPublish; } } if (hasChanges) { await this.git.publish(this.getCommitAuthor(options.userContext), filePaths); } await this.cache.updateContent({ documents: documents.map((document) => ({ ...document, ...this.getGitChange(document, 'published', options.userContext) })), assets: assets.map((asset) => ({ ...asset, ...this.getGitChange(asset, 'published', options.userContext) })), deletedDocumentIds: [], deletedAssetIds: [] }); } } private getGitChange(object: Document | Asset, status: DocumentStatus, userContext?: User): GitChange | null { return { status, updatedAt: new Date().toISOString(), updatedBy: userContext?.email ? (object.updatedBy ?? []).concat(userContext.email) : object.updatedBy ?? [] }; } private async getChangesMap() { const changesMap: Record = {}; if (!this.syncRemote) { return changesMap; } const entries = await this.git.commitLog(); const diff = await this.git.diff(); for (const commitEntry of entries) { for (const { status, filePath, fromFilePath } of commitEntry.changes) { if (changesMap[filePath]) { if (commitEntry.author !== DEFAULT_AUTHOR_EMAIL && !changesMap[filePath]?.updatedBy.includes(commitEntry.author)) { // TODO RND-2251 - taking only last author as a workaround changesMap[filePath]!.updatedBy = [commitEntry.author]; } changesMap[filePath]!.updatedAt = commitEntry.timestamp.toISOString(); if (diff.includes(filePath)) { const prevStatus = changesMap[filePath]!.status; if (prevStatus === 'added' && status === 'deleted') { delete changesMap[filePath]; } else if (prevStatus !== 'added') { changesMap[filePath]!.status = status; } } } else if (fromFilePath && changesMap[fromFilePath] && !diff.includes(fromFilePath)) { const prevStatus = changesMap[fromFilePath]!.status; changesMap[filePath] = { updatedBy: commitEntry.author === DEFAULT_AUTHOR_EMAIL ? [] : [commitEntry.author], updatedAt: commitEntry.timestamp.toISOString(), status: diff.includes(filePath) ? (prevStatus === 'added' ? 'added' : status) : 'published', fromFilePath: changesMap[fromFilePath]!.fromFilePath ?? fromFilePath }; delete changesMap[fromFilePath]; } else { changesMap[filePath] = { updatedBy: commitEntry.author === DEFAULT_AUTHOR_EMAIL ? [] : [commitEntry.author], updatedAt: commitEntry.timestamp.toISOString(), status: diff.includes(filePath) ? status : 'published', fromFilePath }; } } } return changesMap; } }