/*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ import { CharCode } from '../../../../vs/base/common/charCode'; import * as strings from '../../../../vs/base/common/strings'; import { IViewLineTokens, LineTokens, } from '../../../../vs/editor/common/core/lineTokens'; import { TokenizationResult2 } from '../../../../vs/editor/common/core/token'; import { IState, LanguageId } from '../../../../vs/editor/common/modes'; import { NULL_STATE, nullTokenize2, } from '../../../../vs/editor/common/modes/nullMode'; export interface IReducedTokenizationSupport { getInitialState(): IState; tokenize2( line: string, hasEOL: boolean, state: IState, offsetDelta: number ): TokenizationResult2; } const fallback: IReducedTokenizationSupport = { getInitialState: () => NULL_STATE, tokenize2: ( buffer: string, hasEOL: boolean, state: IState, deltaOffset: number ) => nullTokenize2(LanguageId.Null, buffer, state, deltaOffset), }; export function tokenizeToString( text: string, tokenizationSupport: IReducedTokenizationSupport = fallback ): string { return _tokenizeToString(text, tokenizationSupport || fallback); } export function tokenizeLineToHTML( text: string, viewLineTokens: IViewLineTokens, colorMap: string[], startOffset: number, endOffset: number, tabSize: number, useNbsp: boolean ): string { let result = `
`; let charIndex = startOffset; let tabsCharDelta = 0; for ( let tokenIndex = 0, tokenCount = viewLineTokens.getCount(); tokenIndex < tokenCount; tokenIndex++ ) { const tokenEndIndex = viewLineTokens.getEndOffset(tokenIndex); if (tokenEndIndex <= startOffset) { continue; } let partContent = ''; for (; charIndex < tokenEndIndex && charIndex < endOffset; charIndex++) { const charCode = text.charCodeAt(charIndex); switch (charCode) { case CharCode.Tab: let insertSpacesCount = tabSize - ((charIndex + tabsCharDelta) % tabSize); tabsCharDelta += insertSpacesCount - 1; while (insertSpacesCount > 0) { partContent += useNbsp ? ' ' : ' '; insertSpacesCount--; } break; case CharCode.LessThan: partContent += '<'; break; case CharCode.GreaterThan: partContent += '>'; break; case CharCode.Ampersand: partContent += '&'; break; case CharCode.Null: partContent += '�'; break; case CharCode.UTF8_BOM: case CharCode.LINE_SEPARATOR: case CharCode.PARAGRAPH_SEPARATOR: case CharCode.NEXT_LINE: partContent += '\ufffd'; break; case CharCode.CarriageReturn: // zero width space, because carriage return would introduce a line break partContent += '​'; break; case CharCode.Space: partContent += useNbsp ? ' ' : ' '; break; default: partContent += String.fromCharCode(charCode); } } result += `${partContent}`; if (tokenEndIndex > endOffset || charIndex >= endOffset) { break; } } result += `
`; return result; } function _tokenizeToString( text: string, tokenizationSupport: IReducedTokenizationSupport ): string { let result = `
`; let lines = strings.splitLines(text); let currentState = tokenizationSupport.getInitialState(); for (let i = 0, len = lines.length; i < len; i++) { let line = lines[i]; if (i > 0) { result += `
`; } let tokenizationResult = tokenizationSupport.tokenize2( line, true, currentState, 0 ); LineTokens.convertToEndOffset(tokenizationResult.tokens, line.length); let lineTokens = new LineTokens(tokenizationResult.tokens, line); let viewLineTokens = lineTokens.inflate(); let startOffset = 0; for (let j = 0, lenJ = viewLineTokens.getCount(); j < lenJ; j++) { const type = viewLineTokens.getClassName(j); const endIndex = viewLineTokens.getEndOffset(j); result += `${strings.escape( line.substring(startOffset, endIndex) )}`; startOffset = endIndex; } currentState = tokenizationResult.endState; } result += `
`; return result; }