import { Ast } from 'thingtalk'; import BaseTokenizer from './tokenizer/base'; import { Phrase, Concatenation, Replaceable } from '../utils/template-string'; import { EntityMap, AnyEntity } from '../utils/entity-utils'; export interface UnitPreferenceDelegate { timezone: string; getPreferredUnit(type: string): string | undefined; } export interface NormalizedParameterCanonical { default: string; projection_pronoun?: string[]; base: Array; base_projection: Array; argmin: Array; argmax: Array; filter_phrase: Array; enum_value: Record>; enum_filter: Record>; projection: Array; } /** * Base class for all code that is specific to a certain natural language * in Genie. */ export default class LanguagePack { ARGUMENT_NAME_OVERRIDES: { [key: string]: string[]; }; IGNORABLE_TOKENS: { [key: string]: string[]; }; _NO_SPACE_TOKENS: Set; _NO_SPACE_AFTER_TOKENS: Set; NO_IDEA: string[]; CHANGE_SUBJECT_TEMPLATES: string[]; SINGLE_DEVICE_TEMPLATES: Array<[string, RegExp | null]>; DEFINITE_ARTICLE_REGEXP: RegExp | undefined; MUST_CAPITALIZE_TOKEN: Set; ABBREVIATIONS: any; protected _tokenizer: BaseTokenizer | undefined; /** * The actual locale string to use, which can be a subvariant of * the language implementing this language pack. */ readonly locale: string; private _gt; gettext: (x: string) => string; _: (x: string) => string; constructor(locale: string); private _loadTranslations; /** * Return an instance of the tokenizer used by this language. */ getTokenizer(): BaseTokenizer; private _toTemplatePhrases; /** * Apply load-time transformations to the canonical annotation of a function. This normalizes * the form to the expected sets of POS, and adds any automatically generated * plural/gender/case forms as necessary. */ preprocessFunctionCanonical(canonical: unknown, forItem: 'query' | 'action' | 'stream', forSide: 'user' | 'agent', isList: boolean): Replaceable[]; private _ensureDefaultEnumValues; /** * Apply load-time transformations to the canonical annotation of a parameter. This normalizes * the form to the expected sets of POS, and adds any automatically generated * plural/gender/case forms as necessary. */ preprocessParameterCanonical(fromArgument: Ast.ArgumentDef, forSide: 'user' | 'agent'): NormalizedParameterCanonical; /** * Apply final touches to a newly generated synthetic sentence * * This function should correct coreferences, conjugations and other * grammar/readability issues that are too inconvenient to prevent * using the templates. */ postprocessSynthetic(sentence: string, program: unknown, rng: (() => number) | null, forTarget: 'user' | 'agent'): string; /** * Convert a tokenized sentence back into a correctly spaced, correctly * punctuated sentence. * * This is a low-level method called by {@link LanguagePack.detokenizeSentence}. * It can be used to detokenize one token at a time. */ detokenize(sentence: string, prevtoken: string | null, token: string): string; /** * Convert a tokenized sentence back into a correctly spaced, correctly * punctuated sentence. * * This is used for sentences presented to an MTurk worker for paraphrasing, * and it is used for the agent replies before they are shown to the user. */ detokenizeSentence(tokens: string[]): string; /** * Retrieve the list of units to use for a given base unit. This defaults * to metric units, but subclasses can override to choose a different * unit. * * The best unit for a given value (i.e., the one with the fewest digits) * will be chosen to display. If there are ties, the first unit will be chosen. */ protected _getPossibleUnits(baseUnit: string): string[]; private _getBestUnit; private _measureFormatFallback; private _measureToString; private _numberToString; /** * Convert a date object to a user-visible string, displaying only the date part. * * @param {Date} date - the time to display * @return {string} the formatted time */ private _dateToString; /** * Convert a date object to a user-visible string, displaying _only_ the time part. * * @param {Date} date - the time to display * @return {string} the formatted time */ private _timeToString; /** * Convert a date object to a user-visible string, displaying both the date and the time part. * * @param {Date} date - the time to display * @return {string} the formatted time */ private _dateAndTimeToString; getDefaultTemperatureUnit(): string; protected displayPhoneNumber(phone: string): string; protected displayEntity(token: string, entityValue: AnyEntity, delegate: UnitPreferenceDelegate, format?: string): string; /** * Post-process a sentence generated by the neural NLG for display to * the user. * * This includes true-casing, detokenizing, and replacing entity tokens * with actual values. */ postprocessNLG(answer: string, entities: EntityMap, delegate: UnitPreferenceDelegate): string; /** * Convert a word or phrase to plural form. * * This function should return `undefined` if there is no plural form * of the given phrase. */ pluralize(phrase: string): string | undefined; /** * Convert a word or verb phrase to past tense. * * This function should return `undefined` if there is no past tense * of the given phrase. */ toVerbPast(phrase: string): string | undefined; /** * Convert a phrase from the side of the user to the side of the agent. * * This function takes a phrase that talks about "my devices" (uttered by * the user) and converts to a phrase that talks about "your devices" * uttered by the agent. */ toAgentSideUtterance(phrase: string): string; /** * Filter out words that cannot be in the dataset, because they would be * either tokenized/preprocessed out or they are unlikely to be used with * voice. */ isGoodWord(word: string): boolean; /** * Filter out phrases that should not be used as a parameter on their own. * * This is mainly used to remove phrases that would be syntatically * ambiguous, and would not be immediately recognized as a parameter. * A good rule of thumb is to filter out all phrases that consist entirely * of stop words. */ isGoodSentence(sentence: string): boolean; /** * Check if a numeric phrase is valid for the given language. * * This covers ASCII digits as well as language-specific number systems, * like Arabic digits. */ isGoodNumber(number: string): boolean; /** * Check if a phrase looks like a person name. * * This is a coarse check that is used to override * {@link LanguagePack.isGoodWord} to account for foreign person * names and loan words. */ isGoodPersonName(word: string): boolean; /** * Check if a phrase looks like a social media user name. * * This is a coarse check that is used to override * {@link LanguagePack.isGoodWord} to account for foreign person * names and loan words. */ isGoodUserName(word: string): boolean; /** * Add a definite article ("the") to the given phrase. * * If the language has no concept of definite articles, this function * must return `undefined`. */ addDefiniteArticle(phrase: string): string | undefined; }