/**
* ## NILOD - Node.js Interactive Lines Of Dialog
*
* @author Gustavo Ramos Rehermann (Gustavo6046)
* @exports NILOD
* @namespace NILOD
*/
/*
* NILOD - Node.js Interactive Lines Of Dialog
*
* inspired by COBE 2
*
* Made by Gustavo R. Rehermann, published
* under the MIT License.
*/
const FlakeId = require('flake-idgen');
const crypto = require('crypto');
const levenshtein = require('fast-levenshtein').get;
let words = ['above_mentioned', 'above_listed', 'before_mentioned', 'aforementioned', 'abundance', 'accelerate', 'accentuate', 'accommodation', 'accompany', 'accomplish', 'accorded', 'accordingly', 'accrue', 'accurate', 'acquiesce', 'acquire', 'additional', 'address', 'addressees', 'adjustment', 'admissible', 'advantageous', 'advise', 'aggregate', 'aircraft', 'alleviate', 'allocate', 'alternatively', 'ameliorate', 'and/or', 'anticipate', 'applicant', 'application', 'apparent', 'apprehend', 'appreciable', 'appropriate', 'approximate', 'ascertain', 'attain', 'attempt', 'authorize', 'beg', 'belated', 'beneficial', 'bestow', 'beverage', 'capability', 'caveat', 'cease', 'chauffeur', 'clearly', 'obviously', 'combined', 'commence', 'complete', 'component', 'comprise', 'conceal', 'concerning', 'consequently', 'consolidate', 'constitutes', 'contains', 'convene', 'corridor', 'currently', 'deem', 'delete', 'demonstrate', 'depart', 'designate', 'desire', 'determine', 'disclose', 'different', 'discontinue', 'disseminate', 'duly', 'authorized', 'signed', 'each...apiece', 'economical', 'elect', 'eliminate', 'elucidate', 'emphasize', 'employ', 'encounter', 'endeavor', 'end', 'result', 'product', 'enquiry', 'ensure', 'entitlement', 'enumerate', 'equipments', 'equitable', 'equivalent', 'establish', 'evaluate', 'evidenced', 'evident', 'evince', 'excluding', 'exclusively', 'exhibit', 'expedite', 'expeditious', 'expend', 'expertise', 'expiration', 'facilitate', 'fauna', 'feasible', 'females', 'finalize', 'flora', 'following', 'forfeit', 'formulate', 'forward', 'frequently', 'function', 'furnish', 'grant', 'herein', 'heretofore', 'herewith', 'thereof', 'wherefore', 'wherein', 'however', 'identical', 'identify', 'immediately', 'impacted', 'implement', 'inasmuch', 'inception', 'indicate', 'indication', 'initial', 'initiate', 'interface', 'irregardless', 'liaison', '_ly', 'doubtless', 'fast', 'ill', 'much', 'seldom', 'thus', 'magnitude', 'maintain', 'majority', 'maximum', 'merge', 'methodology', 'minimize', 'minimum', 'modify', 'monitor', 'moreover', 'multiple', 'necessitate', 'nevertheless', 'notify', 'not...unless', 'not...except', 'not...until', 'notwithstanding', 'numerous', 'objective', 'obligate', 'observe', 'obtain', 'operate', 'optimum', 'option', 'orientate', '...out', 'calculate', 'cancel,', 'distribute', 'segregate', 'separate', 'overall', 'parameters', 'participate', 'particulars', 'perchance', 'perform', 'permit', 'perspire', 'peruse', 'place', 'portion', 'possess', 'potentiality', 'practicable', 'preclude', 'preowned', 'previously', 'prioritize', 'proceed', 'procure', 'proficiency', 'promulgate', 'provide', 'purchase', 'reflect', 'regarding', 'relocate', 'remain', 'remainder', 'remuneration', 'render', 'represents', 'request', 'require', 'requirement', 'reside', 'residence', 'respectively', 'retain', 'retire', 'rigorous', 'selection', 'separate', 'shall', 'solicit', 'state_of_the_art', 'strategize', 'subject', 'submit', 'subsequent', 'subsequently', 'substantial', 'sufficient', 'terminate', 'therefore', 'therein', 'timely', 'transpire', 'transmit', 'type', 'validate', 'variation', 'very', 'viable', 'warrant', 'whereas', 'whosoever', 'whomsoever', 'witnessed'];
function choice(l) {
return l[Math.floor(Math.random() * l.length)];
}
function randomWordId(size) {
size = Math.round(Math.max(+size, 1));
let v = new Array(size).fill(0).map(() => choice(words));
return v.join('-');
}
/**
* The default falloff gradient used by NILOD
* to weight the weakening of distance between
* the position of compared tokens.
*
* @param {integer} distance The distance of both tokens in their respective sentences' orders.
* @returns {number} The weakening weight; maxes out at 1 (distance = 0), tends toward 0 as distance increases.
*/
function defaultOrderDistanceGradient(distance) {
return 1 / (1 + distance);
}
/**
* Levenshtein _similarity_ between two strings.
* @param {string} s String 1.
* @param {string} t String 2.
* @returns {number} The Levenshtein _similarity_.
*/
function levenshteinSimilarity(s, t) {
return 1 - (levenshtein(s, t) / Math.max(s.length, t.length));
}
function _tokenizePrepare(text) {
return text
.replace(/([^\sa-z0-9])([a-z0-9])/gi, (match) => `${match[0]} ${match[1]}`)
.replace(/([a-z0-9])([^\sa-z0-9])/gi, (match) => `${match[0]} ${match[1]}`);
}
/**
* Splits text into tokens (heterogeneous clusters of letters/numbers and symbols separated by spaces).
*
* @param {string} text The input document, string or text to tokenize.
* @returns {string} The tokenized result.
*/
function tokenize(text) {
return _tokenizePrepare(text).split(' ').filter((w) => w != '');
}
function flaker(options) {
const flakeGen = new FlakeId(options);
return () => {
let buf = Buffer.concat([flakeGen.next(), crypto.randomBytes(3)]);
return buf.toString('hex');
};
}
const flake = flaker();
function pickEdge(edges) {
let possib = [];
edges.forEach((e) => {
for (let i = 0; i < e.count; i++)
possib.push(e);
});
return possib[Math.floor(possib.length * Math.random())];
}
/**
* @typedef NDNode
* A NILOD node.
*
* @property {integer[]} words A bag-of-words-encoded list of words that compose this node.
* @property {string[]} nextEdges A list of IDs of edges that stem from this node.
* @property {string[]} prevEdges A list of IDs of edges that 'point to' this node.
*/
/**
* @typedef RatedSample
* @property {number} rating The rating of this sample.
* @property {OutputHistory} attempt The generated sentence (sample) wrapped by this sample.
*/
/**
* @typedef GenerationResults
* The results of generation.
*
* @property {RatedSample} best The predicted best sample, out of the samples generated.
* @property {RatedSample} worst The predicted worst sample, out of the samples generated. _Laughable._
* @property {string} bestResult The predicted best sample's result sentence as a string.
* @property {RatedSample[]} scoreboard A sorted list of samples, in descending order of rating.
*/
/**
* @typedef NDEdge
* A NILOD edge.
*
* @property {string} from The graph node the edge stems *from*.
* @property {string} to The graph node the edge ends in*to*.
* @property {string} fromWord The particular word of the graph node the edge stems from.
* @property {string} toWord The particular word of the graph node the edge ends onto.
* @property {string} separator The string that delimits the fromWord and toWord, i.e. the separator between both nodes as per this edge.
* @property {integer} count The number of occurences of this particular edge that were found during training. Used as a probabilistic measure during sentence generation.
*/
/**
* @typedef SampleOutput
* A record of a sentence that was once generated by this NILOD model.
* This format is used as a return value by sentence generating
* functions, like {@link NilodModel.tryOnce},
*
* @property {boolean} success Whether this attempt was successful (i.e. no errors).
* @property {?string} seedNode The ID of the node that was used as the seed (i.e. the starting point for this sentence).
* @property {?string} seedText The string representation of the seed node (see above).
* @property {?string} seedQuery The original query from the which the node was deduced (or randomly picked, in case this value is null, which also means there was actually no specific query).
* @property {?string} output The output of this query, or null if unsuccessful.
* @property {string[]} outputWords The words that were joined to compose the output property.
* @property {?string} outputID The ID of this sentence generation, used when rating it.
* @property {?string} reasonCode An upper-case, shorthand string that allows for quick mnemonic identification and comparison of error codes.
* @property {?string} reasonMessage A human-readable reason describing the error.
* @property {Number} reasonNumber An integer used to identify this particular reason.
* @property {function(number)} applyRating A method that allows rating this particular generated sentence. Takes a single argument (a rating, usually between -1 and 1).
*/
/**
* The model that englobates (mostly) everything NILOD.
*/
class NilodModel {
/**
* @typedef NilodGraph
* The JSON graph structure of a NILOD model.
* @type {object}
* @property {string[]} outputHistory A history of past generated sentences.
* @property {Object<string, NDNode>} nodes The nodes, where the keys are the nodes' IDs.
* @property {Object<string, NDEdge>} edges The edges, where the keys are the edges' IDs.
* @property {string[]} wordDictionary A bag of words, used to losslessly compress the graph.
*/
/**
* @param {NilodGraph|string} [jGraph] A NILOD model save to be loaded. Omit for an empty model.
*/
constructor(jGraph = null) {
function _convertNode(words) {
return {
prevEdges: [],
nextEdges: [],
words: words
};
}
if (jGraph == null)
jGraph = {
outputHistory: {},
nodes: {'BEGIN': _convertNode([0, 0]), 'END': _convertNode([0, 0])},
edges: {},
wordDictionary: ['']
};
else if (typeof jGraph === 'string')
jGraph = JSON.parse(jGraph);
/**
* The internal graph structure, which can
* be used to export a JSON save.
* @type {NilodGraph}
*/
this._obj = jGraph;
/**
* The nodes, where the keys are the nodes' IDs.
* @type {Object<string, NDNode>}
*/
this.nodes = jGraph.nodes;
/**
* The edges, where the keys are the edges' IDs.
* @type {Object<string, NDEdge>}
*/
this.edges = jGraph.edges;
/**
* A history of past generated sentences.
* @type {string[]}
*/
this.outputHistory = jGraph.outputHistory;
/**
* A bag of words, used to losslessly compress the graph.
* @type {string[]}
*/
this.wordDictionary = jGraph.wordDictionary || [];
}
/**
* Retrieves a word from the specified index in the word bag.
* @param {integer} index A word bag index.
* @returns {string} The word in the word bag.
*/
getWord(index) {
return this.wordDictionary[index];
}
/**
* Adds a word to the word bag, if it is not in the bag yet, and returns its index.
* @param {string} word A word index.
* @returns {integer} The word's index in the word bag.
*/
encodeWord(word) {
let index;
if ((index = this.wordDictionary.indexOf(word)) === -1)
return this.wordDictionary.push(word) - 1;
else
return index;
}
/**
* Returns a map, from the ID of nodes, to the nodes, from
* all nodes in the model's internal graph.
*/
getNodes() {
return new Map(Object.entries(this.nodes));
}
/**
* Returns a list of all nodes containing the words.
*
* @param {string|integer} word The word to be searched.
* @returns {?NDNode} The first node found, or null if none.
*/
findNodeWith(word) {
let res = null;
let optRes = null;
if (typeof word === 'number')
word = this.getWord(word);
Array.from(this.getNodes().values()).some((node) => {
if (this.getWord(node.words[0]).toUpperCase() === word.toUpperCase() || this.getWord(node.words[1]).toUpperCase() === word.toUpperCase()) {
res = node;
return true;
}
else if (this.getWord(node.words[0]).toUpperCase().includes(word.toUpperCase()) || this.getWord(node.words[1]).toUpperCase().includes(word.toUpperCase())) {
optRes = node;
}
});
return res || optRes;
}
/**
* Returns a list of all nodes containing both words.
*
* @param {string|integer} a The first word in the node.
* @param {string|integer} b The second word in the node.
* @returns {?NDNode} The node found, or null if none.
*/
findNode(a, b) {
let res = null;
let optRes = null;
if (typeof a === 'number')
a = this.getWord(a);
if (typeof b === 'number')
b = this.getWord(b);
Array.from(this.getNodes().values()).some((node) => {
if (this.getWord(node.words[0]).toUpperCase() === a.toUpperCase() && this.getWord(node.words[1]).toUpperCase() === b.toUpperCase()) {
res = node;
return true;
}
else if (this.getWord(node.words[0]).toUpperCase().includes(a.toUpperCase()) && this.getWord(node.words[1]).toUpperCase().includes(b.toUpperCase())) {
optRes = node;
}
});
return res || optRes;
}
/**
* Finds the edge between the specified nodes.
* @param {string} from The ID of the previous node from this edge.
* @param {string} to The ID of the next node from this edge.
* @returns {?NDEdge} The edge found, or null if none.
*/
findEdge(from, to) {
let res = null;
this.allEdgesForward(from).some((e) => {
if (e.from === from && e.to === to) {
res = e;
return true;
}
});
return res;
}
/**
* Returns the graph, optionally serialized as JSON.
* @param {boolean} json Whether to return the graph in JSON encoding or as a plain object.
* @returns {(object|string)} The graph, duh.
*/
save(json = true) {
if (json)
return JSON.stringify(this._obj);
else
return this._obj;
}
/**
* Finds all edges that link a word to another, regardless
* of whichever nodes such words come from. The arguments
* may or may not be encoded using the bag-of-words.
*
* @param {string|number} word1 The 'from' word.
* @param {string|number} word2 The 'to' word.
* @returns {NDEdge[]}
*/
findEdgesBetween(word1, word2) {
if (word1 == null || word2 == null)
return [];
let res = [];
if (typeof word1 === 'string')
word1 = this.encodeWord(word1);
if (typeof word2 === 'string')
word2 = this.encodeWord(word1);
Object.values(this.edges).forEach((e) => {
if (e.fromWord === word1 && e.toWord === word2)
res.push(e);
});
return res;
}
/**
* Retrieves a list of all edges that stem from a given node.
* @param {string} nodeId The ID of the node from the which to look for stemming edges.
* @returns {NDEdge[]}
*/
allEdgesBackward(nodeId) {
if (nodeId == null)
return [];
return this.nodes[nodeId].prevEdges.map((e) => this.edges[e]);
}
/**
* Retrieves a list of all edges that 'point to' a given node.
* @param {string} nodeId The ID of the node to look behind, for edges 'pointing to' it.
* @returns {NDEdge[]}
*/
allEdgesForward(nodeId) {
if (nodeId == null)
return [];
return this.nodes[nodeId].nextEdges.map((e) => this.edges[e]);
}
/**
* Trains this model's Markov-like generator graph.
*
* @param {string[]} sentences A list of sentences from the which to train the model's generator.
* @returns {void}
*/
train(sentences) {
let resNodes = [];
let resEdges = [];
let edgeIndex = new Map();
let nodeIndex = new Map();
let nodeIdIndex = new Map();
if (typeof sentences === 'string')
sentences = [sentences];
this.getNodes().forEach((node, id) => {
nodeIndex.set(JSON.stringify(node), {
id: id,
node: node,
edge: ' ',
});
nodeIdIndex.set(id, node);
});
Object.values(this.edges).forEach((e) => {
edgeIndex.set(JSON.stringify([e.from, e.to, e.edge]), e);
});
function connect(from, to, fromWord, toWord, edata) {
if (edgeIndex.has(JSON.stringify([from, to])))
edgeIndex.get(JSON.stringify([from, to])).count++;
else {
let newEdge = {
id: flake(),
from: from,
to: to,
fromWord: fromWord,
toWord: toWord,
edge: edata,
count: 1,
// fromWord: fromWord,
// toWord: toWord,
};
resEdges.push(newEdge);
edgeIndex.set(JSON.stringify([from, to]), newEdge);
nodeIdIndex.get(to).prevEdges.push(newEdge.id);
nodeIdIndex.get(from).nextEdges.push(newEdge.id);
}
}
sentences.forEach((sent) => {
if (!sent.match(/[a-zA-Z\d'"]/))
return;
let firstNode = null;
let lastNode = null;
let words = [];
let sub = '';
let edge = '';
let isSeparating = false;
let beginEdge = (sent.match(/^[^a-zA-Z\d'"]+/) || [''])[0];
let endEdge = (sent.match(/[^a-zA-Z\d'"]+$/) || [''])[0];
Array.from(sent).forEach((ltr) => {
if (ltr.match(/[a-zA-Z\d'"]/) && isSeparating) {
isSeparating = false;
if (sub !== '')
words.push(sub + edge);
edge = '';
sub = ltr;
}
else {
if (ltr.match(/[a-zA-Z\d'"]/))
sub += ltr;
else {
edge += ltr;
isSeparating = true;
}
}
});
if (sub != '')
words.push(sub);
for (let index = 0; index <= words.length - 2; index++) {
let w = words[index + 1];
let context = words.slice(index, index + 2).map(
(w) => w.match(/^[a-zA-Z\d'"]+/)[0]
);
let newNode;
if (nodeIndex.has(JSON.stringify(context)))
newNode = nodeIndex.get(JSON.stringify(context));
else {
newNode = {
id: flake(), node: context, word: context.slice(-1)[0], edge: (w.match(/[^a-zA-Z\d'"]+$/) || [''])[0],
nextEdges: [],
prevEdges: [],
};
nodeIndex.set(JSON.stringify(context), newNode);
nodeIdIndex.set(newNode.id, newNode);
resNodes.push(newNode);
}
if (index > 0)
connect(lastNode.id, newNode.id, lastNode.word, newNode.word, lastNode.edge);
else
firstNode = newNode;
lastNode = newNode;
}
if (firstNode != null)
connect('BEGIN', firstNode.id, '', firstNode.node[0], beginEdge);
if (lastNode != null)
connect(lastNode.id, 'END', lastNode.word, '', endEdge);
});
resNodes.forEach((n) => {
this.nodes[n.id] = {
words: n.node.map((w) => this.encodeWord(w)),
nextEdges: n.nextEdges,
prevEdges: n.prevEdges
};
});
resEdges.forEach((e) => {
e.edge = this.encodeWord(e.edge);
e.fromWord = this.encodeWord(e.fromWord);
e.toWord = this.encodeWord(e.fromWord);
this.edges[e.id] = e;
});
}
/**
* Generates a random sentence based on the
* given Markov data.
*
* Returns the result, in the {@link NDSampleOutput} record format.
*
* @param {Object} [config] The settings for this attempt.
* @param {string} [config.begin] The seed query for this sentence generation attempt. Random by default.
* @param {integer} [config.maxWords=64] The maximum amount of words to be composed in the output.
* @param {integer} [config.maxChars=512] The maximum amount of characters to form the output.
* @returns {NDSampleOutput}
*/
tryOnce(config = {}) {
let begin = config.begin || null;
let seedQuery = begin;
let maxWords = config.maxWords !== undefined ? config.maxWords : 64;
let maxChars = config.maxChars !== undefined ? config.maxChars : 512;
let traversed = [];
if (Object.keys(this.nodes) === 0)
return {
success: false,
seedNode: null,
seedText: null,
seedQuery: null,
output: null,
outputWords: [],
outputIDs: [],
reasonCode: 'EMPTY',
reasonMessage: 'Can\'t generate a sentence from an empty NILOD model!',
reasonNumber: 1
};
if (begin == null)
begin = Object.keys(this.nodes)[Math.floor(Object.keys(this.nodes).length * Math.random())];
else if (begin.id !== undefined) {
begin = begin.id;
if (!this.nodes.has(begin))
return {
success: false,
seedNode: null,
seedText: null,
seedQuery: null,
output: null,
outputWords: [],
outputIDs: [],
reasonCode: 'NOT_FOUND',
reasonMessage: 'Sentence seed (node ID) not found in NILOD model!',
reasonNumber: 2
};
}
else if (typeof begin === 'string' && this.nodes[begin] == null) {
let words = [];
let sub = '';
let edge = '';
let isSeparating = false;
Array.from(begin).forEach((ltr) => {
if (ltr.match(/[a-zA-Z\d'"]/) && isSeparating) {
isSeparating = false;
words.push({
word: sub,
edge: edge
});
sub = ltr;
edge = '';
}
else {
if (ltr.match(/[a-zA-Z\d'"]/))
sub += ltr;
else {
edge += ltr;
isSeparating = true;
}
}
});
if (sub != '')
words.push({
word: sub,
edge: ''
});
if (words.length > 2) {
let start = Math.floor((words.length - 1) * Math.random());
words = words.slice(start, start + 2);
}
console.log(words);
if (words.length === 2)
begin = this.findNode(words[0].word, words[1].word).id;
else if (words.length === 1) {
let n = this.findNodeWith(words[0].word);
begin = n && n.id || null;
}
else
begin = null;
if (begin == null)
return {
success: false,
seedNode: null,
seedText: null,
seedQuery: null,
output: null,
outputWords: [],
outputIDs: [],
reasonCode: 'NOT_FOUND',
reasonMessage: 'Sentence seed (generic string) invalid or not found in NILOD model!',
reasonNumber: 2
};
}
/////////////////////////////////////////////////
// all checks done, let's begin generating! :D //
/////////////////////////////////////////////////
let curr = begin;
let rwords = this.nodes[curr].words.map((w) => this.getWord(w));
let brwords = this.nodes[curr].words.map((w) => this.getWord(w));
let resIDs = [curr];
let res = '';
for (let index = 0; index < rwords.length; index++) {
res += rwords[index];
if (index < rwords.length) {
let edges = this.findEdgesBetween(rwords[index], rwords[index + 1]);
if (edges.length > 0) {
let pe = pickEdge(edges);
let pdg = this.getWord(pe.edge);
res += pdg;
traversed.push(pe);
}
}
}
let seedText = res;
let max;
// step 1. expand backwards
let backHist = [];
let rlength = res.length;
// (traverse backward text)
for (max = (maxWords != null ? maxWords : 8192); rwords.length < max;) {
let possib = this.allEdgesBackward(curr);
if (possib.length === 0)
break;
let re = pickEdge(possib);
let rdg = this.getWord(re.edge);
let word = this.getWord(this.nodes[re.from].words[0]);
if (rlength + word.length + rdg <= maxChars) {
backHist.unshift({wa: this.getWord(this.nodes[re.to].words[0]), w: word, e: rdg});
rlength += word.length + rdg.length;
}
else
break;
rwords.unshift(word);
curr = re.from;
resIDs.unshift(curr);
traversed.unshift(re);
}
// (append 1st 'seed edge' to backHist)
if (backHist.length > 0) {
let edges = this.findEdgesBetween(backHist.slice(-1)[0].wa, brwords[0]);
let lastBackEdge = '';
if (edges.length > 0) {
let pe = pickEdge(edges);
lastBackEdge = this.getWord(pe.edge);
}
else {
// console.log.apply(console, [backHist, brwords].map(util.inspect.bind(util)));
}
// (and synthesize backward text)
let backres = backHist[0].e;
backHist.forEach((h, i) => {
if (i === 1)
backres += h.w + h.e;
else {
backres += h.w;
if (i > 1)
backres += backHist[i - 1].e;
}
});
backres += lastBackEdge;
res = backres + res;
}
curr = begin;
// step 2. expand forwards
for (max = (maxWords != null ? maxWords : 8192); rwords.length < max;) {
let possib = this.allEdgesForward(curr);
if (possib.length === 0)
break;
let re = pickEdge(possib);
let rdg = this.getWord(re.edge);
let word = this.getWord(this.nodes[re.to].words.slice(-1)[0]);
if (res.length + word.length + rdg.length <= maxChars) {
res += rdg + word;
}
else
break;
rwords.push(word);
curr = re.to;
resIDs.push(curr);
traversed.push(re);
}
let attemptID = randomWordId(5) + '-' + Math.floor(10000 * Math.random());
// ...and finally, return! :D
let resolution = {
attemptID: attemptID,
success: true,
seedNode: curr,
seedText: seedText,
seedQuery: seedQuery,
output: res,
outputWords: rwords,
outputIDs: resIDs,
traversedEdges: traversed,
reasonCode: 'SUCCESS',
reasonMessage: 'Sentence generated successfully!',
reasonNumber: 0,
rating: 0,
ratingCount: 0,
applyRating: (rate) => {
this.applyRating(attemptID, rate);
resolution.rating = this.outputHistory[attemptID].rating;
resolution.ratingCount = this.outputHistory[attemptID].ratingCount;
},
setRating: (rate) => {
this.setRating(attemptID, rate);
resolution.rating = this.outputHistory[attemptID].rating;
resolution.ratingCount = 1;
}
};
this.outputHistory[attemptID] = {
output: res,
rating: resolution.rating,
ratingCount: 0
};
return resolution;
}
/**
* Apply a rating vote to an output sentence.
* @param {string} attemptID The ID of the output sentence.
* @param {number} rating The rating, in no required scale, though a scale between -1 and 1 is preferable.
* @returns {number} The new resulting averaged rating.
*/
applyRating(attemptID, rating) {
let newAvg = (rating + this.outputHistory[attemptID].rating * this.outputHistory[attemptID].ratingCount++) / this.outputHistory[attemptID].ratingCount;
this.outputHistory[attemptID].rating = newAvg;
return newAvg;
}
/**
* Sets an output sentence's rating.
* @param {string} attemptID The ID of the output sentence.
* @param {number} rating The new rating, in no required scale, though a scale between -1 and 1 is preferable.
*/
setRating(attemptID, rating) {
this.outputHistory[attemptID].rating = rating;
this.outputHistory[attemptID].ratingCount = 1;
}
/**
* Predicts a rating to a given sentence or text, based on output
* history (and order in the tokenized text).
*
* @param {string} text The text to rate.
* @param {object} config The rating configuration
* @param {function(string, string): number} config.wordSimilarity A word similarity function; a function that returns 1 for equal strings, and gradually lower numbers for gradually more different strings. Defaults to Levenshtein-based similarity.
* @param {function(number): number} config.orderDistanceGradient A gradient function; computes the falloff of weight of a word, when compared with a word N tokens apart in the comparison sentence. Should be 1 for distance 0, and lower numbers (usually tending toward 0) for higher distances.
* @return {number} The determined rating.
*/
rateText(text, config = {}) {
let rating = 0;
function addRating(rate) {
rating = (rating * (addRating.total++) + rate) / addRating.total;
}
addRating.total = 0;
let wordSimilarity = config.wordSimilarity || levenshteinSimilarity;
let similarityIndex = new Map();
// f(0) MUST be 1, and f(n) SHOULD tend toward 0 as n increases.
let odg = config.orderDistanceGradient || defaultOrderDistanceGradient;
let testedWords = tokenize(text);
Object.values(this.outputHistory).forEach((sent) => {
if (sent.output === null || sent.rating === 0) return;
let denom = 0;
let compareWords = tokenize(sent.output);
let weightVal = [];
testedWords.forEach((wordA, i) => {
compareWords.forEach((wordB, j) => {
let gradDist = odg(
2 * Math.abs(i * compareWords.length - j * testedWords.length) / (testedWords.length + compareWords.length)
);
let simi;
let encA = this.encodeWord(wordA);
let encB = this.encodeWord(wordB);
let simKey = encA > encB ? encB + '|' + encA : encA + '|' + encB;
if (similarityIndex.has(simKey))
simi = similarityIndex.get(simKey);
else {
simi = wordSimilarity(wordA, wordB);
similarityIndex.set(simKey, simi);
}
let score = gradDist * simi;
// process.stdout.write(`\rS(${wordA}, ${wordB}) == ${score} `);
weightVal.push(score);
denom += gradDist;
});
});
addRating(weightVal.reduce((a, b) => a + b, 0) / denom * sent.rating);
});
return rating;
}
/**
* Attempts to generate a good-looking sentence, by manufacturing
* N random sentences, and giving each one a rating, which in turn is
* predicted from previously generated sentences (output history).
*
* @param {integer} samples
* The number of samples (random sentences) to generate and rate.
* @param {object} config
* A configuration object that can hold properties for the config arguments of both
* {@link NILOD.tryOnce} (for per-sample generation) and {@link NILOD.rateText} (for sample rating).
* Please see the respective methods' _config_ arguments for more details on such properties.
* This config in particular may also contain an optional _sampleCallback_ property, called once
* after each sample is generated.
*
* @returns {?GenerationResults} The results of generation, or null if unsuccessful.
*/
generate(samples = 30, config = {}) {
let attempts = samples * 5;
let allSamples = [];
while (attempts > 0 && samples > 0) {
let newSample = this.tryOnce(config);
if (!newSample.success) {
attempts--;
}
else {
allSamples.push(newSample);
if (config.sampleCallback && config.sampleCallback.bind && config.sampleCallback.call)
config.sampleCallback(newSample, allSamples.length - 1);
samples--;
}
}
if (attempts <= 0) {
console.warn('Attempts extinguished!');
return null;
}
allSamples.forEach((sample, sampInd) => {
let rating = this.rateText(sample.output);
allSamples[sampInd] = {
attempt: sample,
id: sample.attemptID,
rating: rating
};
});
allSamples = allSamples.sort((a, b) => b.rating - a.rating);
return {
best: allSamples[0],
worst: allSamples.slice(-1)[0],
scoreboard: allSamples
};
}
/**
* Loads a NILOD model from a save.
*
* @param {(string|object)} data The save, either as a plain object or a JSON string, of the model.
* @returns {NilodModel} The loaded modek.
*/
static load(data) {
if (typeof data === 'string')
data = JSON.parse(data);
return new NilodModel(data);
}
}
/**
* NILOD exports. When called, a shorthand for 'new NILOD.{@link NilodModel}(jGraph)'.
*
* @param {NilodGraph} [jGraph] A NILOD model save to be loaded. Omit for an empty model.
* @returns {NilodModel} A new NILOD model.
*/
function NILOD(jGraph = null) {
return new NilodModel(jGraph);
}
Object.assign(NILOD, {
NilodModel: NilodModel,
load: NilodModel.load,
levenshteinSimilarity: levenshteinSimilarity,
defaultDistGradient: defaultOrderDistanceGradient,
tokenize: tokenize,
});
module.exports = NILOD;