import Algorithm from './Algorithm' import Message from './Message' import Window from './Window' import fs from 'fs' import CronTask from '../model/CronTask' import AlgorithmType from '../enum/AlgorithmType' import RealTimeWindow from './RealTimeWindow' import TimeSpliceSize from '../enum/TimeSpliceSize' import HistoryWindow from './HistoryWindow' import WindowSize from '../enum/WindowSize' import FileUtil from '../util/FileUtil' import DateUtil from '../util/DateUtil' import logger from '../logger/Logger' import eventManager from './EventManager' const tempFolder = "data/temp/" const archiveFoler = "data/archive/" export default abstract class BaseAlgorithm implements Algorithm { // 当前的时间窗口 currentWindow!: Window // 上一个时间窗口 lastWindow: Window | undefined algorithmType: string windowSize: string name: string input: string // 算法的计算操作,当数据来时会更新metric的内容 abstract compute(content: any, window: Window): any /** * * @param algorithmType algorithmType 算法类型 计算实时数据还是历史数据 * @param windowSize TimeSpliceSize 时间切片长度类型 [5min|30min|1h|1d|unlimited] * @param name * @param input */ constructor(algorithmType: string, windowSize: string, name: string, input: string) { this.algorithmType = algorithmType this.windowSize = windowSize this.name = name this.input = input this.initWindow() } getName(): string { return this.name } getInput(): string { return this.input } /** * 算法销毁时,发出事件 */ onDestory() { eventManager.emitEvent(this.getName(), this.currentWindow) } initWindow() { if (this.algorithmType == AlgorithmType.REALTIME) { this.currentWindow = new RealTimeWindow(this.windowSize) let filePath = tempFolder + this.getName() + "/" + this.currentWindow.getPriod() + ".json" if (fs.existsSync(filePath)) { let content = fs.readFileSync(filePath, 'utf-8') logger.info("恢复数据") this.recoverMetrics(content) } this.startTimer() } else { this.currentWindow = new HistoryWindow() } } // 从持久化文件中恢复metric的内容 recoverMetrics(content: string) { let { temp, result } = JSON.parse(content) this.currentWindow.setTemp(temp) this.currentWindow.setResult(result) } // 定时保存算法窗口里的数据到磁盘 startTimer() { // 每隔2分钟保存 new CronTask("0 */5 * * * *", () => { let folder = tempFolder + this.getName() FileUtil.createFolderIfNotExist(folder) // 使用算法名和时间段作为唯一标示 let filePath = folder + "/" + this.currentWindow.getPriod() + ".json" // logger.info("定期执行保存") let data = { temp: this.currentWindow.getTemp(), result: this.currentWindow.getResult() } fs.writeFileSync(filePath, JSON.stringify(data, null, 4), { flag: "w+" }) }) } archive(window: Window) { logger.info(this.getName() + " 窗口销毁时,归档算法的数据") let folder = archiveFoler + this.getName() FileUtil.createFolderIfNotExist(folder) let fileName = folder + "/" + window.getPriod() + ".json" fs.writeFileSync(fileName, JSON.stringify(window.getResult(), null, 4)) // 向事件通道发出窗口关闭事件 eventManager.emitEvent(this.getName(), window) } getDeltaTime(): number { let t = 0 switch (this.windowSize) { case TimeSpliceSize.FIVE_MINUTES: t = WindowSize.FIVE_MINUTES break case TimeSpliceSize.HALF_HOUR: t = WindowSize.HALF_HOUR break case TimeSpliceSize.ONE_HOUR: t = WindowSize.ONE_HOUR break case TimeSpliceSize.ONE_DAY: t = WindowSize.ONE_DAY break default: break; } return t } async onMessage(message: Message): Promise { let promise = new Promise(resolve => { const content = message.getContent() const msgTime = message.getTimestamp() const currentTimestamp = DateUtil.getCurrentTimestamp() // 历史窗口来多少数据计算多少数据,由于数据是有限的,所以窗口没有范围 if (this.algorithmType == AlgorithmType.HISTORY) { this.compute(content, this.currentWindow) resolve(this.currentWindow.getResult()) return } // 实时窗口数据是无限的,数据分窗口处理 // 如果消息不在当前窗口 if (msgTime < this.currentWindow.getBegin()) { // 判断是否是上个窗口的延迟数据 if (this.lastWindow) { if (msgTime >= this.lastWindow.getBegin() && msgTime < this.lastWindow.getEnd()) { this.compute(content, this.lastWindow) } } else { // logger.info("延迟数据不处理", this.input(), msgTime, content["id"]) } } if (msgTime >= this.currentWindow.getBegin() && msgTime < this.currentWindow.getEnd()) { this.compute(content, this.currentWindow) } // 第一条超过时间的窗口的消息到达时,会创建新窗口 if (msgTime >= this.currentWindow.getEnd() && msgTime <= currentTimestamp) { logger.info(this.getName() + "创建新的窗口", this.currentWindow.getBegin(), this.currentWindow.getEnd(), msgTime) // 消息时间超过时间窗口 this.lastWindow = this.currentWindow this.currentWindow = new RealTimeWindow(this.windowSize) this.compute(content, this.currentWindow) } if (this.lastWindow) { // 如果延迟数据超过了五分钟,不处理,归档窗口的数据 if ((currentTimestamp - this.lastWindow.getEnd()) > 5 * 60) { logger.info(this.getName() + "归档数据") this.archive(this.lastWindow) this.lastWindow = undefined } } resolve(this.currentWindow.getResult()) }) return promise } }