import {Accept, Application, Component, Configurable, Exception, Inject} from '@lakutata/core' import {RemoteSocket, Server, Socket} from 'socket.io' import { CONNECT_PATH, FETCH_SERVICES_EVENT, MAX_BUFFER_SIZE, SERVICE_APP_ID, SERVICE_APP_NAME, SERVICE_APP_URL, SERVICE_CHANNEL_PRE_PUBLISH_EVENT, SERVICE_CHANNEL_PROXY_PUBLISH_EVENT, SERVICE_CHANNEL_PUBLISH_EVENT, SERVICE_CHANNEL_SUBSCRIBE_EVENT, SERVICE_DIRECT_INVOKE_PREPARE_EVENT, SERVICE_INVOKE_EVENT, SERVICE_INVOKE_ID, SERVICE_OFFLINE_EVENT, SERVICE_ONLINE_EVENT, SERVICE_PING_EVENT, SERVICE_PROXY_INVOKE_EVENT, SERVICE_UPDATE_EVENT, TRANSPORTS } from '../../constants/SocketConstant' import {RegistryAdapterComponent} from './RegistryAdapterComponent' import {ExtendedError} from 'socket.io/dist/namespace' import {TClientHandshakeRequest, TClientHandshakeRequestSchema} from '../../types/TClientHandshakeRequest' import {IValidClientHandshakeRequest} from '../../interfaces/IValidClientHandshakeRequest' import {IncorrectTokenException} from '../../exceptions/IncorrectTokenException' import {Logger} from '@lakutata/core/build/plugins/Logger' import {DisconnectReason} from 'socket.io/dist/socket' import {TInvokePattern} from '../../types/TInvokePattern' import {TPattern} from '../../types/TPattern' import {ServiceRegistryInitializationException} from '../../exceptions/ServiceRegistryInitializationException' import {IServiceClientInfo} from '../../interfaces/IServiceClientInfo' import {IServiceClientInstance} from '../../interfaces/IServiceClientInstance' import {IPatRun} from '../../interfaces/IPatRun' import {ServiceInvokeTimeoutException} from '../../exceptions/ServiceInvokeTimeoutException' import {format} from 'util' import {PatternNotFoundException} from '../../exceptions/PatternNotFoundException' import {TChannelEvent} from '../../types/TChannelEvent' import {ON_ANY_EVENT_RESERVED_SYMBOL} from '../../constants/ModuleConstant' import {IServiceInfoObject} from '../../interfaces/IServiceInfoObject' import {NoAvailableServiceException} from '../../exceptions/NoAvailableServiceException' import {createServer, Server as HttpServer} from 'http' import random from 'random' import {IServiceInfo} from '../../interfaces/IServiceInfo' export class RegistryComponent extends Component { protected get Logger(): Logger | undefined { return this.logger() ? this.app.Logger : undefined } @Inject(Application) protected readonly app: Application @Inject('registryAdapter') protected readonly registryAdapter: RegistryAdapterComponent @Configurable() protected readonly port: () => number @Configurable() protected readonly token: () => string @Configurable() protected readonly logger: () => boolean @Configurable() protected readonly cleanDeathSocketsInterval: number protected readonly httpServer: HttpServer = createServer() /** * 服务客户端信息Map * @protected */ protected readonly serviceClientMap: Map = new Map() protected server: Server /** * 获取服务列表 * @protected */ protected get services(): IServiceInfoObject { const services: IServiceInfoObject = {} this.serviceClientMap.forEach((instance: IServiceClientInstance) => { if (!services[instance.appId]) { services[instance.appId] = [] } services[instance.appId].push({ socketId: instance.socketId, appId: instance.appId, appName: instance.appName, appURL: instance.appURL, events: instance.events, latency: instance.latency, tasks: instance.tasks }) }) return services } /** * 初始化函数 * @protected */ protected async initialize(): Promise { return new Promise((resolve, reject) => { try { this.server = new Server({ transports: TRANSPORTS, path: CONNECT_PATH, maxHttpBufferSize: MAX_BUFFER_SIZE, pingTimeout: 60000, allowUpgrades: true, httpCompression: true, perMessageDeflate: false, cookie: false, serveClient: false, adapter: this.registryAdapter.adapterConstructor as any }) .attach(this.httpServer) .use(async (socket: Socket, next: (err?: ExtendedError) => void) => { try { await this.validateClientHandshake(socket.handshake) return next() } catch (e) { return next(e as ExtendedError) } }) .on('connect', async (socket: Socket) => { try { return await this.serviceClientOnlineHandler(socket) } catch (e) { this.app.Logger.error('Service client socket connection error:', (e as Error).message) } }) .on(SERVICE_ONLINE_EVENT, (data: IServiceClientInfo) => { this.serviceClientMap.set(data.socketId, this.generateServiceClientInstanceByServiceClientInfo(data)) }) .on(SERVICE_OFFLINE_EVENT, (data: IServiceClientInfo) => { this.serviceClientMap.delete(data.socketId) }) .on(SERVICE_UPDATE_EVENT, (data: IServiceClientInfo) => { this.serviceClientMap.set(data.socketId, this.generateServiceClientInstanceByServiceClientInfo(data)) }) return this.httpServer.listen(this.port(), '0.0.0.0', () => { setInterval(() => this.cleanDeathSockets(), this.cleanDeathSocketsInterval) return resolve() }) } catch (e) { return reject(this.generateException(ServiceRegistryInitializationException, (e as Error).message)) } }) } /** * 清除残留的死连接 * @protected */ protected async cleanDeathSockets(): Promise { try { const checkSocketOnlinePromises: Promise[] = [] this.serviceClientMap.forEach((instance: IServiceClientInstance, socketId: string) => { checkSocketOnlinePromises.push(new Promise(resolve => { this.server.in(socketId).fetchSockets().then(remoteSockets => { return resolve(remoteSockets.length > 0 ? null : socketId) }).catch(() => { return resolve(null) }) })) }) const deathSockets: string[] = (await Promise.all(checkSocketOnlinePromises)).filter(value => value !== null) as string[] deathSockets.forEach(deathSocket => this.serviceClientMap.delete(deathSocket)) } catch (e) { this.app.Logger.error('Clean death sockets error:', (e as Error).message) } } /** * 将服务客户端传入的patterns转换为patternManager * @param patterns * @protected */ protected convertPatternsToPatternManager(patterns: TPattern[]): IPatRun { const patternManager: IPatRun = require('patrun')() patterns.forEach(pattern => patternManager.add(pattern, pattern)) return patternManager } /** * 将服务客户端信息转换为服务客户端实例 * @param info * @protected */ protected generateServiceClientInstanceByServiceClientInfo(info: IServiceClientInfo): IServiceClientInstance { return { socketId: info.socketId, appId: info.appId, appName: info.appName, appURL: info.appURL, pm: this.convertPatternsToPatternManager(info.patterns), events: new Map(Object.entries(info.events)), latency: info.latency, tasks: info.tasks } } /** * 服务客户端上线处理方法 * @param socket * @protected */ protected async serviceClientOnlineHandler(socket: Socket): Promise { //验证socket连接参数 const validClientHandshakeRequest: IValidClientHandshakeRequest = await this.validateClientHandshake(socket.handshake) const clientAppId: string = validClientHandshakeRequest.appId const clientAppName: string = validClientHandshakeRequest.appName const clientAppURL: string = validClientHandshakeRequest.appURL const clientAddress: string = socket?.conn?.remoteAddress ? socket.conn.remoteAddress : 'Unknown' const patterns: TPattern[] = validClientHandshakeRequest.patterns const events: { [key: string]: string[] } = validClientHandshakeRequest.events //每个appId作为一个room,连接的socket加入至指定的room种 await socket.join(clientAppId) await socket.join(socket.id) //将信息同步至其他registry伙伴节点 const serviceClientOnline: IServiceClientInfo = { socketId: socket.id, appId: clientAppId, appName: clientAppName, appURL: clientAppURL, patterns: patterns, events: events, latency: 0, tasks: 0 } this.serviceClientMap.set(serviceClientOnline.socketId, this.generateServiceClientInstanceByServiceClientInfo(serviceClientOnline)) this.server.serverSideEmit(SERVICE_ONLINE_EVENT, serviceClientOnline) this.Logger?.info('Service client', clientAppId, 'connected', clientAddress) socket .on(SERVICE_PING_EVENT, (pingData: { events: { [key: string]: string[] } latency: number tasks: number }, fn: () => void) => { //更新节点的信息 const serviceClientUpdate: IServiceClientInfo = { socketId: socket.id, appId: clientAppId, appName: clientAppName, appURL: clientAppURL, patterns: patterns, events: pingData.events,//使用ping包内的事件列表 latency: pingData.latency, tasks: pingData.tasks } this.serviceClientMap.set(serviceClientUpdate.socketId, this.generateServiceClientInstanceByServiceClientInfo(serviceClientUpdate)) this.server.serverSideEmit(SERVICE_UPDATE_EVENT, serviceClientUpdate) if (typeof fn === 'function') fn() }) .on('disconnect', (reason: DisconnectReason) => { this.Logger?.notice('Service client', clientAppId, 'disconnected:', reason, clientAddress) this.serviceClientOfflineHandler(clientAppId, socket).catch(serviceClientOfflineError => this.app.Logger.error('Service client offline error:', serviceClientOfflineError.message)) }) .on('error', (socketError: Error) => { //错误记录不管Logger有无开启,均进行输出 this.app.Logger.error('Service client', clientAppId, 'error:', socketError.message, clientAddress) }) return this.serviceClientHandler(socket) } /** * 服务客户端下线处理方法 * @param appId * @param socket * @protected */ protected async serviceClientOfflineHandler(appId: string, socket: Socket): Promise { socket.removeAllListeners()//移除已断联的socket内的所有事件监听 await socket.leave(appId) const serviceClientOffline: IServiceClientInfo = { socketId: socket.id, appId: appId, appName: 'OFFLINE', appURL: '', patterns: [], events: {}, latency: 0, tasks: 0 } this.serviceClientMap.delete(serviceClientOffline.socketId) this.server.serverSideEmit(SERVICE_OFFLINE_EVENT, serviceClientOffline) } /** * 服务客户端连接处理方法 * @param socket * @protected */ protected serviceClientHandler(socket: Socket): void { socket .on(SERVICE_DIRECT_INVOKE_PREPARE_EVENT, async (rawInvokePacket: string, fn: (response: string) => void) => fn(JSON.stringify(await this.getURLsForHttpInvokePacket(rawInvokePacket)))) .on(SERVICE_PROXY_INVOKE_EVENT, async (proxyRawInvokePacket: string, fn: (response: string) => void) => { //远程代理方法调用 const {rawInvokeResult, matchedPattern} = await this.serviceProxyInvokeHandler(proxyRawInvokePacket) if (this.logger()) { //调用日志 const invokeResult: TInvokePattern = this.app.JSON.parse(rawInvokeResult) if (invokeResult.error) { const exceptionInstance: Exception | Error = invokeResult.error //Warning类型的日志不管是否开启Logger均进行输出 this.app.Logger.warning( invokeResult.id, '[FAILED]', `[${invokeResult[SERVICE_INVOKE_ID]}]`, matchedPattern ? matchedPattern : invokeResult.object, `${exceptionInstance.name}:${exceptionInstance.message}`, `${invokeResult.timeSpent}ms/${invokeResult.totalTimeSpent}ms`, invokeResult.chain ) } else { this.Logger?.info( invokeResult.id, '[SUCCESS]', `[${invokeResult[SERVICE_INVOKE_ID]}]`, matchedPattern, `(${invokeResult.timeSpent}ms/${invokeResult.totalTimeSpent}ms)`, invokeResult.chain ) } } if (typeof fn === 'function') return fn(rawInvokeResult) }) .on(FETCH_SERVICES_EVENT, (nullData: any, fn: (response: string) => void) => { //获取服务注册中心已连接的服务信息列表 const appIds: string[] = Object.keys(this.services) const response = {} appIds.forEach(appId => { response[appId] = this.services[appId].map(service => { return Object.assign({}, service, { events: Object.fromEntries(service.events.entries()) }) }) }) return fn(JSON.stringify(response)) }) .on(SERVICE_CHANNEL_PRE_PUBLISH_EVENT, (eventThumbStringPacket: string, fn: (stringifyTargetAppURLMap: string) => void) => { //事件预发布(用于获取需要使用注册中心中转发布和服务本身直接发布的信息) const eventThumbPacket: Partial & { accessibleURLs: string[] } = this.app.JSON.parse(eventThumbStringPacket) const clientAccessibleURLs: string[] = eventThumbPacket.accessibleURLs const services: IServiceInfoObject = this.services const targetAppURLMap: Map = new Map() Object.keys(services).forEach((appId: string) => { let targetApp: string = eventThumbPacket.app ? eventThumbPacket.app : appId if (services[appId].length && targetApp === appId) { const availableServiceApps = services[appId].filter(serviceApp => ( !!(serviceApp.events.get(eventThumbPacket.channel!)?.includes(eventThumbPacket.event!)) || !!(serviceApp.events.get(eventThumbPacket.channel!)?.includes(ON_ANY_EVENT_RESERVED_SYMBOL)) ) ) if (availableServiceApps.length) { switch (eventThumbPacket.mode) { case 'one': { const targetServiceApps: IServiceInfo[] = availableServiceApps.sort((a, b) => { const aWeight: number = a.latency * a.tasks * (clientAccessibleURLs.includes(a.appURL) ? 0.5 : 1) const bWeight: number = b.latency * b.tasks * (clientAccessibleURLs.includes(b.appURL) ? 0.5 : 1) return aWeight - bWeight > 0 ? 1 : -1 }) const targetServiceAppsSize: number = targetServiceApps.length //使用随机数进行接收者的选定,尽可能的均衡负载 const targetIndex: number = random.int(0, Math.ceil((targetServiceAppsSize - 1) * 0.618)) const target: IServiceInfo = targetServiceApps[targetIndex] if (target) targetAppURLMap.set(target.appId, [target.appURL]) } break case 'all': default: { availableServiceApps.forEach(availableServiceApp => { if (!targetAppURLMap.has(availableServiceApp.appId)) { targetAppURLMap.set(availableServiceApp.appId, []) } targetAppURLMap.get(availableServiceApp.appId)!.push(availableServiceApp.appURL) }) } } } } }) //经过上边的服务筛选,已将需要发送事件的目标服务AppURL加入至targetAppURLMap中,将筛选结果返回至客户端 return fn(JSON.stringify([...targetAppURLMap])) }) .on(SERVICE_CHANNEL_PROXY_PUBLISH_EVENT, (proxyEventStringPacket: string) => { //事件代理发布(用于客户端无法建立直连的情况下由服务中心进行代理发布处理) const eventPacket: { data: string; targets: [string, string[]][] } = this.app.JSON.parse(proxyEventStringPacket) const targetAppURLMap: Map = new Map(eventPacket.targets) const services: IServiceInfoObject = this.services targetAppURLMap.forEach((appURLs: string[], appId: string) => { services[appId]?.forEach(target => { if (appURLs.includes(target.appURL)) { this.server.to(target.socketId).emit(SERVICE_CHANNEL_SUBSCRIBE_EVENT, eventPacket.data) //日志记录 if (this.Logger) { const event: TChannelEvent = JSON.parse(eventPacket.data) this.Logger?.info( event.id, '[PROXY_EMITTED]', `[${event.source}]`, 'to', `[${appId}]`, event.channel, event.event, `(${event.mode})` ) } } }) }) }) .on(SERVICE_CHANNEL_PUBLISH_EVENT, (eventStringPacket: string) => { //处理向集群发布事件 const eventPacket: TChannelEvent = this.app.JSON.parse(eventStringPacket) const services: IServiceInfoObject = this.services const targetMap: Map = new Map() Object.keys(services).forEach((appId: string) => { let targetApp: string = eventPacket.app ? eventPacket.app : appId if (services[appId].length && targetApp === appId) { const availableServiceApps = services[appId].filter(serviceApp => ( !!(serviceApp.events.get(eventPacket.channel)?.includes(eventPacket.event)) || !!(serviceApp.events.get(eventPacket.channel)?.includes(ON_ANY_EVENT_RESERVED_SYMBOL)) ) ) if (availableServiceApps.length) { switch (eventPacket.mode) { case 'one': { const targetServiceApps: IServiceInfo[] = availableServiceApps.sort((a, b) => { const aWeight: number = a.latency * a.tasks const bWeight: number = b.latency * b.tasks return aWeight - bWeight > 0 ? 1 : -1 }) const targetServiceAppsSize: number = targetServiceApps.length //使用随机数进行接收者的选定,尽可能的均衡负载 const targetIndex: number = random.int(0, Math.ceil((targetServiceAppsSize - 1) * 0.618)) const target: IServiceInfo = targetServiceApps[targetIndex] if (target) targetMap.set(target.appId, [target.socketId]) } break case 'all': default: { availableServiceApps.forEach(availableServiceApp => { if (!targetMap.has(availableServiceApp.appId)) { targetMap.set(availableServiceApp.appId, []) } targetMap.get(availableServiceApp.appId)!.push(availableServiceApp.socketId) }) } } } } }) targetMap.forEach((socketIds: string[], appId: string) => { socketIds.forEach(socketId => { this.server.to(socketId).emit(SERVICE_CHANNEL_SUBSCRIBE_EVENT, eventStringPacket) //日志记录 this.Logger?.info( eventPacket.id, '[EMITTED]', `[${eventPacket.source}]`, 'to', `[${appId}]`, eventPacket.channel, eventPacket.event, `(${eventPacket.mode})` ) }) }) }) } /** * 查找可用的sockets(为http请求做前置工作) * @param rawInvokePacket * @protected */ protected async getURLsForHttpInvokePacket(rawInvokePacket: string): Promise { const proxyInvokePacket: TInvokePattern = this.app.JSON.parse(rawInvokePacket) const serviceId: string = proxyInvokePacket[SERVICE_INVOKE_ID] const invokeObject: TPattern = proxyInvokePacket.object let availableRemoteSockets = await this.server.to(serviceId).fetchSockets() if (!availableRemoteSockets.length) return [] availableRemoteSockets = availableRemoteSockets.filter(availableRemoteSocket => { const serviceClientInstance: IServiceClientInstance | undefined = this.serviceClientMap.get(availableRemoteSocket.id) const pattern: TPattern | undefined = serviceClientInstance?.pm.find(invokeObject) return !(!serviceClientInstance || !pattern) }) if (!availableRemoteSockets.length) return [] return availableRemoteSockets.map(availableRemoteSocket => { const serviceClientInstance: IServiceClientInstance | undefined = this.serviceClientMap.get(availableRemoteSocket.id) if (serviceClientInstance) return serviceClientInstance.appURL return null }).filter(url => url !== null).sort(() => Math.floor((Math.random() * 10) + 1) % 2 ? 1 : -1) as string[] } /** * 远程代理调用处理方法 * @protected * @param proxyRawInvokePacket */ protected async serviceProxyInvokeHandler(proxyRawInvokePacket: string): Promise<{ rawInvokeResult: string; matchedPattern: TPattern | undefined }> { const proxyStartInvokeAt: number = Date.now() const proxyInvokePacket: TInvokePattern = this.app.JSON.parse(proxyRawInvokePacket) const serviceId: string = proxyInvokePacket[SERVICE_INVOKE_ID] const invokeObject: TPattern = proxyInvokePacket.object let availableRemoteSockets = await this.server.to(serviceId).fetchSockets() if (!availableRemoteSockets.length) { proxyInvokePacket.error = this.generateException(NoAvailableServiceException, `No available service: ${serviceId}`) proxyInvokePacket.retryableError = true proxyInvokePacket.timeSpent = Date.now() - proxyStartInvokeAt return { rawInvokeResult: this.app.JSON.stringify(proxyInvokePacket), matchedPattern: undefined } } const matchedPatternMap: Map = new Map() availableRemoteSockets = availableRemoteSockets.filter(availableRemoteSocket => { const serviceClientInstance: IServiceClientInstance | undefined = this.serviceClientMap.get(availableRemoteSocket.id) const pattern: TPattern | undefined = serviceClientInstance?.pm.find(invokeObject) matchedPatternMap.set(availableRemoteSocket.id, pattern) return !(!serviceClientInstance || !pattern) }) if (!availableRemoteSockets.length) { proxyInvokePacket.error = this.generateException(PatternNotFoundException, format(proxyInvokePacket.object)) proxyInvokePacket.retryableError = true proxyInvokePacket.timeSpent = Date.now() - proxyStartInvokeAt return { rawInvokeResult: this.app.JSON.stringify(proxyInvokePacket), matchedPattern: undefined } } if (availableRemoteSockets.length > 1) { availableRemoteSockets = availableRemoteSockets.sort((a, b) => { //按照延迟、执行任务数进行权重计算排序,数组中最小的socket则是最优的执行socket const aLatency: number = this.serviceClientMap.get(a.id)?.latency !== undefined ? this.serviceClientMap.get(a.id)!.latency : Infinity const aTasks: number = this.serviceClientMap.get(a.id)?.tasks !== undefined ? this.serviceClientMap.get(a.id)!.tasks : Infinity const bLatency: number = this.serviceClientMap.get(b.id)?.latency !== undefined ? this.serviceClientMap.get(b.id)!.latency : Infinity const bTasks: number = this.serviceClientMap.get(b.id)?.tasks !== undefined ? this.serviceClientMap.get(b.id)!.tasks : Infinity const aWeight: number = aLatency * aTasks const bWeight: number = bLatency * bTasks return aWeight - bWeight > 0 ? 1 : -1 }) } const availableRemoteSocketsSize: number = availableRemoteSockets.length //使用随机数的方式进行负载均匀分配 const selectedRemoteSocketIndex: number = random.int(0, Math.ceil((availableRemoteSocketsSize - 1) * 0.618)) const selectedRemoteSocket = availableRemoteSockets[selectedRemoteSocketIndex] try { return { rawInvokeResult: await new Promise((resolve, reject) => { this.server.to(selectedRemoteSocket.id).timeout(proxyInvokePacket.timeout).emit(SERVICE_INVOKE_EVENT, this.app.JSON.stringify(proxyInvokePacket), (timeoutError: Error | null, response: string[]) => { return timeoutError ? reject(this.generateException(ServiceInvokeTimeoutException, format(proxyInvokePacket.object), proxyInvokePacket.timeout)) : resolve(response[0]) }) }), matchedPattern: matchedPatternMap.get(selectedRemoteSocket.id) } } catch (e) { proxyInvokePacket.error = e as ServiceInvokeTimeoutException proxyInvokePacket.retryableError = true proxyInvokePacket.timeSpent = Date.now() - proxyStartInvokeAt return { rawInvokeResult: this.app.JSON.stringify(proxyInvokePacket), matchedPattern: matchedPatternMap.get(selectedRemoteSocket.id) } } } /** * 验证客户端连接 * @protected * @param handshakeRequest */ @Accept(TClientHandshakeRequestSchema, {stripUnknown: false}) protected async validateClientHandshake(handshakeRequest: TClientHandshakeRequest): Promise { if (this.token) { const requestToken: string = handshakeRequest.auth.token if (this.token() !== requestToken) throw this.generateException(IncorrectTokenException, 'Incorrect token') } return { appId: Buffer.from(handshakeRequest.headers[SERVICE_APP_ID] as string, 'base64').toString(), appName: Buffer.from(handshakeRequest.headers[SERVICE_APP_NAME] as string, 'base64').toString(), appURL: Buffer.from(handshakeRequest.headers[SERVICE_APP_URL] as string, 'base64').toString(), patterns: handshakeRequest.auth.patterns ? handshakeRequest.auth.patterns : [], events: handshakeRequest.auth.events ? handshakeRequest.auth.events : {} } } /** * 获取集群内的所有服务信息 * @param appId */ public fetchServices(appId?: string | RegExp): IServiceInfoObject { const services: IServiceInfoObject = this.services if (appId) { const regExp: RegExp = typeof appId === 'string' ? new RegExp(appId) : appId Object.keys(services).forEach((serviceAppId: string) => { if (!regExp.test(serviceAppId)) { delete services[serviceAppId] } }) } return services } }