// Copyright Abridged, Inc. 2021,2024. All Rights Reserved. // Node module: @collabland/aws // This file is licensed under the MIT License. // License text available at https://opensource.org/licenses/MIT import {decorateDefaultCredentialProvider} from '@aws-sdk/client-sts'; import {defaultProvider} from '@aws-sdk/credential-provider-node'; import {AwsCredentialIdentity} from '@aws-sdk/types'; import { AnyType, HttpErrors, getEnv, getEnvVar, loggers, } from '@collabland/common'; import {ApiResponse, Client, Connection} from '@opensearch-project/opensearch'; import { Count, Search, } from '@opensearch-project/opensearch/api/requestParams.js'; import {ConnectionOptions} from '@opensearch-project/opensearch/lib/Connection.js'; import aws4, {type Request} from 'aws4'; import {ClientRequest, ClientRequestArgs, request as httpRequest} from 'http'; import {request as httpsRequest} from 'https'; import {getAWSOpenSearchRegion} from '../common/types.js'; const {sign} = aws4; const {trace, info, error} = loggers('collabland:aws:opensearch'); export interface OpenSearchIndexResponse { _index: string; _type: string; _id: string; _version: number; result: string; _shards: {total: number; successful: number; failed: number}; _seq_no: number; _primary_term: number; } export interface IndexConfig { aliases?: Record; settings?: Record; mappings?: Record; } export interface OpenSearchIndexOptions { index: string; id?: string; filterPath?: string[] | string; indexConfig?: IndexConfig; } export interface OpenSearchServiceOptions extends OpenSearchIndexOptions { region: string; domain: string; index: string; } export class OpenSearchServiceClient { private options: OpenSearchServiceOptions; private credentials: AwsCredentialIdentity; private timer: NodeJS.Timeout; client?: Client; constructor(options: Partial) { const index = getEnvVar( 'OPEN_SEARCH_INDEX', `collabland-${getEnv()}-messages`, )!; this.options = { region: getAWSOpenSearchRegion(), domain: getEnvVar('OPEN_SEARCH_DOMAIN', 'opensearch.collab.land')!, index, ...options, }; info('OpenSearch options: %O', this.options); } private getConnection(region: string) { // eslint-disable-next-line @typescript-eslint/no-this-alias const self = this; return class AWSConnection extends Connection { public constructor(opts?: ConnectionOptions) { super(opts); this.makeRequest = (reqParams: ClientRequestArgs) => this.signedRequest(reqParams); } private signedRequest(reqParams: ClientRequestArgs): ClientRequest { const request = reqParams?.protocol === 'https:' ? httpsRequest : httpRequest; const signedReq = sign( {...reqParams, service: 'es', region} as Request, self.credentials, ); if (trace.enabled) { const req = {...signedReq}; delete req.agent; trace('Signed request: %O', req); } return request(signedReq); } }; } private async refreshCredentials() { let timeout = 1000 * 60 * 30; // default to 30 mins try { trace('Refreshing AWS credentials...'); const provider = decorateDefaultCredentialProvider(defaultProvider)({}); this.credentials = await provider(); trace( 'Credentials: %s %s', this.credentials.accessKeyId, this.credentials.expiration, ); if (this.credentials.expiration) { const ttl = this.credentials.expiration.getTime() - Date.now(); timeout = Math.floor((ttl * 4) / 5); } } catch (err) { error('Fail to get AWS credentials: %O', err); } if (timeout > 0) { trace( 'Setting up a timer to refresh credentials in %s seconds', timeout / 1000, ); this.timer = setTimeout(() => { this.refreshCredentials().catch(err => { error('Fail to get AWS credentials: %O', err); }); }, timeout); } } async init() { info('Initializing OpenSearch service...'); await this.refreshCredentials(); this.client = new Client({ Connection: this.getConnection(this.options.region), node: this.options.domain.startsWith('https://') ? this.options.domain : `https://${this.options.domain}`, }); } async stop() { if (this.timer != null) { // Clear the timer clearInterval(this.timer); } } async createIndex( index: string, request: { aliases?: Record; settings?: Record; mappings?: Record; }, ) { if (this.client == null) return undefined; let res = await this.client.indices.get({index}, {ignore: [404]}); const exists = res.statusCode === 200; trace('Index %s exists: %s', index, exists); if (!exists) { res = await this.client.indices.create({index, body: request}); // There is a racing condition here, so we need to check again if ( res.statusCode === 400 && res.body?.error?.type === 'index_not_found_exception' ) { return undefined; } return this.checkResponse(res); } return undefined; } async indexDocument(document: object, options?: OpenSearchIndexOptions) { if (this.client == null) return undefined; /* PUT //_doc/<_id> POST //_doc/ PUT //_create/<_id> POST //_create/<_id> */ options = options ?? this.options; await this.createIndex(options.index, options.indexConfig ?? {}); let method = 'POST'; if (options.id != null) { method = 'PUT'; } let filterPath = undefined; if (options.filterPath != null) { if (Array.isArray(options.filterPath)) { filterPath = options.filterPath.join(','); } else { filterPath = options.filterPath; } } if (this.client == null) return undefined; const res = await this.client.index({ filter_path: filterPath, method: method, id: options.id ?? '', index: options.index, body: document, }); return this.checkResponse(res); } getDate() { const now = new Date(); const year = leftPad(now.getUTCFullYear(), 4); const month = leftPad(now.getUTCMonth() + 1, 2); const date = leftPad(now.getUTCDate() + 1, 2); const suffix = `${year}${month}${date}`; return suffix; } getYear() { const now = new Date(); const year = leftPad(now.getUTCFullYear(), 4); return year; } async reindexCheckRolesJobs() { if (this.client == null) return undefined; let index = `collabland-${getEnv()}-reindex-jobs`; const suffix = this.getYear(); index = `${index}-${suffix}`; trace('OpenSearch index: %s', index); const textKeyword = { type: 'text', fields: { keyword: { type: 'keyword', ignore_above: 256, }, }, }; await this.createIndex(index, { settings: { number_of_shards: 5, number_of_replicas: 1, 'index.mapping.total_fields.limit': 100000, }, mappings: { dynamic_templates: [ { body_body: { path_match: 'body.body', match_mapping_type: 'object', mapping: { enabled: false, type: 'object', }, }, }, { body_results: { path_match: 'body.results', match_mapping_type: 'object', mapping: { enabled: false, type: 'object', }, }, }, { body_accounts: { path_match: 'body.accounts', match_mapping_type: 'object', mapping: { enabled: false, type: 'object', }, }, }, { errors: { path_match: 'errors', match_mapping_type: 'object', mapping: { enabled: false, type: 'object', }, }, }, ], properties: { body: { properties: { accountId: textKeyword, address: textKeyword, asset: textKeyword, body: { type: 'object', enabled: false, }, chainId: { type: 'long', }, classifier: textKeyword, collectionName: textKeyword, communityId: textKeyword, context: { properties: { applicationId: textKeyword, interactionId: textKeyword, token: textKeyword, }, }, contract: textKeyword, contractAddress: textKeyword, createdTime: { type: 'date', }, errors: { properties: { address: textKeyword, args: textKeyword, argument: textKeyword, code: textKeyword, data: textKeyword, details: { properties: { detail: textKeyword, }, }, errorArgs: textKeyword, errorName: textKeyword, errorSignature: textKeyword, event: textKeyword, fault: textKeyword, message: textKeyword, method: textKeyword, name: textKeyword, operation: textKeyword, position: { type: 'long', }, reason: textKeyword, stack: textKeyword, token: { type: 'long', }, transaction: { properties: { data: textKeyword, to: textKeyword, }, }, value: textKeyword, }, }, event: textKeyword, finish: { type: 'date', }, flowAccountId: textKeyword, guild: { properties: { id: textKeyword, memberCount: { type: 'long', }, name: textKeyword, }, }, headers: { properties: { DelaySeconds: { type: 'long', }, jobError: textKeyword, jobMaxRetries: { type: 'long', }, jobReceivedTime: { type: 'date', }, jobRetries: { type: 'long', }, jobRetryReason: textKeyword, jobScheduledTime: { type: 'long', }, requestId: textKeyword, trigger: textKeyword, }, }, id: textKeyword, maxToken: textKeyword, members: { type: 'long', }, minToken: textKeyword, name: textKeyword, niftyUserId: textKeyword, pk: textKeyword, platform: textKeyword, query: textKeyword, requiresMetadata: { type: 'boolean', }, results: { type: 'object', enabled: false, }, roleId: textKeyword, roles: { properties: { id: textKeyword, name: textKeyword, }, }, rolesAdded: { properties: { id: textKeyword, name: textKeyword, }, }, rolesFailed: textKeyword, rolesRemoved: { properties: { id: textKeyword, name: textKeyword, }, }, scheme: textKeyword, signerAddress: textKeyword, sk: textKeyword, start: { type: 'date', }, status: textKeyword, timestamp: { type: 'long', }, tokenId: textKeyword, type: textKeyword, userId: textKeyword, variables: { properties: { _filter: { properties: { asset: textKeyword, condition: textKeyword, filters: { properties: { condition: textKeyword, maxCount: textKeyword, minCount: textKeyword, traits: { properties: { trait_type: textKeyword, value: textKeyword, }, }, }, }, }, }, }, }, version: textKeyword, walletAddress: textKeyword, wallets: { properties: { accountId: textKeyword, address: textKeyword, classifier: textKeyword, communityId: textKeyword, createdTime: { type: 'date', }, id: textKeyword, pk: textKeyword, platform: textKeyword, signerAddress: textKeyword, sk: textKeyword, userId: textKeyword, }, }, }, }, errors: { type: 'object', enabled: false, }, headers: { properties: { DelaySeconds: { type: 'long', }, jobError: textKeyword, jobMaxRetries: { type: 'long', }, jobReceivedTime: { type: 'date', }, jobRetries: { type: 'long', }, jobRetryReason: textKeyword, jobScheduledTime: { type: 'long', }, requestId: textKeyword, trigger: textKeyword, }, }, id: textKeyword, type: textKeyword, }, }, }); return this.client.reindex({ scroll: '240m', wait_for_completion: false, slices: 'auto', body: { conflicts: 'proceed', source: { index: `collabland-${getEnv()}-jobs-*`, _source: { excludes: [ 'errors.args', 'errors.details', 'errors.value', 'body.errors.args', 'body.errors.details', 'body.errors.value', ], }, query: { bool: { must_not: { bool: { must: [ { match: { 'headers.trigger': 'bkg_bal_check', }, }, { match: { type: '*roles*', }, }, ], }, }, }, }, }, dest: { index, }, }, }); } private checkResponse>( res: ApiResponse, expectedStatusCodes: (number | null)[] = [200, 201], ): T { if (!expectedStatusCodes.includes(res.statusCode)) { const err = HttpErrors(res.statusCode!); if (res.headers != null) { err.headers = res.headers; } err.details = res.body; trace('OpenSearch API error: %O', err); throw err; } trace('OpenSearch API response: %O', res); return res.body as T; } async search(params: Search) { const result = await this.client!.search(params); return this.checkResponse(result); } async count(params: Count) { const result = await this.client!.count(params); return this.checkResponse(result); } } function leftPad(num: number, length: number) { return num.toString().padStart(length, '0'); }