import { Resolver, Mutation, Query, Arg, Ctx, Int, Directive } from 'type-graphql' import { Field, InputType, ObjectType } from 'type-graphql' import axios from 'axios' import { labelStudioApi } from '../utils/label-studio-api-client.js' /** * External Data Source Service * * Import data from external sources into Label Studio * Supports: * - REST API endpoints * - S3/Cloud Storage URLs * - Database queries (via Things Factory connections) * - CSV/JSON file URLs */ // ============================================================================ // Type Definitions // ============================================================================ @InputType() class ExternalDataSourceConfig { @Field({ description: 'Data source type' }) sourceType: 'api' | 'url' | 's3' | 'database' | 'csv' | 'json' @Field({ description: 'Source URL or endpoint' }) sourceUrl: string @Field({ nullable: true, description: 'Authentication header if required' }) authHeader?: string @Field({ nullable: true, description: 'HTTP method for API calls' }) httpMethod?: string @Field({ nullable: true, description: 'Request body for POST/PUT' }) requestBody?: string @Field({ nullable: true, description: 'JSONPath or XPath for data extraction' }) dataPath?: string } @InputType() class ImportFromExternalSourceRequest { @Field(type => Int, { description: 'Label Studio project ID' }) projectId: number @Field(type => ExternalDataSourceConfig, { description: 'External data source configuration' }) source: ExternalDataSourceConfig @Field({ nullable: true, description: 'Field mapping JSON (source -> Label Studio)' }) fieldMapping?: string @Field({ nullable: true, description: 'Image field name in source data' }) imageField?: string @Field({ nullable: true, defaultValue: false, description: 'Auto-generate AI predictions' }) autoGeneratePredictions?: boolean @Field({ nullable: true, description: 'Maximum number of items to import' }) limit?: number } @ObjectType() class ExternalImportResult { @Field(type => Int, { description: 'Total items fetched from source' }) totalFetched: number @Field(type => Int, { description: 'Tasks successfully imported' }) tasksImported: number @Field(type => Int, { description: 'Tasks failed to import' }) tasksFailed: number @Field(type => [Int], { description: 'Imported task IDs' }) taskIds: number[] @Field({ nullable: true, description: 'Error message if any' }) error?: string } @ObjectType() class DataSourcePreview { @Field(type => Int, { description: 'Number of items available' }) itemCount: number @Field({ description: 'Sample data (first item)' }) sampleData: string @Field({ description: 'Detected schema/fields' }) schema: string @Field({ nullable: true, description: 'Preview error if any' }) error?: string } // ============================================================================ // Resolver // ============================================================================ @Resolver() export class ExternalDataSourceService { /** * Preview external data source before import */ @Query(returns => DataSourcePreview, { description: 'Preview external data source to verify connection and data structure' }) @Directive('@privilege(category: "label-studio", privilege: "query")') async previewExternalDataSource( @Arg('source', type => ExternalDataSourceConfig) source: ExternalDataSourceConfig, @Ctx() context: ResolverContext ): Promise { console.log(`[External Data] Previewing source: ${source.sourceType} - ${source.sourceUrl}`) try { const data = await this.fetchFromSource(source, 1) if (!data || data.length === 0) { return { itemCount: 0, sampleData: '{}', schema: '{}', error: 'No data found' } } const firstItem = data[0] const schema = this.detectSchema(firstItem) return { itemCount: data.length, sampleData: JSON.stringify(firstItem, null, 2), schema: JSON.stringify(schema, null, 2) } } catch (error) { console.error('[External Data] Preview failed:', error) return { itemCount: 0, sampleData: '{}', schema: '{}', error: error.message } } } /** * Import data from external source to Label Studio */ @Mutation(returns => ExternalImportResult, { description: 'Import data from external source into Label Studio project' }) @Directive('@privilege(category: "label-studio", privilege: "mutation")') async importFromExternalSource( @Arg('input') input: ImportFromExternalSourceRequest, @Ctx() context: ResolverContext ): Promise { console.log(`[External Data] Importing from ${input.source.sourceType}: ${input.source.sourceUrl}`) try { // 1. Fetch data from external source const rawData = await this.fetchFromSource(input.source, input.limit) console.log(`[External Data] Fetched ${rawData.length} items`) if (rawData.length === 0) { return { totalFetched: 0, tasksImported: 0, tasksFailed: 0, taskIds: [], error: 'No data fetched from source' } } // 2. Transform data to Label Studio format const fieldMapping = input.fieldMapping ? JSON.parse(input.fieldMapping) : {} const imageField = input.imageField || 'image' const tasks = rawData.map(item => this.transformToLabelStudioTask(item, fieldMapping, imageField)) // 3. Import tasks to Label Studio const result: ExternalImportResult = { totalFetched: rawData.length, tasksImported: 0, tasksFailed: 0, taskIds: [] } // Import in batches const BATCH_SIZE = 50 for (let i = 0; i < tasks.length; i += BATCH_SIZE) { const batch = tasks.slice(i, i + BATCH_SIZE) try { const importResult = await labelStudioApi.importTasks(input.projectId, batch) // Label Studio import returns task IDs if (importResult.task_count) { result.tasksImported += importResult.task_count } else if (Array.isArray(importResult)) { result.tasksImported += importResult.length result.taskIds.push(...importResult.map((t: any) => t.id)) } console.log(`[External Data] Imported batch ${i / BATCH_SIZE + 1}: ${batch.length} tasks`) } catch (batchError) { console.error(`[External Data] Batch import failed:`, batchError) result.tasksFailed += batch.length } } console.log( `[External Data] Import completed: ${result.tasksImported} imported, ${result.tasksFailed} failed` ) return result } catch (error) { console.error('[External Data] Import failed:', error) throw new Error(`Failed to import from external source: ${error.message}`) } } /** * Fetch data from external source */ private async fetchFromSource( source: ExternalDataSourceConfig, limit?: number ): Promise { switch (source.sourceType) { case 'api': return await this.fetchFromApi(source, limit) case 'url': case 'json': return await this.fetchFromUrl(source, limit) case 'csv': return await this.fetchFromCsv(source, limit) case 's3': throw new Error('S3 source not yet implemented. Use URL with S3 presigned URL instead.') case 'database': throw new Error('Database source not yet implemented. Use API endpoint instead.') default: throw new Error(`Unsupported source type: ${source.sourceType}`) } } /** * Fetch from REST API */ private async fetchFromApi(source: ExternalDataSourceConfig, limit?: number): Promise { const method = (source.httpMethod || 'GET').toUpperCase() const headers: any = { 'Content-Type': 'application/json' } if (source.authHeader) { headers['Authorization'] = source.authHeader } const config: any = { method, url: source.sourceUrl, headers, timeout: 30000 } if (method === 'POST' || method === 'PUT') { config.data = source.requestBody ? JSON.parse(source.requestBody) : {} } const response = await axios(config) let data = response.data // Extract data using JSONPath if specified if (source.dataPath) { data = this.extractDataByPath(data, source.dataPath) } // Ensure data is an array if (!Array.isArray(data)) { data = [data] } // Apply limit if (limit && data.length > limit) { data = data.slice(0, limit) } return data } /** * Fetch from URL (JSON file) */ private async fetchFromUrl(source: ExternalDataSourceConfig, limit?: number): Promise { const headers: any = {} if (source.authHeader) { headers['Authorization'] = source.authHeader } const response = await axios.get(source.sourceUrl, { headers }) let data = response.data // Extract data using JSONPath if specified if (source.dataPath) { data = this.extractDataByPath(data, source.dataPath) } // Ensure data is an array if (!Array.isArray(data)) { data = [data] } // Apply limit if (limit && data.length > limit) { data = data.slice(0, limit) } return data } /** * Fetch from CSV */ private async fetchFromCsv(source: ExternalDataSourceConfig, limit?: number): Promise { const headers: any = {} if (source.authHeader) { headers['Authorization'] = source.authHeader } const response = await axios.get(source.sourceUrl, { headers }) const csvText = response.data // Simple CSV parsing (for production, use a proper CSV library) const lines = csvText.trim().split('\n') const headerLine = lines[0] const headers_csv = headerLine.split(',').map((h: string) => h.trim()) const data = [] const maxLines = limit ? Math.min(lines.length, limit + 1) : lines.length for (let i = 1; i < maxLines; i++) { const line = lines[i] const values = line.split(',').map((v: string) => v.trim()) const obj: any = {} headers_csv.forEach((header, index) => { obj[header] = values[index] }) data.push(obj) } return data } /** * Extract data by JSON path (simple implementation) */ private extractDataByPath(data: any, path: string): any { const parts = path.split('.') let current = data for (const part of parts) { if (current && typeof current === 'object' && part in current) { current = current[part] } else { throw new Error(`Path "${path}" not found in data`) } } return current } /** * Transform raw data item to Label Studio task format */ private transformToLabelStudioTask( item: any, fieldMapping: Record, imageField: string ): any { // Apply field mapping const mappedData: any = {} if (Object.keys(fieldMapping).length > 0) { // Use explicit mapping for (const [sourceField, targetField] of Object.entries(fieldMapping)) { if (item[sourceField] !== undefined) { mappedData[targetField] = item[sourceField] } } } else { // No mapping, use data as-is Object.assign(mappedData, item) } // Ensure image field exists const imageUrl = mappedData[imageField] || item[imageField] || item.url || item.image_url if (!imageUrl) { console.warn('[External Data] No image URL found in item:', item) } return { data: { image: imageUrl, ...mappedData }, meta: { source: 'external-import', originalData: JSON.stringify(item) } } } /** * Detect schema from data item */ private detectSchema(item: any): Record { const schema: Record = {} for (const [key, value] of Object.entries(item)) { schema[key] = typeof value } return schema } }