# Media Understanding Pipeline

**Source:** `src/media/parse.ts`, `src/media/image-ops.ts`, `src/media/fetch.ts`  
**Category:** Media Processing  
**Purpose:** Multi-modal input processing for images, audio, and attachments

## Overview

The Media Understanding Pipeline processes multi-modal inputs (images, audio, documents) for AI agent consumption. It handles URL fetching, format normalization, size optimization, and metadata extraction with provider-specific constraints.

```
┌─────────────────────────────────────────────────────────────┐
│                    User Input                               │
│  "Analyze this image: MEDIA:https://example.com/photo.jpg"  │
└─────────────────────┬───────────────────────────────────────┘
                      │ parse
                      v
┌─────────────────────────────────────────────────────────────┐
│              Media Token Parser                             │
│  - Extract MEDIA: tokens from text                          │
│  - Distinguish URLs vs local paths                          │
│  - Handle quoted paths, fenced code blocks                  │
└─────────────────────┬───────────────────────────────────────┘
                      │
                      v
┌─────────────────────────────────────────────────────────────┐
│             Media Fetcher & Validator                       │
│  - Fetch from URLs (with size limits)                       │
│  - Read local files (sandbox-aware)                         │
│  - Validate MIME types                                      │
│  - Detect file format from content                          │
└─────────────────────┬───────────────────────────────────────┘
                      │
                      v
┌─────────────────────────────────────────────────────────────┐
│            Format Normalizer / Optimizer                    │
│  - Convert HEIC → JPEG (iOS photos)                         │
│  - Resize large images (provider limits)                    │
│  - Optimize compression (quality/size tradeoff)             │
│  - Apply EXIF orientation correction                        │
└─────────────────────┬───────────────────────────────────────┘
                      │
                      v
┌─────────────────────────────────────────────────────────────┐
│               Base64 Encoder                                │
│  - Convert to data URL for API consumption                  │
│  - Attach metadata (MIME type, size, dimensions)            │
└─────────────────────┬───────────────────────────────────────┘
                      │
                      v
┌─────────────────────────────────────────────────────────────┐
│              AI Provider API                                │
│  image_url: { url: "data:image/jpeg;base64,..." }           │
└─────────────────────────────────────────────────────────────┘
```

## Media Token Parsing

### Token Extraction Algorithm

```typescript
const MEDIA_TOKEN_RE = /\bMEDIA:\s*`?([^\n]+)`?/gi;

interface MediaParseResult {
  text: string; // Original text with MEDIA: lines removed
  mediaUrls?: string[]; // Extracted media sources
  mediaUrl?: string; // Legacy: first URL
  audioAsVoice?: boolean; // [[audio_as_voice]] tag detected
}

function splitMediaFromOutput(raw: string): MediaParseResult {
  const trimmedRaw = raw.trimEnd();

  // Parse fenced code blocks to exclude MEDIA: tokens inside them
  const fenceSpans = parseFenceSpans(trimmedRaw);

  const lines = trimmedRaw.split("\n");
  const keptLines: string[] = [];
  const media: string[] = [];
  let foundMediaToken = false;

  let lineOffset = 0;
  for (const line of lines) {
    // Skip MEDIA lines inside code blocks
    if (isInsideFence(fenceSpans, lineOffset)) {
      keptLines.push(line);
      lineOffset += line.length + 1;
      continue;
    }

    // Check if line starts with MEDIA:
    const trimmedStart = line.trimStart();
    if (!trimmedStart.startsWith("MEDIA:")) {
      keptLines.push(line);
      lineOffset += line.length + 1;
      continue;
    }

    // Extract all MEDIA: tokens from line
    const matches = Array.from(line.matchAll(MEDIA_TOKEN_RE));

    if (matches.length === 0) {
      keptLines.push(line);
      lineOffset += line.length + 1;
      continue;
    }

    // Process each match
    const pieces: string[] = [];
    let cursor = 0;

    for (const match of matches) {
      const start = match.index ?? 0;
      pieces.push(line.slice(cursor, start));

      const payload = match[1];
      const candidates = extractMediaCandidates(payload);

      // Validate candidates
      let validCount = 0;
      const invalidParts: string[] = [];

      for (const candidate of candidates) {
        if (isValidMedia(candidate)) {
          media.push(normalizeMediaSource(candidate));
          validCount++;
          foundMediaToken = true;
        } else {
          invalidParts.push(candidate);
        }
      }

      // Fallback: treat whole payload as single path
      if (validCount === 0 && looksLikeLocalPath(payload)) {
        const fallback = normalizeMediaSource(cleanCandidate(payload));
        if (isValidMedia(fallback, { allowSpaces: true })) {
          media.push(fallback);
          foundMediaToken = true;
        }
      }

      cursor = start + match[0].length;
    }

    pieces.push(line.slice(cursor));
    const cleanedLine = pieces.join("").trim();

    if (cleanedLine) {
      keptLines.push(cleanedLine);
    }

    lineOffset += line.length + 1;
  }

  // Detect [[audio_as_voice]] tag
  const audioTagResult = parseAudioTag(keptLines.join("\n"));

  return {
    text: keptLines.join("\n").trim(),
    mediaUrls: media.length > 0 ? media : undefined,
    mediaUrl: media[0],
    ...(audioTagResult.hadTag ? { audioAsVoice: audioTagResult.audioAsVoice } : {}),
  };
}
```

### Media Validation

```typescript
function isValidMedia(
  candidate: string,
  opts?: { allowSpaces?: boolean; allowBareFilename?: boolean },
): boolean {
  if (!candidate || candidate.length > 4096) {
    return false;
  }

  // No spaces unless explicitly allowed (path with spaces)
  if (!opts?.allowSpaces && /\s/.test(candidate)) {
    return false;
  }

  // HTTP(S) URLs always valid
  if (/^https?:\/\//i.test(candidate)) {
    return true;
  }

  // Local path patterns
  if (isLikelyLocalPath(candidate)) {
    return true;
  }

  // Bare filenames (e.g., "image.png") only when opted in
  if (opts?.allowBareFilename && hasFileExtension(candidate)) {
    return true;
  }

  return false;
}

function isLikelyLocalPath(candidate: string): boolean {
  return (
    candidate.startsWith("/") || // Absolute Unix
    candidate.startsWith("./") || // Relative
    candidate.startsWith("../") || // Parent relative
    candidate.startsWith("~") || // Home directory
    candidate.startsWith("\\") || // Windows UNC
    /^[a-zA-Z]:[\\/]/.test(candidate) || // Windows drive
    (!/^[a-zA-Z][a-zA-Z0-9+.-]*:/.test(candidate) && // Not a URL scheme
      (candidate.includes("/") || candidate.includes("\\")))
  );
}
```

### Quote Unwrapping

```typescript
function unwrapQuoted(value: string): string | undefined {
  const trimmed = value.trim();

  if (trimmed.length < 2) {
    return undefined;
  }

  const first = trimmed[0];
  const last = trimmed[trimmed.length - 1];

  // Must match quote type
  if (first !== last) {
    return undefined;
  }

  if (first !== '"' && first !== "'" && first !== "`") {
    return undefined;
  }

  return trimmed.slice(1, -1).trim();
}

// Usage in token parsing
const payload = match[1];
const unwrapped = unwrapQuoted(payload);
const parts = unwrapped
  ? [unwrapped] // Treated as single path with spaces
  : payload.split(/\s+/).filter(Boolean); // Space-separated list
```

## Image Processing

### Format Detection

```typescript
function sniffImageFormat(buffer: Buffer): ImageFormat | null {
  // JPEG: FF D8 FF
  if (buffer.length >= 3 && buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) {
    return "jpeg";
  }

  // PNG: 89 50 4E 47 0D 0A 1A 0A
  if (
    buffer.length >= 8 &&
    buffer[0] === 0x89 &&
    buffer.toString("ascii", 1, 4) === "PNG" &&
    buffer[5] === 0x0d &&
    buffer[6] === 0x0a &&
    buffer[7] === 0x1a &&
    buffer[8] === 0x0a
  ) {
    return "png";
  }

  // GIF: 47 49 46 38
  if (buffer.length >= 4 && buffer.toString("ascii", 0, 3) === "GIF") {
    return "gif";
  }

  // WEBP: RIFF....WEBP
  if (
    buffer.length >= 12 &&
    buffer.toString("ascii", 0, 4) === "RIFF" &&
    buffer.toString("ascii", 8, 12) === "WEBP"
  ) {
    return "webp";
  }

  // HEIC/HEIF: ftyp heuristic
  if (buffer.length >= 8) {
    const ftyp = buffer.toString("ascii", 4, 8);
    if (["heic", "heix", "heim", "heis"].includes(ftyp)) {
      return "heic";
    }
  }

  return null;
}
```

### EXIF Orientation Correction

**Problem:** iOS photos and many cameras store orientation in EXIF metadata rather than rotating pixels. Must apply rotation before sending to AI APIs.

```typescript
/**
 * EXIF Orientation Values:
 * 1 = Normal (0°)
 * 2 = Flip Horizontal
 * 3 = Rotate 180°
 * 4 = Flip Vertical
 * 5 = Rotate 270° CW + Flip H
 * 6 = Rotate 90° CW
 * 7 = Rotate 90° CW + Flip H
 * 8 = Rotate 270° CW
 */
function readJpegExifOrientation(buffer: Buffer): number | null {
  // Check JPEG magic bytes
  if (buffer.length < 2 || buffer[0] !== 0xff || buffer[1] !== 0xd8) {
    return null;
  }

  let offset = 2;

  while (offset < buffer.length - 4) {
    // Look for APP1 marker (EXIF)
    if (buffer[offset] === 0xff && buffer[offset + 1] === 0xe1) {
      const exifStart = offset + 4;

      // Verify "Exif\0\0" header
      if (
        buffer.length > exifStart + 6 &&
        buffer.toString("ascii", exifStart, exifStart + 4) === "Exif" &&
        buffer[exifStart + 4] === 0 &&
        buffer[exifStart + 5] === 0
      ) {
        const tiffStart = exifStart + 6;
        const byteOrder = buffer.toString("ascii", tiffStart, tiffStart + 2);
        const isLittleEndian = byteOrder === "II";

        // Read IFD0 offset
        const ifd0Offset = readU16(tiffStart + 4, isLittleEndian);
        const ifd0Start = tiffStart + ifd0Offset;

        const numEntries = readU16(ifd0Start, isLittleEndian);

        for (let i = 0; i < numEntries; i++) {
          const entryOffset = ifd0Start + 2 + i * 12;
          const tag = readU16(entryOffset, isLittleEndian);

          // Orientation tag = 0x0112
          if (tag === 0x0112) {
            const orientation = readU16(entryOffset + 8, isLittleEndian);
            return orientation >= 1 && orientation <= 8 ? orientation : null;
          }
        }
      }

      return null;
    }

    // Skip to next marker
    if (buffer[offset] === 0xff) {
      offset += 2 + buffer.readUInt16BE(offset + 2);
    } else {
      offset++;
    }
  }

  return null;
}

async function normalizeExifOrientation(buffer: Buffer): Promise<Buffer> {
  if (prefersSips()) {
    // macOS sips backend
    return await sipsApplyOrientation(buffer);
  }

  // Sharp backend
  try {
    const sharp = await loadSharp();
    return await sharp(buffer).rotate().toBuffer();
  } catch {
    return buffer; // Return original on error
  }
}
```

### Image Resizing

**Problem:** AI providers have maximum image dimension limits (e.g., 2048px max side).

```typescript
interface ResizeParams {
  buffer: Buffer;
  maxSide: number;
  quality: number;
  withoutEnlargement?: boolean;
}

async function resizeToJpeg(params: ResizeParams): Promise<Buffer> {
  // Step 1: Normalize EXIF orientation BEFORE resize
  const normalized = await normalizeExifOrientation(params.buffer);

  if (prefersSips()) {
    // macOS sips backend
    if (params.withoutEnlargement !== false) {
      // Check dimensions first
      const meta = await getImageMetadata(normalized);
      if (meta) {
        const maxDim = Math.max(meta.width, meta.height);
        if (maxDim > 0 && maxDim <= params.maxSide) {
          // No resize needed, just convert
          return await sipsConvertToJpeg(normalized);
        }
      }
    }

    return await sipsResizeToJpeg({
      buffer: normalized,
      maxSide: params.maxSide,
      quality: params.quality,
    });
  }

  // Sharp backend
  const sharp = await loadSharp();

  return await sharp(normalized)
    .rotate() // Auto-rotate from EXIF
    .resize({
      width: params.maxSide,
      height: params.maxSide,
      fit: "inside", // Preserve aspect ratio
      withoutEnlargement: params.withoutEnlargement !== false,
    })
    .jpeg({
      quality: params.quality,
      mozjpeg: true, // Better compression
    })
    .toBuffer();
}
```

### Iterative Optimization

**Algorithm:** Find smallest image meeting size constraint through grid search.

```typescript
async function optimizeImageToPng(
  buffer: Buffer,
  maxBytes: number,
): Promise<{
  buffer: Buffer;
  optimizedSize: number;
  resizeSide: number;
  compressionLevel: number;
}> {
  const sides = [2048, 1536, 1280, 1024, 800];
  const compressionLevels = [6, 7, 8, 9]; // PNG compression 0-9

  let smallest: {
    buffer: Buffer;
    size: number;
    resizeSide: number;
    compressionLevel: number;
  } | null = null;

  // Grid search over size/compression combinations
  for (const side of sides) {
    for (const compressionLevel of compressionLevels) {
      try {
        const out = await resizeToPng({
          buffer,
          maxSide: side,
          compressionLevel,
          withoutEnlargement: true,
        });

        const size = out.length;

        // Track smallest valid result
        if (!smallest || size < smallest.size) {
          smallest = { buffer: out, size, resizeSide: side, compressionLevel };
        }

        // Early exit if under limit
        if (size <= maxBytes) {
          return {
            buffer: out,
            optimizedSize: size,
            resizeSide: side,
            compressionLevel,
          };
        }
      } catch {
        // Continue trying other combinations
      }
    }
  }

  // Return smallest found even if over limit
  if (smallest) {
    return {
      buffer: smallest.buffer,
      optimizedSize: smallest.size,
      resizeSide: smallest.resizeSide,
      compressionLevel: smallest.compressionLevel,
    };
  }

  throw new Error("Failed to optimize PNG image");
}
```

### Backend Selection (Sharp vs sips)

```typescript
function prefersSips(): boolean {
  return (
    process.env.SOPHIACLAW_IMAGE_BACKEND === "sips" ||
    (process.env.SOPHIACLAW_IMAGE_BACKEND !== "sharp" && isBun() && process.platform === "darwin")
  );
}

function isBun(): boolean {
  return typeof (process.versions as { bun?: string }).bun === "string";
}

// Rationale:
// - Bun on macOS: prefer sips (native, no Sharp native module issues)
// - Node on macOS: prefer Sharp (better performance)
// - Explicit env var always wins
```

## Media Fetching

### URL Fetch with Size Limits

```typescript
interface FetchMediaOptions {
  maxBytes?: number;
  timeoutMs?: number;
  allowedMimeTypes?: string[];
  userAgent?: string;
}

async function fetchMedia(
  url: string,
  options: FetchMediaOptions = {},
): Promise<{
  buffer: Buffer;
  mimeType: string;
  size: number;
  filename?: string;
}> {
  const maxBytes = options.maxBytes ?? 25 * 1024 * 1024; // 25MB default

  const controller = new AbortController();
  const timeout = setTimeout(() => controller.abort(), options.timeoutMs ?? 30000);

  try {
    const response = await fetch(url, {
      signal: controller.signal,
      headers: {
        "User-Agent": options.userAgent ?? "SophiaClaw/1.0 Media Fetcher",
      },
    });

    if (!response.ok) {
      throw new Error(`HTTP ${response.status}: ${response.statusText}`);
    }

    // Check Content-Length header
    const contentLength = response.headers.get("content-length");
    if (contentLength && Number(contentLength) > maxBytes) {
      throw new Error(`Media too large: ${contentLength} bytes (max: ${maxBytes})`);
    }

    // Stream with size limit
    const chunks: Buffer[] = [];
    let totalBytes = 0;

    for await (const chunk of response.body as any) {
      totalBytes += chunk.length;
      chunks.push(Buffer.from(chunk));

      if (totalBytes > maxBytes) {
        throw new Error(`Media exceeds size limit: ${totalBytes} > ${maxBytes}`);
      }
    }

    const buffer = Buffer.concat(chunks);
    const mimeType = response.headers.get("content-type") ?? "application/octet-stream";

    // Validate MIME type if allowlist provided
    if (
      options.allowedMimeTypes &&
      !options.allowedMimeTypes.some((type) => mimeType.startsWith(type))
    ) {
      throw new Error(`Disallowed MIME type: ${mimeType}`);
    }

    return {
      buffer,
      mimeType,
      size: buffer.length,
      filename: extractFilenameFromUrl(url),
    };
  } finally {
    clearTimeout(timeout);
  }
}
```

### MIME Type Sniffing

```typescript
function sniffMimeType(buffer: Buffer): string | null {
  // Image formats
  if (buffer.length >= 3 && buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) {
    return "image/jpeg";
  }

  if (buffer.length >= 8 && buffer.toString("ascii", 1, 4) === "PNG") {
    return "image/png";
  }

  if (buffer.length >= 4 && buffer.toString("ascii", 0, 3) === "GIF") {
    return "image/gif";
  }

  // Audio formats
  if (
    buffer.length >= 4 &&
    buffer.toString("ascii", 0, 4) === "RIFF" &&
    buffer.toString("ascii", 8, 12) === "WAVE"
  ) {
    return "audio/wav";
  }

  if (buffer.length >= 3 && buffer[0] === 0xff && (buffer[1] & 0xe0) === 0xe0) {
    return "audio/mpeg"; // MP3
  }

  // Video formats
  if (
    buffer.length >= 4 &&
    buffer.toString("ascii", 0, 4) === "RIFF" &&
    buffer.toString("ascii", 8, 12) === "WEBP"
  ) {
    return "image/webp";
  }

  if (
    buffer.length >= 8 &&
    buffer.toString("ascii", 4, 8) === "ftyp" &&
    ["isom", "iso2", "avc1", "mp42"].includes(buffer.toString("ascii", 8, 12).trim())
  ) {
    return "video/mp4";
  }

  return null;
}
```

## Security Considerations

### Sandbox Validation

```typescript
interface SandboxConfig {
  allowedDirs: string[];
  deniedPatterns: RegExp[];
  maxFileSize: number;
}

function validateMediaAccess(
  filePath: string,
  sandbox: SandboxConfig,
): { allowed: boolean; error?: string } {
  // Resolve ~ and environment variables
  const resolved = resolveUserPath(filePath);
  const absolute = path.resolve(resolved);

  // Must be within allowed directories
  const inAllowedDir = sandbox.allowedDirs.some((dir) => {
    const resolvedDir = path.resolve(dir);
    return absolute.startsWith(resolvedDir + path.sep) || absolute === resolvedDir;
  });

  if (!inAllowedDir) {
    return {
      allowed: false,
      error: `Path outside allowed directories: ${absolute}`,
    };
  }

  // Check denied patterns
  for (const pattern of sandbox.deniedPatterns) {
    if (pattern.test(absolute)) {
      return {
        allowed: false,
        error: `Path matches denied pattern: ${absolute}`,
      };
    }
  }

  // Check file size
  try {
    const stats = fs.statSync(absolute);
    if (stats.size > sandbox.maxFileSize) {
      return {
        allowed: false,
        error: `File too large: ${stats.size} bytes (max: ${sandbox.maxFileSize})`,
      };
    }
  } catch (err) {
    return {
      allowed: false,
      error: `Cannot access file: ${err instanceof Error ? err.message : "Unknown"}`,
    };
  }

  return { allowed: true };
}
```

## Testing Strategy

### Image Processing Tests

```typescript
describe("normalizeExifOrientation", () => {
  it("rotates image with EXIF orientation 6 (90° CW)", async () => {
    const buffer = await createTestJpeg({
      width: 100,
      height: 200,
      orientation: 6, // Rotate 90° CW
    });

    const normalized = await normalizeExifOrientation(buffer);
    const meta = await getImageMetadata(normalized);

    // After rotation: width=200, height=100
    expect(meta?.width).toBe(200);
    expect(meta?.height).toBe(100);
  });

  it("handles non-JPEG images gracefully", async () => {
    const pngBuffer = await createTestPng({ width: 100, height: 100 });

    const normalized = await normalizeExifOrientation(pngBuffer);

    // PNG returned unchanged (no EXIF)
    expect(normalized).toEqual(pngBuffer);
  });
});

describe("optimizeImageToPng", () => {
  it("finds combination under size limit", async () => {
    const largeBuffer = await createTestJpeg({
      width: 4000,
      height: 4000,
      quality: 95,
    });

    const result = await optimizeImageToPng(largeBuffer, 500000); // 500KB limit

    expect(result.optimizedSize).toBeLessThanOrEqual(500000);
    expect(result.resizeSide).toBeLessThanOrEqual(2048);
  });
});
```

## Related Documentation

- [Attachment Normalizer](/docs/algorithms/media/attachment-normalizer.md)
- [MIME Type Detection](/docs/media/mime-detection.md)
- [Provider Image Limits](/docs/providers/image-limits.md)