# Attachment Normalizer

**Source:** `src/media/parse.ts`, `src/media/sniff-mime-from-base64.ts`, `src/media/constants.ts`  
**Category:** Media Processing  
**Purpose:** Normalize diverse attachment formats into provider-compatible structures

## Overview

The Attachment Normalizer processes user-submitted attachments from multiple sources (messaging platforms, file uploads, base64 data URLs) and converts them into a standardized format suitable for AI provider APIs. It handles MIME type detection, format conversion, and size optimization.

```
┌─────────────────────────────────────────────────────────────┐
│               User Attachment Input                         │
│  - WhatsApp: Image with caption                             │
│  - Telegram: Document + photo                               │
│  - Slack: File upload                                       │
│  - Direct: base64 data URL                                  │
└─────────────────────┬───────────────────────────────────────┘
                      │
                      v
┌─────────────────────────────────────────────────────────────┐
│              Attachment Normalizer                          │
│                                                             │
│  ┌───────────────────────────────────────────────────────┐ │
│  │  Format Detection                                     │ │
│  │  - Sniff MIME from magic bytes                        │ │
│  │  - Parse data URLs                                    │ │
│  │  - Extract from platform metadata                     │ │
│  └───────────────────────────────────────────────────────┘ │
│                          │                                  │
│  ┌───────────────────────▼───────────────────────────────┐ │
│  │  Format Conversion                                    │ │
│  │  - HEIC → JPEG (iOS photos)                           │ │
│  │  - WEBP → JPEG (provider compatibility)               │ │
│  │  - PNG → JPEG (when no alpha needed)                  │ │
│  └───────────────────────────────────────────────────────┘ │
│                          │                                  │
│  ┌───────────────────────▼───────────────────────────────┐ │
│  │  Size Optimization                                    │ │
│  │  - Resize to provider limits                          │ │
│  │  - Compress to target quality                         │ │
│  │  - Iterative quality reduction                        │ │
│  └───────────────────────────────────────────────────────┘ │
└─────────────────────┬───────────────────────────────────────┘
                      │
                      v
┌─────────────────────────────────────────────────────────────┐
│              Normalized Attachment                          │
│  {                                                          │
│    type: "image" | "audio" | "document" | "video"           │
│    mimeType: "image/jpeg"                                   │
│    data: Buffer | base64 string                             │
│    size?: number                                            │
│    dimensions?: { width, height }                           │
│    duration?: number                                        │
│    filename?: string                                        │
│  }                                                          │
└─────────────────────────────────────────────────────────────┘
```

## Input Format Detection

### Data URL Parsing

```typescript
interface ParsedDataUrl {
  mimeType: string;
  base64: boolean;
  data: Buffer;
  filename?: string;
}

function parseDataUrl(dataUrl: string): ParsedDataUrl | null {
  // Format: data:[<mime>][;base64],<data>
  const match = dataUrl.match(/^data:([^;,]+)(;base64)?,(.*)$/);

  if (!match) {
    return null;
  }

  const mimeType = match[1];
  const isBase64 = match[2] === ";base64";
  const rawData = match[3];

  // Decode based on encoding
  const data = isBase64
    ? Buffer.from(rawData, "base64")
    : Buffer.from(decodeURIComponent(rawData), "utf-8");

  return {
    mimeType,
    base64: isBase64,
    data,
  };
}

// Examples:
// data:image/jpeg;base64,/9j/4AAQSkZ... → JPEG image
// data:text/plain,Hello%20World       → Plain text (URL-encoded)
// data:audio/wav;base64,UklGRi...     → WAV audio
```

### Magic Byte Detection

```typescript
function sniffMimeType(buffer: Buffer): string | null {
  // JPEG: FF D8 FF
  if (buffer.length >= 3 && buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) {
    return "image/jpeg";
  }

  // PNG: 89 50 4E 47 0D 0A 1A 0A
  if (
    buffer.length >= 8 &&
    buffer[0] === 0x89 &&
    buffer.toString("ascii", 1, 4) === "PNG" &&
    buffer[5] === 0x0d &&
    buffer[6] === 0x0a &&
    buffer[7] === 0x1a &&
    buffer[8] === 0x0a
  ) {
    return "image/png";
  }

  // GIF: 47 49 46 38 39 61 or 38 37 61
  if (
    buffer.length >= 6 &&
    buffer.toString("ascii", 0, 3) === "GIF" &&
    (buffer.toString("ascii", 3, 6) === "89a" || buffer.toString("ascii", 3, 6) === "87a")
  ) {
    return "image/gif";
  }

  // WEBP: RIFF....WEBP
  if (
    buffer.length >= 12 &&
    buffer.toString("ascii", 0, 4) === "RIFF" &&
    buffer.toString("ascii", 8, 12) === "WEBP"
  ) {
    return "image/webp";
  }

  // HEIC/HEIF: ftyp + brands
  if (buffer.length >= 12 && buffer.toString("ascii", 4, 8) === "ftyp") {
    const brands = buffer.toString("ascii", 8, 12);
    if (["heic", "heix", "heim", "heis", "mif1"].includes(brands)) {
      return "image/heic";
    }
  }

  // WAV: RIFF....WAVE
  if (
    buffer.length >= 12 &&
    buffer.toString("ascii", 0, 4) === "RIFF" &&
    buffer.toString("ascii", 8, 12) === "WAVE"
  ) {
    return "audio/wav";
  }

  // MP3: FF E2 or FF FB or FF F3
  if (
    buffer.length >= 2 &&
    buffer[0] === 0xff &&
    (buffer[1] === 0xe2 || buffer[1] === 0xfb || buffer[1] === 0xf3)
  ) {
    return "audio/mpeg";
  }

  // MP4: 00 00 00 18 66 74 79 70
  if (
    buffer.length >= 8 &&
    buffer[0] === 0x00 &&
    buffer[1] === 0x00 &&
    buffer[2] === 0x00 &&
    buffer.toString("ascii", 4, 8) === "ftyp"
  ) {
    return "video/mp4";
  }

  // PDF: 25 50 44 46 2D ( %PDF-)
  if (buffer.length >= 5 && buffer.toString("ascii", 0, 5) === "%PDF-") {
    return "application/pdf";
  }

  return null;
}
```

### Base64 MIME Sniffing

**Problem:** User submits raw base64 without data URL prefix.

```typescript
function sniffMimeFromBase64(base64String: string): string | null {
  try {
    // Decode first 12 bytes (enough for most magic bytes)
    const prefix = base64String.slice(0, 16); // ~12 bytes when decoded
    const buffer = Buffer.from(prefix, "base64");

    return sniffMimeType(buffer);
  } catch {
    return null;
  }
}

// Usage in normalization
async function normalizeAttachment(attachment: AttachmentInput): Promise<NormalizedAttachment> {
  let { mimeType, data } = attachment;

  // If MIME missing, sniff from content
  if (!mimeType && data) {
    mimeType =
      sniffMimeType(data) ??
      sniffMimeFromBase64(attachment.base64 ?? "") ??
      "application/octet-stream";
  }

  // ... continue with normalization
}
```

## Format Conversion

### HEIC to JPEG Conversion

**Problem:** iOS devices capture photos in HEIC format, but most AI providers only accept JPEG/PNG.

```typescript
async function convertHeicToJpeg(buffer: Buffer): Promise<Buffer> {
  if (prefersSips()) {
    // macOS sips backend (no sharp native module needed)
    return await withTempDir(async (dir) => {
      const input = path.join(dir, "in.heic");
      const output = path.join(dir, "out.jpg");

      await fs.writeFile(input, buffer);

      await runExec("/usr/bin/sips", ["-s", "format", "jpeg", input, "--out", output], {
        timeoutMs: 20_000,
        maxBuffer: 1024 * 1024,
      });

      return await fs.readFile(output);
    });
  }

  // Sharp backend
  const sharp = await loadSharp();
  return await sharp(buffer).jpeg({ quality: 90, mozjpeg: true }).toBuffer();
}

function shouldConvertHeic(attachment: AttachmentInput): boolean {
  return (
    attachment.mimeType?.toLowerCase().includes("heic") ||
    attachment.mimeType?.toLowerCase().includes("heif") ||
    attachment.mimeType?.toLowerCase() === "image/heif"
  );
}
```

### WEBP to JPEG Conversion

```typescript
async function convertWebpToJpeg(buffer: Buffer): Promise<Buffer> {
  const sharp = await loadSharp();

  return await sharp(buffer).jpeg({ quality: 90, mozjpeg: true }).toBuffer();
}

// When to convert:
// 1. Provider doesn't support WEBP (most don't)
// 2. Image has no alpha channel (transparency)
async function shouldConvertWebp(buffer: Buffer, provider: string): Promise<boolean> {
  // Provider support check
  const providersWithoutWebp = ["anthropic", "openai"];
  if (!providersWithoutWebp.includes(provider)) {
    return false;
  }

  // Check for alpha channel (if has alpha, keep as PNG instead)
  const hasAlpha = await checkAlphaChannel(buffer);
  return !hasAlpha; // Convert only if no transparency
}
```

### PNG with Alpha Handling

```typescript
async function handlePngWithAlpha(
  buffer: Buffer,
  provider: string,
): Promise<{ buffer: Buffer; mimeType: string }> {
  const hasAlpha = await checkAlphaChannel(buffer);

  if (!hasAlpha) {
    // No transparency - can convert to JPEG for smaller size
    const sharp = await loadSharp();
    const jpegBuffer = await sharp(buffer).jpeg({ quality: 90, mozjpeg: true }).toBuffer();

    return { buffer: jpegBuffer, mimeType: "image/jpeg" };
  }

  // Has transparency - keep as PNG or convert to PNG if not already
  return { buffer, mimeType: "image/png" };
}

async function checkAlphaChannel(buffer: Buffer): Promise<boolean> {
  try {
    const sharp = await loadSharp();
    const meta = await sharp(buffer).metadata();
    return meta.hasAlpha ?? false;
  } catch {
    return false;
  }
}
```

## Size Optimization

### Provider-Specific Limits

```typescript
const PROVIDER_LIMITS: Record<string, ProviderLimits> = {
  anthropic: {
    maxImageSizeMB: 5,
    maxImageDimension: 7990, // Max side length
    supportedFormats: ["image/jpeg", "image/png", "image/gif", "image/webp"],
  },
  openai: {
    maxImageSizeMB: 20,
    maxImageDimension: 4096,
    supportedFormats: ["image/jpeg", "image/png", "image/gif", "image/webp"],
  },
  gemini: {
    maxImageSizeMB: 20,
    maxImageDimension: 3072, // Recommended
    supportedFormats: ["image/jpeg", "image/png", "image/webp", "image/heic"],
  },
};

interface OptimizationResult {
  buffer: Buffer;
  mimeType: string;
  resizeSide?: number;
  quality?: number;
  originalSize: number;
  optimizedSize: number;
}

async function optimizeForProvider(
  buffer: Buffer,
  mimeType: string,
  provider: string,
): Promise<OptimizationResult> {
  const limits = PROVIDER_LIMITS[provider] ?? PROVIDER_LIMITS.anthropic;
  const originalSize = buffer.length;

  // Step 1: Check size limit
  if (originalSize <= limits.maxImageSizeMB * 1024 * 1024) {
    // Already under limit - check dimensions
    const meta = await getImageMetadata(buffer);

    if (!meta || Math.max(meta.width, meta.height) <= limits.maxImageDimension) {
      // No optimization needed
      return {
        buffer,
        mimeType,
        originalSize,
        optimizedSize: originalSize,
      };
    }
  }

  // Step 2: Convert unsupported formats
  let workingBuffer = buffer;
  let workingMimeType = mimeType;

  if (!limits.supportedFormats.includes(mimeType)) {
    if (mimeType.includes("heic") || mimeType.includes("heif")) {
      workingBuffer = await convertHeicToJpeg(buffer);
      workingMimeType = "image/jpeg";
    } else if (mimeType.includes("webp")) {
      workingBuffer = await convertWebpToJpeg(buffer);
      workingMimeType = "image/jpeg";
    }
  }

  // Step 3: Resize to fit dimension limit
  const meta = await getImageMetadata(workingBuffer);
  const maxSide = meta ? Math.max(meta.width, meta.height) : limits.maxImageDimension;

  if (maxSide > limits.maxImageDimension) {
    workingBuffer = await resizeToJpeg({
      buffer: workingBuffer,
      maxSide: limits.maxImageDimension,
      quality: 90,
    });
  }

  // Step 4: Iterative quality reduction if still over size
  let quality = 90;
  while (workingBuffer.length > limits.maxImageSizeMB * 1024 * 1024 && quality >= 35) {
    workingBuffer = await resizeToJpeg({
      buffer: workingBuffer,
      maxSide: limits.maxImageDimension,
      quality,
    });
    quality -= 5;
  }

  return {
    buffer: workingBuffer,
    mimeType: workingMimeType,
    resizeSide: maxSide,
    quality,
    originalSize,
    optimizedSize: workingBuffer.length,
  };
}
```

### Iterative Quality Reduction

```typescript
const IMAGE_REDUCE_QUALITY_STEPS = [85, 75, 65, 55, 45, 35] as const;

async function reduceImageQuality(
  buffer: Buffer,
  targetSizeBytes: number,
): Promise<{ buffer: Buffer; quality: number }> {
  let currentBuffer = buffer;

  for (const quality of IMAGE_REDUCE_QUALITY_STEPS) {
    currentBuffer = await resizeToJpeg({
      buffer: currentBuffer,
      maxSide: 2048, // Keep same dimensions
      quality,
    });

    if (currentBuffer.length <= targetSizeBytes) {
      return { buffer: currentBuffer, quality };
    }
  }

  // Return smallest even if over target
  return { buffer: currentBuffer, quality: 35 };
}
```

## Normalization Pipeline

### Full Pipeline Implementation

```typescript
async function normalizeAttachment(
  input: AttachmentInput,
  options: NormalizeOptions = {},
): Promise<NormalizedAttachment> {
  const provider = options.provider ?? "anthropic";

  // Step 1: Parse data URL or extract from platform format
  let data: Buffer;
  let mimeType: string | undefined;
  let filename: string | undefined;

  if (input.dataUrl) {
    const parsed = parseDataUrl(input.dataUrl);
    if (!parsed) {
      throw new Error("Invalid data URL format");
    }
    data = parsed.data;
    mimeType = parsed.mimeType;
    filename = input.filename;
  } else if (input.buffer) {
    data = input.buffer;
    mimeType = input.mimeType;
    filename = input.filename;
  } else if (input.base64) {
    // Sniff MIME from base64 prefix
    mimeType = sniffMimeFromBase64(input.base64) ?? input.mimeType;
    data = Buffer.from(input.base64, "base64");
    filename = input.filename;
  } else {
    throw new Error("No attachment data provided");
  }

  // Step 2: Detect MIME if not provided
  if (!mimeType) {
    mimeType = sniffMimeType(data) ?? "application/octet-stream";
  }

  // Step 3: Convert HEIC/HEIF to JPEG
  if (mimeType.includes("heic") || mimeType.includes("heif")) {
    data = await convertHeicToJpeg(data);
    mimeType = "image/jpeg";
  }

  // Step 4: Apply EXIF orientation correction (JPEG only)
  if (mimeType === "image/jpeg") {
    data = await normalizeExifOrientation(data);
  }

  // Step 5: Optimize for provider
  const optimization = await optimizeForProvider(data, mimeType, provider);
  data = optimization.buffer;
  mimeType = optimization.mimeType;

  // Step 6: Extract metadata
  let dimensions: { width: number; height: number } | undefined;
  if (mimeType.startsWith("image/")) {
    const meta = await getImageMetadata(data);
    if (meta) {
      dimensions = { width: meta.width, height: meta.height };
    }
  }

  // Step 7: Return normalized attachment
  return {
    type: classifyAttachmentType(mimeType),
    mimeType,
    data,
    base64: data.toString("base64"),
    size: data.length,
    dimensions,
    filename,
    optimized: optimization.originalSize !== optimization.optimizedSize,
  };
}

function classifyAttachmentType(mimeType: string): AttachmentType {
  if (mimeType.startsWith("image/")) return "image";
  if (mimeType.startsWith("audio/")) return "audio";
  if (mimeType.startsWith("video/")) return "video";
  return "document";
}
```

### Platform-Specific Extraction

```typescript
interface PlatformAttachment {
  platform: "whatsapp" | "telegram" | "slack" | "discord";
  raw: any;
}

function extractFromPlatform(platformAttachment: PlatformAttachment): AttachmentInput {
  switch (platformAttachment.platform) {
    case "whatsapp": {
      const raw = platformAttachment.raw;
      return {
        mimeType: raw.mimetype,
        filename: raw.filename,
        base64: raw.body, // WhatsApp provides base64 directly
      };
    }

    case "telegram": {
      const raw = platformAttachment.raw;
      return {
        mimeType: raw.mime_type,
        filename: raw.file_name,
        fileId: raw.file_id, // Need to download via Bot API
      };
    }

    case "slack": {
      const raw = platformAttachment.raw;
      return {
        mimeType: raw.mimetype,
        filename: raw.name,
        url: raw.url_private, // Need to download with auth
      };
    }

    default:
      throw new Error(`Unsupported platform: ${platformAttachment.platform}`);
  }
}
```

## Security Considerations

### File Type Validation

```typescript
const ALLOWED_ATTACHMENT_TYPES = [
  "image/jpeg",
  "image/png",
  "image/gif",
  "image/webp",
  "audio/wav",
  "audio/mpeg",
  "video/mp4",
  "application/pdf",
  "text/plain",
];

function validateAttachmentType(mimeType: string): { valid: boolean; error?: string } {
  if (!ALLOWED_ATTACHMENT_TYPES.includes(mimeType)) {
    return {
      valid: false,
      error: `Unsupported attachment type: ${mimeType}`,
    };
  }

  // Check for potentially dangerous types
  const dangerousTypes = [
    "application/x-executable",
    "application/x-msdownload",
    "application/x-shellscript",
  ];

  if (dangerousTypes.some((type) => mimeType.includes(type))) {
    return {
      valid: false,
      error: "Executable attachments are not allowed",
    };
  }

  return { valid: true };
}
```

### Size Limits

```typescript
const MAX_ATTACHMENT_SIZE_BYTES = 25 * 1024 * 1024; // 25MB

function validateAttachmentSize(sizeBytes: number): { valid: boolean; error?: string } {
  if (sizeBytes > MAX_ATTACHMENT_SIZE_BYTES) {
    return {
      valid: false,
      error: `Attachment too large: ${(sizeBytes / 1024 / 1024).toFixed(2)}MB (max: 25MB)`,
    };
  }

  return { valid: true };
}
```

## Testing Strategy

### Format Detection Tests

```typescript
describe("sniffMimeType", () => {
  it("detects JPEG from magic bytes", () => {
    const jpegHeader = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10]);
    expect(sniffMimeType(jpegHeader)).toBe("image/jpeg");
  });

  it("detects PNG from magic bytes", () => {
    const pngHeader = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
    expect(sniffMimeType(pngHeader)).toBe("image/png");
  });

  it("detects HEIC from ftyp header", () => {
    const heicHeader = Buffer.from([
      0x00,
      0x00,
      0x00,
      0x18, // size
      0x66,
      0x74,
      0x79,
      0x70, // 'ftyp'
      0x68,
      0x65,
      0x69,
      0x63, // 'heic'
    ]);
    expect(sniffMimeType(heicHeader)).toBe("image/heic");
  });
});

describe("parseDataUrl", () => {
  it("parses base64 data URL", () => {
    const dataUrl = "data:image/jpeg;base64,SGVsbG8gV29ybGQ=";
    const result = parseDataUrl(dataUrl);

    expect(result?.mimeType).toBe("image/jpeg");
    expect(result?.base64).toBe(true);
    expect(result?.data.toString()).toBe("Hello World");
  });

  it("parses URL-encoded data URL", () => {
    const dataUrl = "data:text/plain,Hello%20World";
    const result = parseDataUrl(dataUrl);

    expect(result?.mimeType).toBe("text/plain");
    expect(result?.base64).toBe(false);
    expect(result?.data.toString()).toBe("Hello World");
  });
});
```

## Related Documentation

- [Media Pipeline](/docs/algorithms/media/media-pipeline.md)
- [Image Operations](/docs/media/image-ops.md)
- [Provider Limits](/docs/providers/image-limits.md)