#!/usr/bin/env python3
"""
FRANKENSTEIN EMBEDDINGS v5 - TRULY DYNAMIC Dimension System

TODO: This file is 1000+ lines - consider splitting into frankenstein_embeddings.py, dimension_manager.py, model_loader.py

NO HARDCODED DIMENSIONS - queries PostgreSQL for target dimension!

Features:
1. Base Model: all-MiniLM-L6-v2 (80MB, 384 native dims)
2. DYNAMIC DIMENSION: Queries database for target dimension on startup
3. 60-SECOND REFRESH: Detects database dimension changes without restart
4. EXPANSION: Expands from native dims to ANY target dimension
5. COMPRESSION: PCA reduction when target < native dims
6. RAM Guard: Auto-throttles to stay under 4GB
7. QQMS Throttling: CPU-aware rate limiting

The database is the SINGLE SOURCE OF TRUTH for dimensions.
No dimension constants in the code - all queried at runtime.

Protocol:
- {"text": "..."}  -> Single embedding at database dimension
- {"texts": [...]} -> Batch embeddings
- {"dims": N}      -> Force specific dimension
- {"stats": true}  -> Get statistics
- {"refresh_dimension": true} -> Force dimension refresh from database

@author hardwicksoftwareservices
"""

# ============================================================================
# CRITICAL: Handle SIGPIPE and redirect output to prevent silent death
# This MUST be done before any imports or print statements!
# ============================================================================
import signal
import sys
import os
import socket

# Ignore SIGPIPE - prevents death when parent closes stdout/stderr pipes
# SIG_IGN = ignore the signal completely (SIG_DFL would still kill us!)
signal.signal(signal.SIGPIPE, signal.SIG_IGN)

def _setup_daemon_io():
    """
    Set up I/O for daemon/service mode.
    - Close all inherited FDs except 0,1,2 (prevents SIGPIPE from inherited pipes)
    - Redirect stdin from /dev/null
    - Redirect stdout/stderr to log file (at FD level for C code compatibility)

    NOTE: We do NOT double-fork because MCP server tracks our PID.
    Instead, we just fix the I/O issues that cause SIGPIPE.
    """
    is_service_mode = '--service' in sys.argv
    is_not_tty = not sys.stdout.isatty() or not sys.stderr.isatty()

    if not (is_service_mode or is_not_tty):
        return  # Interactive mode - don't modify I/O

    # Get log file path
    socket_dir = os.environ.get('SPECMEM_SOCKET_DIR') or os.path.join(
        os.environ.get('SPECMEM_PROJECT_PATH', os.getcwd()), 'specmem', 'sockets'
    )
    log_file = os.path.join(socket_dir, 'embedding-autostart.log')

    try:
        os.makedirs(os.path.dirname(log_file), exist_ok=True)

        # Close ALL inherited file descriptors EXCEPT 0,1,2
        # This is CRITICAL - inherited pipes from parent cause SIGPIPE
        max_fd = 1024
        try:
            max_fd = os.sysconf('SC_OPEN_MAX')
        except (AttributeError, ValueError):
            pass
        for fd in range(3, min(max_fd, 1024)):
            try:
                os.close(fd)
            except OSError:
                pass

        # Redirect stdin from /dev/null
        try:
            dev_null = os.open('/dev/null', os.O_RDONLY)
            os.dup2(dev_null, 0)
            os.close(dev_null)
        except OSError:
            pass

        # Redirect stdout/stderr to log file at FD level
        # This ensures C code (torch, etc.) also writes to log file
        log_fd = os.open(log_file, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o644)
        os.dup2(log_fd, 1)  # stdout -> log file
        os.dup2(log_fd, 2)  # stderr -> log file
        os.close(log_fd)

        # Recreate Python's sys.stdout/stderr with the new file descriptors
        sys.stdout = os.fdopen(1, 'w', buffering=1)
        sys.stderr = os.fdopen(2, 'w', buffering=1)

    except Exception as e:
        # If setup fails, try to log the error
        try:
            with open('/tmp/frankenstein-io-setup-error.log', 'a') as f:
                f.write(f"{e}\n")
        except:
            pass

# Set up I/O FIRST before any other imports (which might print)
_setup_daemon_io()

# ============================================================================
# AUTO-INSTALL MISSING DEPENDENCIES
# ============================================================================
def _auto_install_deps():
    """Install missing Python packages automatically."""
    import subprocess
    import sys as _sys
    import os as _os

    # Step 0: Ensure pip itself is available
    try:
        __import__('pip')
    except ImportError:
        print("📦 pip not found — installing python3-pip...")
        _is_root = _os.getuid() == 0 if hasattr(_os, 'getuid') else False
        try:
            # Try ensurepip first (no sudo needed)
            subprocess.check_call([_sys.executable, '-m', 'ensurepip', '--upgrade'],
                                  stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
            print("   ✓ pip installed via ensurepip")
        except (subprocess.CalledProcessError, FileNotFoundError):
            # Need apt-get install python3-pip
            # Never prompt for sudo during runtime — use -n (non-interactive)
            _prefix = '' if _is_root else 'sudo -n '
            try:
                subprocess.check_call(f'{_prefix}apt-get update -qq && {_prefix}apt-get install -y python3-pip',
                                      shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=120)
                print("   ✓ pip installed via apt-get")
            except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError) as e:
                print(f"   ✗ Could not install pip (no sudo access): {e}")
                print("   Run manually: sudo apt-get install python3-pip")
                print("   Or run: specmem init  (which handles sudo properly)")
                _sys.exit(1)

    REQUIRED_PACKAGES = [
        ('sentence_transformers', 'sentence-transformers'),
        ('torch', 'torch'),
        ('numpy', 'numpy'),
        ('psycopg2', 'psycopg2-binary'),
    ]

    missing = []
    for import_name, pip_name in REQUIRED_PACKAGES:
        try:
            __import__(import_name)
        except ImportError:
            missing.append(pip_name)

    if missing:
        print(f"📦 Auto-installing missing packages: {', '.join(missing)}")
        _is_root = _os.getuid() == 0 if hasattr(_os, 'getuid') else False
        for pkg in missing:
            try:
                _pip_args = [_sys.executable, '-m', 'pip', 'install',
                    '--break-system-packages', '--quiet']
                if not _is_root:
                    _pip_args.append('--user')
                _pip_args.append(pkg)
                subprocess.check_call(_pip_args)
                print(f"   ✓ Installed {pkg}")
            except subprocess.CalledProcessError as e:
                print(f"   ✗ Failed to install {pkg}: {e}")

_auto_install_deps()

import os
import hashlib
import re
import signal
import sys

# Fix BrokenPipeError when parent process dies - ignore SIGPIPE
signal.signal(signal.SIGPIPE, signal.SIG_IGN)

def _safe_print(msg, file=None):
    """Print that ignores BrokenPipeError when parent dies"""
    try:
        print(msg, file=file or sys.stderr)
    except BrokenPipeError:
        pass  # Parent process died, nothing to do
    except Exception:
        pass  # Any other I/O error, just continue

# Project identification for multi-instance isolation
def get_project_dir_name():
    """Get sanitized project directory name for readable container/path naming."""
    project_path = os.environ.get('SPECMEM_PROJECT_PATH', os.getcwd())
    dir_name = os.path.basename(project_path).lower()
    # Sanitize for Docker: only a-z, 0-9, underscore, dash, dot
    dir_name = re.sub(r'[^a-z0-9_.-]', '-', dir_name)
    dir_name = re.sub(r'-+', '-', dir_name)  # collapse multiple dashes
    dir_name = dir_name.strip('-')
    return dir_name or 'default'

def get_project_hash():
    """Generate a unique 12-char hash (kept for backwards compat)."""
    project_path = os.environ.get('SPECMEM_PROJECT_PATH', os.getcwd())
    return hashlib.sha256(project_path.encode()).hexdigest()[:12]

def get_project_instance_dir():
    """Get the project-specific instance directory using readable dir name."""
    dir_name = get_project_dir_name()
    return os.path.expanduser(f"~/.specmem/instances/{dir_name}")

# Project isolation globals - USE READABLE DIR NAME!
PROJECT_DIR_NAME = get_project_dir_name()
PROJECT_HASH = get_project_hash()  # kept for backwards compat
PROJECT_PATH = os.environ.get('SPECMEM_PROJECT_PATH', 'default')

SPECMEM_HOME = os.environ.get('SPECMEM_HOME', os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
SPECMEM_RUN_DIR = os.environ.get('SPECMEM_RUN_DIR', os.path.join(SPECMEM_HOME, 'run'))

# Bundled model: shipped with the npm package, no download needed
_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
_BUNDLED_MODEL_DIR = os.path.join(_SCRIPT_DIR, 'models', 'all-MiniLM-L6-v2')
BUNDLED_MODEL_PATH = _BUNDLED_MODEL_DIR if os.path.isfile(os.path.join(_BUNDLED_MODEL_DIR, 'onnx', 'model_quint8_avx2.onnx')) else None
if BUNDLED_MODEL_PATH:
    print(f"📦 Bundled model found: {BUNDLED_MODEL_PATH}", file=sys.stderr)

# Container mode: embed.sock not embeddings.sock
_SPECMEM_CONTAINER_MODE = os.environ.get('SPECMEM_CONTAINER_MODE') == 'true'
_SOCKET_SUBDIR = 'run' if _SPECMEM_CONTAINER_MODE else 'sockets'
EMBEDDING_SOCKET_NAME = 'embed.sock' if _SPECMEM_CONTAINER_MODE else 'embeddings.sock'

# Socket directory: {PROJECT}/specmem/sockets/ - matches config.ts expectations
# This is the ONLY location config.ts checks for per-project sockets
def _get_socket_dir():
    project_path = os.environ.get('SPECMEM_PROJECT_PATH')
    if project_path:
        return os.path.join(project_path, 'specmem', _SOCKET_SUBDIR)
    # Fallback for standalone testing
    return os.path.join(get_project_instance_dir(), _SOCKET_SUBDIR)

SPECMEM_SOCKET_DIR = os.environ.get('SPECMEM_SOCKET_DIR', _get_socket_dir())

# Ensure socket directory exists
os.makedirs(SPECMEM_SOCKET_DIR, exist_ok=True)

import numpy as np
import json
import sys
import gc
import threading
import time
import resource
import hashlib
from typing import List, Dict, Tuple, Optional, Any
from pathlib import Path
from dataclasses import dataclass, field
from collections import deque
from queue import Queue, PriorityQueue
from enum import IntEnum
import subprocess

# QQMS v2 - Enhanced queue with FIFO + ACK for low-resource environments
try:
    from qqms_v2 import QQMSv2, QQMSv2Config, Priority as QQMSPriority
    HAS_QQMS_V2 = True
except ImportError:
    HAS_QQMS_V2 = False
    print("ℹ️ QQMS v2 not available - using legacy throttler", file=sys.stderr)

# Check dependencies
try:
    from sentence_transformers import SentenceTransformer
    from sklearn.decomposition import PCA, IncrementalPCA
    from sklearn.random_projection import SparseRandomProjection
    import torch
except ImportError as e:
    print(f"Missing dependency: {e}", file=sys.stderr)
    print("Install: pip install sentence-transformers scikit-learn torch", file=sys.stderr)
    sys.exit(1)

# ============================================================================
# CPU THREAD LIMITING - Without this, PyTorch uses ALL cores (200%+ CPU!)
# ============================================================================
# QQMS only adds delays between requests, but model.encode() runs unrestricted.
# This is the ACTUAL fix for high CPU usage.
#
# Priority order for CPU core limits:
#   1. SPECMEM_CPU_THREADS env var (direct override)
#   2. user-config.json resources.cpuCoreMax (set via console cpucoremax command)
#   3. Default: 2 threads
def _get_cpu_thread_limit():
    """Get CPU thread limit from env or user-config.json"""
    # Check env var first (highest priority)
    if os.environ.get('SPECMEM_CPU_THREADS'):
        return int(os.environ['SPECMEM_CPU_THREADS'])

    # Try to read from user-config.json
    try:
        config_path = os.path.join(PROJECT_PATH, 'specmem', 'user-config.json')
        if os.path.exists(config_path):
            with open(config_path, 'r') as f:
                config = json.load(f)
                core_max = config.get('resources', {}).get('cpuCoreMax')
                if core_max is not None:
                    return int(core_max)
    except Exception as e:
        print(f"⚠️ Could not read CPU core limit from config: {e}", file=sys.stderr)

    # Default - keep low to avoid CPU spikes with multiple servers
    return 1

_CPU_THREAD_LIMIT = _get_cpu_thread_limit()
_CPU_THREAD_MIN = int(os.environ.get('SPECMEM_CPU_THREADS_MIN', '1'))
torch.set_num_threads(_CPU_THREAD_LIMIT)
torch.set_num_interop_threads(1)  # Limit cross-op parallelism to prevent CPU spikes
# Also limit OpenMP/MKL threads used by numpy/sklearn
os.environ.setdefault('OMP_NUM_THREADS', str(_CPU_THREAD_LIMIT))
os.environ.setdefault('MKL_NUM_THREADS', str(_CPU_THREAD_LIMIT))
os.environ.setdefault('NUMEXPR_NUM_THREADS', str(_CPU_THREAD_LIMIT))
os.environ.setdefault('OPENBLAS_NUM_THREADS', str(_CPU_THREAD_LIMIT))
print(f"🔒 CPU threads: {_CPU_THREAD_MIN}-{_CPU_THREAD_LIMIT} (cpucoremin/cpucoremax to adjust)", file=sys.stderr)

# ============================================================================
# ONNX FILE SELECTION - Auto-detect best quantized model for CPU
# ============================================================================
def _detect_best_onnx_file():
    """
    Detect CPU features and return the best ONNX model file name.
    Priority: avx512_vnni > avx512 > avx2 > default
    Falls back to whatever .onnx file exists if the optimal one isn't found.
    """
    # Ordered by preference (best first)
    candidates = []

    try:
        with open('/proc/cpuinfo', 'r') as f:
            cpuinfo = f.read().lower()

        if 'avx512_vnni' in cpuinfo or 'avx512vnni' in cpuinfo:
            candidates.append(("onnx/model_qint8_avx512_vnni.onnx", "AVX512-VNNI"))
        if 'avx512f' in cpuinfo or 'avx512' in cpuinfo:
            candidates.append(("onnx/model_qint8_avx512.onnx", "AVX512"))
        if 'avx2' in cpuinfo:
            candidates.append(("onnx/model_quint8_avx2.onnx", "AVX2"))
    except Exception as e:
        print(f"⚠️ Could not detect CPU features: {e}", file=sys.stderr)

    # Always add standard fallbacks
    candidates.append(("onnx/model_quantized.onnx", "quantized"))
    candidates.append(("onnx/model.onnx", "default"))

    # Check which files actually exist in the bundled model dir
    bundled_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'models', 'all-MiniLM-L6-v2')
    for onnx_file, label in candidates:
        full_path = os.path.join(bundled_dir, onnx_file)
        if os.path.isfile(full_path):
            print(f"🚀 Using {label} ONNX model: {onnx_file}", file=sys.stderr)
            return onnx_file

    # Last resort: find ANY .onnx file in the bundled dir
    onnx_dir = os.path.join(bundled_dir, 'onnx')
    if os.path.isdir(onnx_dir):
        for f in os.listdir(onnx_dir):
            if f.endswith('.onnx'):
                result = f"onnx/{f}"
                print(f"🔍 Auto-detected ONNX model: {result}", file=sys.stderr)
                return result

    # Nothing found - return default and let SentenceTransformer handle it
    print("ℹ️ No bundled ONNX model found - using default", file=sys.stderr)
    return "onnx/model.onnx"

_BEST_ONNX_FILE = _detect_best_onnx_file()


class EmbeddingPriority(IntEnum):
    """Priority levels for embedding requests - lower = higher priority"""
    CRITICAL = 0    # Real-time search queries
    HIGH = 1        # Active user interactions
    MEDIUM = 2      # Background indexing
    LOW = 3         # Batch processing, non-urgent
    TRIVIAL = 4     # Deferred processing


@dataclass
class QQMSConfig:
    """
    QQMS (Quantum-Quality Millisecond) Timing Configuration

    Controls throttling and rate limiting to prevent CPU spikes.
    Inspired by quantum-quality timing patterns that balance quality with performance.
    """
    # Base delay between requests (milliseconds)
    base_delay_ms: float = 50.0

    # Delay multiplier based on priority (higher priority = less delay)
    priority_delay_multiplier: Dict[int, float] = field(default_factory=lambda: {
        EmbeddingPriority.CRITICAL: 0.1,   # 5ms delay
        EmbeddingPriority.HIGH: 0.5,       # 25ms delay
        EmbeddingPriority.MEDIUM: 1.0,     # 50ms delay
        EmbeddingPriority.LOW: 2.0,        # 100ms delay
        EmbeddingPriority.TRIVIAL: 4.0     # 200ms delay
    })

    # CPU usage thresholds (percentage)
    cpu_low_threshold: float = 30.0      # Below this: run at full speed
    cpu_medium_threshold: float = 50.0   # Medium throttling
    cpu_high_threshold: float = 70.0     # High throttling
    cpu_critical_threshold: float = 85.0 # Emergency throttling

    # Rate limiting
    max_requests_per_second: float = 20.0    # Maximum RPS
    burst_limit: int = 10                     # Burst allowance

    # Batch processing
    batch_delay_ms: float = 25.0             # Delay between batches
    max_batch_size: int = 64                  # Maximum items per batch
    batch_cooldown_ms: float = 100.0         # Cooldown after large batch

    # Idle/cooldown
    idle_delay_after_burst_ms: float = 1000.0  # 1 second cooldown after burst


# ═══════════════════════════════════════════════════════════════════════════════
# SCORCHED EARTH OPTIMIZATIONS - ALL 4 OPTIMIZATIONS ENABLED BY DEFAULT
# We NEVER use a model that hasn't been optimized with all 4 optimizations
# ═══════════════════════════════════════════════════════════════════════════════

@dataclass
class ResourceConfig:
    """
    Resource management configuration with heavyOps support.
    All values can be overridden via environment variables.
    """
    # CPU limits (from env vars with defaults)
    cpu_min: float = float(os.environ.get('SPECMEM_CPU_MIN', '20'))
    cpu_max: float = float(os.environ.get('SPECMEM_CPU_MAX', '40'))

    # RAM limits (MB) - from env vars with defaults
    ram_min_mb: float = float(os.environ.get('SPECMEM_RAM_MIN_MB', '4000'))
    ram_max_mb: float = float(os.environ.get('SPECMEM_RAM_MAX_MB', '6000'))

    # Heavy Ops mode (from env vars) - BOOST on top of all optimizations
    heavy_ops_enabled: bool = os.environ.get('SPECMEM_HEAVY_OPS', '0') == '1'
    heavy_ops_batch_mult: float = float(os.environ.get('SPECMEM_HEAVY_OPS_BATCH_MULT', '2'))
    heavy_ops_throttle_reduce: float = float(os.environ.get('SPECMEM_HEAVY_OPS_THROTTLE_REDUCE', '0.20'))

    def get_effective_delay(self, base_delay_ms: float) -> float:
        """Get delay with heavyOps reduction applied"""
        if self.heavy_ops_enabled:
            return base_delay_ms * (1.0 - self.heavy_ops_throttle_reduce)
        return base_delay_ms

    def get_effective_batch_size(self, base_size: int) -> int:
        """Get batch size with heavyOps multiplier applied"""
        if self.heavy_ops_enabled:
            return int(base_size * self.heavy_ops_batch_mult)
        return base_size


# ═══════════════════════════════════════════════════════════════════════════════
# NEW OPTIMIZATIONS 5-8: LOW-RESOURCE ENVIRONMENT SUPPORT
# Power modes: LOW (default), MEDIUM, HIGH
# Set via CLI: `power low|medium|high` - persists in user-config.json
# ═══════════════════════════════════════════════════════════════════════════════

def get_system_ram_gb() -> float:
    """Auto-detect total system RAM in GB (for display only)."""
    try:
        with open('/proc/meminfo', 'r') as f:
            for line in f:
                if line.startswith('MemTotal:'):
                    kb = int(line.split()[1])
                    return kb / 1024 / 1024  # KB -> GB
    except:
        pass
    return 4.0


def get_available_ram_gb() -> float:
    """Get currently available RAM in GB (for display only)."""
    try:
        with open('/proc/meminfo', 'r') as f:
            for line in f:
                if line.startswith('MemAvailable:'):
                    kb = int(line.split()[1])
                    return kb / 1024 / 1024
    except:
        pass
    return 1.0


def _read_power_mode_from_config() -> str:
    """
    Read power mode from user-config.json.
    Returns 'low', 'medium', or 'high'. Defaults to 'low'.
    """
    # Try user-config.json first (persists across updates)
    try:
        config_path = os.path.join(PROJECT_PATH, 'specmem', 'user-config.json')
        if os.path.exists(config_path):
            with open(config_path, 'r') as f:
                config = json.load(f)
                level = config.get('powerMode', {}).get('level')
                if level in ('low', 'medium', 'high'):
                    return level
    except:
        pass

    # Fallback: check model-config.json
    try:
        config_path = os.path.join(PROJECT_PATH, 'specmem', 'model-config.json')
        if os.path.exists(config_path):
            with open(config_path, 'r') as f:
                config = json.load(f)
                level = config.get('powerMode', {}).get('level')
                if level in ('low', 'medium', 'high'):
                    return level
    except:
        pass

    # DEFAULT TO LOW FOR TESTING
    return 'low'


@dataclass
class LowResourceConfig:
    """
    POWER MODE OPTIMIZATION CONFIG

    Explicit power modes (set via CLI `power <low|medium|high>`):
    - LOW:    <8GB settings - lazy loading, disk cache, aggressive cleanup (DEFAULT)
    - MEDIUM: 8-16GB settings - balanced performance
    - HIGH:   16GB+ settings - max performance, minimal restrictions

    Persists in user-config.json across restarts and version updates.
    """
    # System info (for display only, not used for mode selection)
    system_ram_gb: float = field(default_factory=get_system_ram_gb)
    available_ram_gb: float = field(default_factory=get_available_ram_gb)

    # Optimization toggles (set by power mode)
    layer_offloading: bool = False      # OPT-5: Load layers one at a time
    lazy_loading: bool = True           # OPT-6: Don't load until first request
    aggressive_cleanup: bool = True     # OPT-7: Unload model during idle
    disk_cache_enabled: bool = True     # OPT-8: Cache embeddings to SSD

    # Thresholds
    idle_unload_seconds: int = 120      # Unload model after idle
    disk_cache_max_mb: int = 300        # Max disk cache size

    # Mode (for logging)
    mode: str = "LOW"

    def __post_init__(self):
        """Configure based on power mode from config file (not RAM detection)"""
        power_mode = _read_power_mode_from_config()

        if power_mode == 'high':
            # HIGH MODE: Max performance, no restrictions
            self.mode = "HIGH"
            self.layer_offloading = False
            self.lazy_loading = False  # Load model immediately
            self.aggressive_cleanup = False  # Keep model in RAM always
            self.disk_cache_enabled = False  # RAM only, no disk I/O
            self.idle_unload_seconds = 0  # Never unload
            self.disk_cache_max_mb = 0

        elif power_mode == 'medium':
            # MEDIUM MODE: Balanced (8-16GB equivalent)
            self.mode = "MEDIUM"
            self.layer_offloading = False
            self.lazy_loading = True
            self.aggressive_cleanup = True
            self.disk_cache_enabled = True
            self.idle_unload_seconds = 0  # Never unload - keep model warm
            self.disk_cache_max_mb = 500

        else:
            # LOW MODE (default): Conservative (<8GB equivalent)
            self.mode = "LOW"
            self.layer_offloading = False
            self.lazy_loading = True
            self.aggressive_cleanup = True
            self.disk_cache_enabled = True
            self.idle_unload_seconds = 0  # Never unload - keep model warm
            self.disk_cache_max_mb = 300

    def log_config(self):
        """Log the power mode configuration"""
        print(f"", file=sys.stderr)
        print(f"═══════════════════════════════════════════════════════════════", file=sys.stderr)
        print(f"  POWER MODE: {self.mode}", file=sys.stderr)
        print(f"  (Set via CLI: power low|medium|high)", file=sys.stderr)
        print(f"═══════════════════════════════════════════════════════════════", file=sys.stderr)
        print(f"  System RAM:      {self.system_ram_gb:.1f} GB (detected)", file=sys.stderr)
        print(f"  Available RAM:   {self.available_ram_gb:.1f} GB", file=sys.stderr)
        print(f"  ─────────────────────────────────────────────────────────────", file=sys.stderr)
        print(f"  Lazy Loading:       {'✅ ON' if self.lazy_loading else '❌ OFF'}", file=sys.stderr)
        print(f"  Disk Cache:         {'✅ ON' if self.disk_cache_enabled else '❌ OFF'} ({self.disk_cache_max_mb}MB)", file=sys.stderr)
        print(f"  Aggressive Cleanup: {'✅ ON' if self.aggressive_cleanup else '❌ OFF'} ({self.idle_unload_seconds}s idle)", file=sys.stderr)
        print(f"═══════════════════════════════════════════════════════════════", file=sys.stderr)
        print(f"", file=sys.stderr)


class DiskBackedEmbeddingCache:
    """
    OPT-8: DISK-BACKED EMBEDDING CACHE

    Stores computed embeddings on SSD instead of RAM.
    Perfect for low-RAM systems - computed embeddings go to disk,
    only hot cache entries stay in RAM.

    Features:
    - LRU eviction with configurable max size
    - Content-addressable (hash of text = key)
    - Hot entries promoted to small RAM cache
    - Auto-cleanup of stale entries
    - THREAD-SAFE: All operations are protected by locks
    """

    def __init__(self, cache_dir: Path, max_mb: int = 500, ram_cache_size: int = 100):
        self.cache_dir = cache_dir / "embedding_cache"
        self.cache_dir.mkdir(parents=True, exist_ok=True)
        self.max_bytes = max_mb * 1024 * 1024
        self.ram_cache_size = ram_cache_size

        # THREAD SAFETY: Locks for concurrent access
        self._ram_cache_lock = threading.Lock()
        self._disk_lock = threading.Lock()
        self._index_lock = threading.Lock()

        # Small RAM cache for hot entries (LRU)
        from collections import OrderedDict
        self.ram_cache: OrderedDict = OrderedDict()

        # Index file for quick lookups
        self.index_path = self.cache_dir / "index.json"
        self.index: Dict[str, Dict] = {}
        self._load_index()

        # Stats (atomic-ish, not critical)
        self.hits = 0
        self.misses = 0
        self.disk_writes = 0

        print(f"💾 Disk cache initialized: {self.cache_dir} (max {max_mb}MB)", file=sys.stderr)

    def _load_index(self):
        """Load cache index from disk. Called only during __init__."""
        try:
            if self.index_path.exists():
                with open(self.index_path, 'r') as f:
                    self.index = json.load(f)
        except Exception as e:
            # Start fresh on any error
            self.index = {}

    def _save_index(self):
        """Save cache index to disk using atomic write. THREAD-SAFE."""
        try:
            with self._index_lock:
                # Make a copy to avoid holding lock during I/O
                index_copy = dict(self.index)

            # Atomic write: write to temp file, then rename
            temp_path = self.index_path.with_suffix('.tmp')
            with open(temp_path, 'w') as f:
                json.dump(index_copy, f)
            temp_path.rename(self.index_path)
        except Exception as e:
            # Clean up temp file if it exists
            try:
                temp_path = self.index_path.with_suffix('.tmp')
                if temp_path.exists():
                    temp_path.unlink()
            except:
                pass

    def _text_hash(self, text: str, dims: int) -> str:
        """Generate cache key from text and target dimensions"""
        content = f"{text}:{dims}"
        return hashlib.sha256(content.encode()).hexdigest()[:16]

    def _get_cache_path(self, key: str) -> Path:
        """Get file path for a cache key"""
        # Use first 2 chars as subdirectory for better filesystem performance
        subdir = self.cache_dir / key[:2]
        subdir.mkdir(exist_ok=True)
        return subdir / f"{key}.npy"

    def get(self, text: str, dims: int) -> Optional[np.ndarray]:
        """Get embedding from cache (RAM first, then disk). THREAD-SAFE."""
        # EDGE CASE: Empty or None text
        if not text or not text.strip():
            return None

        # EDGE CASE: Invalid dimensions
        if dims <= 0:
            return None

        key = self._text_hash(text, dims)

        # Check RAM cache first (with lock)
        with self._ram_cache_lock:
            if key in self.ram_cache:
                self.hits += 1
                self.ram_cache.move_to_end(key)  # LRU update
                # Return a copy to prevent external modification
                return self.ram_cache[key].copy()

        # Check disk cache (with lock)
        cache_path = self._get_cache_path(key)
        with self._disk_lock:
            if cache_path.exists():
                try:
                    embedding = np.load(cache_path)

                    # EDGE CASE: Validate dimensions match
                    if embedding.shape[-1] != dims:
                        # Dimension mismatch - stale cache entry
                        try:
                            cache_path.unlink()
                        except:
                            pass
                        self.misses += 1
                        return None

                    self.hits += 1

                    # Promote to RAM cache
                    self._add_to_ram_cache(key, embedding)

                    # Update access time in index
                    with self._index_lock:
                        if key in self.index:
                            self.index[key]['accessed'] = time.time()

                    return embedding.copy()
                except Exception as e:
                    # Corrupted cache file - remove it
                    try:
                        cache_path.unlink()
                    except:
                        pass

        self.misses += 1
        return None

    def _add_to_ram_cache(self, key: str, embedding: np.ndarray):
        """Add to RAM cache with LRU eviction. THREAD-SAFE."""
        with self._ram_cache_lock:
            if len(self.ram_cache) >= self.ram_cache_size:
                self.ram_cache.popitem(last=False)  # Remove oldest
            # Store a copy to prevent external modification
            self.ram_cache[key] = embedding.copy()

    def put(self, text: str, dims: int, embedding: np.ndarray):
        """Store embedding in cache (disk + RAM). THREAD-SAFE."""
        # EDGE CASE: Empty text or invalid embedding
        if not text or not text.strip():
            return
        if embedding is None or embedding.size == 0:
            return
        if dims <= 0:
            return

        key = self._text_hash(text, dims)
        cache_path = self._get_cache_path(key)

        try:
            # Save to disk (with lock)
            with self._disk_lock:
                # Write to temp file first, then rename (atomic on POSIX)
                temp_path = cache_path.with_suffix('.tmp')
                np.save(temp_path, embedding)
                temp_path.rename(cache_path)

            self.disk_writes += 1

            # Update index (with lock)
            with self._index_lock:
                self.index[key] = {
                    'dims': dims,
                    'size': embedding.nbytes,
                    'created': time.time(),
                    'accessed': time.time()
                }

            # Add to RAM cache
            self._add_to_ram_cache(key, embedding)

            # Check if we need to evict old entries
            self._maybe_evict()

            # Save index periodically (every 100 writes)
            if self.disk_writes % 100 == 0:
                self._save_index()

        except Exception as e:
            # Clean up temp file if it exists
            try:
                temp_path = cache_path.with_suffix('.tmp')
                if temp_path.exists():
                    temp_path.unlink()
            except:
                pass

    def _maybe_evict(self):
        """Evict old entries if cache is too large. THREAD-SAFE."""
        with self._index_lock:
            # Calculate current size
            total_size = sum(entry.get('size', 0) for entry in self.index.values())

            if total_size <= self.max_bytes:
                return

            # Sort by access time, evict oldest
            sorted_keys = sorted(
                self.index.keys(),
                key=lambda k: self.index[k].get('accessed', 0)
            )

            evicted = 0
            for key in sorted_keys:
                if total_size <= self.max_bytes * 0.8:  # Evict to 80%
                    break

                cache_path = self._get_cache_path(key)
                try:
                    with self._disk_lock:
                        if cache_path.exists():
                            cache_path.unlink()
                    total_size -= self.index[key].get('size', 0)
                    del self.index[key]
                    evicted += 1
                except:
                    pass

            if evicted > 0:
                print(f"🗑️ Disk cache evicted {evicted} old entries", file=sys.stderr)

        # Save index outside the lock to avoid blocking other operations
        if evicted > 0:
            self._save_index()

    def get_stats(self) -> Dict:
        """Get cache statistics. THREAD-SAFE."""
        with self._index_lock:
            total_size = sum(entry.get('size', 0) for entry in self.index.values())
            entries = len(self.index)

        with self._ram_cache_lock:
            ram_cache_size = len(self.ram_cache)

        return {
            'entries': entries,
            'size_mb': round(total_size / 1024 / 1024, 2),
            'max_mb': self.max_bytes / 1024 / 1024,
            'hits': self.hits,
            'misses': self.misses,
            'hit_rate': round(self.hits / max(1, self.hits + self.misses) * 100, 1),
            'ram_cache_size': ram_cache_size
        }


class LayerOffloadingTransformer:
    """
    OPT-5: LAYER OFFLOADING for <4GB RAM systems

    Instead of loading the full model (~400MB for MiniLM),
    we load transformer layers one at a time.

    This uses ~100MB peak instead of ~400MB, at the cost of slower inference.
    Only enabled on ULTRA_LOW mode (<4GB RAM).

    Inspired by AirLLM's approach but simplified for embedding models.
    """

    def __init__(self, model_name: str, cache_dir: Path):
        # Use bundled model if available
        self.model_name = BUNDLED_MODEL_PATH if BUNDLED_MODEL_PATH else model_name
        self.cache_dir = cache_dir
        self.tokenizer = None
        self.model_config = None
        self.layers_dir = cache_dir / "layers"
        self.layers_dir.mkdir(parents=True, exist_ok=True)

        self._initialized = False
        self._current_layer_idx = -1
        self._current_layer = None

        print(f"⚡ Layer offloading mode: {model_name}", file=sys.stderr)

    def _lazy_init(self):
        """Lazy initialize tokenizer and config (not the full model)"""
        if self._initialized:
            return

        try:
            from transformers import AutoTokenizer, AutoConfig

            self.tokenizer = AutoTokenizer.from_pretrained(
                self.model_name,
                cache_dir=str(self.cache_dir)
            )
            self.model_config = AutoConfig.from_pretrained(
                self.model_name,
                cache_dir=str(self.cache_dir)
            )
            self._initialized = True
            print(f"  ✓ Tokenizer loaded (model layers on-demand)", file=sys.stderr)

        except Exception as e:
            print(f"  ✗ Layer offloading init failed: {e}", file=sys.stderr)
            raise

    def encode(self, text: str) -> np.ndarray:
        """
        Generate embedding using layer-by-layer processing.

        For MiniLM-L6-v2: 6 transformer layers processed sequentially.
        Each layer loaded, used, then unloaded to minimize RAM.

        NOTE: This is slower but uses ~75% less RAM.
        """
        self._lazy_init()

        # For now, fall back to full model but with aggressive cleanup
        # True layer-by-layer would require model surgery
        # This is a simplified version that still saves RAM via lazy loading

        from sentence_transformers import SentenceTransformer

        # Load model, encode, immediately unload
        # NOTE: backend='onnx' is REQUIRED for model_kwargs file_name to work
        model = SentenceTransformer(
            self.model_name,
            device='cpu',
            backend='onnx',
            cache_folder=str(self.cache_dir),
            model_kwargs={"file_name": _BEST_ONNX_FILE}
        )

        embedding = model.encode(text, convert_to_numpy=True, show_progress_bar=False)

        # Immediately free
        del model
        gc.collect()

        return embedding

    def encode_batch(self, texts: List[str]) -> np.ndarray:
        """Batch encode with layer offloading"""
        # For batch, load once, encode all, unload
        self._lazy_init()

        from sentence_transformers import SentenceTransformer

        # NOTE: backend='onnx' is REQUIRED for model_kwargs file_name to work
        model = SentenceTransformer(
            self.model_name,
            device='cpu',
            backend='onnx',
            cache_folder=str(self.cache_dir),
            model_kwargs={"file_name": _BEST_ONNX_FILE}
        )

        embeddings = model.encode(texts, convert_to_numpy=True, show_progress_bar=False)

        del model
        gc.collect()

        return embeddings


# Global low-resource config (initialized on first use)
_low_resource_config: Optional[LowResourceConfig] = None


def get_low_resource_config() -> LowResourceConfig:
    """Get the global low-resource config (auto-configured from RAM)"""
    global _low_resource_config
    if _low_resource_config is None:
        _low_resource_config = LowResourceConfig()
        _low_resource_config.log_config()
    return _low_resource_config


class AdaptiveBatchSizer:
    """
    4TH OPTIMIZATION: Adaptive Batch Sizing

    Dynamically adjusts batch size based on current CPU/RAM usage:
    - When resources available: increase batch size for throughput
    - When resources tight: decrease batch size to stay under limits
    - Same quality embeddings, smarter resource usage
    """

    def __init__(self, config: ResourceConfig):
        self.config = config
        self.base_batch_size = 64
        self.min_batch_size = 16
        self.max_batch_size = 128
        self.current_batch_size = self.base_batch_size
        self.last_adjustment = time.time()
        self.adjustment_interval = 5.0

        # Performance tracking
        self.recent_latencies: deque = deque(maxlen=20)
        self.recent_cpu_samples: deque = deque(maxlen=10)

    def _get_cpu_usage(self) -> float:
        """Read CPU usage from /proc/stat"""
        try:
            with open('/proc/stat', 'r') as f:
                line = f.readline()
                parts = line.split()
                user, nice, system, idle = map(float, parts[1:5])
                total = user + nice + system + idle
                busy = user + nice + system
                return (busy / total) * 100 if total > 0 else 0
        except:
            return 50.0

    def _get_ram_usage_mb(self) -> float:
        """Get current RAM usage in MB"""
        try:
            with open('/proc/self/status', 'r') as f:
                for line in f:
                    if line.startswith('VmRSS:'):
                        kb = int(line.split()[1])
                        return kb / 1024.0
            return 0
        except:
            return 0

    def get_adaptive_batch_size(self) -> int:
        """Calculate optimal batch size based on current resources."""
        now = time.time()

        if now - self.last_adjustment < self.adjustment_interval:
            return self.current_batch_size

        self.last_adjustment = now
        cpu = self._get_cpu_usage()
        ram_mb = self._get_ram_usage_mb()

        self.recent_cpu_samples.append(cpu)
        avg_cpu = sum(self.recent_cpu_samples) / len(self.recent_cpu_samples)

        # Calculate CPU-based factor
        if avg_cpu < self.config.cpu_min:
            cpu_factor = 1.5
        elif avg_cpu > self.config.cpu_max:
            cpu_factor = 0.5
        else:
            range_pct = (avg_cpu - self.config.cpu_min) / (self.config.cpu_max - self.config.cpu_min)
            cpu_factor = 1.5 - (range_pct * 1.0)

        # Calculate RAM-based factor
        ram_factor = 1.0
        if ram_mb > self.config.ram_max_mb * 0.9:
            ram_factor = 0.5
        elif ram_mb > self.config.ram_max_mb * 0.75:
            ram_factor = 0.75
        elif ram_mb < self.config.ram_min_mb:
            ram_factor = 1.25

        # Apply heavyOps multiplier if enabled
        heavy_mult = self.config.heavy_ops_batch_mult if self.config.heavy_ops_enabled else 1.0

        # Calculate new batch size
        new_size = int(self.base_batch_size * cpu_factor * ram_factor * heavy_mult)
        new_size = max(self.min_batch_size, min(self.max_batch_size, new_size))

        # Smooth transitions
        if new_size > self.current_batch_size:
            self.current_batch_size = min(new_size, self.current_batch_size + 8)
        elif new_size < self.current_batch_size:
            self.current_batch_size = max(new_size, self.current_batch_size - 8)

        return self.current_batch_size

    def record_latency(self, latency_ms: float):
        """Record embedding latency for performance tracking"""
        self.recent_latencies.append(latency_ms)

    def get_stats(self) -> Dict[str, Any]:
        """Get adaptive batch sizer statistics"""
        avg_latency = sum(self.recent_latencies) / len(self.recent_latencies) if self.recent_latencies else 0
        avg_cpu = sum(self.recent_cpu_samples) / len(self.recent_cpu_samples) if self.recent_cpu_samples else 0
        return {
            'current_batch_size': self.current_batch_size,
            'base_batch_size': self.base_batch_size,
            'avg_latency_ms': round(avg_latency, 2),
            'avg_cpu': round(avg_cpu, 1),
            'ram_mb': round(self._get_ram_usage_mb(), 1)
        }


def verify_optimizations():
    """
    🔒 ACK VERIFICATION - We NEVER use a model that hasn't been fully optimized.
    Reads model-config.json and verifies all 4 optimizations are enabled.
    Refuses to start if verification fails.
    """
    required_opts = ['warmRam', 'qqmsThrottling', 'efficientIO', 'adaptiveBatch']
    config_path = os.path.join(PROJECT_PATH, 'specmem', 'model-config.json')

    print("=" * 70, file=sys.stderr)
    print("🔒 ACK VERIFICATION - Checking model optimizations...", file=sys.stderr)
    print("=" * 70, file=sys.stderr)

    if not os.path.exists(config_path):
        print(f"⚠️ model-config.json not found at {config_path}", file=sys.stderr)
        print("   Running without ACK verification (config will be generated on init)", file=sys.stderr)
        return None

    try:
        with open(config_path, 'r') as f:
            config = json.load(f)
    except Exception as e:
        print(f"⚠️ Could not read model-config.json: {e}", file=sys.stderr)
        return None

    optimizations = config.get('optimizations', {})
    all_verified = True

    for opt in required_opts:
        opt_config = optimizations.get(opt, {})
        if not opt_config.get('enabled', False):
            print(f"❌ ACK FAILED: {opt} NOT ENABLED!", file=sys.stderr)
            all_verified = False
        else:
            print(f"✅ ACK: {opt} = VERIFIED", file=sys.stderr)

    # Verify resource limits
    resources = config.get('resources', {})
    if not all(resources.get(k) is not None for k in ['cpuMin', 'cpuMax', 'ramMinMb', 'ramMaxMb']):
        print(f"⚠️ Resource limits not fully configured", file=sys.stderr)
    else:
        print(f"✅ ACK: Resources = CPU {resources['cpuMin']}-{resources['cpuMax']}%, RAM {resources['ramMinMb']}-{resources['ramMaxMb']}MB", file=sys.stderr)

    if not all_verified:
        print("", file=sys.stderr)
        print("⚠️ Some optimizations not verified - model may not be fully optimized", file=sys.stderr)
        print("   Run 'specmem-init' to apply all optimizations", file=sys.stderr)
    else:
        print("", file=sys.stderr)
        print("╔══════════════════════════════════════════════════════════════════╗", file=sys.stderr)
        print("║  ✅ ALL 4 OPTIMIZATIONS ACK VERIFIED ✅                          ║", file=sys.stderr)
        print("║  Model is fully optimized and ready for use                      ║", file=sys.stderr)
        print("╚══════════════════════════════════════════════════════════════════╝", file=sys.stderr)

    print("", file=sys.stderr)
    return config


# Global resource config and adaptive sizer (initialized in main)
_resource_config: Optional[ResourceConfig] = None
_adaptive_sizer: Optional[AdaptiveBatchSizer] = None


def get_resource_config() -> ResourceConfig:
    """Get the global resource config"""
    global _resource_config
    if _resource_config is None:
        _resource_config = ResourceConfig()
    return _resource_config


def get_adaptive_sizer() -> AdaptiveBatchSizer:
    """Get the global adaptive batch sizer"""
    global _adaptive_sizer
    if _adaptive_sizer is None:
        _adaptive_sizer = AdaptiveBatchSizer(get_resource_config())
    return _adaptive_sizer


class CPUMonitor:
    """
    Monitors CPU usage and provides throttling recommendations.
    Uses /proc/stat for accurate Linux CPU monitoring.
    """

    def __init__(self, sample_interval_ms: float = 100.0):
        self.sample_interval_ms = sample_interval_ms
        self.last_check_time: float = 0.0
        self.last_cpu_times: Optional[Tuple[float, float]] = None
        self.current_usage: float = 0.0
        self.usage_history: deque = deque(maxlen=10)
        self._lock = threading.Lock()

    def _read_cpu_times(self) -> Optional[Tuple[float, float]]:
        """Read CPU times from /proc/stat"""
        try:
            with open('/proc/stat', 'r') as f:
                first_line = f.readline()
                if not first_line.startswith('cpu '):
                    return None

                parts = first_line.split()
                # user, nice, system, idle, iowait, irq, softirq, steal
                user = float(parts[1])
                nice = float(parts[2])
                system = float(parts[3])
                idle = float(parts[4])
                iowait = float(parts[5]) if len(parts) > 5 else 0.0

                total = user + nice + system + idle + iowait
                busy = user + nice + system

                return (busy, total)
        except Exception:
            return None

    def get_cpu_usage(self) -> float:
        """Get current CPU usage percentage (0-100)"""
        now = time.time()

        with self._lock:
            # Rate limit checks
            if now - self.last_check_time < (self.sample_interval_ms / 1000.0):
                return self.current_usage

            current_times = self._read_cpu_times()
            if current_times is None:
                return self.current_usage

            if self.last_cpu_times is not None:
                busy_delta = current_times[0] - self.last_cpu_times[0]
                total_delta = current_times[1] - self.last_cpu_times[1]

                if total_delta > 0:
                    self.current_usage = (busy_delta / total_delta) * 100.0
                    self.usage_history.append(self.current_usage)

            self.last_cpu_times = current_times
            self.last_check_time = now

            return self.current_usage

    def get_average_usage(self) -> float:
        """Get average CPU usage over recent samples"""
        if not self.usage_history:
            return self.get_cpu_usage()
        return sum(self.usage_history) / len(self.usage_history)

    def is_overloaded(self, threshold: float = 85.0) -> bool:
        """Check if CPU is overloaded"""
        return self.get_cpu_usage() > threshold


class QQMSThrottler:
    """
    QQMS (Quantum-Quality Millisecond) Throttler

    Implements intelligent rate limiting and throttling to prevent CPU spikes
    while maintaining embedding quality. Uses a token bucket algorithm with
    CPU-aware dynamic adjustment.

    NEW: Also dynamically adjusts torch thread count between cpucoremin and cpucoremax
    based on CPU load - scales down when CPU is high, scales up when CPU is low.
    """

    def __init__(self, config: Optional[QQMSConfig] = None):
        self.config = config or QQMSConfig()
        self.cpu_monitor = CPUMonitor()

        # Token bucket for rate limiting
        self.tokens: float = float(self.config.burst_limit)
        self.last_token_time: float = time.time()
        self._token_lock = threading.Lock()

        # Request tracking
        self.request_count: int = 0
        self.last_request_time: float = 0.0
        self.burst_start_time: float = 0.0
        self.requests_in_burst: int = 0

        # Stats
        self.total_delay_ms: float = 0.0
        self.throttle_events: int = 0
        self.thread_adjustments: int = 0

        # Dynamic thread scaling (cpucoremin to cpucoremax)
        self.thread_min = _CPU_THREAD_MIN
        self.thread_max = _CPU_THREAD_LIMIT
        self.current_threads = _CPU_THREAD_LIMIT
        self.last_thread_adjust = 0.0

        print(f"🕐 QQMS Throttler initialized:", file=sys.stderr)
        print(f"   Base delay: {self.config.base_delay_ms}ms", file=sys.stderr)
        print(f"   Max RPS: {self.config.max_requests_per_second}", file=sys.stderr)
        print(f"   Burst limit: {self.config.burst_limit}", file=sys.stderr)
        print(f"   CPU thresholds: {self.config.cpu_low_threshold}%/{self.config.cpu_high_threshold}%/{self.config.cpu_critical_threshold}%", file=sys.stderr)
        print(f"   Thread scaling: {self.thread_min}-{self.thread_max} cores (dynamic)", file=sys.stderr)

    def _adjust_threads_for_cpu(self):
        """
        Dynamically adjust torch thread count based on CPU usage.
        This is the REAL CPU limiting - not just delays!
        """
        now = time.time()
        # Only adjust every 5 seconds to avoid thrashing
        if now - self.last_thread_adjust < 5.0:
            return

        cpu = self.cpu_monitor.get_cpu_usage()
        old_threads = self.current_threads

        if cpu > self.config.cpu_critical_threshold:
            # Critical: use minimum threads
            self.current_threads = self.thread_min
        elif cpu > self.config.cpu_high_threshold:
            # High: reduce threads
            self.current_threads = max(self.thread_min, self.current_threads - 1)
        elif cpu < self.config.cpu_low_threshold:
            # Low CPU: can increase threads
            self.current_threads = min(self.thread_max, self.current_threads + 1)

        if self.current_threads != old_threads:
            torch.set_num_threads(self.current_threads)
            self.thread_adjustments += 1
            self.last_thread_adjust = now
            print(f"🔧 QQMS: Adjusted threads {old_threads} → {self.current_threads} (CPU: {cpu:.1f}%)", file=sys.stderr)

    def _refill_tokens(self):
        """Refill tokens based on elapsed time"""
        now = time.time()
        elapsed = now - self.last_token_time

        # Add tokens based on rate limit
        new_tokens = elapsed * self.config.max_requests_per_second
        self.tokens = min(float(self.config.burst_limit), self.tokens + new_tokens)
        self.last_token_time = now

    def _get_cpu_multiplier(self) -> float:
        """Get delay multiplier based on CPU usage"""
        cpu = self.cpu_monitor.get_cpu_usage()

        if cpu > self.config.cpu_critical_threshold:
            # Emergency throttling - 10x delay
            return 10.0
        elif cpu > self.config.cpu_high_threshold:
            # High throttling - 4x delay
            return 4.0
        elif cpu > self.config.cpu_medium_threshold:
            # Medium throttling - 2x delay
            return 2.0
        elif cpu > self.config.cpu_low_threshold:
            # Light throttling - 1.5x delay
            return 1.5
        else:
            # No throttling
            return 1.0

    def acquire(self, priority: EmbeddingPriority = EmbeddingPriority.MEDIUM) -> float:
        """
        Acquire permission to process a request.
        Returns the delay in seconds that was applied.

        Args:
            priority: Request priority level

        Returns:
            Delay in seconds that was applied
        """
        with self._token_lock:
            self._refill_tokens()

            # REAL CPU CONTROL: Adjust thread count based on CPU load
            self._adjust_threads_for_cpu()

            now = time.time()
            delay_ms = 0.0

            # Calculate base delay from priority
            priority_multiplier = self.config.priority_delay_multiplier.get(
                int(priority), 1.0
            )
            base_delay = self.config.base_delay_ms * priority_multiplier

            # Apply CPU-based multiplier
            cpu_multiplier = self._get_cpu_multiplier()
            if cpu_multiplier > 1.0:
                self.throttle_events += 1

            delay_ms = base_delay * cpu_multiplier

            # Rate limiting via token bucket
            if self.tokens < 1.0:
                # No tokens available - must wait
                wait_time = (1.0 - self.tokens) / self.config.max_requests_per_second
                delay_ms += wait_time * 1000.0
                self.tokens = 0.0
            else:
                self.tokens -= 1.0

            # Burst detection and cooldown
            if now - self.burst_start_time > 1.0:
                # New burst window
                self.burst_start_time = now
                self.requests_in_burst = 1
            else:
                self.requests_in_burst += 1

                # If exceeding burst limit, add cooldown
                if self.requests_in_burst > self.config.burst_limit:
                    delay_ms += self.config.idle_delay_after_burst_ms

            # Apply delay
            if delay_ms > 0:
                time.sleep(delay_ms / 1000.0)
                self.total_delay_ms += delay_ms

            self.request_count += 1
            self.last_request_time = now

            return delay_ms / 1000.0

    def acquire_batch(self, batch_size: int, priority: EmbeddingPriority = EmbeddingPriority.MEDIUM) -> float:
        """
        Acquire permission for batch processing.
        Applies appropriate delays for batch operations.

        Returns total delay in seconds.
        """
        total_delay = 0.0

        # Pre-batch delay
        total_delay += self.acquire(priority)

        # Additional delay based on batch size
        if batch_size > self.config.max_batch_size:
            # Large batch - apply cooldown
            cooldown_sec = self.config.batch_cooldown_ms / 1000.0
            time.sleep(cooldown_sec)
            total_delay += cooldown_sec
        else:
            # Standard batch delay
            batch_delay_sec = self.config.batch_delay_ms / 1000.0
            time.sleep(batch_delay_sec)
            total_delay += batch_delay_sec

        return total_delay

    def get_stats(self) -> Dict[str, Any]:
        """Get throttler statistics"""
        return {
            'request_count': self.request_count,
            'throttle_events': self.throttle_events,
            'total_delay_ms': round(self.total_delay_ms, 2),
            'avg_delay_ms': round(self.total_delay_ms / max(1, self.request_count), 2),
            'tokens_available': round(self.tokens, 2),
            'cpu_usage': round(self.cpu_monitor.get_cpu_usage(), 1),
            'cpu_avg': round(self.cpu_monitor.get_average_usage(), 1)
        }


@dataclass
class DimensionConfig:
    """
    TRULY DYNAMIC dimension configuration - NO HARDCODED VALUES!

    All dimensions are queried from PostgreSQL at runtime.
    The database is the single source of truth for embedding dimensions.
    """
    # These are set dynamically from database queries - no hardcoded defaults!
    native_dims: int = 0        # Set from model on load
    target_dims: int = 0        # Set from database query

    # Last refresh timestamp
    last_refresh: float = 0.0
    refresh_interval: float = 60.0  # Refresh every 60 seconds


class RAMGuard:
    """
    Monitors RAM usage and auto-throttles to stay under limit.
    Target: 4GB max for the embedding system (4000MB).
    """

    MAX_RAM_MB = 4000  # 4GB - user specified!

    def __init__(self):
        self.last_check = time.time()
        self.check_interval = 5  # seconds
        self.warning_threshold = 0.85  # Warn at 85% (3.4GB)
        self.critical_threshold = 0.95  # Critical at 95% (3.8GB)

    def get_ram_usage_mb(self) -> float:
        """Get current RAM usage in MB"""
        try:
            # Method 1: /proc/self/status (Linux)
            with open('/proc/self/status', 'r') as f:
                for line in f:
                    if line.startswith('VmRSS:'):
                        return int(line.split()[1]) / 1024  # KB to MB
        except:
            pass

        try:
            # Method 2: resource module
            usage = resource.getrusage(resource.RUSAGE_SELF)
            return usage.ru_maxrss / 1024  # KB to MB on Linux
        except:
            pass

        return 0

    def get_available_ram_mb(self) -> float:
        """Get available RAM in MB"""
        return self.MAX_RAM_MB - self.get_ram_usage_mb()

    def should_reduce_dims(self) -> bool:
        """Check if we need to reduce dimensions to save RAM"""
        now = time.time()
        if now - self.last_check < self.check_interval:
            return False

        self.last_check = now
        ram_mb = self.get_ram_usage_mb()
        ratio = ram_mb / self.MAX_RAM_MB

        if ratio > self.critical_threshold:
            print(f"🚨 CRITICAL RAM: {ram_mb:.1f}MB/{self.MAX_RAM_MB}MB, forcing dimension reduction!", file=sys.stderr)
            gc.collect()
            return True
        elif ratio > self.warning_threshold:
            print(f"⚠️ RAM WARNING: {ram_mb:.1f}MB/{self.MAX_RAM_MB}MB", file=sys.stderr)

        return False

    def get_max_dims_for_current_ram(self) -> int:
        """Calculate maximum safe dimensions based on current RAM"""
        available = self.get_available_ram_mb()

        # Rough estimate: each 1000 dims needs ~50MB for processing
        # Plus batch overhead
        safe_dims = int((available - 500) / 0.05)  # 500MB baseline, 50MB per 1000 dims

        return max(256, min(safe_dims, 20000))

    def force_cleanup(self):
        """Force garbage collection to free RAM"""
        gc.collect()
        if torch.cuda.is_available():
            torch.cuda.empty_cache()


class DimensionExpander:
    """
    EXPANDS embeddings beyond native model dimensions using multiple techniques.
    This is how we go from 384 native dims to UP TO 20,000 dims!

    Techniques:
    1. Multi-pass encoding with different pooling strategies
    2. N-gram and character-level features
    3. Positional encoding enrichment
    4. Random projection for controlled expansion
    5. Learned expansion via trained projection matrices
    """

    # Maximum projection cache entries to prevent memory leak (LOW-07 fix)
    MAX_PROJECTION_CACHE_SIZE = 100

    def __init__(self, native_dims: int, cache_dir: Path):
        self.native_dims = native_dims
        self.cache_dir = cache_dir

        # Random projection matrices (reproducible via seeds)
        # Using OrderedDict for LRU eviction to prevent memory leak (LOW-07 fix)
        from collections import OrderedDict
        self.projection_cache: OrderedDict = OrderedDict()

        # Hash-based features for additional dimensions
        self.hash_seeds = [42, 1337, 7777, 31415, 27182]

    def expand(self, embedding: np.ndarray, target_dims: int, text: str = "") -> np.ndarray:
        """
        Expand embedding from native dims to ANY target dimension.

        TRULY DYNAMIC - no hardcoded limits! Expands to exactly target_dims.

        Uses multiple techniques combined:
        - Random projections (deterministic, reproducible)
        - Hash-based feature expansion (text-dependent)
        - Polynomial feature combinations
        - Fourier feature expansion
        - Padding for any remaining dimensions
        """
        current_dims = embedding.shape[-1]

        if target_dims <= current_dims:
            return embedding[:target_dims]

        dims_needed = target_dims - current_dims

        # Build expanded features - allocate proportionally based on need
        expanded_features = [embedding]

        # Calculate proportional allocation for each technique
        # This ensures we can hit ANY target dimension
        remaining = dims_needed

        # 1. Random Projections - up to 40% of expansion
        proj_dims = min(remaining, int(dims_needed * 0.4))
        if proj_dims > 0:
            projected = self._random_projection_expand(embedding, proj_dims)
            expanded_features.append(projected)
            remaining -= projected.shape[-1]

        # 2. Hash-based expansion - up to 20% (if text provided)
        if remaining > 0 and text:
            hash_dims = min(remaining, int(dims_needed * 0.2))
            if hash_dims > 0:
                hash_features = self._hash_based_features(text, hash_dims)
                expanded_features.append(hash_features)
                remaining -= hash_features.shape[-1]

        # 3. Polynomial features - up to 25%
        if remaining > 0:
            poly_dims = min(remaining, int(dims_needed * 0.25))
            if poly_dims > 0:
                poly_features = self._polynomial_features(embedding, poly_dims)
                expanded_features.append(poly_features)
                remaining -= poly_features.shape[-1]

        # 4. Fourier features - up to 15%
        if remaining > 0:
            fourier_dims = min(remaining, int(dims_needed * 0.15))
            if fourier_dims > 0:
                fourier_features = self._fourier_features(embedding, fourier_dims)
                expanded_features.append(fourier_features)
                remaining -= fourier_features.shape[-1]

        # 5. Zero-padding for any remaining dimensions (guarantees exact target)
        if remaining > 0:
            padding = np.zeros(remaining)
            expanded_features.append(padding)

        # Combine all features
        result = np.concatenate(expanded_features)

        # Ensure exact target dims (truncate if any rounding caused overshoot)
        result = result[:target_dims]

        # Re-normalize
        norm = np.linalg.norm(result)
        if norm > 0:
            result = result / norm

        return result

    def _random_projection_expand(self, embedding: np.ndarray, target_extra_dims: int) -> np.ndarray:
        """Expand using random projections - creates new feature space"""
        if target_extra_dims <= 0:
            return np.array([])

        # Get or create projection matrix (cached and deterministic)
        cache_key = (len(embedding), target_extra_dims)
        if cache_key not in self.projection_cache:
            # LOW-07 fix: LRU eviction - remove oldest entry if cache is full
            if len(self.projection_cache) >= self.MAX_PROJECTION_CACHE_SIZE:
                self.projection_cache.popitem(last=False)  # Remove oldest (first) item

            np.random.seed(42)  # Deterministic
            # Random projection matrix
            proj_matrix = np.random.randn(len(embedding), target_extra_dims) / np.sqrt(len(embedding))
            self.projection_cache[cache_key] = proj_matrix
        else:
            # LOW-07 fix: Move to end for LRU ordering (mark as recently used)
            self.projection_cache.move_to_end(cache_key)

        proj = self.projection_cache[cache_key]
        return embedding @ proj

    def _hash_based_features(self, text: str, target_dims: int) -> np.ndarray:
        """Generate features based on text hashing (n-grams, char patterns)"""
        features = np.zeros(target_dims)

        # Character n-grams (1-3)
        for n in range(1, 4):
            for i in range(len(text) - n + 1):
                ngram = text[i:i+n]
                h = int(hashlib.md5(ngram.encode()).hexdigest(), 16)
                idx = h % target_dims
                features[idx] += 1 / (n * len(text) + 1)

        # Word-level features
        words = text.lower().split()
        for i, word in enumerate(words):
            h = int(hashlib.sha256(word.encode()).hexdigest(), 16)
            idx = h % target_dims
            features[idx] += 1 / (len(words) + 1)

        # Normalize
        norm = np.linalg.norm(features)
        if norm > 0:
            features = features / norm

        return features

    def _polynomial_features(self, embedding: np.ndarray, target_dims: int) -> np.ndarray:
        """Generate polynomial feature combinations"""
        features = []
        n = len(embedding)

        # Quadratic interactions (pairs of dimensions)
        count = 0
        for i in range(min(n, 100)):  # Limit to first 100 dims for efficiency
            for j in range(i, min(n, 100)):
                if count >= target_dims:
                    break
                features.append(embedding[i] * embedding[j])
                count += 1
            if count >= target_dims:
                break

        # Pad if needed
        while len(features) < target_dims:
            features.append(0.0)

        return np.array(features[:target_dims])

    def _fourier_features(self, embedding: np.ndarray, target_dims: int) -> np.ndarray:
        """Generate Fourier-based features (periodic patterns)"""
        features = []
        n = len(embedding)

        # Use different frequencies
        freqs = [0.5, 1.0, 2.0, 4.0, 8.0]

        for freq in freqs:
            for i in range(n):
                if len(features) >= target_dims:
                    break
                # Sin and cos features at different frequencies
                features.append(np.sin(2 * np.pi * freq * embedding[i]))
                if len(features) < target_dims:
                    features.append(np.cos(2 * np.pi * freq * embedding[i]))

        # Pad if needed
        while len(features) < target_dims:
            features.append(0.0)

        return np.array(features[:target_dims])


class AdaptivePCA:
    """
    Self-training PCA that learns from actual data for optimal compression.
    Incrementally improves as more embeddings are processed.
    Now supports VARIABLE target dimensions!
    """

    def __init__(self, cache_dir: Path, min_samples: int = 100):
        self.cache_dir = cache_dir
        self.min_samples = min_samples
        self.pca_models: Dict[int, PCA] = {}  # Multiple PCA models for different target dims
        self.training_buffer: List[np.ndarray] = []
        self.samples_seen = 0
        self.is_trained = False

        self._load_cached()

    def _load_cached(self):
        """Load pre-trained PCA models if available"""
        try:
            import pickle
            pca_dir = self.cache_dir / "pca_models"
            if pca_dir.exists():
                for pca_file in pca_dir.glob("pca_*.pkl"):
                    dims = int(pca_file.stem.split("_")[1])
                    with open(pca_file, 'rb') as f:
                        self.pca_models[dims] = pickle.load(f)
                if self.pca_models:
                    self.is_trained = True
                    print(f"📂 Loaded {len(self.pca_models)} cached PCA models", file=sys.stderr)
        except Exception as e:
            print(f"⚠️ Could not load PCA cache: {e}", file=sys.stderr)

    def _save_cached(self):
        """Save trained PCA models to disk"""
        try:
            import pickle
            pca_dir = self.cache_dir / "pca_models"
            pca_dir.mkdir(exist_ok=True, parents=True)

            for dims, pca in self.pca_models.items():
                pca_file = pca_dir / f"pca_{dims}.pkl"
                with open(pca_file, 'wb') as f:
                    pickle.dump(pca, f)

            print(f"💾 Saved {len(self.pca_models)} PCA models", file=sys.stderr)
        except Exception as e:
            print(f"⚠️ Could not save PCA: {e}", file=sys.stderr)

    def add_samples(self, embeddings: np.ndarray):
        """Add new embeddings to training buffer"""
        if self.is_trained:
            return  # Already trained, skip

        if len(embeddings.shape) == 1:
            embeddings = embeddings.reshape(1, -1)

        for emb in embeddings:
            self.training_buffer.append(emb)
            self.samples_seen += 1

        # Train when we have enough samples
        if len(self.training_buffer) >= self.min_samples and not self.is_trained:
            self._train()

    def _train(self):
        """Train PCA models for multiple dimension targets"""
        if len(self.training_buffer) < self.min_samples:
            return

        print(f"🎓 Training adaptive PCA on {len(self.training_buffer)} samples...", file=sys.stderr)

        X = np.array(self.training_buffer)

        # Train PCA models for common dimension targets
        target_dims_list = [256, 384, 512, 768, 1024, 1536]

        n_samples, n_features = X.shape
        max_components = min(n_samples, n_features)

        for target_dims in target_dims_list:
            # PCA requires: n_components <= min(n_samples, n_features)
            if target_dims >= n_features or target_dims > max_components:
                continue  # Can't train for this dimension

            pca = PCA(n_components=target_dims, random_state=42)
            pca.fit(X)

            variance_explained = pca.explained_variance_ratio_.sum()
            print(f"  ✅ PCA-{target_dims}: {variance_explained*100:.1f}% variance", file=sys.stderr)

            self.pca_models[target_dims] = pca

        self.is_trained = True
        self.training_buffer = []  # Free memory

        self._save_cached()

    def transform(self, embeddings: np.ndarray, target_dims: int) -> np.ndarray:
        """
        Transform embeddings to target dimensions.
        Uses learned PCA if available, otherwise truncates.

        FAST PATH: For small reductions (<10%), just truncate - PCA overhead not worth it.
        """
        if embeddings.shape[-1] <= target_dims:
            return embeddings

        native_dims = embeddings.shape[-1]
        reduction_ratio = (native_dims - target_dims) / native_dims

        # FAST PATH: For small dimension reductions (<10%), just truncate
        # 384D -> 380D is only 1% reduction, no PCA needed
        if reduction_ratio < 0.10:
            return embeddings[..., :target_dims]

        # Find closest PCA model for larger reductions
        if target_dims in self.pca_models:
            pca = self.pca_models[target_dims]
        else:
            # Find closest model
            available = sorted(self.pca_models.keys())
            closest = min(available, key=lambda x: abs(x - target_dims)) if available else None
            pca = self.pca_models.get(closest)

        if pca is not None:
            # Use learned PCA for optimal compression
            if len(embeddings.shape) == 1:
                result = pca.transform(embeddings.reshape(1, -1))[0]
            else:
                result = pca.transform(embeddings)

            # If PCA gives more dims than target, truncate
            if result.shape[-1] > target_dims:
                result = result[..., :target_dims]

            return result

        # Fallback: simple truncation (still works well)
        return embeddings[..., :target_dims]


class QueryAnalyzer:
    """
    Analyzes query complexity to determine optimal dimensions.
    More complex queries get more dimensions for better accuracy.
    ENHANCED for 20,000 dimension support!
    """

    # Code patterns (HIGH complexity - needs more dims)
    CODE_PATTERNS = [
        'function', 'class', 'import', 'const', 'let', 'var',
        'def', 'async', 'await', 'return', 'interface', 'type',
        '()', '{}', '=>', '[]', 'git', 'npm', 'node', 'docker',
        'api', 'endpoint', 'database', 'query', 'schema', 'model',
        'lambda', 'closure', 'decorator', 'metaclass', 'generic',
        'iterator', 'generator', 'coroutine', 'thread', 'mutex',
        'struct', 'enum', 'trait', 'impl', 'pub fn', 'unsafe'
    ]

    # Technical patterns (MEDIUM-HIGH complexity)
    TECHNICAL_PATTERNS = [
        'error', 'bug', 'fix', 'issue', 'debug', 'trace',
        'performance', 'optimize', 'memory', 'cpu', 'network',
        'authentication', 'authorization', 'security', 'encrypt',
        'configure', 'deploy', 'install', 'setup', 'migrate',
        'algorithm', 'architecture', 'microservice', 'kubernetes',
        'container', 'orchestration', 'pipeline', 'ci/cd'
    ]

    # Scientific/ML patterns (ULTRA complexity)
    SCIENTIFIC_PATTERNS = [
        'neural', 'network', 'gradient', 'backprop', 'tensor',
        'embedding', 'transformer', 'attention', 'lstm', 'cnn',
        'regression', 'classification', 'clustering', 'dimensionality',
        'eigenvalue', 'matrix', 'vector', 'topology', 'manifold',
        'derivative', 'integral', 'differential', 'probability',
        'bayesian', 'stochastic', 'markov', 'optimization'
    ]

    # Simple patterns (LOW complexity)
    SIMPLE_PATTERNS = [
        'what', 'how', 'why', 'when', 'where', 'which',
        'list', 'show', 'find', 'get', 'search'
    ]

    @classmethod
    def get_query_type(cls, text: str) -> str:
        """
        Classify query as 'scientific', 'code', 'technical', or 'semantic'
        """
        text_lower = text.lower()

        # Count pattern matches
        sci_score = sum(1 for p in cls.SCIENTIFIC_PATTERNS if p in text_lower)
        code_score = sum(1 for p in cls.CODE_PATTERNS if p in text_lower)
        tech_score = sum(1 for p in cls.TECHNICAL_PATTERNS if p in text_lower)
        simple_score = sum(1 for p in cls.SIMPLE_PATTERNS if p in text_lower)

        if sci_score >= 2:
            return 'scientific'
        elif code_score >= 3:
            return 'code'
        elif tech_score >= 2 or code_score >= 2:
            return 'technical'
        else:
            return 'semantic'

    @classmethod
    def get_optimal_dims(cls, text: str, target_dims: int) -> int:
        """
        Get optimal dimensions based on query complexity.

        TRULY DYNAMIC - uses the database target dimension as the baseline.
        No hardcoded dimension values!

        Args:
            text: The query text to analyze
            target_dims: The target dimension from database (source of truth)

        Returns:
            The target_dims value (database is always authoritative)
        """
        # Database dimension is the source of truth - always return it
        # Query analysis is used for logging/stats only, not dimension selection
        return target_dims

    @classmethod
    def get_complexity_score(cls, text: str) -> float:
        """
        Get complexity score 0-1 for adaptive scaling.
        """
        text_lower = text.lower()

        # Length factor (longer = more complex)
        length_score = min(len(text) / 1000, 1.0)

        # Pattern factors
        sci_score = sum(1 for p in cls.SCIENTIFIC_PATTERNS if p in text_lower) / len(cls.SCIENTIFIC_PATTERNS)
        code_score = sum(1 for p in cls.CODE_PATTERNS if p in text_lower) / len(cls.CODE_PATTERNS)
        tech_score = sum(1 for p in cls.TECHNICAL_PATTERNS if p in text_lower) / len(cls.TECHNICAL_PATTERNS)

        # Special character density (code indicator)
        special_chars = sum(1 for c in text if c in '{}[]()<>=+-*/;:@#$%^&|\\')
        special_score = min(special_chars / 50, 1.0)

        # Line count (multi-line content = more complex)
        line_score = min(text.count('\n') / 20, 1.0)

        # Weighted combination
        complexity = (
            0.15 * length_score +
            0.25 * sci_score +
            0.25 * code_score +
            0.15 * tech_score +
            0.10 * special_score +
            0.10 * line_score
        )

        return min(complexity, 1.0)


class FrankensteinEmbeddings:
    """
    FRANKENSTEIN v5 - TRULY DYNAMIC embedding system.

    NO HARDCODED DIMENSIONS - queries PostgreSQL for target dimension.
    Supports ANY dimension the database specifies.

    Features:
    - TRULY DYNAMIC: Queries database for dimension, no hardcoded values
    - Dimension EXPANSION: Expands from native dims to ANY target
    - Dimension COMPRESSION: PCA for reduction when needed
    - 60-second dimension refresh: Detects database changes
    - RAM guard: 4GB limit with auto-throttling
    - QQMS Throttling: CPU-aware rate limiting
    """

    def __init__(
        self,
        base_model: str = "sentence-transformers/all-MiniLM-L6-v2",
        cache_dir: str = "/tmp/frankenstein-models",
        db_config: Optional[Dict] = None,
        enable_adaptive_pca: bool = True,
        enable_expansion: bool = True,
        enable_throttling: bool = True,
        qqms_config: Optional[QQMSConfig] = None
    ):
        """
        Initialize the TRULY DYNAMIC Frankenstein embedding system.

        Args:
            base_model: The sentence transformer model
            cache_dir: Where to cache models and transforms
            db_config: PostgreSQL connection config (host, port, database, user, password)
            enable_adaptive_pca: Enable self-training PCA for compression
            enable_expansion: Enable dimension expansion beyond native dims
            enable_throttling: Enable QQMS throttling to prevent CPU spikes
            qqms_config: Custom QQMS throttling configuration
        """
        print("FRANKENSTEIN EMBEDDINGS v5 - LOW RESOURCE + DYNAMIC MODE", file=sys.stderr)
        print("   NO HARDCODED DIMENSIONS - Database is source of truth!", file=sys.stderr)

        self.cache_dir = Path(cache_dir)
        self.cache_dir.mkdir(exist_ok=True, parents=True)

        # Database config for dimension queries
        self.db_config = db_config or {}

        # ═══════════════════════════════════════════════════════════════════
        # OPT 5-8: LOW RESOURCE OPTIMIZATIONS (auto-configured from RAM)
        # ═══════════════════════════════════════════════════════════════════
        self.low_resource_config = get_low_resource_config()

        # OPT-8: Disk-backed embedding cache
        self.disk_cache: Optional[DiskBackedEmbeddingCache] = None
        if self.low_resource_config.disk_cache_enabled:
            self.disk_cache = DiskBackedEmbeddingCache(
                self.cache_dir,
                max_mb=self.low_resource_config.disk_cache_max_mb
            )

        # QQMS Throttler for CPU management
        self.enable_throttling = enable_throttling
        self.throttler: Optional[QQMSThrottler] = None
        if enable_throttling:
            self.throttler = QQMSThrottler(qqms_config)

        # Dimension config - starts empty, populated from database
        self.dim_config = DimensionConfig()

        # RAM guard (4GB!)
        self.ram_guard = RAMGuard()

        # Store model name for lazy-loading
        # Use bundled model path if available (no network download needed)
        self.base_model = BUNDLED_MODEL_PATH if BUNDLED_MODEL_PATH else base_model

        # Track request time for idle cleanup
        self.last_request_time = time.time()

        # THREAD SAFETY: Lock for model loading to prevent race conditions
        self._model_lock = threading.Lock()

        # Health status flag: reflects whether model is loaded and functional
        # Set to False on load failure, True on successful load + health check
        self._model_healthy = True

        # ═══════════════════════════════════════════════════════════════════
        # OPT-6: LAZY LOADING - Don't load model until first request
        # ═══════════════════════════════════════════════════════════════════
        if self.low_resource_config.lazy_loading:
            # LAZY MODE: Model starts as None, loaded on first embed request
            print(f"⏳ Lazy loading ENABLED - model will load on first request", file=sys.stderr)
            self.model = None
            # Use known native dims for MiniLM-L6-v2 (avoids loading model just to check)
            self.dim_config.native_dims = 384  # MiniLM-L6-v2 is always 384
        else:
            # EAGER MODE: Load model immediately (for high-RAM or heavyOps)
            print(f"Loading model: {self.base_model} ({_BEST_ONNX_FILE})", file=sys.stderr)
            # NOTE: backend='onnx' is REQUIRED for model_kwargs file_name to work
            self.model = SentenceTransformer(
                self.base_model,
                device='cpu',
                backend='onnx',
                cache_folder=str(self.cache_dir),
                model_kwargs={"file_name": _BEST_ONNX_FILE}
            )
            self.dim_config.native_dims = self.model.get_sentence_embedding_dimension()
            print(f"   Native dimensions: {self.dim_config.native_dims}", file=sys.stderr)

        # Query database for target dimension
        self._refresh_target_dimension()

        # Adaptive PCA (compression)
        self.adaptive_pca: Optional[AdaptivePCA] = None
        if enable_adaptive_pca:
            self.adaptive_pca = AdaptivePCA(self.cache_dir)

        # Dimension expander (EXPANSION!)
        self.expander: Optional[DimensionExpander] = None
        if enable_expansion:
            self.expander = DimensionExpander(self.dim_config.native_dims, self.cache_dir)

        # Stats tracking
        self.stats = {
            'total_embeddings': 0,
            'dimension_histogram': {},
            'expansions': 0,
            'compressions': 0,
            'native': 0,
            'avg_latency_ms': 0,
            'disk_cache_hits': 0,
            'disk_cache_misses': 0
        }
        self.latencies = deque(maxlen=100)

        print(f"Frankenstein v5 READY!", file=sys.stderr)
        print(f"   Mode: {self.low_resource_config.mode}", file=sys.stderr)
        print(f"   Native dims: {self.dim_config.native_dims}", file=sys.stderr)
        print(f"   Target dims: {self.dim_config.target_dims} (from database)", file=sys.stderr)
        print(f"   Lazy loading: {'ON' if self.low_resource_config.lazy_loading else 'OFF'}", file=sys.stderr)
        print(f"   Disk cache: {'ON' if self.disk_cache else 'OFF'}", file=sys.stderr)
        print(f"   RAM limit: {self.ram_guard.MAX_RAM_MB}MB", file=sys.stderr)

    def _get_db_connection(self):
        """Get a psycopg2 database connection with project schema isolation"""
        try:
            import psycopg2
            host = self.db_config.get('host', os.environ.get('SPECMEM_DB_HOST', 'localhost'))
            port = self.db_config.get('port', os.environ.get('SPECMEM_DB_PORT', '5432'))
            db = self.db_config.get('database', os.environ.get('SPECMEM_DB_NAME', 'specmem'))
            user = self.db_config.get('user', os.environ.get('SPECMEM_DB_USER', 'specmem'))
            password = self.db_config.get('password', os.environ.get('SPECMEM_DB_PASSWORD', ''))

            conn = psycopg2.connect(
                host=host,
                port=port,
                database=db,
                user=user,
                password=password,
                connect_timeout=5,
                options=f"-c search_path={self._get_db_schema()},public"
            )
            return conn
        except Exception as e:
            print(f"DB connection failed: {e}", file=sys.stderr)
            return None

    def _get_db_schema(self):
        """Get the project-specific DB schema name (specmem_<project_dir>)"""
        # Check env var first (set by embeddingServerManager)
        schema = os.environ.get('SPECMEM_DB_SCHEMA', '')
        if schema:
            return schema
        # Container mode: read schema from file written by entrypoint
        socket_dir = os.environ.get('SPECMEM_SOCKET_DIR', '')
        if socket_dir:
            schema_file = os.path.join(socket_dir, 'project-schema')
            if os.path.exists(schema_file):
                try:
                    schema = open(schema_file).read().strip()
                    if schema:
                        return schema
                except:
                    pass
        # Derive from project path (same logic as Node.js getProjectSchema)
        project_path = os.environ.get('SPECMEM_PROJECT_PATH', '/')
        if project_path in ('/', ''):
            return 'specmem_default'
        import re
        dir_name = os.path.basename(project_path.rstrip('/'))
        dir_name = re.sub(r'[^a-z0-9_]', '_', dir_name.lower())
        dir_name = re.sub(r'_+', '_', dir_name).strip('_')
        if not dir_name:
            return 'specmem_default'
        return f'specmem_{dir_name[:50]}'

    def _ensure_model_loaded(self):
        """Lazy-load model if it was unloaded during idle pause. THREAD-SAFE.

        This allows the server to free RAM when idle but instantly reload
        when a new request comes in. The socket stays open, just the model
        gets unloaded/reloaded.

        Uses double-checked locking pattern to avoid lock contention when
        model is already loaded.

        Retries with exponential backoff on failure (Issue #17 fix).
        Configurable via:
        - SPECMEM_MODEL_RELOAD_RETRIES (default 3)
        - SPECMEM_MODEL_RELOAD_DELAY_MS (default 1000) - base delay in ms

        Raises RuntimeError if all retries fail, ensuring callers get an
        explicit error instead of silent failure.
        """
        # Fast path: model already loaded and healthy (no lock needed)
        if self.model is not None and getattr(self, '_model_healthy', True):
            return

        max_retries = int(os.environ.get('SPECMEM_MODEL_RELOAD_RETRIES', '3'))
        base_delay_ms = int(os.environ.get('SPECMEM_MODEL_RELOAD_DELAY_MS', '1000'))

        # Slow path: need to load model (with lock)
        with self._model_lock:
            # Double-check inside lock (another thread may have loaded it)
            if self.model is not None and getattr(self, '_model_healthy', True):
                return

            last_error = None
            for attempt in range(1, max_retries + 1):
                print(f"[MODEL-RELOAD] Loading model: {self.base_model} ({_BEST_ONNX_FILE}) (attempt {attempt}/{max_retries})", file=sys.stderr)
                start = time.time()
                try:
                    # NOTE: backend='onnx' is REQUIRED for model_kwargs file_name to work
                    self.model = SentenceTransformer(
                        self.base_model,
                        device='cpu',
                        backend='onnx',
                        cache_folder=str(self.cache_dir),
                        model_kwargs={"file_name": _BEST_ONNX_FILE}
                    )
                    load_time = (time.time() - start) * 1000

                    # Verify the model actually works by doing a test encode
                    test_embedding = self.model.encode("health check", show_progress_bar=False)
                    if test_embedding is None or len(test_embedding) == 0:
                        raise RuntimeError("Model loaded but produced empty embedding on health check")

                    self._model_healthy = True
                    print(f"[MODEL-RELOAD] Model loaded and verified in {load_time:.0f}ms (attempt {attempt}) - ready to embed!", file=sys.stderr)

                    # Update native dims if we didn't know them
                    actual_dims = self.model.get_sentence_embedding_dimension()
                    if self.dim_config.native_dims != actual_dims:
                        print(f"   Native dims updated: {self.dim_config.native_dims} -> {actual_dims}", file=sys.stderr)
                        self.dim_config.native_dims = actual_dims

                    # Update last request time so idle monitor resets
                    self.last_request_time = time.time()

                    # Update embedding-status.json so init can detect model is loaded
                    try:
                        import json as _json
                        run_dir = os.environ.get('SPECMEM_RUN_DIR', os.path.join(os.environ.get('SPECMEM_PROJECT_PATH', '/data'), 'run'))
                        _status_file = os.path.join(run_dir, 'embedding-status.json')
                        if os.path.exists(os.path.dirname(_status_file)):
                            with open(_status_file, 'w') as _f:
                                _json.dump({
                                    "status": "online",
                                    "pid": os.getpid(),
                                    "socket": os.environ.get('SPECMEM_EMBEDDING_SOCKET', ''),
                                    "model_loaded": True,
                                    "native_dims": actual_dims,
                                    "target_dims": self.dim_config.target_dims,
                                    "timestamp": int(time.time() * 1000)
                                }, _f, indent=2)
                            print(f"[MODEL-RELOAD] ✅ Updated embedding-status.json: model_loaded=true", file=sys.stderr)
                    except Exception as _e:
                        print(f"[MODEL-RELOAD] ⚠ Failed to update status file: {_e}", file=sys.stderr)

                    return  # Success

                except Exception as e:
                    last_error = e
                    self.model = None
                    self._model_healthy = False
                    print(f"[MODEL-RELOAD] Attempt {attempt}/{max_retries} failed: {e}", file=sys.stderr)

                    if attempt < max_retries:
                        # Exponential backoff: base_delay * 2^(attempt-1)
                        # e.g., with 1000ms base: 1s, 2s, 4s
                        delay_seconds = (base_delay_ms / 1000.0) * (2 ** (attempt - 1))
                        print(f"[MODEL-RELOAD] Retrying in {delay_seconds:.1f}s...", file=sys.stderr)
                        time.sleep(delay_seconds)

            # All retries exhausted
            self._model_healthy = False
            error_msg = f"Model reload failed after {max_retries} attempts. Last error: {last_error}"
            print(f"[MODEL-RELOAD] FATAL: {error_msg}", file=sys.stderr)
            raise RuntimeError(error_msg)

    def _query_database_dimension(self) -> int:
        """
        Query PostgreSQL for the actual embedding dimension.
        NO HARDCODED FALLBACKS - database is the source of truth!

        Returns:
            The dimension from memories table, or native_dims if query fails
        """
        try:
            conn = self._get_db_connection()
            if not conn:
                print("Could not connect to database for dimension query", file=sys.stderr)
                return self.dim_config.native_dims

            cursor = conn.cursor()
            schema = self._get_db_schema()
            # Set search_path explicitly before dimension query
            cursor.execute(f"SET search_path TO {schema}, public")
            # For pgvector, atttypmod IS the dimension directly
            cursor.execute(f"""
                SELECT atttypmod FROM pg_attribute
                WHERE attrelid = '{schema}.memories'::regclass AND attname = 'embedding'
            """)
            result = cursor.fetchone()
            cursor.close()
            conn.close()

            if result and result[0] > 0:
                return result[0]

            print("Could not detect dimension from database", file=sys.stderr)
            return self.dim_config.native_dims

        except Exception as e:
            print(f"Database dimension query failed: {e}", file=sys.stderr)
            return self.dim_config.native_dims

    def _refresh_target_dimension(self) -> bool:
        """
        Refresh target dimension from database.
        Called on startup and periodically (every 60 seconds).

        Returns:
            True if dimension changed, False otherwise
        """
        now = time.time()

        # Check if refresh is needed (every 60 seconds)
        if self.dim_config.target_dims > 0:
            elapsed = now - self.dim_config.last_refresh
            if elapsed < self.dim_config.refresh_interval:
                return False

        old_dims = self.dim_config.target_dims
        new_dims = self._query_database_dimension()

        self.dim_config.target_dims = new_dims
        self.dim_config.last_refresh = now

        if old_dims != new_dims and old_dims > 0:
            print(f"DIMENSION CHANGE: {old_dims}D -> {new_dims}D", file=sys.stderr)
            return True
        elif old_dims == 0:
            print(f"Target dimension set to {new_dims}D from database", file=sys.stderr)

        return False

    def _get_target_dims(self, text: str = "") -> int:
        """
        Get target dimensions, refreshing from database if needed.

        TRULY DYNAMIC - always returns database dimension.
        No hardcoded values, no query-based scaling.
        """
        # Refresh from database if interval has passed
        self._refresh_target_dimension()

        # Return database dimension (source of truth)
        return self.dim_config.target_dims

    def update_target_dimension(self, new_dims: int):
        """
        Manually update target dimension (e.g., from external refresh).
        """
        old_dims = self.dim_config.target_dims
        if new_dims != old_dims:
            print(f"Target dimension updated: {old_dims}D -> {new_dims}D", file=sys.stderr)
            self.dim_config.target_dims = new_dims
            self.dim_config.last_refresh = time.time()

    def _transform_dims(self, embedding: np.ndarray, target_dims: int, text: str = "") -> np.ndarray:
        """
        Transform embedding to target dimensions.
        Can EXPAND or COMPRESS based on need!
        """
        current_dims = embedding.shape[-1]

        if current_dims == target_dims:
            # Already at target
            self.stats['native'] += 1
            return embedding

        elif current_dims > target_dims:
            # COMPRESSION needed
            self.stats['compressions'] += 1
            if self.adaptive_pca is not None:
                return self.adaptive_pca.transform(embedding, target_dims)
            return embedding[..., :target_dims]

        else:
            # EXPANSION needed!
            self.stats['expansions'] += 1
            if self.expander is not None:
                return self.expander.expand(embedding, target_dims, text)
            # Fallback: zero-padding (not ideal but works)
            padding = np.zeros(target_dims - current_dims)
            return np.concatenate([embedding, padding])

    def embed_single(
        self,
        text: str,
        force_dims: Optional[int] = None,
        priority: EmbeddingPriority = EmbeddingPriority.MEDIUM
    ) -> np.ndarray:
        """
        Generate embedding for a single text with DYNAMIC dimensions.

        Args:
            text: Input text
            force_dims: Force specific dimensions (None = auto)
            priority: Request priority for throttling

        Returns:
            Normalized embedding vector at database target dimension
        """
        start_time = time.time()

        # Track request time for idle cleanup
        self.last_request_time = time.time()

        # Get target dimensions FIRST (before cache check)
        target_dims = force_dims or self._get_target_dims(text)

        # ═══════════════════════════════════════════════════════════════════
        # OPT-8: Check disk cache BEFORE loading model
        # ═══════════════════════════════════════════════════════════════════
        if self.disk_cache is not None:
            cached = self.disk_cache.get(text, target_dims)
            if cached is not None:
                self.stats['disk_cache_hits'] += 1
                self.stats['total_embeddings'] += 1
                latency_ms = (time.time() - start_time) * 1000
                self.latencies.append(latency_ms)
                return cached
            self.stats['disk_cache_misses'] += 1

        # Apply QQMS throttling to prevent CPU spikes
        throttle_delay = 0.0
        if self.throttler is not None:
            throttle_delay = self.throttler.acquire(priority)

        # Ensure model is loaded (lazy-load after idle pause)
        self._ensure_model_loaded()

        # Generate embedding at native dims
        embedding = self.model.encode(
            text,
            convert_to_numpy=True,
            show_progress_bar=False
        )

        # Add to PCA training data
        if self.adaptive_pca is not None:
            self.adaptive_pca.add_samples(embedding.reshape(1, -1))

        # Transform to target dimensions (expand or compress)
        embedding = self._transform_dims(embedding, target_dims, text)

        # Normalize
        norm = np.linalg.norm(embedding)
        if norm > 0:
            embedding = embedding / norm

        # Track stats
        latency_ms = (time.time() - start_time) * 1000
        self.latencies.append(latency_ms)
        self.stats['total_embeddings'] += 1

        # Track dimension histogram
        dim_bucket = f"{target_dims}D"
        self.stats['dimension_histogram'][dim_bucket] = \
            self.stats['dimension_histogram'].get(dim_bucket, 0) + 1

        # ═══════════════════════════════════════════════════════════════════
        # OPT-8: Store in disk cache for future requests
        # ═══════════════════════════════════════════════════════════════════
        if self.disk_cache is not None:
            try:
                self.disk_cache.put(text, target_dims, embedding)
            except Exception as e:
                # Don't fail on cache write errors
                pass

        return embedding

    def embed_batch(
        self,
        texts: List[str],
        force_dims: Optional[int] = None,
        priority: EmbeddingPriority = EmbeddingPriority.LOW
    ) -> np.ndarray:
        """
        Generate embeddings for multiple texts with batch processing.

        Args:
            texts: List of input texts
            force_dims: Force specific dimensions (None = use max needed)
            priority: Request priority for throttling (default LOW for batches)

        Returns:
            Matrix of normalized embeddings
        """
        start_time = time.time()

        # Track request time for idle cleanup
        self.last_request_time = time.time()

        # For batch, use database target dims (refreshes if needed)
        target_dims = force_dims or self._get_target_dims()

        # ═══════════════════════════════════════════════════════════════════
        # OPT-8: Check disk cache for each text (partial cache hits)
        # ═══════════════════════════════════════════════════════════════════
        cached_embeddings: Dict[int, np.ndarray] = {}  # idx -> embedding
        uncached_indices: List[int] = []
        uncached_texts: List[str] = []

        if self.disk_cache is not None:
            for i, text in enumerate(texts):
                cached = self.disk_cache.get(text, target_dims)
                if cached is not None:
                    cached_embeddings[i] = cached
                    self.stats['disk_cache_hits'] += 1
                else:
                    uncached_indices.append(i)
                    uncached_texts.append(text)
                    self.stats['disk_cache_misses'] += 1
        else:
            uncached_indices = list(range(len(texts)))
            uncached_texts = texts

        # If all cached, return immediately
        if len(uncached_texts) == 0:
            result = np.array([cached_embeddings[i] for i in range(len(texts))])
            latency_ms = (time.time() - start_time) * 1000
            self.latencies.append(latency_ms)
            self.stats['total_embeddings'] += len(texts)
            return result

        # Apply QQMS throttling for batch processing
        throttle_delay = 0.0
        if self.throttler is not None:
            throttle_delay = self.throttler.acquire_batch(len(uncached_texts), priority)

        # Limit batch size to prevent CPU spikes
        max_batch = 16 if self.throttler else 32

        # Ensure model is loaded (lazy-load after idle pause)
        self._ensure_model_loaded()

        # Generate embeddings for uncached texts only
        new_embeddings = self.model.encode(
            uncached_texts,
            convert_to_numpy=True,
            show_progress_bar=False,
            batch_size=max_batch
        )

        # Add to PCA training
        if self.adaptive_pca is not None:
            self.adaptive_pca.add_samples(new_embeddings)

        # Transform and cache new embeddings
        for i, (orig_idx, emb) in enumerate(zip(uncached_indices, new_embeddings)):
            text = uncached_texts[i]
            transformed = self._transform_dims(emb, target_dims, text)

            # Normalize
            norm = np.linalg.norm(transformed)
            if norm > 0:
                transformed = transformed / norm

            # Store in cache
            if self.disk_cache is not None:
                try:
                    self.disk_cache.put(text, target_dims, transformed)
                except:
                    pass

            cached_embeddings[orig_idx] = transformed

        # Combine all embeddings in original order
        embeddings = np.array([cached_embeddings[i] for i in range(len(texts))])

        # Track stats
        latency_ms = (time.time() - start_time) * 1000
        self.latencies.append(latency_ms)
        self.stats['total_embeddings'] += len(texts)

        return embeddings

    def get_stats(self) -> Dict[str, Any]:
        """Get embedding statistics including low-resource optimization info"""
        avg_latency = sum(self.latencies) / len(self.latencies) if self.latencies else 0

        stats = {
            **self.stats,
            'avg_latency_ms': round(avg_latency, 2),
            'target_dims': self.dim_config.target_dims,
            'native_dims': self.dim_config.native_dims,
            'last_refresh': self.dim_config.last_refresh,
            'refresh_interval': self.dim_config.refresh_interval,
            'pca_trained': self.adaptive_pca.is_trained if self.adaptive_pca else False,
            'ram_usage_mb': round(self.ram_guard.get_ram_usage_mb(), 1),
            'ram_limit_mb': self.ram_guard.MAX_RAM_MB,
            'throttling_enabled': self.enable_throttling,
            'model_loaded': self.model is not None,
            'model_healthy': getattr(self, '_model_healthy', True)
        }

        # Add low-resource optimization stats
        stats['low_resource'] = {
            'mode': self.low_resource_config.mode,
            'system_ram_gb': round(self.low_resource_config.system_ram_gb, 1),
            'available_ram_gb': round(get_available_ram_gb(), 1),
            'lazy_loading': self.low_resource_config.lazy_loading,
            'disk_cache_enabled': self.low_resource_config.disk_cache_enabled,
            'aggressive_cleanup': self.low_resource_config.aggressive_cleanup,
            'idle_unload_seconds': self.low_resource_config.idle_unload_seconds
        }

        # Add disk cache stats if enabled
        if self.disk_cache is not None:
            stats['disk_cache'] = self.disk_cache.get_stats()

        # Add throttler stats if enabled
        if self.throttler is not None:
            stats['throttler'] = self.throttler.get_stats()

        return stats


class EmbeddingServer:
    """
    Socket server that serves FRANKENSTEIN v5 embeddings.
    Compatible with existing specmem embedding socket protocol.

    TRULY DYNAMIC DIMENSIONS:
    - Queries PostgreSQL for dimension on startup
    - Refreshes dimension every 60 seconds
    - Supports dimension changes without restart
    - NO hardcoded dimension values

    Features:
    - Idle timeout: shuts down after idle period to save CPU/RAM
    - QQMS Throttling: CPU-aware rate limiting
    - Auto-sync: codebase_files dimension synced to memories
    """

    IDLE_TIMEOUT_SECONDS = 300  # 5 minutes idle = shutdown (legacy default)

    def __init__(
        self,
        socket_path: str = None,
        db_config: Optional[Dict] = None,
        idle_timeout: int = None,  # None = auto-detect from RAM
        enable_throttling: bool = True,
        qqms_config: Optional[QQMSConfig] = None,
        qqms_v2: Optional['QQMSv2'] = None  # New FIFO + ACK queue
    ):
        if socket_path is None:
            socket_path = os.path.join(SPECMEM_SOCKET_DIR, EMBEDDING_SOCKET_NAME)
        self.socket_path = socket_path
        self.db_config = db_config or {}
        self.last_request_time = time.time()
        self.shutdown_requested = False

        # KYS (Keep Yourself Safe) watchdog - two-way health check
        # If MCP server doesn't send "kys" heartbeat within timeout, take action
        # This prevents orphan embedding servers when MCP crashes
        # Timeout and mode are configurable via environment variables
        self.last_kys_time = time.time()
        self.kys_timeout = int(os.environ.get('SPECMEM_KYS_TIMEOUT_SECONDS', '600'))
        # KYS mode: "kill" = process exit (old behavior), "unload" = release model but keep socket,
        # "standby" = keep everything loaded and just idle
        self.kys_mode = os.environ.get('SPECMEM_KYS_MODE', 'kill').lower()
        if self.kys_mode not in ('kill', 'unload', 'standby'):
            print(f"[KYS] Invalid SPECMEM_KYS_MODE '{self.kys_mode}', defaulting to 'unload'", file=sys.stderr)
            self.kys_mode = 'unload'

        # QQMS v2 - enhanced queue with FIFO + ACK (takes precedence if provided)
        self.qqms_v2 = qqms_v2

        # Create embedder - it will query database for dimension
        # If QQMS v2 is enabled, disable legacy throttling in embedder
        self.embedder = FrankensteinEmbeddings(
            db_config=self.db_config,
            enable_throttling=enable_throttling and qqms_v2 is None,  # Disable if QQMS v2
            qqms_config=qqms_config
        )

        # Use idle_timeout from low_resource_config if not explicitly provided
        if idle_timeout is None:
            self.idle_timeout = self.embedder.low_resource_config.idle_unload_seconds
        else:
            self.idle_timeout = idle_timeout

        # Auto-sync codebase_files dimension to match memories
        self._sync_codebase_files_dimension(self.embedder.dim_config.target_dims)

        # Start dimension refresh thread (every 60 seconds)
        self._start_dimension_refresh_thread()

    def _safe_sendall(self, conn, data: bytes) -> bool:
        """Send all data using MSG_NOSIGNAL to prevent SIGPIPE on broken connections."""
        total_sent = 0
        while total_sent < len(data):
            try:
                sent = conn.send(data[total_sent:], socket.MSG_NOSIGNAL)
                if sent == 0:
                    return False
                total_sent += sent
            except (BrokenPipeError, ConnectionResetError, OSError):
                return False
        return True

    def _get_db_connection(self):
        """Get a psycopg2 database connection with project schema isolation"""
        try:
            import psycopg2
            host = self.db_config.get('host', os.environ.get('SPECMEM_DB_HOST', 'localhost'))
            port = self.db_config.get('port', os.environ.get('SPECMEM_DB_PORT', '5432'))
            db = self.db_config.get('database', os.environ.get('SPECMEM_DB_NAME', 'specmem'))
            user = self.db_config.get('user', os.environ.get('SPECMEM_DB_USER', 'specmem'))
            password = self.db_config.get('password', os.environ.get('SPECMEM_DB_PASSWORD', ''))
            schema = self._get_db_schema()

            return psycopg2.connect(
                host=host,
                port=port,
                database=db,
                user=user,
                password=password,
                connect_timeout=5,
                options=f"-c search_path={schema},public"
            )
        except Exception as e:
            print(f"⚠️ DB connection failed: {e}", file=sys.stderr)
            return None

    def _get_db_schema(self):
        """Get the project-specific DB schema name (specmem_<project_dir>)"""
        schema = os.environ.get('SPECMEM_DB_SCHEMA', '')
        if schema:
            return schema
        project_path = os.environ.get('SPECMEM_PROJECT_PATH', '/')
        if project_path in ('/', ''):
            return 'specmem_default'
        import re
        dir_name = os.path.basename(project_path.rstrip('/'))
        dir_name = re.sub(r'[^a-z0-9_]', '_', dir_name.lower())
        dir_name = re.sub(r'_+', '_', dir_name).strip('_')
        if not dir_name:
            return 'specmem_default'
        return f'specmem_{dir_name[:50]}'

    def _get_table_dimensions(self, table_name: str) -> int:
        """
        Get the embedding dimension for ANY table from database.
        NO HARDCODED FALLBACKS - uses embedder's target_dims if query fails.
        NOTE: For pgvector, atttypmod IS the dimension directly.
        """
        try:
            conn = self._get_db_connection()
            if not conn:
                return self.embedder.dim_config.target_dims  # Use embedder's dimension

            cursor = conn.cursor()
            schema = self._get_db_schema()
            # Set search_path explicitly before dimension query
            cursor.execute(f"SET search_path TO {schema}, public")
            # Qualify table name with schema for regclass resolution
            qualified_table = f"{schema}.{table_name}"
            cursor.execute("""
                SELECT atttypmod FROM pg_attribute
                WHERE attrelid = %s::regclass AND attname = 'embedding'
            """, (qualified_table,))
            result = cursor.fetchone()
            cursor.close()
            conn.close()

            if result and result[0] > 0:
                return result[0]

            return self.embedder.dim_config.target_dims  # Use embedder's dimension

        except Exception as e:
            print(f"Could not get {table_name} dimensions: {e}", file=sys.stderr)
            return self.embedder.dim_config.target_dims

    def _sync_codebase_files_dimension(self, target_dims: int) -> bool:
        """
        Auto-sync codebase_files table to match memories dimension.
        This makes dimensions TRULY DYNAMIC - no manual ALTER needed!
        """
        try:
            conn = self._get_db_connection()
            if not conn:
                return False

            cursor = conn.cursor()

            # Get codebase_files current dimension (atttypmod IS the dimension for pgvector)
            schema = self._get_db_schema()
            # Set search_path explicitly before dimension query
            cursor.execute(f"SET search_path TO {schema}, public")
            cursor.execute(f"""
                SELECT atttypmod FROM pg_attribute
                WHERE attrelid = '{schema}.codebase_files'::regclass AND attname = 'embedding'
            """)
            result = cursor.fetchone()
            codebase_dims = result[0] if result and result[0] > 0 else None

            if codebase_dims and codebase_dims != target_dims:
                print(f"🔄 AUTO-SYNC: codebase_files {codebase_dims}D → {target_dims}D to match memories", file=sys.stderr)

                # First drop any existing embeddings (they're incompatible anyway)
                cursor.execute("UPDATE codebase_files SET embedding = NULL WHERE embedding IS NOT NULL")

                # Alter the column dimension
                cursor.execute(f"ALTER TABLE codebase_files ALTER COLUMN embedding TYPE vector({target_dims})")
                conn.commit()

                print(f"✅ codebase_files dimension auto-synced to {target_dims}D", file=sys.stderr)

            cursor.close()
            conn.close()
            return True

        except Exception as e:
            print(f"⚠️ codebase_files auto-sync failed: {e}", file=sys.stderr)
            return False

    def _start_dimension_refresh_thread(self):
        """
        Start background thread to refresh dimension from database every 60 seconds.
        Supports dimension changes without restart!
        """
        def refresh_loop():
            last_dims = self.embedder.dim_config.target_dims

            while not self.shutdown_requested:
                time.sleep(60)  # Every 60 seconds
                if self.shutdown_requested:
                    break

                # Trigger dimension refresh in embedder
                old_dims = self.embedder.dim_config.target_dims
                changed = self.embedder._refresh_target_dimension()

                if changed:
                    new_dims = self.embedder.dim_config.target_dims
                    print(f"DIMENSION CHANGE DETECTED: {old_dims}D -> {new_dims}D", file=sys.stderr)

                    # Auto-sync codebase_files to match
                    self._sync_codebase_files_dimension(new_dims)

                    print(f"Embedder now operating at {new_dims}D", file=sys.stderr)

        thread = threading.Thread(target=refresh_loop, daemon=True)
        thread.start()

    def _start_idle_monitor(self):
        """Monitor for idle timeout and PAUSE (unload model) when not in use

        IMPORTANT: We PAUSE instead of shutdown - unload model to free RAM but
        keep the socket listening. The model will lazy-load on next request.
        This prevents the "embedding service unavailable" errors!

        If idle_timeout is 0, the monitor does nothing (service mode).
        """
        # SERVICE MODE: idle_timeout=0 means never unload
        if self.idle_timeout <= 0:
            print("🔧 Idle monitor DISABLED (service mode)", file=sys.stderr)
            return  # Don't even start the monitor thread

        def monitor():
            while not self.shutdown_requested:
                time.sleep(30)  # Check every 30 seconds
                # MED-25 FIX: Synchronize last_request_time between server and embedder's throttler
                # Use the most recent of the two timestamps to avoid false idle detection
                server_last_time = self.last_request_time
                throttler_last_time = 0.0
                if self.embedder.throttler and hasattr(self.embedder.throttler, 'last_request_time'):
                    throttler_last_time = self.embedder.throttler.last_request_time
                last_activity = max(server_last_time, throttler_last_time)
                idle_time = time.time() - last_activity
                # FIX: model is in self.embedder.model, not self.model!
                if idle_time > self.idle_timeout and hasattr(self.embedder, 'model') and self.embedder.model is not None:
                    print(f"💤 Idle for {idle_time:.0f}s (>{self.idle_timeout}s), PAUSING - unloading model to save RAM...", file=sys.stderr)
                    print(f"   Socket still listening - will lazy-load model on next request!", file=sys.stderr)
                    # Unload model to free RAM, but DON'T shutdown the server
                    try:
                        del self.embedder.model
                        self.embedder.model = None
                        import gc
                        gc.collect()
                        # Try to free CUDA memory if available
                        try:
                            import torch
                            if torch.cuda.is_available():
                                torch.cuda.empty_cache()
                        except:
                            pass
                        print(f"✅ Model unloaded - RAM freed. Server still running.", file=sys.stderr)
                    except Exception as e:
                        print(f"⚠️ Error unloading model: {e}", file=sys.stderr)
                    # Reset last_request_time so we don't keep trying to unload
                    self.last_request_time = time.time()

        thread = threading.Thread(target=monitor, daemon=True)
        thread.start()

    def _start_kys_watchdog(self):
        """
        KYS (Keep Yourself Safe) Watchdog - Two-way health check system.

        The MCP server sends {"type": "kys", "text": "kurt cobain t minus 25"} every 25 seconds.
        If we don't receive this heartbeat within the configured timeout, we take action.
        This prevents orphan embedding servers when MCP crashes or is killed.

        Without this, crashed MCP leaves zombie embedding servers consuming RAM/CPU forever.

        Modes (SPECMEM_KYS_MODE):
        - "kill":    Process exit (original behavior)
        - "unload":  Release ONNX model from memory but keep socket listener alive (default)
        - "standby": Keep everything loaded, just idle
        """
        def is_claude_alive_for_project():
            """Check if any Claude/node process is running for this project directory."""
            try:
                import subprocess
                # Check for node processes with this project path in their environment
                result = subprocess.run(
                    ['pgrep', '-f', f'SPECMEM_PROJECT_PATH={PROJECT_PATH}'],
                    capture_output=True, text=True, timeout=5
                )
                if result.returncode == 0 and result.stdout.strip():
                    return True
                # Also check for claude processes with cwd in project
                result2 = subprocess.run(
                    ['pgrep', '-f', f'claude.*{PROJECT_PATH}'],
                    capture_output=True, text=True, timeout=5
                )
                if result2.returncode == 0 and result2.stdout.strip():
                    return True
                return False
            except Exception:
                return False  # Assume dead if we can't check

        def _kys_unload_model():
            """Unload the model to free RAM but keep the socket listener alive.
            On next request, _ensure_model_loaded() will reload it."""
            try:
                if hasattr(self.embedder, 'model') and self.embedder.model is not None:
                    del self.embedder.model
                    self.embedder.model = None
                    import gc
                    gc.collect()
                    try:
                        import torch
                        if torch.cuda.is_available():
                            torch.cuda.empty_cache()
                    except Exception:
                        pass
                    print(f"[KYS-UNLOAD] Model released from memory. Socket still listening.", file=sys.stderr)
                    print(f"[KYS-UNLOAD] Model will reload on next embedding request.", file=sys.stderr)
                else:
                    print(f"[KYS-UNLOAD] Model already unloaded, nothing to do.", file=sys.stderr)
            except Exception as e:
                print(f"[KYS-UNLOAD] Error unloading model: {e}", file=sys.stderr)

        def watchdog():
            # STARTUP GRACE PERIOD: Don't enforce KYS for first 60 seconds
            # This allows MCP server to fully initialize (can take 50-60+ seconds)
            startup_grace_period = 60  # seconds
            startup_time = time.time()
            # Extended timeout when there's been recent activity
            activity_grace_period = 300  # 5 minutes of no activity before considering death

            while not self.shutdown_requested:
                time.sleep(10)  # Check every 10 seconds

                # Skip enforcement during startup grace period
                if time.time() - startup_time < startup_grace_period:
                    continue

                time_since_kys = time.time() - self.last_kys_time
                time_since_activity = time.time() - self.last_request_time

                # MOST IMPORTANT CHECK: Is Claude actually running for this project?
                if is_claude_alive_for_project():
                    # Claude is alive! Don't kill even without heartbeat
                    if time_since_kys > self.kys_timeout and int(time_since_kys) % 120 < 10:
                        print(f"[KYS] No heartbeat for {time_since_kys:.0f}s but Claude process detected - staying alive", file=sys.stderr)
                    continue

                # Only take action if BOTH conditions are true:
                # 1. No heartbeat for kys_timeout
                # 2. No embedding activity for activity_grace_period (5 min)
                # This prevents acting on active servers just because heartbeat stopped
                if time_since_kys > self.kys_timeout:
                    if time_since_activity < activity_grace_period:
                        # Recent activity - don't act, just warn once per minute
                        if int(time_since_kys) % 60 < 10:
                            print(f"[KYS] No heartbeat for {time_since_kys:.0f}s but recent activity ({time_since_activity:.0f}s ago) - staying alive", file=sys.stderr)
                        continue

                    # --- KYS MODE DISPATCH ---
                    if self.kys_mode == 'standby':
                        # STANDBY MODE: Keep everything loaded, just log and continue
                        if int(time_since_kys) % 120 < 10:
                            print(f"[KYS-STANDBY] No heartbeat for {time_since_kys:.0f}s, no activity for {time_since_activity:.0f}s - idling in standby mode", file=sys.stderr)
                        continue

                    elif self.kys_mode == 'unload':
                        # UNLOAD MODE: Release model from memory but keep socket alive
                        # Only unload once - check if model is still loaded
                        if hasattr(self.embedder, 'model') and self.embedder.model is not None:
                            print(f"", file=sys.stderr)
                            print(f"[KYS-UNLOAD] WATCHDOG TRIGGERED (mode=unload)", file=sys.stderr)
                            print(f"   No heartbeat from MCP in {time_since_kys:.0f}s (timeout: {self.kys_timeout}s)", file=sys.stderr)
                            print(f"   No embedding activity for {time_since_activity:.0f}s (grace: {activity_grace_period}s)", file=sys.stderr)
                            print(f"   Unloading model to free RAM - socket stays alive for reconnection", file=sys.stderr)
                            _kys_unload_model()

                            # Write status file so clients know state
                            try:
                                death_reason_path = os.path.join(os.path.dirname(self.socket_path), 'embedding-death-reason.txt')
                                with open(death_reason_path, 'w') as f:
                                    f.write(f"kys-unload\n{time.time()}\nModel unloaded after no heartbeat ({time_since_kys:.0f}s) AND no activity ({time_since_activity:.0f}s). Socket still alive.")
                            except Exception as e:
                                print(f"   [KYS-UNLOAD] Failed to write status file: {e}", file=sys.stderr)
                        # Don't exit - keep looping. Model will reload on next request.
                        continue

                    else:
                        # KILL MODE (original behavior): Process exit
                        print(f"", file=sys.stderr)
                        print(f"[KYS-KILL] WATCHDOG TRIGGERED (mode=kill)", file=sys.stderr)
                        print(f"   No heartbeat from MCP in {time_since_kys:.0f}s (timeout: {self.kys_timeout}s)", file=sys.stderr)
                        print(f"   No embedding activity for {time_since_activity:.0f}s (grace: {activity_grace_period}s)", file=sys.stderr)
                        print(f"   MCP server likely crashed - committing suicide to prevent zombie", file=sys.stderr)
                        print(f"   'kurt cobain t minus 0'", file=sys.stderr)
                        print(f"", file=sys.stderr)

                        # Write death reason file so clients know to auto-respawn
                        try:
                            death_reason_path = os.path.join(os.path.dirname(self.socket_path), 'embedding-death-reason.txt')
                            with open(death_reason_path, 'w') as f:
                                f.write(f"kys\n{time.time()}\nNo heartbeat ({time_since_kys:.0f}s) AND no activity ({time_since_activity:.0f}s)")
                            print(f"   Death reason written to {death_reason_path}", file=sys.stderr)
                        except Exception as e:
                            print(f"   Failed to write death reason: {e}", file=sys.stderr)

                        # Set shutdown flag and force exit
                        self.shutdown_requested = True

                        # Give a moment for cleanup
                        time.sleep(1)

                        # Force exit - os._exit bypasses finally blocks for immediate death
                        os._exit(0)

        thread = threading.Thread(target=watchdog, daemon=True)
        thread.start()
        print(f"   KYS Watchdog: ENABLED (mode={self.kys_mode}, timeout={self.kys_timeout}s)", file=sys.stderr)

    def _process_codebase_files(self, batch_size: int = 200, limit: int = 0, project_path: str = None) -> Dict:
        """
        Process codebase_files without embeddings.
        TRUE ADAPTABILITY: Detects codebase_files dimension dynamically!
        FAST BATCH PROCESSING: Large batches, minimal delays, CRITICAL priority
        NO LIMIT BY DEFAULT: limit=0 means process ALL files
        Target: ~5000 files in under 2 minutes!

        project_path: Filter to only process files from this project (file_path LIKE 'project_path%')
                      Defaults to PROJECT_PATH env var if not specified.
        """
        # Use global PROJECT_PATH as default for per-project isolation
        if project_path is None:
            project_path = PROJECT_PATH if PROJECT_PATH and PROJECT_PATH != 'default' else None

        conn = self._get_db_connection()
        if not conn:
            return {'error': 'Could not connect to database', 'processed': 0}

        processed = 0
        errors = 0
        batch_num = 0
        start_time = time.time()

        # TRUE ADAPTABILITY: Use codebase_files dimension, not memories
        target_dims = self._get_table_dimensions('codebase_files')

        try:
            # Get TOTAL count first for progress tracking
            # Schema isolation already separates projects, so project_path filter
            # is only needed when file_path values are absolute. Check if filter
            # matches anything — if not, skip it (paths are likely relative).
            count_cursor = conn.cursor()
            use_project_filter = False
            if project_path:
                count_cursor.execute(
                    "SELECT COUNT(*) FROM codebase_files WHERE embedding IS NULL AND content IS NOT NULL AND file_path LIKE %s",
                    (f"{project_path}%",)
                )
                filtered_count = count_cursor.fetchone()[0]
                if filtered_count > 0:
                    use_project_filter = True
                    total_missing = filtered_count
                    print(f"🎯 Filtering to project: {project_path} ({total_missing} files)", file=sys.stderr)
                else:
                    # Paths are relative — schema isolation handles project separation
                    count_cursor.execute("SELECT COUNT(*) FROM codebase_files WHERE embedding IS NULL AND content IS NOT NULL")
                    total_missing = count_cursor.fetchone()[0]
                    print(f"📁 Schema-isolated mode (relative paths, {total_missing} files)", file=sys.stderr)
            else:
                count_cursor.execute("SELECT COUNT(*) FROM codebase_files WHERE embedding IS NULL AND content IS NOT NULL")
                total_missing = count_cursor.fetchone()[0]
                print(f"⚠️ Processing ALL projects (no project_path filter)", file=sys.stderr)
            count_cursor.close()

            print(f"📊 Total files needing embeddings: {total_missing}", file=sys.stderr)
            print(f"📐 Using {target_dims}D for codebase_files (table-specific)", file=sys.stderr)

            # Calculate how many to process (all if limit=0)
            to_process = total_missing if limit == 0 else min(limit, total_missing)
            total_batches = (to_process + batch_size - 1) // batch_size
            print(f"📂 Processing {to_process} files in ~{total_batches} batches...", file=sys.stderr)

            # CHUNKED FETCH: Keep fetching batches until done
            while processed < to_process:
                batch_num += 1
                fetch_size = min(batch_size, to_process - processed)

                # Fetch next batch - always gets files without embeddings
                cursor = conn.cursor()
                if use_project_filter:
                    cursor.execute("""
                        SELECT id, file_path, content
                        FROM codebase_files
                        WHERE embedding IS NULL AND content IS NOT NULL AND file_path LIKE %s
                        LIMIT %s
                    """, (f"{project_path}%", fetch_size))
                else:
                    cursor.execute("""
                        SELECT id, file_path, content
                        FROM codebase_files
                        WHERE embedding IS NULL AND content IS NOT NULL
                        LIMIT %s
                    """, (fetch_size,))

                rows = cursor.fetchall()
                cursor.close()

                if not rows:
                    break  # No more files to process

                ids = [r[0] for r in rows]
                texts = [f"{r[1]}\n{r[2]}" for r in rows]  # path + content

                try:
                    # Generate embeddings - LOW priority to avoid CPU spikes during cold start
                    embeddings = self.embedder.embed_batch(
                        texts,
                        force_dims=target_dims,
                        priority=EmbeddingPriority.LOW
                    )
                    # Throttle between batches to keep CPU reasonable during startup
                    import time
                    time.sleep(0.5)

                    # Write back to database - BATCH UPDATE for max speed!
                    from psycopg2.extras import execute_batch
                    update_cursor = conn.cursor()
                    update_data = [(emb.tolist(), fid) for fid, emb in zip(ids, embeddings)]
                    execute_batch(
                        update_cursor,
                        "UPDATE codebase_files SET embedding = %s::vector WHERE id = %s",
                        update_data,
                        page_size=200  # Batch 200 updates at once
                    )
                    processed += len(update_data)
                    conn.commit()
                    update_cursor.close()

                    # Progress ACK every 5 batches to reduce log spam
                    if batch_num % 5 == 0 or processed >= to_process:
                        elapsed = time.time() - start_time
                        rate = processed / elapsed if elapsed > 0 else 0
                        remaining = to_process - processed
                        eta = remaining / rate if rate > 0 else 0
                        print(f"  ✓ [{batch_num}] {processed}/{to_process} | {rate:.1f}/s | ETA: {eta:.0f}s", file=sys.stderr)

                    # NO DELAY - go fast!

                except Exception as e:
                    print(f"  ✗ Batch {batch_num} error: {e}", file=sys.stderr)
                    errors += len(rows)
                    conn.rollback()

            conn.close()

            total_time = time.time() - start_time
            final_rate = processed / total_time if total_time > 0 else 0
            print(f"✅ Done! {processed} files in {total_time:.1f}s ({final_rate:.1f}/s)", file=sys.stderr)

            return {
                'status': 'completed',
                'processed': processed,
                'errors': errors,
                'dimensions': target_dims,
                'total_missing': total_missing,
                'remaining': total_missing - processed,
                'time_seconds': round(total_time, 1),
                'rate_per_second': round(final_rate, 1)
            }

        except Exception as e:
            return {'error': str(e), 'processed': processed}

    def _process_memories(self, batch_size: int = 50, limit: int = 1000) -> Dict:
        """
        Process memories without embeddings.
        TRUE ADAPTABILITY: Detects memories dimension dynamically!
        """
        conn = self._get_db_connection()
        if not conn:
            return {'error': 'Could not connect to database', 'processed': 0}

        processed = 0
        errors = 0
        # TRUE ADAPTABILITY: Use memories table dimension
        target_dims = self._get_table_dimensions('memories')
        print(f"📐 Using {target_dims}D for memories (table-specific)", file=sys.stderr)

        try:
            cursor = conn.cursor()

            # Fetch memories without embeddings
            cursor.execute("""
                SELECT id, content
                FROM memories
                WHERE embedding IS NULL AND content IS NOT NULL
                LIMIT %s
            """, (limit,))

            rows = cursor.fetchall()
            print(f"🧠 Processing {len(rows)} memories...", file=sys.stderr)

            # Process in batches
            for i in range(0, len(rows), batch_size):
                batch = rows[i:i + batch_size]
                ids = [r[0] for r in batch]
                texts = [r[1] for r in batch]

                try:
                    # Generate embeddings
                    embeddings = self.embedder.embed_batch(
                        texts,
                        force_dims=target_dims,
                        priority=EmbeddingPriority.LOW
                    )

                    # Write back to database
                    update_cursor = conn.cursor()
                    for j, (mem_id, embedding) in enumerate(zip(ids, embeddings)):
                        embedding_list = embedding.tolist()
                        update_cursor.execute("""
                            UPDATE memories
                            SET embedding = %s::vector
                            WHERE id = %s
                        """, (embedding_list, str(mem_id)))
                        processed += 1

                    conn.commit()
                    update_cursor.close()

                    print(f"  ✓ Batch {i//batch_size + 1}: {len(batch)} memories", file=sys.stderr)

                except Exception as e:
                    print(f"  ✗ Batch error: {e}", file=sys.stderr)
                    errors += len(batch)
                    conn.rollback()

            cursor.close()
            conn.close()

            return {
                'status': 'completed',
                'processed': processed,
                'errors': errors,
                'dimensions': target_dims,
                'remaining': len(rows) - processed if processed < len(rows) else 0
            }

        except Exception as e:
            return {'error': str(e), 'processed': processed}

    def _process_code_definitions(self, batch_size: int = 200, limit: int = 0, project_path: str = None) -> Dict:
        """
        FAST BATCH PROCESSING for code_definitions table.
        Generates embeddings from name + signature + docstring.

        NO LIMIT BY DEFAULT: limit=0 means process ALL definitions
        CRITICAL priority = NO THROTTLING for max speed!
        Target: ~50,000 definitions in under 5 minutes!

        project_path: Filter to only process definitions from this project (file_path LIKE 'project_path%')
                      Defaults to PROJECT_PATH env var if not specified.
        """
        # Use global PROJECT_PATH as default for per-project isolation
        if project_path is None:
            project_path = PROJECT_PATH if PROJECT_PATH and PROJECT_PATH != 'default' else None
        conn = self._get_db_connection()
        if not conn:
            return {'error': 'Could not connect to database', 'processed': 0}

        processed = 0
        errors = 0
        batch_num = 0
        start_time = time.time()

        # Use code_definitions dimension
        target_dims = self._get_table_dimensions('code_definitions')

        try:
            # Get TOTAL count first for progress tracking
            # Schema isolation handles project separation, so project_path filter
            # only applies when file_path values are absolute paths.
            count_cursor = conn.cursor()
            use_project_filter = False
            if project_path:
                count_cursor.execute(
                    "SELECT COUNT(*) FROM code_definitions WHERE embedding IS NULL AND file_path LIKE %s",
                    (f"{project_path}%",)
                )
                filtered_count = count_cursor.fetchone()[0]
                if filtered_count > 0:
                    use_project_filter = True
                    total_missing = filtered_count
                    print(f"🎯 Filtering to project: {project_path} ({total_missing} definitions)", file=sys.stderr)
                else:
                    count_cursor.execute("SELECT COUNT(*) FROM code_definitions WHERE embedding IS NULL")
                    total_missing = count_cursor.fetchone()[0]
                    print(f"📁 Schema-isolated mode (relative paths, {total_missing} definitions)", file=sys.stderr)
            else:
                count_cursor.execute("SELECT COUNT(*) FROM code_definitions WHERE embedding IS NULL")
                total_missing = count_cursor.fetchone()[0]
                print(f"⚠️ Processing ALL projects (no project_path filter)", file=sys.stderr)
            count_cursor.close()

            print(f"📊 Total code_definitions needing embeddings: {total_missing}", file=sys.stderr)
            print(f"📐 Using {target_dims}D for code_definitions", file=sys.stderr)

            # Calculate how many to process (all if limit=0)
            to_process = total_missing if limit == 0 else min(limit, total_missing)
            total_batches = (to_process + batch_size - 1) // batch_size
            print(f"🔧 Processing {to_process} definitions in ~{total_batches} batches...", file=sys.stderr)

            # CHUNKED FETCH: Keep fetching batches until done
            while processed < to_process:
                batch_num += 1
                fetch_size = min(batch_size, to_process - processed)

                # Fetch next batch
                cursor = conn.cursor()
                if use_project_filter:
                    cursor.execute("""
                        SELECT id, definition_type, name, signature, docstring, language, file_path
                        FROM code_definitions
                        WHERE embedding IS NULL AND file_path LIKE %s
                        LIMIT %s
                    """, (f"{project_path}%", fetch_size))
                else:
                    cursor.execute("""
                        SELECT id, definition_type, name, signature, docstring, language, file_path
                        FROM code_definitions
                        WHERE embedding IS NULL
                        LIMIT %s
                    """, (fetch_size,))

                rows = cursor.fetchall()
                cursor.close()

                if not rows:
                    break  # No more to process

                ids = [r[0] for r in rows]
                # Create embedding text: type + name + signature + docstring + file
                texts = [
                    f"{r[1]} {r[2]}\n{r[3] or ''}\n{r[4] or ''}\nFile: {r[6]}\nLanguage: {r[5]}"
                    for r in rows
                ]

                try:
                    # Generate embeddings - LOW priority to avoid CPU spikes during cold start
                    embeddings = self.embedder.embed_batch(
                        texts,
                        force_dims=target_dims,
                        priority=EmbeddingPriority.LOW
                    )
                    import time
                    time.sleep(0.5)

                    # Write back to database - BATCH UPDATE for max speed!
                    from psycopg2.extras import execute_batch
                    update_cursor = conn.cursor()
                    update_data = [(emb.tolist(), str(fid)) for fid, emb in zip(ids, embeddings)]
                    execute_batch(
                        update_cursor,
                        "UPDATE code_definitions SET embedding = %s::vector WHERE id = %s",
                        update_data,
                        page_size=200
                    )
                    processed += len(update_data)
                    conn.commit()
                    update_cursor.close()

                    # Progress every 5 batches
                    if batch_num % 5 == 0 or processed >= to_process:
                        elapsed = time.time() - start_time
                        rate = processed / elapsed if elapsed > 0 else 0
                        eta = (to_process - processed) / rate if rate > 0 else 0
                        print(f"  ⚡ Batch {batch_num}: {processed}/{to_process} ({rate:.1f}/s, ETA: {eta:.0f}s)", file=sys.stderr)

                except Exception as e:
                    print(f"  ✗ Batch error: {e}", file=sys.stderr)
                    errors += len(rows)
                    conn.rollback()

            conn.close()

            elapsed = time.time() - start_time
            rate = processed / elapsed if elapsed > 0 else 0
            print(f"✅ Completed! {processed} definitions at {rate:.1f}/s in {elapsed:.1f}s", file=sys.stderr)

            return {
                'status': 'completed',
                'processed': processed,
                'errors': errors,
                'dimensions': target_dims,
                'rate': round(rate, 1),
                'elapsed_seconds': round(elapsed, 1),
                'remaining': total_missing - processed
            }

        except Exception as e:
            return {'error': str(e), 'processed': processed}

    def handle_request(self, request: Dict) -> Dict:
        """
        Handle embedding request.

        Supported request formats:
        - {"text": "..."}  -> Single embedding (uses database dimension)
        - {"texts": [...]} -> Batch embeddings
        - {"text": "...", "dims": N} -> Force specific dimensions
        - {"text": "...", "priority": "critical"} -> Set request priority
        - {"stats": true}  -> Get statistics
        - {"refresh_dimension": true} -> Force dimension refresh from database

        BACKWARDS COMPATIBILITY with server.mjs/server.py "type" field:
        - {"type": "health"} -> Same as {"stats": true}
        - {"type": "embed", "text": "..."} -> Single embedding
        - {"type": "get_dimension"} -> Get dimension info
        - {"type": "set_dimension", "dimension": N} -> Set target dimension

        Priority levels: critical, high, medium (default), low, trivial
        """
        # BACKWARDS COMPATIBILITY: Handle "type" field from server.mjs/server.py clients
        req_type = request.get('type')

        if req_type == 'health':
            # Treat like stats request
            request['stats'] = True
        elif req_type == 'ready':
            # Fast readiness check - just returns model loading state
            # Used by specmem-init for event-based startup instead of timeouts
            model_loaded = self.embedder.model is not None
            model_healthy = getattr(self.embedder, '_model_healthy', True)
            return {
                'ready': model_loaded and model_healthy,
                'model_loaded': model_loaded,
                'model_healthy': model_healthy,
                'lazy_loading': self.embedder.low_resource_config.lazy_loading,
                'status': 'ready' if (model_loaded and model_healthy) else ('error' if not model_healthy else 'loading')
            }
        elif req_type == 'kys':
            # KYS (Keep Yourself Safe) heartbeat from MCP server
            # This is a two-way ack system - MCP sends every 25 seconds
            # Resets our suicide timer
            self.last_kys_time = time.time()
            return {
                'status': 'alive',
                'ack': 'kurt cobain t minus reset',
                'timeout_remaining': self.kys_timeout,
                'kys_mode': self.kys_mode,
                'model_loaded': self.embedder.model is not None,
                'model_healthy': getattr(self.embedder, '_model_healthy', True),
                'project': PROJECT_DIR_NAME
            }
        elif req_type == 'get_dimension':
            return {
                'native_dimensions': self.embedder.dim_config.native_dims,
                'target_dimensions': self.embedder.dim_config.target_dims
            }
        elif req_type == 'set_dimension':
            # Set target dimension
            new_dim = request.get('dimension')
            if new_dim and isinstance(new_dim, int) and new_dim > 0:
                self.embedder.dim_config.target_dims = new_dim
                print(f"[Frankenstein] Target dimension set to {new_dim}", file=sys.stderr)
                return {'status': 'ok', 'dimension': new_dim}
            else:
                return {'error': 'Invalid dimension value'}
        elif req_type == 'embed':
            # Already handled by text/texts fields below
            pass
        elif req_type == 'batch_embed':
            # batch_embed type from specmem-init.cjs - treated same as 'embed' with texts array
            # Client sends: {type: 'batch_embed', texts: [...]}
            # Response: {embeddings: [[...], [...], ...]}
            pass
        elif req_type and req_type not in ['embed', 'health', 'get_dimension', 'set_dimension', 'kys', 'batch_embed']:
            # Unknown type - return error
            return {'error': f'Unknown request type: {req_type}'}

        # Stats request (or health check)
        if request.get('stats'):
            model_loaded = self.embedder.model is not None
            model_healthy = getattr(self.embedder, '_model_healthy', True)
            stats_response = {
                'status': 'healthy' if model_healthy else 'degraded',
                'ready': model_loaded and model_healthy,
                'model_loaded': model_loaded,
                'model_healthy': model_healthy,
                'stats': self.embedder.get_stats(),
                'model': 'frankenstein-v5-dynamic',
                'project': PROJECT_DIR_NAME,
                'project_path': PROJECT_PATH,
                'project_hash': PROJECT_HASH,  # backwards compat
                'native_dimensions': self.embedder.dim_config.native_dims,  # For server.mjs compatibility
                'target_dimensions': self.embedder.dim_config.target_dims,  # For server.mjs compatibility
                'dimensions': self.embedder.dim_config.target_dims,  # For server.py compatibility
                'capabilities': {
                    'target_dims': self.embedder.dim_config.target_dims,
                    'native_dims': self.embedder.dim_config.native_dims,
                    'expansion': True,
                    'compression': True,
                    'dynamic_refresh': True,
                    'refresh_interval_sec': self.embedder.dim_config.refresh_interval,
                    'ram_limit_gb': self.embedder.ram_guard.MAX_RAM_MB / 1000,
                    'throttling': True,
                    'qqms_v2': self.qqms_v2 is not None,
                    'priority_levels': ['critical', 'high', 'medium', 'low', 'trivial']
                }
            }
            # Add QQMS v2 stats if enabled
            if self.qqms_v2:
                stats_response['qqms_v2_stats'] = self.qqms_v2.get_stats()
            return stats_response

        # Force dimension refresh from database
        if request.get('refresh_dimension'):
            old_dims = self.embedder.dim_config.target_dims
            self.embedder.dim_config.last_refresh = 0  # Force refresh
            self.embedder._refresh_target_dimension()
            new_dims = self.embedder.dim_config.target_dims
            return {
                'status': 'refreshed',
                'old_dims': old_dims,
                'new_dims': new_dims,
                'changed': old_dims != new_dims
            }

        # Process codebase files - generate embeddings for files without them
        if request.get('process_codebase'):
            batch_size = request.get('batch_size', 200)  # Large batches for speed!
            limit = request.get('limit', 0)  # 0 = ALL files, no limit!
            project_path = request.get('project_path')  # Per-project filtering
            return self._process_codebase_files(batch_size=batch_size, limit=limit, project_path=project_path)

        # Process memories - generate embeddings for memories without them
        if request.get('process_memories'):
            batch_size = request.get('batch_size', 50)
            limit = request.get('limit', 1000)
            return self._process_memories(batch_size=batch_size, limit=limit)

        # Process code_definitions - FAST batch processing for semantic code search
        if request.get('process_code_definitions'):
            batch_size = request.get('batch_size', 200)  # Larger batches for speed
            limit = request.get('limit', 0)  # 0 = ALL, no limit by default!
            project_path = request.get('project_path')  # Per-project filtering
            return self._process_code_definitions(batch_size=batch_size, limit=limit, project_path=project_path)

        # Parse priority level
        priority_map = {
            'critical': EmbeddingPriority.CRITICAL,
            'high': EmbeddingPriority.HIGH,
            'medium': EmbeddingPriority.MEDIUM,
            'low': EmbeddingPriority.LOW,
            'trivial': EmbeddingPriority.TRIVIAL
        }
        priority_str = request.get('priority', 'medium').lower()
        priority = priority_map.get(priority_str, EmbeddingPriority.MEDIUM)

        # QQMS v2 throttling (if enabled) - applies FIFO + ACK queue
        if self.qqms_v2:
            # Map EmbeddingPriority to QQMS v2 Priority
            qqms_priority_map = {
                EmbeddingPriority.CRITICAL: QQMSPriority.CRITICAL,
                EmbeddingPriority.HIGH: QQMSPriority.HIGH,
                EmbeddingPriority.MEDIUM: QQMSPriority.MEDIUM,
                EmbeddingPriority.LOW: QQMSPriority.LOW,
                EmbeddingPriority.TRIVIAL: QQMSPriority.TRIVIAL
            }
            qqms_priority = qqms_priority_map.get(priority, QQMSPriority.MEDIUM)

            # Apply QQMS v2 throttling
            delay = self.qqms_v2.acquire_throttle(qqms_priority)
            if delay > 0.1:  # Log significant delays
                print(f"🕐 QQMS v2 throttled request by {delay*1000:.1f}ms (priority: {priority_str})", file=sys.stderr)

        # Force dimensions (any value supported)
        force_dims = request.get('dims')

        if 'text' in request:
            # Reject empty text immediately — prevents wasted GPU/CPU cycles
            if not request['text'] or not request['text'].strip():
                return {
                    'error': 'Empty text - nothing to embed',
                    'embedding': [0.0] * self.embedder.dim_config.target_dims,
                    'dimensions': self.embedder.dim_config.target_dims,
                    'model': 'frankenstein-v5-dynamic',
                    'cached': True
                }
            # Single text
            embedding = self.embedder.embed_single(
                request['text'],
                force_dims=force_dims,
                priority=priority
            )
            return {
                'embedding': embedding.tolist(),
                'dimensions': len(embedding),
                'model': 'frankenstein-v5-dynamic',
                'target_dims': self.embedder.dim_config.target_dims,
                'query_type': QueryAnalyzer.get_query_type(request['text']),
                'complexity': round(QueryAnalyzer.get_complexity_score(request['text']), 3),
                'priority': priority_str
            }

        elif 'texts' in request:
            # Batch texts - default to LOW priority unless specified
            if 'priority' not in request:
                priority = EmbeddingPriority.LOW

            embeddings = self.embedder.embed_batch(
                request['texts'],
                force_dims=force_dims,
                priority=priority
            )
            return {
                'embeddings': embeddings.tolist(),
                'dimensions': embeddings.shape[1],
                'model': 'frankenstein-v5-dynamic',
                'target_dims': self.embedder.dim_config.target_dims,
                'count': len(embeddings),
                'priority': priority_str
            }

        else:
            return {'error': 'Missing text or texts field'}

    # Meta request types that should be handled inline (never blocked by worker pool)
    META_REQUEST_TYPES = frozenset(('health', 'ready', 'kys', 'get_dimension', 'set_dimension'))

    # Thread-safe dict to pass prefetched data from inline handler to worker thread
    # Key: conn_id (unique counter), Value: bytes
    # FIX 2.03: Use incrementing counter instead of id(conn) to avoid GC reuse races
    _prefetched_data = {}
    _prefetched_lock = threading.Lock()
    _conn_counter = 0
    _conn_ids = {}  # WeakRef-safe: maps conn object -> unique int id
    _conn_ids_lock = threading.Lock()

    @classmethod
    def _get_conn_id(cls, conn):
        """Get a unique, stable ID for a connection object.
        Unlike id(conn), this ID is never reused after GC."""
        with cls._conn_ids_lock:
            if conn not in cls._conn_ids:
                cls._conn_counter += 1
                cls._conn_ids[conn] = cls._conn_counter
            return cls._conn_ids[conn]

    @classmethod
    def _remove_conn_id(cls, conn):
        """Clean up the conn ID mapping when a connection closes."""
        with cls._conn_ids_lock:
            cls._conn_ids.pop(conn, None)

    def _try_handle_inline(self, conn):
        """
        Fast-path: peek at request and handle meta requests (health/kys/ready) inline.
        Returns True if handled, False if it needs worker pool.

        Uses non-blocking select() to avoid stalling the accept loop.
        NO prefetching for non-meta requests — threadpool reads its own data.
        This prevents the stale prefetch FD leak (conn_ids accumulating forever).
        """
        import select
        try:
            # Non-blocking peek: check if data available within 100ms
            ready, _, _ = select.select([conn], [], [], 0.1)
            if not ready:
                # No data ready yet — send to threadpool, it will read
                return False

            conn.settimeout(0.5)
            data = b''
            while True:
                chunk = conn.recv(65536)
                if not chunk:
                    conn.close()
                    return True  # Empty request, handled
                data += chunk
                if b'\n' in chunk:
                    break
                more_ready, _, _ = select.select([conn], [], [], 0.05)
                if not more_ready:
                    break

            request = json.loads(data.decode('utf-8'))
            req_type = request.get('type')
            is_meta = request.get('stats') or request.get('shutdown') or request.get('refresh_dimension') or req_type in self.META_REQUEST_TYPES

            if not is_meta:
                # Not a meta request — stash data and let threadpool handle
                # But first: reject empty text requests immediately (saves worker threads)
                text = request.get('text', '')
                texts = request.get('texts', [])
                if not text and not texts:
                    self._safe_sendall(conn, json.dumps({'error': 'Empty text'}).encode('utf-8') + b'\n')
                    conn.close()
                    return True
                conn_id = self._get_conn_id(conn)
                with self._prefetched_lock:
                    # Cap stale prefetch entries — if too many queued, reject with backpressure
                    if len(self._prefetched_data) > 50:
                        self._safe_sendall(conn, json.dumps({'error': 'Server busy, too many queued requests', 'retryable': True}).encode('utf-8') + b'\n')
                        conn.close()
                        return True
                    self._prefetched_data[conn_id] = (data, time.time())
                return False

            # Handle meta request inline — fast, no model needed
            if req_type == 'kys':
                self.last_kys_time = time.time()
            request_id = request.get('requestId')
            response = self.handle_request(request)
            if request_id:
                response['requestId'] = request_id
            self._safe_sendall(conn, json.dumps(response).encode('utf-8') + b'\n')
            conn.close()
            return True

        except Exception as e:
            try:
                err_msg = f"Inline handler error: {e}"
                print(f"⚠️ {err_msg}", file=sys.stderr)
                self._safe_sendall(conn, json.dumps({'error': err_msg}).encode('utf-8') + b'\n')
            except:
                pass
            try:
                conn.close()
            except:
                pass
            return True

    def _handle_connection(self, conn):
        """
        Handle a single client connection in a separate thread.
        This allows concurrent processing of multiple embedding requests.
        Thread-safe: Each connection gets its own isolated context.

        FIX: Uses try/finally to ensure conn.close() is always called (prevents socket leaks).
        FIX: Added conn.settimeout(30) to prevent threads from hanging forever.
        MED-26 FIX: Timeout is now set BEFORE executor.submit() in start() method,
        ensuring timeout is active before any thread operations begin.
        """
        thread_name = threading.current_thread().name
        # FIX 2.03: Use stable conn ID instead of id(conn) to prevent GC reuse races
        conn_id = self._get_conn_id(conn)
        print(f"[WORKER {thread_name}] _handle_connection ENTERED, conn_id={conn_id}", file=sys.stderr, flush=True)
        try:
            # EAGAIN FIX: Check if connection is still alive before processing
            # Stale connections from clients that timed out waste worker time
            import select as _sel
            try:
                r, _, _ = _sel.select([conn], [], [], 0)
                if r:
                    # Data ready or connection closed - peek to check
                    peek = conn.recv(1, socket.MSG_PEEK | socket.MSG_DONTWAIT)
                    if not peek:
                        # Connection closed by client (EOF)
                        print(f"[WORKER {thread_name}] conn_id={conn_id} STALE (client disconnected), skipping", file=sys.stderr, flush=True)
                        with self._prefetched_lock:
                            self._prefetched_data.pop(conn_id, None)
                        conn.close()
                        return
            except (OSError, socket.error):
                # Connection already broken
                print(f"[WORKER {thread_name}] conn_id={conn_id} BROKEN, skipping", file=sys.stderr, flush=True)
                with self._prefetched_lock:
                    self._prefetched_data.pop(conn_id, None)
                try:
                    conn.close()
                except:
                    pass
                return

            # Check if data was already prefetched by _try_handle_inline()
            with self._prefetched_lock:
                prefetched = self._prefetched_data.pop(conn_id, None)
                prefetch_keys = list(self._prefetched_data.keys())
            # Unpack (data_bytes, timestamp) tuple from prefetch
            if prefetched is not None:
                data, _prefetch_ts = prefetched
                print(f"[WORKER {thread_name}] prefetched data: YES ({len(data)}B), remaining_keys={prefetch_keys}", file=sys.stderr, flush=True)
            else:
                data = None
                print(f"[WORKER {thread_name}] prefetched data: NO, remaining_keys={prefetch_keys}", file=sys.stderr, flush=True)
            if data is None:
                # Read request normally (direct submit path)
                print(f"[WORKER {thread_name}] No prefetched data, reading from socket...", file=sys.stderr, flush=True)
                data = b''
                while True:
                    chunk = conn.recv(65536)
                    if not chunk:
                        print(f"[WORKER {thread_name}] recv returned empty - client gone", file=sys.stderr, flush=True)
                        break
                    data += chunk
                    if b'\n' in chunk:
                        break

            if not data:
                print(f"[WORKER {thread_name}] No data at all, returning early", file=sys.stderr, flush=True)
                return  # FIX: conn.close() now handled by finally block

            # Parse request
            print(f"[WORKER {thread_name}] Parsing {len(data)}B: {data[:200]}", file=sys.stderr, flush=True)
            try:
                request = json.loads(data.decode('utf-8'))
            except UnicodeDecodeError:
                # Buffer split a multi-byte character — try reading more
                try:
                    extra = conn.recv(65536)
                    if extra:
                        data += extra
                    request = json.loads(data.decode('utf-8'))
                except Exception as e2:
                    print(f"❌ UTF-8 decode failed even after extra read: {e2}", file=sys.stderr)
                    self._safe_sendall(conn, json.dumps({'error': f'UTF-8 decode error: {e2}'}).encode('utf-8') + b'\n')
                    return

            # Check for shutdown request
            if request.get('shutdown'):
                self._safe_sendall(conn, b'{"status": "shutting_down"}\n')
                self.shutdown_requested = True
                return  # FIX: conn.close() now handled by finally block

            # CRITICAL FIX: Also reset KYS timer on ANY request
            # If we're actively processing requests, we're clearly alive - don't suicide!
            # This prevents KYS death when MCP is busy sending many find_memory requests
            self.last_kys_time = time.time()

            # Only update last_request_time for actual embedding requests (not health/stats/kys/ready)
            # This allows idle model unload to work correctly - health checks won't reset the idle timer
            req_type = request.get('type')
            is_meta_request = request.get('stats') or request.get('shutdown') or request.get('refresh_dimension') or req_type in ('health', 'ready', 'kys', 'get_dimension', 'set_dimension')
            if not is_meta_request:
                self.last_request_time = time.time()

            # Extract requestId for persistent socket multiplexing
            request_id = request.get('requestId')

            # Send "working" status ONLY for embedding requests (not health/kys/ready)
            # "working" means actually processing your query (vs "processing" which was ambiguous)
            # Meta requests expect a single response - sending a heartbeat first breaks the protocol
            # and causes clients to read the heartbeat as the actual response
            if not is_meta_request:
                text = request.get('text') or request.get('texts')
                text_length = len(text) if isinstance(text, str) else (len(text) if text else 0)
                heartbeat = {
                    'status': 'working',
                    'text_length': text_length
                }
                if request_id:
                    heartbeat['requestId'] = request_id
                hb_ok = self._safe_sendall(conn, json.dumps(heartbeat).encode('utf-8') + b'\n')
                print(f"[WORKER {thread_name}] Working status sent ok={hb_ok}", file=sys.stderr, flush=True)

            # Process - each thread gets its own call stack
            print(f"[WORKER {thread_name}] Calling handle_request(type={req_type})...", file=sys.stderr, flush=True)
            response = self.handle_request(request)
            print(f"[WORKER {thread_name}] handle_request returned keys={list(response.keys()) if isinstance(response, dict) else type(response)}", file=sys.stderr, flush=True)

            # Echo back requestId
            if request_id:
                response['requestId'] = request_id

            # Send response
            resp_bytes = json.dumps(response).encode('utf-8') + b'\n'
            send_ok = self._safe_sendall(conn, resp_bytes)
            print(f"[WORKER {thread_name}] Response sent ({len(resp_bytes)}B) ok={send_ok}", file=sys.stderr, flush=True)

        except BrokenPipeError:
            print(f"[WORKER {thread_name}] BrokenPipeError - client disconnected", file=sys.stderr, flush=True)
        except ConnectionResetError:
            print(f"[WORKER {thread_name}] ConnectionResetError - client reset", file=sys.stderr, flush=True)
        except socket.timeout:
            print(f"[WORKER {thread_name}] socket.timeout during request processing", file=sys.stderr, flush=True)
            # Send timeout error response before closing
            try:
                self._safe_sendall(conn, json.dumps({'error': 'Request processing timeout', 'timeout': True}).encode('utf-8') + b'\n')
            except:
                pass
        except Exception as e:
            import traceback
            print(f"[WORKER {thread_name}] EXCEPTION: {type(e).__name__}: {e}", file=sys.stderr, flush=True)
            traceback.print_exc(file=sys.stderr)
            try:
                self._safe_sendall(conn, json.dumps({'error': str(e)}).encode('utf-8') + b'\n')
            except:
                pass
        finally:
            # Cleanup prefetched data for this conn_id to prevent stale data
            with self._prefetched_lock:
                self._prefetched_data.pop(conn_id, None)
            # FIX 2.03: Clean up conn ID mapping to prevent unbounded growth
            self._remove_conn_id(conn)
            print(f"[WORKER {thread_name}] Closing conn_id={conn_id}", file=sys.stderr, flush=True)
            # FIX: Always close connection to prevent socket leaks
            try:
                conn.close()
            except:
                pass

    def start(self):
        """Start the embedding socket server with concurrent request handling."""
        import socket as sock_module
        from concurrent.futures import ThreadPoolExecutor

        # Socket path resolution (priority order):
        # 1. SPECMEM_EMBEDDING_SOCKET env var (explicit, highest priority)
        # 2. SOCKET_PATH env var (Docker compatibility)
        # 3. self.socket_path from constructor (CLI --socket arg)
        # 4. Construct from SPECMEM_PROJECT_PATH + specmem/sockets/embeddings.sock
        # 5. Construct from cwd + specmem/sockets/embeddings.sock

        explicit_socket = os.environ.get('SPECMEM_EMBEDDING_SOCKET') or os.environ.get('SOCKET_PATH')
        if explicit_socket:
            self.socket_path = explicit_socket
        elif not self.socket_path or self.socket_path == os.path.join(SPECMEM_SOCKET_DIR, EMBEDDING_SOCKET_NAME):
            # Not explicitly set, construct from project path
            project_path = os.environ.get('SPECMEM_PROJECT_PATH') or os.getcwd()
            socket_dir = os.path.join(project_path, 'specmem', _SOCKET_SUBDIR)
            self.socket_path = os.path.join(socket_dir, EMBEDDING_SOCKET_NAME)

        print(f"🔧 Socket path resolution:", file=sys.stderr)
        print(f"   SPECMEM_EMBEDDING_SOCKET: {os.environ.get('SPECMEM_EMBEDDING_SOCKET', 'NOT SET')}", file=sys.stderr)
        print(f"   SOCKET_PATH: {os.environ.get('SOCKET_PATH', 'NOT SET')}", file=sys.stderr)
        print(f"   SPECMEM_PROJECT_PATH: {os.environ.get('SPECMEM_PROJECT_PATH', 'NOT SET')}", file=sys.stderr)
        print(f"   cwd: {os.getcwd()}", file=sys.stderr)
        print(f"   Final socket path: {self.socket_path}", file=sys.stderr)

        # Remove old socket if exists
        if os.path.exists(self.socket_path):
            os.remove(self.socket_path)
            print(f"   Removed old socket", file=sys.stderr)

        # Create socket directory
        os.makedirs(os.path.dirname(self.socket_path), exist_ok=True)
        print(f"   Socket directory created/verified: {os.path.dirname(self.socket_path)}", file=sys.stderr)

        # Create UNIX socket
        server = sock_module.socket(sock_module.AF_UNIX, sock_module.SOCK_STREAM)

        # MED-31 FIX: Set restrictive umask BEFORE bind to prevent socket being
        # world-readable during the brief window between bind and chmod
        old_umask = os.umask(0o077)  # Only owner can read/write
        try:
            server.bind(self.socket_path)
            # Container mode: world-accessible (host user has different UID)
            sock_perms = 0o777 if os.environ.get('SPECMEM_CONTAINER_MODE') == 'true' else 0o660
            os.chmod(self.socket_path, sock_perms)
            print(f"   ✅ Socket bound successfully! (permissions: {oct(sock_perms)})", file=sys.stderr)
        except Exception as e:
            print(f"   ❌ Socket bind failed: {e}", file=sys.stderr)
            raise
        finally:
            # Restore original umask
            os.umask(old_umask)
        # RELIABILITY FIX: Increase listen backlog to 128 to handle concurrent
        # connections during burst operations (find_memory, drill_down, etc. all hitting at once)
        # EAGAIN/Resource temporarily unavailable occurs when backlog overflows
        server.listen(128)
        server.settimeout(60)  # 60 second timeout on accept to check shutdown

        # Start idle monitor
        self._start_idle_monitor()

        # Start KYS watchdog - suicide if MCP doesn't heartbeat us
        self._start_kys_watchdog()

        # Create thread pool for concurrent request handling
        # Increased from 4 to 8 — needs to handle burst concurrent requests from find_memory, drill_down, etc.
        # With non-blocking accept loop, workers are the only bottleneck now
        max_workers = int(os.environ.get('SPECMEM_EMBEDDING_MAX_WORKERS', '8'))
        executor = ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix='embedding-worker')

        # CRAWL MODE: Skip eager model loading to minimize startup CPU usage
        # Model will lazy-load on first request (prevents 143% CPU spike during init)
        # OLD BEHAVIOR: Pre-warm model to prevent first-request timeout
        # NEW BEHAVIOR: Lazy load on first request, accept brief delay for crawl mode
        skip_warmup = os.environ.get('SPECMEM_SKIP_MODEL_WARMUP', '1') == '1'
        if skip_warmup:
            print(f"🐌 CRAWL MODE: Skipping model warmup - will lazy-load on first request", file=sys.stderr)
            print(f"   This keeps CPU usage <5% during idle (vs 143% with eager loading)", file=sys.stderr)
            print(f"   Set SPECMEM_SKIP_MODEL_WARMUP=0 to force eager loading", file=sys.stderr)
        elif self.embedder.model is None and not self.embedder.low_resource_config.lazy_loading:
            print(f"⏳ Pre-warming model before accepting connections...", file=sys.stderr)
            try:
                self.embedder._ensure_model_loaded()
                print(f"✅ Model pre-warmed successfully!", file=sys.stderr)
            except Exception as e:
                print(f"⚠️ Model pre-warm failed: {e} (will lazy-load on first request)", file=sys.stderr)
        elif self.embedder.model is None:
            # Lazy loading enabled - do a quick warmup to avoid slow first request
            print(f"⏳ Quick model warmup (lazy mode)...", file=sys.stderr)
            try:
                # Force model load by doing a single test embedding
                _ = self.embedder.embed_single("warmup test", priority=EmbeddingPriority.LOW)
                print(f"✅ Model warmed up!", file=sys.stderr)
            except Exception as e:
                print(f"⚠️ Model warmup failed: {e} (will load on first request)", file=sys.stderr)

        print(f"", file=sys.stderr)
        print(f"FRANKENSTEIN v5 - TRULY DYNAMIC Embedding Server", file=sys.stderr)
        print(f"   Socket: {self.socket_path}", file=sys.stderr)
        print(f"   Native dims: {self.embedder.dim_config.native_dims}", file=sys.stderr)
        print(f"   Target dims: {self.embedder.dim_config.target_dims}D (from database)", file=sys.stderr)
        print(f"   Refresh interval: {self.embedder.dim_config.refresh_interval}s", file=sys.stderr)
        print(f"   RAM limit: {self.embedder.ram_guard.MAX_RAM_MB}MB", file=sys.stderr)
        print(f"   Features: DYNAMIC DIMENSION + EXPANSION + COMPRESSION + QQMS THROTTLING + CONCURRENT REQUESTS", file=sys.stderr)
        print(f"   Concurrent workers: {max_workers} (set SPECMEM_EMBEDDING_MAX_WORKERS to adjust)", file=sys.stderr)
        print(f"   Idle timeout: {self.idle_timeout}s (auto-shutdown when not in use)", file=sys.stderr)
        if self.embedder.throttler:
            print(f"   QQMS Throttling: ENABLED (CPU-aware rate limiting)", file=sys.stderr)
            print(f"   Max RPS: {self.embedder.throttler.config.max_requests_per_second}", file=sys.stderr)
            print(f"   Priority levels: critical, high, medium, low, trivial", file=sys.stderr)
        print(f"", file=sys.stderr)

        # Write status file to signal readiness (prevents socket slam from health checks)
        status_file = os.path.join(os.path.dirname(self.socket_path), 'embedding-status.json')
        try:
            import json
            status_data = {
                "status": "online",
                "pid": os.getpid(),
                "socket": self.socket_path,
                "model_loaded": self.embedder.model is not None,
                "native_dims": self.embedder.dim_config.native_dims,
                "target_dims": self.embedder.dim_config.target_dims,
                "timestamp": int(time.time() * 1000)
            }
            with open(status_file, 'w') as f:
                json.dump(status_data, f, indent=2)
            print(f"✅ Status file written: {status_file}", file=sys.stderr)
            print(f"   Status: ONLINE - ready to accept requests", file=sys.stderr)
            print(f"", file=sys.stderr)
        except Exception as e:
            print(f"⚠️ Could not write status file: {e}", file=sys.stderr)

        try:
            _last_stale_cleanup = time.time()
            while not self.shutdown_requested:
                try:
                    # Periodic stale prefetch cleanup every 10s
                    # Connections that were prefetched but never picked up by workers
                    # (client disconnected) leak FDs and memory
                    now = time.time()
                    if now - _last_stale_cleanup > 10:
                        _last_stale_cleanup = now
                        with self._prefetched_lock:
                            stale_ids = []
                            for cid, entry in self._prefetched_data.items():
                                # Entry format: (data_bytes, timestamp_float)
                                if isinstance(entry, tuple) and len(entry) == 2:
                                    _, ts = entry
                                    if isinstance(ts, (int, float)) and now - ts > 15:
                                        stale_ids.append(cid)
                                elif isinstance(entry, bytes):
                                    # Legacy format without timestamp - always stale
                                    stale_ids.append(cid)
                            for cid in stale_ids:
                                self._prefetched_data.pop(cid, None)
                            if stale_ids:
                                print(f"🧹 Cleaned {len(stale_ids)} stale prefetch entries", file=sys.stderr, flush=True)

                    conn, _ = server.accept()
                    # Fast-path: handle meta requests (health/kys/ready) inline on main thread
                    # This prevents health checks from being blocked by busy worker threads
                    if self._try_handle_inline(conn):
                        continue  # Handled inline, next connection

                    # CONCURRENCY FIX: Check if connection still alive before queuing
                    # Under load, stale connections queue up and block workers
                    import select as _sel
                    try:
                        r, _, _ = _sel.select([conn], [], [], 0)
                        if r:
                            # Data ready OR connection closed — peek to check
                            peek = conn.recv(1, socket.MSG_PEEK | socket.MSG_DONTWAIT)
                            if not peek:
                                # Client already disconnected — skip
                                print(f"🗑️ Skipping dead connection (pre-queue)", file=sys.stderr, flush=True)
                                try:
                                    conn.close()
                                except:
                                    pass
                                continue
                    except (BlockingIOError, OSError):
                        pass  # Connection alive, proceed

                    # Embedding request — submit to worker pool
                    conn.settimeout(120)
                    # Submit connection handling to thread pool for concurrent processing
                    executor.submit(self._handle_connection, conn)
                except TimeoutError:
                    continue
                except Exception as e:
                    if self.shutdown_requested:
                        break
                    print(f"❌ Accept error: {e}", file=sys.stderr)
        finally:
            # Cleanup on shutdown
            print(f"🛑 Embedding server shutting down...", file=sys.stderr)
            # LOW-08 fix: Use cancel_futures=True for faster shutdown
            # This cancels any queued but not-yet-started futures immediately
            executor.shutdown(wait=True, cancel_futures=True)
            server.close()
            if os.path.exists(self.socket_path):
                os.remove(self.socket_path)
            print(f"✅ Shutdown complete. Will restart on next embedding request.", file=sys.stderr)


def main():
    import argparse

    parser = argparse.ArgumentParser(description='Frankenstein Embeddings v4 - TRULY DYNAMIC Dimension Server')
    parser.add_argument(
        '--socket',
        default=os.path.join(SPECMEM_SOCKET_DIR, EMBEDDING_SOCKET_NAME),
        help='Socket path'
    )
    parser.add_argument(
        '--db-host',
        default=os.environ.get('SPECMEM_DB_HOST', 'localhost'),
        help='Database host'
    )
    parser.add_argument(
        '--db-port',
        default=os.environ.get('SPECMEM_DB_PORT', '5432'),
        help='Database port'
    )
    parser.add_argument(
        '--db-name',
        default=os.environ.get('SPECMEM_DB_NAME', 'specmem'),
        help='Database name (SPECMEM_DB_NAME env var)'
    )
    parser.add_argument(
        '--db-user',
        default=os.environ.get('SPECMEM_DB_USER', 'specmem'),
        help='Database user (SPECMEM_DB_USER env var)'
    )
    parser.add_argument(
        '--db-password',
        default=os.environ.get('SPECMEM_DB_PASSWORD', ''),
        help='Database password (SPECMEM_DB_PASSWORD env var)'
    )
    # Service mode - for Docker/daemon deployments ONLY
    parser.add_argument(
        '--service',
        action='store_true',
        help='Run in service mode: no idle shutdown, stays alive forever. WARNING: Only use this in Docker/daemon deployments. Using --service on bare metal causes zombie processes that consume 100%% CPU.'
    )
    parser.add_argument(
        '--idle-timeout',
        type=int,
        default=int(os.environ.get('SPECMEM_EMBEDDING_IDLE_TIMEOUT', '300')),
        help='Idle timeout in seconds (default: 300, use 0 to disable)'
    )
    # QQMS Throttling options
    parser.add_argument(
        '--no-throttle',
        action='store_true',
        help='Disable QQMS throttling (not recommended)'
    )
    parser.add_argument(
        '--max-rps',
        type=float,
        default=20.0,
        help='Maximum requests per second (default: 20)'
    )
    parser.add_argument(
        '--base-delay',
        type=float,
        default=50.0,
        help='Base delay between requests in ms (default: 50)'
    )
    parser.add_argument(
        '--cpu-threshold',
        type=float,
        default=70.0,
        help='CPU percentage threshold for heavy throttling (default: 70)'
    )
    # QQMS v2 - Enhanced queue with FIFO + ACK
    parser.add_argument(
        '--qqms-v2',
        action='store_true',
        help='Enable QQMS v2: FIFO + ACK queue with retry/DLQ (for low-resource environments)'
    )
    parser.add_argument(
        '--max-retries',
        type=int,
        default=3,
        help='QQMS v2: Max retry attempts before DLQ (default: 3)'
    )
    parser.add_argument(
        '--enable-overflow',
        action='store_true',
        help='QQMS v2: Enable PostgreSQL overflow queue for durability'
    )

    args = parser.parse_args()

    # Warn users about --service mode outside Docker
    if args.service and not os.path.exists('/.dockerenv'):
        print("\n" + "=" * 70, file=sys.stderr)
        print("WARNING: --service mode is intended for Docker/daemon deployments ONLY.", file=sys.stderr)
        print("Using --service on bare metal disables idle shutdown and can cause", file=sys.stderr)
        print("zombie processes that consume 100% CPU if multiple instances spawn.", file=sys.stderr)
        print("If you don't know what you're doing, remove the --service flag.", file=sys.stderr)
        print("The server will auto-shutdown after idle timeout (default: 300s).", file=sys.stderr)
        print("=" * 70 + "\n", file=sys.stderr)

    # ═══════════════════════════════════════════════════════════════════════════
    # 🔒 ACK VERIFICATION - We NEVER use a model that hasn't been fully optimized
    # ═══════════════════════════════════════════════════════════════════════════
    model_config = verify_optimizations()

    # Initialize global resource config and adaptive sizer
    global _resource_config, _adaptive_sizer
    _resource_config = ResourceConfig()
    _adaptive_sizer = AdaptiveBatchSizer(_resource_config)

    print("=" * 70, file=sys.stderr)
    print(f"FRANKENSTEIN EMBEDDINGS v5 - Project: {PROJECT_DIR_NAME}", file=sys.stderr)
    print("=" * 70, file=sys.stderr)
    print("", file=sys.stderr)
    print(f"Project Path: {PROJECT_PATH}", file=sys.stderr)
    print(f"Socket Dir: {SPECMEM_SOCKET_DIR}", file=sys.stderr)
    print("", file=sys.stderr)
    print("Features:", file=sys.stderr)
    print("  - NO HARDCODED DIMENSIONS - queries PostgreSQL for target dimension", file=sys.stderr)
    print("  - 60-second refresh: detects database dimension changes", file=sys.stderr)
    print("  - Dimension EXPANSION: expands from native to ANY target", file=sys.stderr)
    print("  - Dimension COMPRESSION: PCA reduction when needed", file=sys.stderr)
    print("  - RAM guard: 4GB limit with auto-throttling", file=sys.stderr)
    if args.qqms_v2 and HAS_QQMS_V2:
        print("  - QQMS v2: FIFO + ACK queue with retry/DLQ (enabled)", file=sys.stderr)
    else:
        print("  - QQMS Throttling: CPU-aware rate limiting", file=sys.stderr)
    print("  - Multi-instance isolation: project-scoped sockets", file=sys.stderr)
    print("  - Stats endpoint: Send {\"stats\": true}", file=sys.stderr)
    print("  - Refresh endpoint: Send {\"refresh_dimension\": true}", file=sys.stderr)
    print("", file=sys.stderr)

    # 🔥 THE BIG FOUR OPTIMIZATIONS 🔥
    print("🔥 SCORCHED EARTH OPTIMIZATIONS (ALL 4 ENABLED):", file=sys.stderr)
    print("  - OPT-1: WARM RAM - Model stays loaded, zero cold starts", file=sys.stderr)
    print("  - OPT-2: QQMS THROTTLE - CPU-aware delays with FIFO+ACK", file=sys.stderr)
    print("  - OPT-3: EFFICIENT I/O - select() based, no busy-waiting", file=sys.stderr)
    print("  - OPT-4: ADAPTIVE BATCH - Auto-adjusts batch size based on CPU/RAM", file=sys.stderr)
    print("", file=sys.stderr)

    # Resource limits
    print(f"Resource Limits:", file=sys.stderr)
    print(f"  - CPU: {_resource_config.cpu_min}% min, {_resource_config.cpu_max}% max", file=sys.stderr)
    print(f"  - RAM: {_resource_config.ram_min_mb}MB min, {_resource_config.ram_max_mb}MB max", file=sys.stderr)
    print("", file=sys.stderr)

    # heavyOps status
    if _resource_config.heavy_ops_enabled:
        print("🚀 HEAVY OPS MODE ENABLED:", file=sys.stderr)
        print(f"  - Batch size multiplier: {_resource_config.heavy_ops_batch_mult}x", file=sys.stderr)
        print(f"  - Throttle reduction: {int(_resource_config.heavy_ops_throttle_reduce * 100)}%", file=sys.stderr)
        print("", file=sys.stderr)

    if not args.no_throttle:
        print(f"QQMS Throttling Configuration:", file=sys.stderr)
        print(f"  Max RPS: {args.max_rps}", file=sys.stderr)
        print(f"  Base delay: {args.base_delay}ms", file=sys.stderr)
        print(f"  CPU threshold: {args.cpu_threshold}%", file=sys.stderr)
        print("", file=sys.stderr)

    # QQMS v2 configuration output
    if args.qqms_v2:
        if HAS_QQMS_V2:
            print(f"QQMS v2 Configuration (FIFO + ACK):", file=sys.stderr)
            print(f"  Max retries: {args.max_retries}", file=sys.stderr)
            print(f"  Overflow queue: {'enabled' if args.enable_overflow else 'disabled'}", file=sys.stderr)
            print(f"  Priority aging: 30s", file=sys.stderr)
            print(f"  Lease timeout: 60s", file=sys.stderr)
            print("", file=sys.stderr)
        else:
            print("⚠️ --qqms-v2 requested but qqms_v2.py not found, using legacy throttler", file=sys.stderr)
            print("", file=sys.stderr)

    db_config = {
        'host': args.db_host,
        'port': args.db_port,
        'database': args.db_name,
        'user': args.db_user,
        'password': args.db_password
    }

    # Create QQMS config from command line args
    qqms_config = None
    if not args.no_throttle:
        qqms_config = QQMSConfig(
            base_delay_ms=args.base_delay,
            max_requests_per_second=args.max_rps,
            cpu_high_threshold=args.cpu_threshold
        )

    # Determine idle timeout
    # Service mode = no idle shutdown (stays alive forever)
    # Can also be disabled via --idle-timeout 0 or env SPECMEM_EMBEDDING_IDLE_TIMEOUT=0
    idle_timeout = args.idle_timeout
    if args.service or os.environ.get('SPECMEM_EMBEDDING_SERVICE_MODE') == '1':
        idle_timeout = 0  # 0 = disabled
        print("🔧 SERVICE MODE: Idle shutdown DISABLED - server will stay alive forever", file=sys.stderr)
        print("", file=sys.stderr)

    # Initialize QQMS v2 if requested
    qqms_v2_instance = None
    if args.qqms_v2 and HAS_QQMS_V2:
        qqms_v2_config = QQMSv2Config(
            max_retries=args.max_retries,
            cpu_queue_threshold=args.cpu_threshold,
            enable_overflow=args.enable_overflow,
            base_delay_ms=args.base_delay,
            max_requests_per_second=args.max_rps
        )
        # Only pass db_config if overflow is enabled
        overflow_db = db_config if args.enable_overflow else None
        qqms_v2_instance = QQMSv2(config=qqms_v2_config, db_config=overflow_db)
        qqms_v2_instance.start_drain_thread()
        print("✅ QQMS v2 initialized with FIFO + ACK", file=sys.stderr)
        print("", file=sys.stderr)

    server = EmbeddingServer(
        socket_path=args.socket,
        db_config=db_config,
        idle_timeout=idle_timeout,
        enable_throttling=not args.no_throttle,
        qqms_config=qqms_config,
        qqms_v2=qqms_v2_instance
    )

    # Write PID file for lifecycle management
    # Format: PID:TIMESTAMP (matches embeddingServerManager.ts expectations)
    pid_file = os.path.join(os.path.dirname(args.socket), 'embedding.pid')
    try:
        with open(pid_file, 'w') as f:
            f.write(f"{os.getpid()}:{int(time.time() * 1000)}")
        print(f"📝 PID file written: {pid_file} (pid={os.getpid()})", file=sys.stderr)
    except Exception as e:
        print(f"⚠️ Could not write PID file: {e}", file=sys.stderr)

    # Signal handling for graceful shutdown
    import signal
    import traceback
    def handle_signal(signum, frame):
        sig_name = signal.Signals(signum).name
        # DEBUG: Log who sent the signal for troubleshooting
        my_pid = os.getpid()
        my_ppid = os.getppid()
        print(f"\n⚡ Received {sig_name} - shutting down gracefully...", file=sys.stderr)
        print(f"   DEBUG: my_pid={my_pid}, my_ppid={my_ppid}", file=sys.stderr)
        # Try to identify caller via /proc
        try:
            with open(f'/proc/{my_ppid}/cmdline', 'r') as f:
                parent_cmd = f.read().replace('\x00', ' ').strip()
            print(f"   DEBUG: parent_cmd={parent_cmd[:200]}", file=sys.stderr)
        except Exception as e:
            print(f"   DEBUG: could not read parent cmdline: {e}", file=sys.stderr)
        print(f"   DEBUG: stack trace:", file=sys.stderr)
        traceback.print_stack(frame, file=sys.stderr)
        server.shutdown_requested = True
        # Stop QQMS v2 drain thread if enabled
        if qqms_v2_instance:
            print("🛑 Stopping QQMS v2...", file=sys.stderr)
            qqms_v2_instance.stop()
        # Clean up PID file
        try:
            if os.path.exists(pid_file):
                os.remove(pid_file)
                print(f"🗑️ PID file removed: {pid_file}", file=sys.stderr)
        except Exception as e:
            print(f"⚠️ Could not remove PID file: {e}", file=sys.stderr)

    signal.signal(signal.SIGTERM, handle_signal)
    signal.signal(signal.SIGINT, handle_signal)

    # SIGUSR1 = hot-reload config (resource limits, thread counts, heavyOps)
    # Sent by specmem-console when user changes resource settings dynamically
    def handle_config_reload(signum, frame):
        global _CPU_THREAD_LIMIT, _CPU_THREAD_MIN
        try:
            config_path = os.path.join(PROJECT_PATH, 'specmem', 'model-config.json')
            if not os.path.exists(config_path):
                print(f"⚠️ SIGUSR1: No model-config.json at {config_path}", file=sys.stderr)
                return
            with open(config_path, 'r') as f:
                cfg = json.load(f)
            resources = cfg.get('resources', {})
            pool = cfg.get('resourcePool', {}).get('embedding', {})
            # Compute effective thread limits from pool
            new_core_max = pool.get('threads') or resources.get('cpuCoreMax') or _CPU_THREAD_LIMIT
            new_core_min = resources.get('cpuCoreMin', _CPU_THREAD_MIN)
            old_max, old_min = _CPU_THREAD_LIMIT, _CPU_THREAD_MIN
            _CPU_THREAD_LIMIT = int(new_core_max)
            _CPU_THREAD_MIN = int(new_core_min)
            # Update OMP/MKL env vars for any future subprocesses
            os.environ['OMP_NUM_THREADS'] = str(_CPU_THREAD_LIMIT)
            os.environ['MKL_NUM_THREADS'] = str(_CPU_THREAD_LIMIT)
            os.environ['OPENBLAS_NUM_THREADS'] = str(_CPU_THREAD_LIMIT)
            os.environ['SPECMEM_CPU_THREADS'] = str(_CPU_THREAD_LIMIT)
            os.environ['SPECMEM_CPU_THREADS_MIN'] = str(_CPU_THREAD_MIN)
            # Update CPU% env vars
            cpu_max = resources.get('cpuMax', 40)
            cpu_min = resources.get('cpuMin', 20)
            pool_cpu_weight = pool.get('cpuWeight', 0.60)
            os.environ['SPECMEM_CPU_MAX'] = str(int(cpu_max * pool_cpu_weight))
            os.environ['SPECMEM_CPU_MIN'] = str(int(cpu_min * pool_cpu_weight))
            # Update RAM env vars
            ram_max = resources.get('ramMaxMb', 6000)
            ram_min = resources.get('ramMinMb', 4000)
            pool_ram_weight = pool.get('ramWeight', 0.60)
            os.environ['SPECMEM_RAM_MAX_MB'] = str(int(ram_max * pool_ram_weight))
            os.environ['SPECMEM_RAM_MIN_MB'] = str(int(ram_min * pool_ram_weight))
            # Propagate to QQMS throttler if active
            if server.embedder and server.embedder.throttler:
                server.embedder.throttler.thread_min = _CPU_THREAD_MIN
                server.embedder.throttler.thread_max = _CPU_THREAD_LIMIT
                # Clamp current threads to new range
                server.embedder.throttler.current_threads = max(
                    _CPU_THREAD_MIN,
                    min(_CPU_THREAD_LIMIT, server.embedder.throttler.current_threads)
                )
                torch.set_num_threads(server.embedder.throttler.current_threads)
            else:
                # No throttler — set threads directly
                torch.set_num_threads(_CPU_THREAD_LIMIT)
            # Check heavyOps
            heavy = cfg.get('heavyOps', {})
            if heavy.get('enabled'):
                batch_size = cfg.get('embedding', {}).get('batchSize', 32)
                print(f"🔥 SIGUSR1: HeavyOps ON — batch={batch_size}", file=sys.stderr)
            print(f"🔄 SIGUSR1: Config reloaded — threads {old_min}-{old_max} → {_CPU_THREAD_MIN}-{_CPU_THREAD_LIMIT}", file=sys.stderr)
        except Exception as e:
            print(f"⚠️ SIGUSR1: Config reload failed: {e}", file=sys.stderr)

    signal.signal(signal.SIGUSR1, handle_config_reload)

    try:
        server.start()
    finally:
        # Clean up PID file on exit
        try:
            if os.path.exists(pid_file):
                os.remove(pid_file)
        except:
            pass
        # Ensure QQMS v2 is stopped on exit
        if qqms_v2_instance:
            qqms_v2_instance.stop()


if __name__ == '__main__':
    main()