#!/usr/bin/env python3
"""Independent CPU tokenizer port for the reviewed native harness only.

No framework import occurs on module import. Execution loads a local tokenizer,
never a model, with CUDA hidden and offline operation. Input/output are bounded
plain JSON; this is not an executable trace, pickle or general RPC endpoint.
"""
import hashlib
import importlib.util
import json
import os
from pathlib import Path
import stat
import sys


def require(value, message):
    if not value: raise ValueError(message)


def main():
    require(sys.flags.isolated and sys.flags.dont_write_bytecode and os.environ.get("CUDA_VISIBLE_DEVICES") == "-1", "Explicit isolated CPU tokenizer process required")
    raw = sys.stdin.buffer.read(2 * 1048576 + 1)
    require(len(raw) <= 2 * 1048576, "Tokenizer input bound exceeded")
    def pairs(items):
        out = {}
        for key, value in items:
            require(key not in out, "Duplicate tokenizer JSON key")
            out[key] = value
        return out
    value = json.loads(raw.decode("utf8"), object_pairs_hook=pairs, parse_constant=lambda _: (_ for _ in ()).throw(ValueError("Nonfinite tokenizer JSON")))
    expected = {"schema", "runtimePath", "runtimeSha256", "checkpointRoot", "model", "binding", "prompt", "promptArtifact", "maxPromptBytes", "maxInputTokens"}
    require(type(value) is dict and set(value) == expected and value["schema"] == "omnius.nol.vllm-independent-tokenizer-input.v1", "Fixed tokenizer input schema required")
    path = Path(value["runtimePath"])
    require(path.is_absolute() and path.resolve(strict=True) == path and not path.is_symlink() and path.stat().st_size <= 1048576, "Exact bounded runtime source required")
    require(hashlib.sha256(path.read_bytes()).hexdigest() == value["runtimeSha256"], "Tokenizer prompt-parser source differs")
    spec = importlib.util.spec_from_file_location("_nol_independent_prompt_parser", path)
    runtime = importlib.util.module_from_spec(spec); spec.loader.exec_module(runtime)
    runtime.integer(value["maxPromptBytes"], 1, 1048576); runtime.integer(value["maxInputTokens"], 1, 256)
    messages, ref = runtime.parse_prompt({"prompt": value["prompt"], "promptArtifact": value["promptArtifact"]}, value["maxPromptBytes"])
    model, binding = value["model"], value["binding"]
    require(type(model) is dict and type(binding) is dict and binding.get("backend") == "vllm" and model.get("modelDigest") == binding.get("modelDigest") and model.get("tokenizerDigest") == binding.get("tokenizerDigest") and model.get("chatTemplateDigest") == binding.get("chatTemplateDigest"), "Full exact tokenizer/model binding required")
    root = Path(value["checkpointRoot"])
    require(root.is_absolute() and root.resolve(strict=True) == root and root.is_dir() and not root.is_symlink(), "Canonical tokenizer checkpoint root required")
    require(type(model.get("files")) is list and 1 <= len(model["files"]) <= 4096, "Bounded complete source inventory required")
    total, names, tokenizers = 0, set(), []
    for entry in model["files"]:
        require(type(entry) is dict and set(entry) == {"path", "role", "artifact"}, "Fixed source file ref required")
        name, artifact = entry["path"], entry["artifact"]
        require(type(name) is str and len(name) <= 1024 and "\\" not in name and not name.startswith("/") and all(part not in ("", ".", "..") for part in name.split("/")) and name not in names, "Normalized distinct tokenizer source paths required")
        names.add(name)
        if entry["role"] == "weights":
            require(name.endswith(".safetensors"), "Unknown source weight layout")
            continue
        require(type(artifact) is dict and set(artifact) == {"uri", "sha256", "sizeBytes", "mediaType"} and artifact["uri"] == "omnius-artifact://sha256/" + artifact["sha256"], "Full source artifact required")
        runtime.integer(artifact["sizeBytes"], 0, 16 * 1048576)
        total += artifact["sizeBytes"]; require(total <= 32 * 1048576, "Nonweight tokenizer inventory exceeds CPU budget")
        local = root / name
        require(local.resolve(strict=True) == local and local.is_file() and not local.is_symlink(), "Tokenizer source escaped the exact checkpoint")
        fd = os.open(local, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
        try:
            before = os.fstat(fd); require(stat.S_ISREG(before.st_mode) and before.st_size == artifact["sizeBytes"], "Source file extent/type differs")
            digest, consumed = hashlib.sha256(), 0
            while True:
                block = os.read(fd, min(1048576, artifact["sizeBytes"] - consumed + 1))
                if not block: break
                consumed += len(block); require(consumed <= artifact["sizeBytes"], "Tokenizer source grew")
                digest.update(block)
            after = os.fstat(fd)
            require(consumed == artifact["sizeBytes"] and digest.hexdigest() == artifact["sha256"] and (before.st_ino, before.st_size, before.st_mtime_ns, before.st_ctime_ns) == (after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns), "Tokenizer source bytes changed")
        finally: os.close(fd)
        if entry["role"] == "tokenizer": tokenizers.append({"path": name, "sha256": artifact["sha256"], "sizeBytes": artifact["sizeBytes"]})
    require({"config.json", "tokenizer.json", "tokenizer_config.json", "chat_template.jinja"} <= names and runtime.sha(runtime.canonical(tokenizers).encode()) == binding["tokenizerDigest"], "Tokenizer source inventory digest differs")
    require(os.environ.get("HF_HUB_OFFLINE") == "1" and os.environ.get("TRANSFORMERS_OFFLINE") == "1", "Offline tokenizer load required")
    wire = os.fdopen(os.dup(1), "wb", buffering=0); os.dup2(2, 1); sys.stdout = sys.stderr
    from transformers import AutoTokenizer
    tokenizer = AutoTokenizer.from_pretrained(str(root), local_files_only=True, trust_remote_code=False, use_fast=True)
    require(getattr(tokenizer, "is_fast", False) and type(tokenizer.chat_template) is str and runtime.sha(tokenizer.chat_template.encode()) == binding["chatTemplateDigest"], "Exact source-native fast tokenizer/template required")
    rendered = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, enable_thinking=False)
    require(type(rendered) is str and len(rendered.encode()) <= value["maxPromptBytes"], "Independent rendered prompt exceeds bound")
    ids = runtime.token_ids(tokenizer.encode(rendered, add_special_tokens=False), value["maxInputTokens"])
    same_ids = runtime.token_ids(tokenizer.apply_chat_template(messages, tokenize=True, return_dict=False, add_generation_prompt=True, enable_thinking=False), value["maxInputTokens"])
    require(ids == same_ids, "Independent template/encode paths differ")
    result = {"schema": "omnius.nol.vllm-tokenization.v1", "binding": binding, "promptArtifact": ref, "renderedPrompt": rendered, "renderedSha256": runtime.sha(rendered.encode()), "renderedBytes": len(rendered.encode()), "tokenIds": ids, "promptTokens": len(ids), "thinking": False}
    encoded = runtime.canonical(result).encode(); require(len(encoded) <= 2 * 1048576, "Tokenizer response bound exceeded")
    wire.write(encoded); wire.flush()


if __name__ == "__main__":
    try: main()
    except BaseException as error:
        sys.stderr.write("Independent tokenizer rejected: " + type(error).__name__ + "\n")
        sys.exit(1)
