# -*- coding: utf-8 -*-
"""原子级中英归一管线（lexicon + semantic + mdcg 集成）· 设计态验收。

管线四件套：
  lexicon/build_standard_en.py  英文标准库构建器（牛肉→cow_meat，构建即验证）
  lexicon/standard_en.json      词条真源（语素拼接自洽 + snake_case）
  semantic/en_normalizer.py     英文 query → 中文语素序列（时态归零→停用词→映射→专有词保留）
  semantic/zh_en_atoms.py       中文文本 → 标准英文原子序列（反方向序列化）

集成面（mdcg.en_zh_terms → expand_query_terms）：
  英→中语素追加为召回词+打分 bigram（en_zh_bigrams）；默认关闭
  （双语双路裁定：md_cg 主链路英文走独立 Jaccard 路，见 semantic/REPRODUCE.md），
  MDCG_EN_ATOMS=1 显式开启；纯中文 query 零触发；
  mdcos 四路 RRF 的 lexical 路同源复用 expand_query_terms，自动受益。

⚠ 与**统一归一层**的关系（2026-09-24 澄清，曾造成本件红灯；2026-09-30 缺省翻关）：
  `semantic/unify.py::unify_query` 是**另一个**开关，缺省**关**（MDCG_UNIFY_QUERY，
  2026-09-30 使用者裁定由「默认 "1"」翻为「未设即关、显式 "1" 才开」）：显式开启时
  检索入口把任意语言的 query 先归一成标准中文原子序列——英文 query 因此在**词法路**
  上就能命中中文节点（与 en_zh_terms 默认关并不矛盾：一个在 query 侧归一，一个在召回
  词扩展侧）。故「跨语词面零交集 ⇒ 召回必须归零」在**缺省态**（两个开关都不设）与
  「两个开关都关」两种形态下都成立；命中只在显式 `MDCG_UNIFY_QUERY=1` 时断言。
  三态都钉住（改口径必红）。

运行：python -m md_cg.test_en_pipeline
"""
from __future__ import annotations

import io
import json
import os
import shutil
import tempfile

from . import mdcg
from .semantic import en_normalizer, zh_en_atoms

PASS = FAIL = 0
FAILS = []


def ok(cond, name=""):
    global PASS, FAIL
    if cond:
        PASS += 1
    else:
        FAIL += 1
        FAILS.append(name)


# ---- 1 · en_normalizer：屈折归零 ----
ok(en_normalizer.strip_tense("Ate") == "eat", "irregular ate→eat")
ok(en_normalizer.strip_tense("went") == "go", "irregular went→go")
ok(en_normalizer.strip_tense("regretted") == "regret", "-ed 剥离+双辅音还原")
ok(en_normalizer.strip_tense("running") == "run", "-ing 剥离+双辅音还原")
ok(en_normalizer.strip_tense("stopped") == "stop", "stopped→stop")
ok(en_normalizer.strip_tense("crossed") == "cross", "ss 保留（正字法例外）")
ok(en_normalizer.strip_tense("called") == "call", "ll 保留（正字法例外）")
ok(en_normalizer.strip_tense("dogs") == "dog", "-s 复数剥离")
ok(en_normalizer.strip_tense("class") == "class", "-ss 例外不剥")
ok(en_normalizer.strip_tense("be") == "be", "短词不动")

# ---- 2 · en_normalizer：停用词 / 专有词 / 复合词 ----
t, d = en_normalizer.normalize_en_query("I eat beef yesterday")
ok(t == ["我", "吃", "牛肉", "昨天"], "端到端映射 beef→牛肉: %s" % t)
t2, _ = en_normalizer.normalize_en_query("Caroline went to Paris")
ok("Caroline" in t2 and "Paris" in t2, "专有词保留原名: %s" % t2)
ok(en_normalizer.normalize_en_query("soul hub")[0] == ["灵枢"], "短语级复合 soul hub→灵枢")
ok(en_normalizer.normalize_en_query("beef")[0] == ["牛肉"], "单词级复合 beef→牛肉")
ok(en_normalizer.normalize_en_query("xyzzy")[0] == ["xyzzy"], "未知词 unknown_keep")
t3, _ = en_normalizer.normalize_en_query("The dogs were running")
ok("the" not in [x.lower() for x in t3] and "were" not in [x.lower() for x in t3],
   "停用词剔除: %s" % t3)

# ---- 2b · 置信分层与第三方暴露面修正（2026-09-15，第三方 LoCoMo 报告发现 5）----
# 置信信号：mapped 词带 source（manual=人工校对层 / cedict=机械反查层）与
# low_confidence（cedict 层=True——第一义项直译语境失配风险，警告不拒绝）
t4, d4 = en_normalizer.normalize_en_query("The scavenger hunt was fun")
ok("寻宝游戏" in t4, "内嵌短语扫描 scavenger hunt→寻宝游戏（长文本内生效）: %s" % t4)
ok(d4 and d4[0].get("action") == "phrase_inline", "phrase_inline 留痕: %s" % d4[:1])
ok(en_normalizer.normalize_en_query("scared")[0] == ["害怕"],
   "手工层修正 scared→害怕（错切链路 scar×CEDICT 创痕）")
ok(en_normalizer.normalize_en_query("shelter")[0] == ["收容所"], "手工层修正 shelter→收容所")
ok(en_normalizer.normalize_en_query("tough")[0] == ["坚强"], "手工层修正 tough→坚强")
ok(en_normalizer.normalize_en_query("three times a week")[0] == ["每周三次"],
   "短语 three times a week→每周三次")
ok(en_normalizer.normalize_en_query("times")[0] == ["次"], "手工层修正 times→次")
t6, d6 = en_normalizer.normalize_en_query("great and thanks")
lowc = [dd for dd in d6 if dd.get("action") == "mapped"]
ok(lowc and all(dd.get("source") == "cedict" and dd.get("low_confidence") is True
                for dd in lowc), "CEDICT 机械层命中=低置信: %s" % d6)
t7, d7 = en_normalizer.normalize_en_query("pottery was great")
mfd = [dd for dd in d7 if dd.get("orig", "").lower() == "pottery"]
ok(mfd and mfd[0].get("source") == "manual" and mfd[0].get("low_confidence") is False,
   "人工校对层=高置信+来源透出: %s" % mfd)
ok(en_normalizer.low_confidence_terms(d6) == ["great", "thanks"],
   "low_confidence_terms 审计提取（去重保序）: %s" % en_normalizer.low_confidence_terms(d6))

# ---- 3 · zh_en_atoms：中文→标准英文原子序列 ----
ok(zh_en_atoms.serialize("我昨天吃牛肉") == "i yester day eat cow meat",
   "serialize 复合词拆原子")
ok(zh_en_atoms.serialize("灵枢记忆库") == "soul hub memory store", "serialize 灵枢记忆库")
ok(zh_en_atoms.segment("吃牛肉") == ["吃", "牛肉"], "贪心最长匹配")

# ---- 4 · mdcg.en_zh_terms：集成单元（双态：默认关 / MDCG_EN_ATOMS=1 开） ----
ok(mdcg.en_zh_terms("I eat beef yesterday") == [],
   "默认关闭：en_zh_terms 不触发（双语双路裁定）")
os.environ["MDCG_EN_ATOMS"] = "1"
ok(mdcg.en_zh_terms("I eat beef yesterday") == ["吃", "牛肉", "昨天"],
   "开启：en_zh_terms 三语素（代词剔除）: %s" % mdcg.en_zh_terms("I eat beef yesterday"))
ok(mdcg.en_zh_terms("我昨天吃了牛肉面") == [], "纯中文零触发")
ok(all(t not in mdcg._EN_ZH_PRONOUNS for t in mdcg.en_zh_terms("She saw my dog")),
   "代词全剔除、实词保留")
os.environ["MDCG_EN_ATOMS"] = "0"
ok(mdcg.en_zh_terms("I eat beef") == [], "显式关闭回退 legacy")
del os.environ["MDCG_EN_ATOMS"]

# ---- 5 · expand_query_terms：召回词融合 + 纯中文回归守卫 ----
os.environ["MDCG_EN_ATOMS"] = "1"
et = mdcg.expand_query_terms("I ate beef yesterday")
ok("牛肉" in et and "昨天" in et, "开启：英文 query 产出中文召回词: %s" % et)
ok("eat" in et, "形态归一仍在（ate→eat）")
del os.environ["MDCG_EN_ATOMS"]
ok("牛肉" not in mdcg.expand_query_terms("I ate beef yesterday"),
   "默认关闭：expand_query_terms 回 legacy（无中文语素）")
_zc = "抑制剂怎么选择效果好"
os.environ["MDCG_EN_ATOMS"] = "1"
zc_on = mdcg.expand_query_terms(_zc)
del os.environ["MDCG_EN_ATOMS"]
zc_off = mdcg.expand_query_terms(_zc)
ok(zc_on == zc_off, "纯中文 query 开关前后零差异（回归守卫）")

# ---- 6 · 端到端：英文 query 召回中文记忆 ----
tmp = tempfile.mkdtemp(prefix="mdcg_en_pipe_")
try:
    cg = mdcg.MdCG(root=tmp)
    cg.add("n_beef", "昨天中午我在家里吃了牛肉面，配了可乐。",
           layer="contextual", verification_basis="data")
    cg.add("n_unrel", "我喜欢在雨天听爵士乐。",
           layer="contextual", verification_basis="data")
    cg.flush()
    os.environ["MDCG_EN_ATOMS"] = "1"
    res, _meta = cg.search("I ate beef yesterday", judge=False)
    ids = [r[0].get("id") for r in res]
    del os.environ["MDCG_EN_ATOMS"]
    ok(len(res) >= 1 and ids[0] == "n_beef", "开启：英文 query top1 召回中文牛肉节点 %s" % ids)
    # 2026-09-30 使用者裁定：统一归一层缺省由开翻为关（semantic/unify.py）。
    # 故「两个开关都不设」= **两个都关** → 词面零交集，召回必须归零；
    # 命中只在**显式** MDCG_UNIFY_QUERY=1 时才成立（英文对照路要用）。
    # （旧断言「缺省态命中」对应的是 2026-09-23 的「默认开」口径，已随缺省翻转改写。）
    res_u, _mu = cg.search("I ate beef yesterday", judge=False)
    ids_u = [r[0].get("id") for r in res_u]
    ok(not res_u or all(r[1] <= 0 for r in res_u),
       "缺省（unify 缺省关 + EN_ATOMS 关）：词面零重叠召回归零 %s" % ids_u)
    os.environ["MDCG_UNIFY_QUERY"] = "1"
    try:
        res1, _m1 = cg.search("I ate beef yesterday", judge=False)
        ids1 = [r[0].get("id") for r in res1]
    finally:
        os.environ.pop("MDCG_UNIFY_QUERY", None)
    ok(len(res1) >= 1 and ids1[0] == "n_beef",
       "显式 unify=1：英文 query 经统一归一后 top1 命中中文节点 %s" % ids1)
    os.environ["MDCG_UNIFY_QUERY"] = "0"
    try:
        res0, _m0 = cg.search("I ate beef yesterday", judge=False)   # 关归一+EN_ATOMS 关
    finally:
        os.environ.pop("MDCG_UNIFY_QUERY", None)
    ok(not res0 or all(r[1] <= 0 for r in res0),
       "unify=0 且 EN_ATOMS 关：词面零重叠召回归零（跨语断点复现）")
finally:
    shutil.rmtree(tmp, ignore_errors=True)

# ---- 7 · 构建自洽：standard_en.json / search_index.json ----
_here = os.path.dirname(os.path.abspath(__file__))
std = json.load(io.open(os.path.join(_here, "lexicon", "standard_en.json"), encoding="utf-8"))
entries = {e["zh"]: e for e in std["entries"]}
ok(len(entries) == 62, "词条数 62（真源守卫）")
ok(entries["牛肉"]["standard_en"] == "cow_meat" and "beef" in entries["牛肉"]["legacy_en"],
   "牛肉=cow_meat（legacy 别名 beef）")
ok(entries["火山"]["standard_en"] == "fire_mountain", "语素序直拼 火山=fire_mountain")
ok(sum(1 for e in std["entries"] if e["class"] == "opaque") == 5, "opaque 化石词 5 条")
si = json.load(io.open(os.path.join(_here, "semantic", "search_index.json"), encoding="utf-8"))
ok(si["meta"]["atom_count"] == 815 and si["meta"]["zh_terms"] == 815,
   "search_index 与 atoms 数一致")
ok(si["en_index"].get("meat") and "牛肉" in si["en_index"]["meat"],
   "en_index 倒排 meat→牛肉")

# ---- 8 · mdcos lexical 路同源受益 ----
from .mdcos import MdCGOS
from .mdcg import expand_query_terms as _eqt
ok(_eqt is mdcg.expand_query_terms, "mdcos 复用同一 expand_query_terms（同源单实现）")

print("en_pipeline: %d assertions all green" % PASS if not FAIL
      else "en_pipeline: %d passed, %d FAILED: %s" % (PASS, FAIL, FAILS))
raise SystemExit(1 if FAIL else 0)