diff --git a/backend/contracts/judgment.py b/backend/contracts/judgment.py new file mode 100644 index 0000000..98b76aa --- /dev/null +++ b/backend/contracts/judgment.py @@ -0,0 +1,87 @@ +"""T08 专项提供方准入契约：逐任务/用途验收与冻结权重修订。 + +Owner：本文件（docs/v2/11-judgment-providers.md §8.1 的 profile 契约）。 +`contracts/prescreen.py` 继续持有运行侧对象（Judgment / JudgmentConfig / +ScreeningRun）；把 profile 契约独立出来有两个理由： + +1. 11 §8.1 规定 profile 必须带 tokenizer / 预算字段，而 `prescreen.py` 有一条 + 静态底线（`test_prescreen_contracts.test_no_credentials_in_contract_module`： + 契约模块不得出现凭据字样）。把预算字段放进独立模块，比放宽那条安全断言更合适。 +2. T08 的准入规则（task_validations）与运行对象生命周期不同：配置可换修订， + 运行不得被就地改写（BR-20）。 + +依赖方向：本模块**不导入** `services/`。 +""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import Field, model_validator + +from contracts.prescreen import ID_MAX, PATH_MAX, Contract + +JudgmentTask = Literal[ + "extraction_grounding", + "experience_applicability", + "trajectory_quality", + "evidence_support", +] +JudgmentUsage = Literal["offline", "shadow", "runtime"] + + +class TaskValidation(Contract): + """逐任务、逐用途的验收登记（docs/v2/11-judgment-providers.md §8.1）。 + + 一条登记只证明"该任务在该用途下通过过验收"，不是模型整体合格，也不能 + 由"服务能连通"或"别的任务通过"顶替（AC-38）。offline 通过与 runtime + 启用必须分开登记，否则离线质检可以直接当生产准入。 + """ + + task: JudgmentTask + rubric_version: str = Field(min_length=1, max_length=ID_MAX) + dataset_ref: str = Field(min_length=1, max_length=PATH_MAX) + report_ref: str = Field(min_length=1, max_length=PATH_MAX) + decision: Literal["passed", "failed", "unverified"] + allowed_usage: JudgmentUsage + # 独立评测来源：同模型自评或其生成标签不算独立评测（AC-41） + independent: bool = False + + +class JudgmentProfile(Contract): + """专项提供方配置（T08）。只描述"能跑什么"，不代表已经跑通。 + + `checkpoint_ref` 必须可复现：远程不可变版本不可得时填 "unresolved"， + 此时运行期不允许隐式换权重（AC-39）。 + """ + + id: str = Field(min_length=1, max_length=ID_MAX) + revision: int = Field(ge=1) + provider: Literal["jev", "local"] + engine: str = Field(min_length=1, max_length=ID_MAX) + model_version: str = Field(min_length=1, max_length=ID_MAX) + adapter_version: str = Field(min_length=1, max_length=ID_MAX) + prompt_version: str = Field(min_length=1, max_length=ID_MAX) + mapping_version: str = Field(min_length=1, max_length=ID_MAX) + checkpoint_ref: str = Field(min_length=1, max_length=ID_MAX) + tokenizer_revision: str = Field(min_length=1, max_length=ID_MAX) + input_token_limit: int = Field(gt=0) + option_token_limit: int = Field(gt=0) + language_scope: str = Field(min_length=1, max_length=ID_MAX) + deployment_mode: Literal["in_process", "internal_http", "external_http"] + # 外发授权：未批准时 jev 一律拒绝发送，且不据此降低本地要求（AC-16/19） + egress_approved: bool = False + readiness: Literal["ready", "unready"] = "unready" + task_validations: list[TaskValidation] = Field(default_factory=list, max_length=32) + is_default: bool = False + + @model_validator(mode="after") + def _check_profile(self) -> JudgmentProfile: + if self.provider == "local" and self.deployment_mode == "external_http": + raise ValueError("本地模型不得走外部出口；外发只属于 jev 路径") + if self.provider == "jev" and self.deployment_mode != "external_http": + raise ValueError("jev 属外部服务，部署模式必须是 external_http") + seen = {(item.task, item.allowed_usage) for item in self.task_validations} + if len(seen) != len(self.task_validations): + raise ValueError("同一任务与用途只能登记一条验收结果，不得重复累计") + return self diff --git a/backend/contracts/prescreen.py b/backend/contracts/prescreen.py index 8024dfa..279709f 100644 --- a/backend/contracts/prescreen.py +++ b/backend/contracts/prescreen.py @@ -110,6 +110,8 @@ class ErrorCode(StrEnum): JUDGMENT_UNAVAILABLE = "JUDGMENT_UNAVAILABLE" JUDGMENT_CAPABILITY_UNSUPPORTED = "JUDGMENT_CAPABILITY_UNSUPPORTED" JUDGMENT_OUTPUT_INVALID = "JUDGMENT_OUTPUT_INVALID" + # E-25：输入超出模型已验证预算（BR-39 / AC-40）。T08 适配层使用。 + JUDGMENT_CONTEXT_EXCEEDED = "JUDGMENT_CONTEXT_EXCEEDED" RUN_BUDGET_EXCEEDED = "RUN_BUDGET_EXCEEDED" @@ -139,6 +141,7 @@ RUN_LEVEL_CODES = frozenset({ ErrorCode.JUDGMENT_UNAVAILABLE, ErrorCode.JUDGMENT_CAPABILITY_UNSUPPORTED, ErrorCode.JUDGMENT_OUTPUT_INVALID, + ErrorCode.JUDGMENT_CONTEXT_EXCEEDED, ErrorCode.RUN_BUDGET_EXCEEDED, }) @@ -703,6 +706,8 @@ class Judgment(Contract): native_metrics: list[NativeMetric] = Field(default_factory=list, max_length=32) error_code: ErrorCode | None = None skip_reason: Literal["disabled", "not_configured", "not_authorized", "unsupported"] | None = None + # 本次实际使用的权重修订（T08，AC-39）：与冻结值不符即拒绝，不做隐式路由。 + actual_checkpoint_ref: str | None = Field(default=None, max_length=ID_MAX) @model_validator(mode="after") def _check_status_consistency(self) -> Judgment: @@ -739,6 +744,8 @@ class JudgmentConfig(Contract): prompt_version: str | None = Field(default=None, max_length=ID_MAX) mapping_version: str | None = Field(default=None, max_length=ID_MAX) disabled_reason: str | None = Field(default=None, max_length=DESC_MAX) + # T08：local 必须固定权重修订与摘要；不可得时为 "unresolved"（11 §8.1）。 + checkpoint_ref: str | None = Field(default=None, max_length=ID_MAX) @model_validator(mode="after") def _check_freeze(self) -> JudgmentConfig: @@ -757,6 +764,7 @@ class JudgmentConfig(Contract): return self + # ---------------------------------------------------------------- 预审运行（T07-c） class Degradation(Contract): """降级记录：未完成的检查必须留痕，不能静默（BR-09）。""" diff --git a/backend/fixtures/judgment/jev_vendor_responses.json b/backend/fixtures/judgment/jev_vendor_responses.json new file mode 100644 index 0000000..f849e9e --- /dev/null +++ b/backend/fixtures/judgment/jev_vendor_responses.json @@ -0,0 +1,29 @@ +{ + "_note": "T08 统一适配 fixture —— Jev 厂商响应形态（虚构字段名，仅用于验证适配契约）。没有凭据时不发送任何请求：本文件只作离线契约用例（docs/v2/11-judgment-providers.md §8.4 步6）。", + "responses": { + "Q-EX-01": { + "vendor": "jev", + "api_version": "2026-06-01", + "checkpoint": "jev-3.2@sha256:7ad401", + "result": {"decision_code": "COND_APPROVE", "narrative": "条件上成立，需补充回款核验。"}, + "metrics": {"confidence": 0.74, "uncertainty": 0.19}, + "usage": {"prompt_tokens": 812, "output_tokens": 96} + }, + "Q-EX-02": { + "vendor": "jev", + "api_version": "2026-06-01", + "checkpoint": "jev-3.2@sha256:7ad401", + "result": {"decision_code": "REJECT", "narrative": "证据不足，不能支持该依据。"}, + "metrics": {"confidence": 0.62}, + "usage": {"prompt_tokens": 790, "output_tokens": 88} + }, + "Q-EX-03": { + "vendor": "jev", + "api_version": "2026-06-01", + "checkpoint": "jev-3.2@sha256:7ad401", + "result": {"decision_code": "NOT_LISTED", "narrative": "厂商新增编码，本适配层未登记映射。"}, + "metrics": {}, + "usage": {"prompt_tokens": 640, "output_tokens": 40} + } + } +} diff --git a/backend/fixtures/judgment/question_bank_extraction.json b/backend/fixtures/judgment/question_bank_extraction.json new file mode 100644 index 0000000..23070fd --- /dev/null +++ b/backend/fixtures/judgment/question_bank_extraction.json @@ -0,0 +1,165 @@ +{ + "_note": "T08 统一适配 fixture —— 中文提炼依据质检题集与本地模型原始输出。虚构演示内容，不是兴邦真实题库；fixture 通过不等于中文效果合格（docs/v2/06-validation.md §6）。", + "rubric_version": "extraction-grounding.v1", + "checkpoint_ref": "laya-1.0-zh@sha256:2f1c8d", + "questions": { + "Q-EX-01": { + "id": "Q-EX-01", + "version": "extraction-grounding.v1#1", + "type": "choice", + "prompt": "材料是否支持『客户结构稳定』这一提炼依据？", + "options": [ + "supported", + "unsupported" + ], + "evidence_ids": [ + "EV-EX-01" + ], + "evidence_text": [ + "近三年主要客户未发生变动，前五大客户集中度由 61% 降至 58%。" + ] + }, + "Q-EX-02": { + "id": "Q-EX-02", + "version": "extraction-grounding.v1#1", + "type": "choice", + "prompt": "材料是否支持『回款能力未下降』这一提炼依据？", + "options": [ + "supported", + "unsupported" + ], + "evidence_ids": [ + "EV-EX-02" + ], + "evidence_text": [ + "主要客户账期由 60 天延长至 95 天，尚未提供银行流水核验。" + ] + }, + "Q-EX-03": { + "id": "Q-EX-03", + "version": "extraction-grounding.v1#1", + "type": "choice", + "prompt": "历史案例 H-018 的适用条件在当前项目是否成立？", + "options": [ + "supported", + "unsupported" + ], + "evidence_ids": [ + "EV-EX-03" + ], + "evidence_text": [ + "案例适用条件为『客户结构稳定且回款可逐笔匹配』，例外为『收入集中或关联回款未穿透』。" + ] + }, + "Q-EX-04": { + "id": "Q-EX-04", + "version": "extraction-grounding.v1#1", + "type": "choice", + "prompt": "两份来源对同一事实的说明是否一致？", + "options": [ + "supported", + "unsupported" + ], + "evidence_ids": [ + "EV-EX-04", + "EV-EX-05" + ], + "evidence_text": [ + "说明称设备清单与发票品名一致；核验记录标注存在三项品名不符。" + ] + }, + "Q-EX-05": { + "id": "Q-EX-05", + "version": "extraction-grounding.v1#1", + "type": "choice", + "prompt": "长材料是否仍支持同一提炼依据？", + "options": [ + "supported", + "unsupported" + ], + "evidence_ids": [ + "EV-EX-06" + ], + "evidence_text": [ + "长材料样例：经营情况说明、账期变化说明、设备权属说明、租赁物清单与核验记录逐段拼接，用于触发上下文预算拒绝路径。" + ], + "evidence_padding": "这是一段刻意加长的正文，用于验证超出已验证上下文预算时必须返回未完成而不是截断后继续发送。", + "evidence_repeat": 12 + } + }, + "local_answers": { + "Q-EX-01": { + "label": "支持", + "mapping": { + "支持": "supported", + "不支持": "unsupported" + }, + "confidence": 0.91, + "evidence_ids": [ + "EV-EX-01" + ], + "actual_checkpoint_ref": "laya-1.0-zh@sha256:2f1c8d" + }, + "Q-EX-02": { + "label": "不符合", + "mapping": { + "支持": "supported", + "不支持": "unsupported" + }, + "confidence": 0.88, + "evidence_ids": [ + "EV-EX-02" + ], + "actual_checkpoint_ref": "laya-1.0-zh@sha256:2f1c8d" + }, + "Q-EX-03": { + "label": "支持", + "mapping": { + "支持": "supported" + }, + "exception_applies": true, + "evidence_ids": [ + "EV-EX-03" + ], + "actual_checkpoint_ref": "laya-1.0-zh@sha256:2f1c8d" + }, + "Q-EX-04": { + "label": "支持", + "mapping": { + "支持": "supported", + "不支持": "unsupported" + }, + "conflicts": [ + "EV-EX-04", + "EV-EX-05" + ], + "evidence_ids": [ + "EV-EX-04", + "EV-EX-05" + ], + "actual_checkpoint_ref": "laya-1.0-zh@sha256:2f1c8d" + }, + "Q-EX-05": { + "label": "支持", + "mapping": { + "支持": "supported", + "不支持": "unsupported" + }, + "evidence_ids": [ + "EV-EX-06" + ], + "actual_checkpoint_ref": "laya-1.0-zh@sha256:2f1c8d" + }, + "Q-EX-06": { + "label": "支持", + "mapping": { + "支持": "supported", + "不支持": "unsupported" + }, + "evidence_ids": [ + "EV-EX-01" + ], + "actual_checkpoint_ref": "laya-1.1-zh@sha256:99be07" + } + } +} diff --git a/backend/services/judgment_adapters.py b/backend/services/judgment_adapters.py new file mode 100644 index 0000000..7d6f46f --- /dev/null +++ b/backend/services/judgment_adapters.py @@ -0,0 +1,251 @@ +"""T08 专项离线质检：统一适配 fixture、逐任务/用途准入、冻结 checkpoint 与上下文预算。 + +依据：docs/v2/11-judgment-providers.md §8、docs/v2/06-validation.md §4/§6； +BR-17/18/19/20/37/38/39、E-10/13/14/15/25、AC-16/17/18/19/20/38/39/40/41。 + +本批**只交付离线 fixture 路径**。没有 Jev 凭据、没有 Laya 权重与授权， +因此： +- live 传输层只留注入点，未注入时明确 `LiveTransportUnavailable`； +- fixture 通过只证明适配代码结构成立，**不证明**中文专项效果合格 + （06-validation.md §6："fixture 通过仅证明适配代码结构"）。 + +四条不可让渡的规则： +1. **逐任务逐用途准入**：`task_validations` 里没有 (task, usage) 的 passed 记录， + 就不执行；服务能连通、别的任务通过、装好了权重，都不算准入（AC-38）。 +2. **权重不可隐式换**：运行冻结 `checkpoint_ref`；提供方实际权重不一致即拒绝， + 不按语言或负载偷偷路由到别的模型（AC-39）。 +3. **超预算不外发**：超出已验证上下文预算时返回 E-25 并保留原证据范围， + 绝不截断后半段照样发送（AC-40 / BR-39）。 +4. **歧义不硬选**：映射未知、中文否定、条件例外、来源矛盾都落 unknown/人工复核， + 不挑一个"看起来正面"的选项（AC-18）。 +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from contracts.judgment import JudgmentProfile +from contracts.prescreen import JudgmentConfig + +FIXTURE_ROOT = Path(__file__).resolve().parent.parent / "fixtures" / "judgment" + +# 离线预算估算：中文按 1 字≈1 token、ASCII 按 4 字符≈1 token 的保守上界。 +# 真实预算必须按选定模型的 tokenizer 实测填写（11 §8.1），这里的估算只用于 +# 离线 fixture 与"宁可拒绝"的方向，不作为生产阈值。 +_CJK_FLOOR = 0x2E80 + + +class ContextBudgetExceeded(RuntimeError): + """超出已验证上下文预算（E-25）。携带原证据范围，供上层保留引用。""" + + def __init__(self, message: str, evidence_ids=()): + super().__init__(message) + self.evidence_ids = list(evidence_ids) + + +class TaskNotValidated(ValueError): + """该任务/用途没有 passed 的验收登记（AC-38）。""" + + +class CheckpointMismatch(ValueError): + """实际权重修订与运行冻结值不一致（AC-39）。""" + + +class EgressNotApproved(PermissionError): + """未获批准的外部出口：不发送（AC-16/19）。""" + + +class LiveTransportUnavailable(RuntimeError): + """没有可用的 live 传输层（本批未接入硬件与授权）。""" + + +def estimate_tokens(text: str) -> int: + """保守估算 token 数：CJK 每字 1，其它每 4 字符 1，向上取整。""" + if not text: + return 0 + cjk = sum(1 for char in text if ord(char) >= _CJK_FLOOR) + other = len(text) - cjk + return cjk + (other + 3) // 4 + + +def load_fixture(name: str) -> dict: + return json.loads((FIXTURE_ROOT / name).read_text(encoding="utf-8")) + + +# ------------------------------------------------------------------ 准入 +def validation_for(profile: JudgmentProfile, task: str, usage: str): + """取出该任务与用途的登记；没有登记返回 None，不做"默认通过"。""" + for item in profile.task_validations: + if item.task == task and item.allowed_usage == usage: + return item + return None + + +def assert_task_allowed(profile: JudgmentProfile, task: str, usage: str) -> None: + """逐任务逐用途准入（AC-38 / AC-41）。 + + offline 与 runtime 分开登记：只有 offline 通过的配置不能进 runtime； + runtime 还要求验收是独立评测——同模型自评或其生成标签不算证据。 + """ + if profile.readiness != "ready": + raise TaskNotValidated("该专项配置尚未就绪，不能执行任何任务") + record = validation_for(profile, task, usage) + if record is None or record.decision != "passed": + raise TaskNotValidated( + f"该配置未在用途 {usage} 通过任务 {task} 的验收；" + "连通、安装或其它任务通过都不能替代该任务准入") + if usage == "runtime" and not record.independent: + raise TaskNotValidated("runtime 准入必须是独立评测结果，同模型自评不能作为效果证据") + + +def assert_checkpoint_frozen(config: JudgmentConfig, profile: JudgmentProfile) -> None: + """运行冻结值优先：配置登记被改动后，旧运行仍按原值执行（AC-20/39）。""" + if profile.checkpoint_ref == "unresolved": + raise CheckpointMismatch("权重修订不可复现（unresolved），不得用于任何执行") + if config.checkpoint_ref and config.checkpoint_ref != profile.checkpoint_ref: + raise CheckpointMismatch( + "登记配置的权重修订与本次运行冻结值不一致；请新建运行而不是就地替换") + + +def assert_within_budget(question: dict, profile: JudgmentProfile) -> None: + """调用前算预算；超限即拒绝，不截断、不外发（AC-40）。 + + 正文（含提示词）与选项分开计：选项预算不能被正文的余量顶掉，反之亦然。 + """ + body = "".join(str(item) for item in question.get("evidence_text") or ()) + prompt = str(question.get("prompt") or "") + options = " ".join(str(item) for item in question.get("options") or ()) + evidence_ids = question.get("evidence_ids") or () + + body_tokens = estimate_tokens(body) + estimate_tokens(prompt) + if body_tokens > profile.input_token_limit: + raise ContextBudgetExceeded( + f"正文约 {body_tokens} token，超出该模型已验证输入预算 " + f"{profile.input_token_limit}；请缩小问题或转人工复核", + evidence_ids=evidence_ids) + + option_tokens = estimate_tokens(options) + if option_tokens > profile.option_token_limit: + raise ContextBudgetExceeded( + f"选项约 {option_tokens} token，超出该模型已验证选项预算 " + f"{profile.option_token_limit}；请缩减候选项或转人工复核", + evidence_ids=evidence_ids) + + +# ------------------------------------------------------------------ 映射 +def _unknown(reason: str) -> dict: + return {"kind": "unknown", "reason_code": reason} + + +def map_local_answer(raw: dict, question: dict, profile: JudgmentProfile) -> dict: + """本地模型原始输出 → 统一契约形状。 + + 歧义一律落 unknown/inconclusive：映射未知、中文否定、条件例外、来源矛盾 + 都不许"挑一个正面结果"（AC-18）。`confidence` 只作为 self_reported 指标 + 保留原语义，**不**换算成任何概率。 + """ + mapping = (raw.get("mapping") or {}) + if raw.get("conflicts"): + return _unknown("inconclusive") + if raw.get("exception_applies"): + return _unknown("inconclusive") + label = raw.get("label") + if not label: + return _unknown("missing_evidence" if not raw.get("evidence_ids") else "inconclusive") + option = mapping.get(label) + if option is None: + # 例如中文否定表述（"不符合"）未登记映射：不得当成正面选项使用 + return _unknown("inconclusive") + metrics = [] + if raw.get("confidence") is not None: + metrics.append({"name": "self_confidence", "value": float(raw["confidence"]), + "semantics": "模型自报置信度，未经校准，与审批概率无关", + "source": "self_reported"}) + return {"kind": question["type"], "value": option, "native_metrics": metrics, + "actual_checkpoint_ref": raw.get("actual_checkpoint_ref")} + + +# jev 厂商字段名与本地完全不同：映射表按 mapping_version 固定，主审不读厂商字段 +# （AC-17）。新增厂商版本要新增一行映射并单独登记验收，不改这一层的行为。 +# 本行只登记 choice 题型的映射：其它题型没有登记就落 unknown，而不是借这一行 +# 顺带支持（06-validation.md §6："阈值和映射按提供方/模型/题型独立登记"）。 +_DEV_VENDOR_CHOICES = { + "APPROVE": "supported", + "REJECT": "unsupported", + # COND_APPROVE（附条件）刻意**不**映射为通过：附条件不等于通过（BR-21） +} + + +def map_jev_answer(raw: dict, question: dict, profile: JudgmentProfile) -> dict: + """Jev 厂商响应 → 统一契约形状（fixture 验证用，不发送任何请求）。""" + if question["type"] != "choice": + return _unknown("inconclusive") + envelope = raw.get("result") or {} + code = envelope.get("decision_code") + value = _DEV_VENDOR_CHOICES.get(code) + if value is None: + return _unknown("inconclusive") + metrics = [] + for name, item in (raw.get("metrics") or {}).items(): + # 厂商原生指标保留语义与来源；不跨提供方套阈值，也不补齐缺失概率（AC-18） + metrics.append({"name": str(name), "value": float(item), + "semantics": "厂商原生指标，定义与兴邦校准口径不一致", + "source": "provider_native"}) + return {"kind": question["type"], "value": value, "native_metrics": metrics, + "actual_checkpoint_ref": raw.get("checkpoint")} + + +MAPPERS = {"local": map_local_answer, "jev": map_jev_answer} + + +# ------------------------------------------------------------------ 适配入口 +def build_provider_call(config: JudgmentConfig, profile: JudgmentProfile, task: str, + usage: str, *, fixture=None, transport=None, call_log=None): + """构造 `judge()` 用的 provider_call：先过准入与冻结，再决定是否真的发送。 + + `fixture(question) -> raw` 是离线路径（不联网）；`transport(target, payload)` + 是 live 路径。两者不能互相冒充：call_log 会记录 mode，测试据此断言 + "local 运行期间没有任何 jev 请求"（AC-17）。 + """ + assert_task_allowed(profile, task, usage) + assert_checkpoint_frozen(config, profile) + mapper = MAPPERS[profile.provider] + + def call(config: JudgmentConfig, question: dict) -> dict: + # 预算在发送之前判定：超限即结束，调用记录里不会出现任何发送 + assert_within_budget(question, profile) + payload = json.dumps({ + "question_id": question["id"], "version": question["version"], + "type": question["type"], "prompt": question.get("prompt", ""), + "options": list(question.get("options") or ()), + "evidence_ids": list(question.get("evidence_ids") or ()), + "checkpoint_ref": config.checkpoint_ref, + "prompt_version": config.prompt_version, + "mapping_version": config.mapping_version, + }, ensure_ascii=False) + + if fixture is not None: + raw = fixture(question) + mode = "fixture" + else: + if config.provider == "jev" and not profile.egress_approved: + raise EgressNotApproved("该外部出口未获批准，已拒绝发送") + if transport is None: + raise LiveTransportUnavailable( + "本批未接入 live 传输层（缺硬件与授权），只能走离线 fixture") + raw = transport(profile.provider, payload) + mode = "live" + + if call_log is not None: + call_log.append({"target": config.provider, "mode": mode, + "question_id": question["id"], "payload": payload}) + + adapted = mapper(raw, question, profile) + actual = adapted.get("actual_checkpoint_ref") + if actual and actual != config.checkpoint_ref: + raise CheckpointMismatch( + f"提供方实际权重 {actual} 与运行冻结值 {config.checkpoint_ref} 不一致") + return adapted + + return call diff --git a/backend/services/judgment_provider.py b/backend/services/judgment_provider.py index 6528985..3ce9301 100644 --- a/backend/services/judgment_provider.py +++ b/backend/services/judgment_provider.py @@ -26,6 +26,14 @@ from contracts.prescreen import ( ) from services import assistant_store as store from services.component_registry import TOOLS +# T08：适配层的策略拒绝用异常表达，在这里统一映射成 Judgment 记录。 +# judgment_adapters 只依赖 contracts，不反向依赖本模块，因此没有循环导入。 +from services.judgment_adapters import ( + CheckpointMismatch, + ContextBudgetExceeded, + EgressNotApproved, + TaskNotValidated, +) DISABLED_REASON_DEFAULT = "本次运行未启用专项判断；主审仍按规则、工具与经验形成意见" @@ -69,7 +77,9 @@ def resolve_config(enabled: bool, profile_id: str | None = None, model_version=profile.get("model_version") or "unresolved", adapter_version=profile.get("adapter_version") or "not_applicable", prompt_version=profile.get("prompt_version") or "not_applicable", - mapping_version=profile.get("mapping_version") or "not_applicable") + mapping_version=profile.get("mapping_version") or "not_applicable", + # T08：权重修订随配置一起冻结；缺失记 unresolved 而不是留空猜测（AC-39） + checkpoint_ref=profile.get("checkpoint_ref") or "unresolved") def allowed_tools(skill_body: dict, registry_tools=None) -> set[str]: @@ -127,6 +137,20 @@ def judge(config: JudgmentConfig, run_id: str, question: dict, raw = provider_call(config, question) except TimeoutError: return _failed(run_id, question, config, ErrorCode.JUDGMENT_UNAVAILABLE) + except ContextBudgetExceeded: + # E-25：超预算不外发、不静默截断；evidence_ids 仍保留原引用范围（AC-40） + return _failed(run_id, question, config, ErrorCode.JUDGMENT_CONTEXT_EXCEEDED) + except TaskNotValidated: + # 未获准的任务不给"未完成"以外的结论（AC-38） + return {**_failed(run_id, question, config, ErrorCode.JUDGMENT_UNAVAILABLE), + "status": "skipped", "error_code": None, "skip_reason": "unsupported"} + except EgressNotApproved: + # 未批准出口：不发送，记录策略拒绝（AC-16/19） + return {**_failed(run_id, question, config, ErrorCode.JUDGMENT_UNAVAILABLE), + "status": "skipped", "error_code": None, "skip_reason": "not_authorized"} + except CheckpointMismatch: + # 实际权重与冻结修订不符：拒绝并记失败，不得静默换模型（AC-39） + return _failed(run_id, question, config, ErrorCode.JUDGMENT_OUTPUT_INVALID) except Exception: return _failed(run_id, question, config, ErrorCode.JUDGMENT_UNAVAILABLE) @@ -179,7 +203,8 @@ def _normalise(run_id: str, question: dict, config: JudgmentConfig, raw: dict) - "provider": config.provider, "profile_id": config.profile_id, "profile_revision": config.profile_revision, "model_version": config.model_version, "native_metrics": metrics, - "error_code": None, "skip_reason": None} + "error_code": None, "skip_reason": None, + "actual_checkpoint_ref": raw.get("actual_checkpoint_ref")} return Judgment(**body).model_dump(mode="json") diff --git a/backend/test_judgment_adapters.py b/backend/test_judgment_adapters.py new file mode 100644 index 0000000..597e04e --- /dev/null +++ b/backend/test_judgment_adapters.py @@ -0,0 +1,403 @@ +"""T08 专项离线质检：统一适配 fixture、逐任务/用途准入、冻结 checkpoint、上下文预算。 + +覆盖 AC-16/17/18/19/20/38/39/40/41（对应 BR-17/18/19/20/37/38/39、E-10/13/14/15/25）。 + +本文件只跑**离线 fixture**：不发网络请求、不接真实权重。fixture 通过只证明 +适配代码结构与守卫成立，不证明中文专项效果合格（docs/v2/06-validation.md §6）。 +""" + +from __future__ import annotations + +import unittest + +from contracts.judgment import JudgmentProfile, TaskValidation +from contracts.prescreen import ErrorCode, JudgmentConfig +from services import judgment_adapters as adapters +from services.judgment_provider import judge + +BANK = "question_bank_extraction.json" +VENDOR = "jev_vendor_responses.json" + +LOCAL_CHECKPOINT = "laya-1.0-zh@sha256:2f1c8d" +JEV_CHECKPOINT = "jev-3.2@sha256:7ad401" + + +def bank() -> dict: + return adapters.load_fixture(BANK) + + +def question(qid: str, **over) -> dict: + body = dict(bank()["questions"][qid]) + body.update(over) + return body + + +def local_answer(qid: str) -> dict: + return bank()["local_answers"][qid] + + +def vendor_answer(qid: str) -> dict: + return adapters.load_fixture(VENDOR)["responses"][qid] + + +def validation(task="extraction_grounding", usage="runtime", + decision="passed", independent=True) -> TaskValidation: + return TaskValidation( + task=task, rubric_version="extraction-grounding.v1", + dataset_ref=f"fixtures/judgment/{BANK}", report_ref="test-evidence/t08/REPORT.md", + decision=decision, allowed_usage=usage, independent=independent) + + +def profile(**over) -> JudgmentProfile: + base = dict( + id="PR-LOCAL-1", revision=1, provider="local", engine="laya", + model_version="laya-1.0-zh", adapter_version="local-adapter.v1", + prompt_version="grounding-prompt.v1", mapping_version="local-map.v1", + checkpoint_ref=LOCAL_CHECKPOINT, tokenizer_revision="tok.v1", + input_token_limit=400, option_token_limit=40, language_scope="zh-CN", + deployment_mode="internal_http", egress_approved=False, readiness="ready", + task_validations=[validation()], is_default=True) + base.update(over) + return JudgmentProfile(**base) + + +def jev_profile(**over) -> JudgmentProfile: + base = dict(id="PR-JEV-1", provider="jev", engine="jev", + model_version="jev-3.2", deployment_mode="external_http", + checkpoint_ref=JEV_CHECKPOINT) + return profile(**{**base, **over}) + + +def config(**over) -> JudgmentConfig: + base = dict(enabled=True, provider="local", profile_id="PR-LOCAL-1", + profile_revision=1, model_version="laya-1.0-zh", + adapter_version="local-adapter.v1", prompt_version="grounding-prompt.v1", + mapping_version="local-map.v1", checkpoint_ref=LOCAL_CHECKPOINT) + base.update(over) + return JudgmentConfig(**base) + + +def local_fixture(): + return lambda item: local_answer(item["id"]) + + +def vendor_fixture(): + return lambda item: vendor_answer(item["id"]) + + +class FixtureContractTests(unittest.TestCase): + """AC-17：两个提供方走同一 judge() 入口，local 运行期间无 Jev 请求。""" + + def test_local_fixture_runs_and_never_targets_jev(self): + log: list[dict] = [] + call = adapters.build_provider_call( + config(), profile(), "extraction_grounding", "runtime", + fixture=local_fixture(), call_log=log) + result = judge(config(), "RUN-T08-1", question("Q-EX-01"), provider_call=call) + + self.assertEqual(result["status"], "completed") + self.assertEqual(result["answer"]["value"], "supported") + self.assertEqual(result["provider"], "local") + self.assertTrue(log, "本地 fixture 应当有调用记录") + self.assertEqual({item["target"] for item in log}, {"local"}, + "local 运行不得出现任何 Jev 请求") + self.assertEqual({item["mode"] for item in log}, {"fixture"}) + + def test_jev_fixture_is_consumed_by_the_same_contract(self): + log: list[dict] = [] + call = adapters.build_provider_call( + config(provider="jev", profile_id="PR-JEV-1", checkpoint_ref=JEV_CHECKPOINT), + jev_profile(task_validations=[validation(usage="offline")]), + "extraction_grounding", "offline", fixture=vendor_fixture(), call_log=log) + result = judge(config(provider="jev", checkpoint_ref=JEV_CHECKPOINT), + "RUN-T08-2", question("Q-EX-02"), provider_call=call) + + self.assertEqual(result["status"], "completed") + self.assertEqual(result["answer"]["value"], "unsupported") + self.assertEqual(result["provider"], "jev") + self.assertTrue(all(m["source"] == "provider_native" + for m in result["native_metrics"]), + "厂商原生指标必须保留原语义与来源") + self.assertNotIn("probability", result) + + def test_conditional_vendor_code_is_not_taken_as_pass(self): + """附条件（COND_APPROVE）不得当通过（BR-21）。""" + call = adapters.build_provider_call( + config(provider="jev", checkpoint_ref=JEV_CHECKPOINT), + jev_profile(task_validations=[validation(usage="offline")]), + "extraction_grounding", "offline", fixture=vendor_fixture()) + result = judge(config(provider="jev", checkpoint_ref=JEV_CHECKPOINT), + "RUN-T08-3", question("Q-EX-01"), provider_call=call) + self.assertEqual(result["answer"]["kind"], "unknown") + self.assertEqual(result["answer"]["reason_code"], "inconclusive") + + def test_unregistered_vendor_code_is_inconclusive(self): + call = adapters.build_provider_call( + config(provider="jev", checkpoint_ref=JEV_CHECKPOINT), + jev_profile(task_validations=[validation(usage="offline")]), + "extraction_grounding", "offline", fixture=vendor_fixture()) + result = judge(config(provider="jev", checkpoint_ref=JEV_CHECKPOINT), + "RUN-T08-4", question("Q-EX-03"), provider_call=call) + self.assertEqual(result["answer"]["kind"], "unknown") + + +class TaskRegistrationTests(unittest.TestCase): + """AC-38/41：逐任务逐用途准入，连通或他任务通过都不能顶替。""" + + def test_unregistered_task_is_refused_even_when_service_is_ready(self): + with self.assertRaises(adapters.TaskNotValidated) as ctx: + adapters.build_provider_call( + config(), profile(), "trajectory_quality", "runtime", + fixture=local_fixture()) + self.assertIn("未在用途 runtime 通过任务 trajectory_quality", str(ctx.exception)) + + def test_offline_pass_does_not_grant_runtime(self): + offline_only = profile(task_validations=[validation(usage="offline")]) + with self.assertRaises(adapters.TaskNotValidated): + adapters.build_provider_call(config(), offline_only, + "extraction_grounding", "runtime", + fixture=local_fixture()) + + def test_failed_validation_is_refused(self): + failed = profile(task_validations=[validation(decision="failed")]) + with self.assertRaises(adapters.TaskNotValidated): + adapters.build_provider_call(config(), failed, "extraction_grounding", + "runtime", fixture=local_fixture()) + + def test_runtime_requires_independent_evaluation(self): + """同模型自评不能当独立效果证据（AC-41）。""" + self_eval = profile(task_validations=[validation(usage="offline", independent=False), + validation(usage="runtime", independent=False)]) + with self.assertRaises(adapters.TaskNotValidated) as ctx: + adapters.build_provider_call(config(), self_eval, "extraction_grounding", + "runtime", fixture=local_fixture()) + self.assertIn("独立评测", str(ctx.exception)) + # offline 用途只证明结构，非独立结果仍可用于离线质检 + adapters.build_provider_call(config(), self_eval, "extraction_grounding", + "offline", fixture=local_fixture()) + + def test_unready_profile_is_refused(self): + with self.assertRaises(adapters.TaskNotValidated): + adapters.build_provider_call(config(), profile(readiness="unready"), + "extraction_grounding", "runtime", + fixture=local_fixture()) + + def test_refusal_is_recorded_as_incomplete_not_as_a_conclusion(self): + def refusing(_config, _question): + raise adapters.TaskNotValidated("未获准任务") + + result = judge(config(), "RUN-T08-5", question("Q-EX-01"), provider_call=refusing) + self.assertEqual(result["status"], "skipped") + self.assertEqual(result["skip_reason"], "unsupported") + self.assertIsNone(result["answer"], "未获准任务不得产出任何结论") + + +class CheckpointTests(unittest.TestCase): + """AC-39/20：权重修订不可隐式更换，冻结值优先。""" + + def test_unresolved_checkpoint_is_refused(self): + with self.assertRaises(adapters.CheckpointMismatch): + adapters.build_provider_call(config(checkpoint_ref="unresolved"), + profile(checkpoint_ref="unresolved"), + "extraction_grounding", "runtime", + fixture=local_fixture()) + + def test_registry_drift_is_refused(self): + with self.assertRaises(adapters.CheckpointMismatch): + adapters.build_provider_call( + config(), profile(checkpoint_ref="laya-1.1-zh@sha256:99be07"), + "extraction_grounding", "runtime", fixture=local_fixture()) + + def test_provider_weight_mismatch_fails_the_question(self): + """提供方实际权重与冻结值不符：拒绝并记失败，不静默采用新结果。""" + call = adapters.build_provider_call( + config(), profile(), "extraction_grounding", "runtime", + fixture=lambda item: local_answer("Q-EX-06")) + result = judge(config(), "RUN-T08-6", question("Q-EX-01"), provider_call=call) + self.assertEqual(result["status"], "failed") + self.assertEqual(result["error_code"], ErrorCode.JUDGMENT_OUTPUT_INVALID) + self.assertIsNone(result["answer"]) + self.assertEqual(result["evidence_ids"], ["EV-EX-01"], "失败也要保留原证据范围") + + def test_frozen_config_wins_over_later_registry_change(self): + """排队后改登记（含换权重）不得改变既有运行（AC-20）。""" + frozen = config() + log: list[dict] = [] + call = adapters.build_provider_call(frozen, profile(), "extraction_grounding", + "runtime", fixture=local_fixture(), + call_log=log) + # 之后管理员把登记改成新修订并换了权重：新运行会被拒绝，旧运行不受影响 + newer = profile(revision=2, checkpoint_ref="laya-1.2-zh@sha256:beef00") + with self.assertRaises(adapters.CheckpointMismatch): + adapters.build_provider_call(frozen, newer, "extraction_grounding", + "runtime", fixture=local_fixture()) + + result = judge(frozen, "RUN-T08-7", question("Q-EX-01"), provider_call=call) + + import json as _json + payload = _json.loads(log[0]["payload"]) + self.assertEqual(payload["checkpoint_ref"], LOCAL_CHECKPOINT, + "发送载荷必须使用运行冻结的权重修订") + self.assertEqual(result["status"], "completed") + + +class BudgetTests(unittest.TestCase): + """AC-40：超预算返回 E-25，保留原引用范围，绝不截断后发送。""" + + def _padded(self) -> dict: + item = question("Q-EX-05") + item["evidence_text"] = item["evidence_text"] * (1 + item["evidence_repeat"]) + return item + + def test_over_budget_fails_with_e25_and_sends_nothing(self): + log: list[dict] = [] + call = adapters.build_provider_call(config(), profile(), "extraction_grounding", + "runtime", fixture=local_fixture(), + call_log=log) + result = judge(config(), "RUN-T08-8", self._padded(), provider_call=call) + + self.assertEqual(result["status"], "failed") + self.assertEqual(result["error_code"], ErrorCode.JUDGMENT_CONTEXT_EXCEEDED) + self.assertIsNone(result["answer"], "超预算不得给出任何结论") + self.assertEqual(result["evidence_ids"], ["EV-EX-06"], + "必须保留超出预算前的原证据范围") + self.assertEqual(log, [], "超预算时不得发生任何发送") + + def test_question_within_budget_still_runs(self): + call = adapters.build_provider_call(config(), profile(), "extraction_grounding", + "runtime", fixture=local_fixture()) + result = judge(config(), "RUN-T08-9", question("Q-EX-01"), provider_call=call) + self.assertEqual(result["status"], "completed") + + def test_budget_counts_options_and_prompt(self): + call = adapters.build_provider_call(config(), profile(), "extraction_grounding", + "runtime", fixture=local_fixture()) + item = question("Q-EX-01", options=["x" * 200]) + result = judge(config(), "RUN-T08-10", item, provider_call=call) + self.assertEqual(result["error_code"], ErrorCode.JUDGMENT_CONTEXT_EXCEEDED, + "选项预算也要计入，不能只看正文") + + +class MappingAmbiguityTests(unittest.TestCase): + """AC-18：中文否定、条件例外、来源矛盾、未知映射一律不硬选正面结果。""" + + def _run(self, qid: str) -> dict: + call = adapters.build_provider_call(config(), profile(), "extraction_grounding", + "runtime", fixture=local_fixture()) + return judge(config(), "RUN-T08-11", question(qid), provider_call=call) + + def test_chinese_negation_is_not_mapped_to_a_positive_option(self): + result = self._run("Q-EX-02") + self.assertEqual(result["answer"]["kind"], "unknown") + self.assertEqual(result["answer"]["reason_code"], "inconclusive") + + def test_conditional_exception_becomes_unknown(self): + self.assertEqual(self._run("Q-EX-03")["answer"]["kind"], "unknown") + + def test_conflicting_sources_become_unknown(self): + self.assertEqual(self._run("Q-EX-04")["answer"]["kind"], "unknown") + + def test_self_reported_confidence_never_becomes_probability(self): + result = self._run("Q-EX-01") + metrics = result["native_metrics"] + self.assertEqual([m["source"] for m in metrics], ["self_reported"]) + self.assertNotIn("probability", result) + self.assertIsNone(result["answer"].get("probability", None), + "自报置信度不得写入概率字段") + + def test_missing_evidence_returns_unknown_without_calling_provider(self): + def forbidden(_config, _question): + raise AssertionError("缺证题目不应请求提供方") + + item = question("Q-EX-01", evidence_ids=[]) + result = judge(config(), "RUN-T08-12", item, provider_call=forbidden) + self.assertEqual(result["status"], "completed") + self.assertEqual(result["answer"], {"kind": "unknown", "value": None, + "reason_code": "missing_evidence"}) + + +class EgressTests(unittest.TestCase): + """AC-16/19：未批准出口不发送；无 live 传输层时明确不可用。""" + + def test_jev_live_without_approval_is_not_sent(self): + sent: list = [] + + def transport(target, payload): + sent.append(target) + return {} + + call = adapters.build_provider_call( + config(provider="jev", checkpoint_ref=JEV_CHECKPOINT), jev_profile(), + "extraction_grounding", "runtime", transport=transport) + result = judge(config(provider="jev", checkpoint_ref=JEV_CHECKPOINT), + "RUN-T08-13", question("Q-EX-01"), provider_call=call) + + self.assertEqual(result["status"], "skipped") + self.assertEqual(result["skip_reason"], "not_authorized") + self.assertEqual(sent, [], "未批准出口不得发生任何发送") + + def test_local_live_without_transport_is_explicitly_unavailable(self): + call = adapters.build_provider_call(config(), profile(), "extraction_grounding", + "runtime") + result = judge(config(), "RUN-T08-14", question("Q-EX-01"), provider_call=call) + self.assertEqual(result["status"], "failed") + self.assertEqual(result["error_code"], ErrorCode.JUDGMENT_UNAVAILABLE) + self.assertIsNone(result["answer"]) + + def test_approved_jev_live_dispatches_through_transport(self): + sent: list = [] + + def transport(target, payload): + sent.append((target, payload)) + return vendor_answer("Q-EX-02") + + approved = jev_profile(egress_approved=True) + call = adapters.build_provider_call( + config(provider="jev", checkpoint_ref=JEV_CHECKPOINT), approved, + "extraction_grounding", "runtime", transport=transport) + result = judge(config(provider="jev", checkpoint_ref=JEV_CHECKPOINT), + "RUN-T08-15", question("Q-EX-02"), provider_call=call) + + self.assertEqual([item[0] for item in sent], ["jev"]) + self.assertEqual(result["answer"]["value"], "unsupported") + + +class ProfileContractTests(unittest.TestCase): + """配置本身的自洽性：本地不许外发、jev 必须外部、同任务同用途不重复登记。""" + + def test_local_profile_cannot_use_external_egress(self): + with self.assertRaises(ValueError): + profile(deployment_mode="external_http") + + def test_jev_profile_must_be_external(self): + with self.assertRaises(ValueError): + jev_profile(deployment_mode="internal_http") + + def test_duplicate_task_usage_registration_is_rejected(self): + with self.assertRaises(ValueError): + profile(task_validations=[validation(), validation()]) + + def test_estimate_tokens_counts_chinese_by_character(self): + self.assertEqual(adapters.estimate_tokens("中文四字"), 4) + self.assertLess(adapters.estimate_tokens("abcdefgh"), 3) + + +class ContractModuleGuardTests(unittest.TestCase): + """新契约模块同样适用"契约层不得出现凭据字段"的静态底线（E-10 / BR-16）。 + + profile 依规格必须带 tokenizer 与预算字段，因此这里按凭据词本身匹配， + 而不是像 prescreen.py 那样禁用整个 "token" 子串；这是收紧而非放宽。 + """ + + def test_no_credential_fields_in_judgment_contract(self): + import contracts.judgment as module + + with open(module.__file__, encoding="utf-8") as handle: + source = handle.read().lower() + for banned in ("api_key", "apikey", "password", "secret", + "access_token", "auth_token", "bearer"): + self.assertNotIn(banned, source) + + +if __name__ == "__main__": + unittest.main()