"""T08 专项离线质检：统一适配 fixture、逐任务/用途准入、冻结 checkpoint、上下文预算。 覆盖 AC-16/17/18/19/20/38/39/40/41（对应 BR-17/18/19/20/37/38/39、E-10/13/14/15/25）。 本文件只跑**离线 fixture**：不发网络请求、不接真实权重。fixture 通过只证明 适配代码结构与守卫成立，不证明中文专项效果合格（docs/v2/06-validation.md §6）。 """ from __future__ import annotations import unittest from contracts.judgment import JudgmentProfile, TaskValidation from contracts.prescreen import ErrorCode, JudgmentConfig from services import judgment_adapters as adapters from services.judgment_provider import judge BANK = "question_bank_extraction.json" VENDOR = "jev_vendor_responses.json" LOCAL_CHECKPOINT = "laya-1.0-zh@sha256:2f1c8d" JEV_CHECKPOINT = "jev-3.2@sha256:7ad401" def bank() -> dict: return adapters.load_fixture(BANK) def question(qid: str, **over) -> dict: body = dict(bank()["questions"][qid]) body.update(over) return body def local_answer(qid: str) -> dict: return bank()["local_answers"][qid] def vendor_answer(qid: str) -> dict: return adapters.load_fixture(VENDOR)["responses"][qid] def validation(task="extraction_grounding", usage="runtime", decision="passed", independent=True) -> TaskValidation: return TaskValidation( task=task, rubric_version="extraction-grounding.v1", dataset_ref=f"fixtures/judgment/{BANK}", report_ref="test-evidence/t08/REPORT.md", decision=decision, allowed_usage=usage, independent=independent) def profile(**over) -> JudgmentProfile: base = dict( id="PR-LOCAL-1", revision=1, provider="local", engine="laya", model_version="laya-1.0-zh", adapter_version="local-adapter.v1", prompt_version="grounding-prompt.v1", mapping_version="local-map.v1", checkpoint_ref=LOCAL_CHECKPOINT, tokenizer_revision="tok.v1", input_token_limit=400, option_token_limit=40, language_scope="zh-CN", deployment_mode="internal_http", egress_approved=False, readiness="ready", task_validations=[validation()], is_default=True) base.update(over) return JudgmentProfile(**base) def jev_profile(**over) -> JudgmentProfile: base = dict(id="PR-JEV-1", provider="jev", engine="jev", model_version="jev-3.2", deployment_mode="external_http", checkpoint_ref=JEV_CHECKPOINT) return profile(**{**base, **over}) def config(**over) -> JudgmentConfig: base = dict(enabled=True, provider="local", profile_id="PR-LOCAL-1", profile_revision=1, model_version="laya-1.0-zh", adapter_version="local-adapter.v1", prompt_version="grounding-prompt.v1", mapping_version="local-map.v1", checkpoint_ref=LOCAL_CHECKPOINT) base.update(over) return JudgmentConfig(**base) def local_fixture(): return lambda item: local_answer(item["id"]) def vendor_fixture(): return lambda item: vendor_answer(item["id"]) class FixtureContractTests(unittest.TestCase): """AC-17：两个提供方走同一 judge() 入口，local 运行期间无 Jev 请求。""" def test_local_fixture_runs_and_never_targets_jev(self): log: list[dict] = [] call = adapters.build_provider_call( config(), profile(), "extraction_grounding", "runtime", fixture=local_fixture(), call_log=log) result = judge(config(), "RUN-T08-1", question("Q-EX-01"), provider_call=call) self.assertEqual(result["status"], "completed") self.assertEqual(result["answer"]["value"], "supported") self.assertEqual(result["provider"], "local") self.assertTrue(log, "本地 fixture 应当有调用记录") self.assertEqual({item["target"] for item in log}, {"local"}, "local 运行不得出现任何 Jev 请求") self.assertEqual({item["mode"] for item in log}, {"fixture"}) def test_jev_fixture_is_consumed_by_the_same_contract(self): log: list[dict] = [] call = adapters.build_provider_call( config(provider="jev", profile_id="PR-JEV-1", checkpoint_ref=JEV_CHECKPOINT), jev_profile(task_validations=[validation(usage="offline")]), "extraction_grounding", "offline", fixture=vendor_fixture(), call_log=log) result = judge(config(provider="jev", checkpoint_ref=JEV_CHECKPOINT), "RUN-T08-2", question("Q-EX-02"), provider_call=call) self.assertEqual(result["status"], "completed") self.assertEqual(result["answer"]["value"], "unsupported") self.assertEqual(result["provider"], "jev") self.assertTrue(all(m["source"] == "provider_native" for m in result["native_metrics"]), "厂商原生指标必须保留原语义与来源") self.assertNotIn("probability", result) def test_conditional_vendor_code_is_not_taken_as_pass(self): """附条件（COND_APPROVE）不得当通过（BR-21）。""" call = adapters.build_provider_call( config(provider="jev", checkpoint_ref=JEV_CHECKPOINT), jev_profile(task_validations=[validation(usage="offline")]), "extraction_grounding", "offline", fixture=vendor_fixture()) result = judge(config(provider="jev", checkpoint_ref=JEV_CHECKPOINT), "RUN-T08-3", question("Q-EX-01"), provider_call=call) self.assertEqual(result["answer"]["kind"], "unknown") self.assertEqual(result["answer"]["reason_code"], "inconclusive") def test_unregistered_vendor_code_is_inconclusive(self): call = adapters.build_provider_call( config(provider="jev", checkpoint_ref=JEV_CHECKPOINT), jev_profile(task_validations=[validation(usage="offline")]), "extraction_grounding", "offline", fixture=vendor_fixture()) result = judge(config(provider="jev", checkpoint_ref=JEV_CHECKPOINT), "RUN-T08-4", question("Q-EX-03"), provider_call=call) self.assertEqual(result["answer"]["kind"], "unknown") class TaskRegistrationTests(unittest.TestCase): """AC-38/41：逐任务逐用途准入，连通或他任务通过都不能顶替。""" def test_unregistered_task_is_refused_even_when_service_is_ready(self): with self.assertRaises(adapters.TaskNotValidated) as ctx: adapters.build_provider_call( config(), profile(), "trajectory_quality", "runtime", fixture=local_fixture()) self.assertIn("未在用途 runtime 通过任务 trajectory_quality", str(ctx.exception)) def test_offline_pass_does_not_grant_runtime(self): offline_only = profile(task_validations=[validation(usage="offline")]) with self.assertRaises(adapters.TaskNotValidated): adapters.build_provider_call(config(), offline_only, "extraction_grounding", "runtime", fixture=local_fixture()) def test_failed_validation_is_refused(self): failed = profile(task_validations=[validation(decision="failed")]) with self.assertRaises(adapters.TaskNotValidated): adapters.build_provider_call(config(), failed, "extraction_grounding", "runtime", fixture=local_fixture()) def test_runtime_requires_independent_evaluation(self): """同模型自评不能当独立效果证据（AC-41）。""" self_eval = profile(task_validations=[validation(usage="offline", independent=False), validation(usage="runtime", independent=False)]) with self.assertRaises(adapters.TaskNotValidated) as ctx: adapters.build_provider_call(config(), self_eval, "extraction_grounding", "runtime", fixture=local_fixture()) self.assertIn("独立评测", str(ctx.exception)) # offline 用途只证明结构，非独立结果仍可用于离线质检 adapters.build_provider_call(config(), self_eval, "extraction_grounding", "offline", fixture=local_fixture()) def test_unready_profile_is_refused(self): with self.assertRaises(adapters.TaskNotValidated): adapters.build_provider_call(config(), profile(readiness="unready"), "extraction_grounding", "runtime", fixture=local_fixture()) def test_refusal_is_recorded_as_incomplete_not_as_a_conclusion(self): def refusing(_config, _question): raise adapters.TaskNotValidated("未获准任务") result = judge(config(), "RUN-T08-5", question("Q-EX-01"), provider_call=refusing) self.assertEqual(result["status"], "skipped") self.assertEqual(result["skip_reason"], "unsupported") self.assertIsNone(result["answer"], "未获准任务不得产出任何结论") class CheckpointTests(unittest.TestCase): """AC-39/20：权重修订不可隐式更换，冻结值优先。""" def test_unresolved_checkpoint_is_refused(self): with self.assertRaises(adapters.CheckpointMismatch): adapters.build_provider_call(config(checkpoint_ref="unresolved"), profile(checkpoint_ref="unresolved"), "extraction_grounding", "runtime", fixture=local_fixture()) def test_registry_drift_is_refused(self): with self.assertRaises(adapters.CheckpointMismatch): adapters.build_provider_call( config(), profile(checkpoint_ref="laya-1.1-zh@sha256:99be07"), "extraction_grounding", "runtime", fixture=local_fixture()) def test_provider_weight_mismatch_fails_the_question(self): """提供方实际权重与冻结值不符：拒绝并记失败，不静默采用新结果。""" call = adapters.build_provider_call( config(), profile(), "extraction_grounding", "runtime", fixture=lambda item: local_answer("Q-EX-06")) result = judge(config(), "RUN-T08-6", question("Q-EX-01"), provider_call=call) self.assertEqual(result["status"], "failed") self.assertEqual(result["error_code"], ErrorCode.JUDGMENT_OUTPUT_INVALID) self.assertIsNone(result["answer"]) self.assertEqual(result["evidence_ids"], ["EV-EX-01"], "失败也要保留原证据范围") def test_frozen_config_wins_over_later_registry_change(self): """排队后改登记（含换权重）不得改变既有运行（AC-20）。""" frozen = config() log: list[dict] = [] call = adapters.build_provider_call(frozen, profile(), "extraction_grounding", "runtime", fixture=local_fixture(), call_log=log) # 之后管理员把登记改成新修订并换了权重：新运行会被拒绝，旧运行不受影响 newer = profile(revision=2, checkpoint_ref="laya-1.2-zh@sha256:beef00") with self.assertRaises(adapters.CheckpointMismatch): adapters.build_provider_call(frozen, newer, "extraction_grounding", "runtime", fixture=local_fixture()) result = judge(frozen, "RUN-T08-7", question("Q-EX-01"), provider_call=call) import json as _json payload = _json.loads(log[0]["payload"]) self.assertEqual(payload["checkpoint_ref"], LOCAL_CHECKPOINT, "发送载荷必须使用运行冻结的权重修订") self.assertEqual(result["status"], "completed") class BudgetTests(unittest.TestCase): """AC-40：超预算返回 E-25，保留原引用范围，绝不截断后发送。""" def _padded(self) -> dict: item = question("Q-EX-05") item["evidence_text"] = item["evidence_text"] * (1 + item["evidence_repeat"]) return item def test_over_budget_fails_with_e25_and_sends_nothing(self): log: list[dict] = [] call = adapters.build_provider_call(config(), profile(), "extraction_grounding", "runtime", fixture=local_fixture(), call_log=log) result = judge(config(), "RUN-T08-8", self._padded(), provider_call=call) self.assertEqual(result["status"], "failed") self.assertEqual(result["error_code"], ErrorCode.JUDGMENT_CONTEXT_EXCEEDED) self.assertIsNone(result["answer"], "超预算不得给出任何结论") self.assertEqual(result["evidence_ids"], ["EV-EX-06"], "必须保留超出预算前的原证据范围") self.assertEqual(log, [], "超预算时不得发生任何发送") def test_question_within_budget_still_runs(self): call = adapters.build_provider_call(config(), profile(), "extraction_grounding", "runtime", fixture=local_fixture()) result = judge(config(), "RUN-T08-9", question("Q-EX-01"), provider_call=call) self.assertEqual(result["status"], "completed") def test_budget_counts_options_and_prompt(self): call = adapters.build_provider_call(config(), profile(), "extraction_grounding", "runtime", fixture=local_fixture()) item = question("Q-EX-01", options=["x" * 200]) result = judge(config(), "RUN-T08-10", item, provider_call=call) self.assertEqual(result["error_code"], ErrorCode.JUDGMENT_CONTEXT_EXCEEDED, "选项预算也要计入，不能只看正文") class MappingAmbiguityTests(unittest.TestCase): """AC-18：中文否定、条件例外、来源矛盾、未知映射一律不硬选正面结果。""" def _run(self, qid: str) -> dict: call = adapters.build_provider_call(config(), profile(), "extraction_grounding", "runtime", fixture=local_fixture()) return judge(config(), "RUN-T08-11", question(qid), provider_call=call) def test_chinese_negation_is_not_mapped_to_a_positive_option(self): result = self._run("Q-EX-02") self.assertEqual(result["answer"]["kind"], "unknown") self.assertEqual(result["answer"]["reason_code"], "inconclusive") def test_conditional_exception_becomes_unknown(self): self.assertEqual(self._run("Q-EX-03")["answer"]["kind"], "unknown") def test_conflicting_sources_become_unknown(self): self.assertEqual(self._run("Q-EX-04")["answer"]["kind"], "unknown") def test_self_reported_confidence_never_becomes_probability(self): result = self._run("Q-EX-01") metrics = result["native_metrics"] self.assertEqual([m["source"] for m in metrics], ["self_reported"]) self.assertNotIn("probability", result) self.assertIsNone(result["answer"].get("probability", None), "自报置信度不得写入概率字段") def test_missing_evidence_returns_unknown_without_calling_provider(self): def forbidden(_config, _question): raise AssertionError("缺证题目不应请求提供方") item = question("Q-EX-01", evidence_ids=[]) result = judge(config(), "RUN-T08-12", item, provider_call=forbidden) self.assertEqual(result["status"], "completed") self.assertEqual(result["answer"], {"kind": "unknown", "value": None, "reason_code": "missing_evidence"}) class EgressTests(unittest.TestCase): """AC-16/19：未批准出口不发送；无 live 传输层时明确不可用。""" def test_jev_live_without_approval_is_not_sent(self): sent: list = [] def transport(target, payload): sent.append(target) return {} call = adapters.build_provider_call( config(provider="jev", checkpoint_ref=JEV_CHECKPOINT), jev_profile(), "extraction_grounding", "runtime", transport=transport) result = judge(config(provider="jev", checkpoint_ref=JEV_CHECKPOINT), "RUN-T08-13", question("Q-EX-01"), provider_call=call) self.assertEqual(result["status"], "skipped") self.assertEqual(result["skip_reason"], "not_authorized") self.assertEqual(sent, [], "未批准出口不得发生任何发送") def test_local_live_without_transport_is_explicitly_unavailable(self): call = adapters.build_provider_call(config(), profile(), "extraction_grounding", "runtime") result = judge(config(), "RUN-T08-14", question("Q-EX-01"), provider_call=call) self.assertEqual(result["status"], "failed") self.assertEqual(result["error_code"], ErrorCode.JUDGMENT_UNAVAILABLE) self.assertIsNone(result["answer"]) def test_approved_jev_live_dispatches_through_transport(self): sent: list = [] def transport(target, payload): sent.append((target, payload)) return vendor_answer("Q-EX-02") approved = jev_profile(egress_approved=True) call = adapters.build_provider_call( config(provider="jev", checkpoint_ref=JEV_CHECKPOINT), approved, "extraction_grounding", "runtime", transport=transport) result = judge(config(provider="jev", checkpoint_ref=JEV_CHECKPOINT), "RUN-T08-15", question("Q-EX-02"), provider_call=call) self.assertEqual([item[0] for item in sent], ["jev"]) self.assertEqual(result["answer"]["value"], "unsupported") class ProfileContractTests(unittest.TestCase): """配置本身的自洽性：本地不许外发、jev 必须外部、同任务同用途不重复登记。""" def test_local_profile_cannot_use_external_egress(self): with self.assertRaises(ValueError): profile(deployment_mode="external_http") def test_jev_profile_must_be_external(self): with self.assertRaises(ValueError): jev_profile(deployment_mode="internal_http") def test_duplicate_task_usage_registration_is_rejected(self): with self.assertRaises(ValueError): profile(task_validations=[validation(), validation()]) def test_estimate_tokens_counts_chinese_by_character(self): self.assertEqual(adapters.estimate_tokens("中文四字"), 4) self.assertLess(adapters.estimate_tokens("abcdefgh"), 3) class ContractModuleGuardTests(unittest.TestCase): """新契约模块同样适用"契约层不得出现凭据字段"的静态底线（E-10 / BR-16）。 profile 依规格必须带 tokenizer 与预算字段，因此这里按凭据词本身匹配， 而不是像 prescreen.py 那样禁用整个 "token" 子串；这是收紧而非放宽。 """ def test_no_credential_fields_in_judgment_contract(self): import contracts.judgment as module with open(module.__file__, encoding="utf-8") as handle: source = handle.read().lower() for banned in ("api_key", "apikey", "password", "secret", "access_token", "auth_token", "bearer"): self.assertNotIn(banned, source) if __name__ == "__main__": unittest.main()