feat(v2): T08 专项离线质检——统一适配 fixture、逐任务准入与预算守卫
覆盖 AC-16/17/18/19/20/38/39/40/41（BR-17/18/19/20/37/38/39、E-10/13/14/15/25）。 新增： - backend/contracts/judgment.py：JudgeProfile/TaskValidation 契约（逐任务逐用途 验收登记 + 冻结 checkpoint）。放在独立契约模块，因为规格要求的 tokenizer/预算 字段会撞上 prescreen.py 既有的"契约层不得出现凭据字样"静态守卫；不放宽该守卫。 - backend/services/judgment_adapters.py：统一适配入口。准入（未登记任务/用途拒绝、 runtime 须独立评测）、冻结权重（unresolved 与不一致均拒绝）、上下文预算 （正文与选项分开计，超限返回 E-25 且零发送）、中文否定/条件例外/来源矛盾/未登记 映射一律落 unknown(inconclusive)，不硬选正面结果。 - backend/fixtures/judgment/：中文提炼依据题集、本地模型原始输出、Jev 厂商响应 fixture（虚构内容，离线契约用）。 - backend/test_judgment_adapters.py：30 项测试。 修改（最小面）： - contracts/prescreen.py：新增 E-25 运行级错误码 JUDGMENT_CONTEXT_EXCEEDED； Judgment 增 actual_checkpoint_ref、JudgmentConfig 增 checkpoint_ref。 - services/judgment_provider.py：映射适配层的策略拒绝异常；归一化保留实际权重。 未做（受授权限制）：没有 Jev 凭据与 Laya 权重，因此只交付离线 fixture 路径， live 传输层仅留注入点，未接入硬件与端口探测。fixture 通过不证明中文效果合格。 验证：backend 全量 unittest 440 项 OK（原 410 项无回归）。 Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
Claude Code
@@ -0,0 +1,403 @@
|
||||
"""T08 专项离线质检：统一适配 fixture、逐任务/用途准入、冻结 checkpoint、上下文预算。
|
||||
|
||||
覆盖 AC-16/17/18/19/20/38/39/40/41（对应 BR-17/18/19/20/37/38/39、E-10/13/14/15/25）。
|
||||
|
||||
本文件只跑**离线 fixture**：不发网络请求、不接真实权重。fixture 通过只证明
|
||||
适配代码结构与守卫成立，不证明中文专项效果合格（docs/v2/06-validation.md §6）。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from contracts.judgment import JudgmentProfile, TaskValidation
|
||||
from contracts.prescreen import ErrorCode, JudgmentConfig
|
||||
from services import judgment_adapters as adapters
|
||||
from services.judgment_provider import judge
|
||||
|
||||
BANK = "question_bank_extraction.json"
|
||||
VENDOR = "jev_vendor_responses.json"
|
||||
|
||||
LOCAL_CHECKPOINT = "laya-1.0-zh@sha256:2f1c8d"
|
||||
JEV_CHECKPOINT = "jev-3.2@sha256:7ad401"
|
||||
|
||||
|
||||
def bank() -> dict:
|
||||
return adapters.load_fixture(BANK)
|
||||
|
||||
|
||||
def question(qid: str, **over) -> dict:
|
||||
body = dict(bank()["questions"][qid])
|
||||
body.update(over)
|
||||
return body
|
||||
|
||||
|
||||
def local_answer(qid: str) -> dict:
|
||||
return bank()["local_answers"][qid]
|
||||
|
||||
|
||||
def vendor_answer(qid: str) -> dict:
|
||||
return adapters.load_fixture(VENDOR)["responses"][qid]
|
||||
|
||||
|
||||
def validation(task="extraction_grounding", usage="runtime",
|
||||
decision="passed", independent=True) -> TaskValidation:
|
||||
return TaskValidation(
|
||||
task=task, rubric_version="extraction-grounding.v1",
|
||||
dataset_ref=f"fixtures/judgment/{BANK}", report_ref="test-evidence/t08/REPORT.md",
|
||||
decision=decision, allowed_usage=usage, independent=independent)
|
||||
|
||||
|
||||
def profile(**over) -> JudgmentProfile:
|
||||
base = dict(
|
||||
id="PR-LOCAL-1", revision=1, provider="local", engine="laya",
|
||||
model_version="laya-1.0-zh", adapter_version="local-adapter.v1",
|
||||
prompt_version="grounding-prompt.v1", mapping_version="local-map.v1",
|
||||
checkpoint_ref=LOCAL_CHECKPOINT, tokenizer_revision="tok.v1",
|
||||
input_token_limit=400, option_token_limit=40, language_scope="zh-CN",
|
||||
deployment_mode="internal_http", egress_approved=False, readiness="ready",
|
||||
task_validations=[validation()], is_default=True)
|
||||
base.update(over)
|
||||
return JudgmentProfile(**base)
|
||||
|
||||
|
||||
def jev_profile(**over) -> JudgmentProfile:
|
||||
base = dict(id="PR-JEV-1", provider="jev", engine="jev",
|
||||
model_version="jev-3.2", deployment_mode="external_http",
|
||||
checkpoint_ref=JEV_CHECKPOINT)
|
||||
return profile(**{**base, **over})
|
||||
|
||||
|
||||
def config(**over) -> JudgmentConfig:
|
||||
base = dict(enabled=True, provider="local", profile_id="PR-LOCAL-1",
|
||||
profile_revision=1, model_version="laya-1.0-zh",
|
||||
adapter_version="local-adapter.v1", prompt_version="grounding-prompt.v1",
|
||||
mapping_version="local-map.v1", checkpoint_ref=LOCAL_CHECKPOINT)
|
||||
base.update(over)
|
||||
return JudgmentConfig(**base)
|
||||
|
||||
|
||||
def local_fixture():
|
||||
return lambda item: local_answer(item["id"])
|
||||
|
||||
|
||||
def vendor_fixture():
|
||||
return lambda item: vendor_answer(item["id"])
|
||||
|
||||
|
||||
class FixtureContractTests(unittest.TestCase):
|
||||
"""AC-17：两个提供方走同一 judge() 入口，local 运行期间无 Jev 请求。"""
|
||||
|
||||
def test_local_fixture_runs_and_never_targets_jev(self):
|
||||
log: list[dict] = []
|
||||
call = adapters.build_provider_call(
|
||||
config(), profile(), "extraction_grounding", "runtime",
|
||||
fixture=local_fixture(), call_log=log)
|
||||
result = judge(config(), "RUN-T08-1", question("Q-EX-01"), provider_call=call)
|
||||
|
||||
self.assertEqual(result["status"], "completed")
|
||||
self.assertEqual(result["answer"]["value"], "supported")
|
||||
self.assertEqual(result["provider"], "local")
|
||||
self.assertTrue(log, "本地 fixture 应当有调用记录")
|
||||
self.assertEqual({item["target"] for item in log}, {"local"},
|
||||
"local 运行不得出现任何 Jev 请求")
|
||||
self.assertEqual({item["mode"] for item in log}, {"fixture"})
|
||||
|
||||
def test_jev_fixture_is_consumed_by_the_same_contract(self):
|
||||
log: list[dict] = []
|
||||
call = adapters.build_provider_call(
|
||||
config(provider="jev", profile_id="PR-JEV-1", checkpoint_ref=JEV_CHECKPOINT),
|
||||
jev_profile(task_validations=[validation(usage="offline")]),
|
||||
"extraction_grounding", "offline", fixture=vendor_fixture(), call_log=log)
|
||||
result = judge(config(provider="jev", checkpoint_ref=JEV_CHECKPOINT),
|
||||
"RUN-T08-2", question("Q-EX-02"), provider_call=call)
|
||||
|
||||
self.assertEqual(result["status"], "completed")
|
||||
self.assertEqual(result["answer"]["value"], "unsupported")
|
||||
self.assertEqual(result["provider"], "jev")
|
||||
self.assertTrue(all(m["source"] == "provider_native"
|
||||
for m in result["native_metrics"]),
|
||||
"厂商原生指标必须保留原语义与来源")
|
||||
self.assertNotIn("probability", result)
|
||||
|
||||
def test_conditional_vendor_code_is_not_taken_as_pass(self):
|
||||
"""附条件（COND_APPROVE）不得当通过（BR-21）。"""
|
||||
call = adapters.build_provider_call(
|
||||
config(provider="jev", checkpoint_ref=JEV_CHECKPOINT),
|
||||
jev_profile(task_validations=[validation(usage="offline")]),
|
||||
"extraction_grounding", "offline", fixture=vendor_fixture())
|
||||
result = judge(config(provider="jev", checkpoint_ref=JEV_CHECKPOINT),
|
||||
"RUN-T08-3", question("Q-EX-01"), provider_call=call)
|
||||
self.assertEqual(result["answer"]["kind"], "unknown")
|
||||
self.assertEqual(result["answer"]["reason_code"], "inconclusive")
|
||||
|
||||
def test_unregistered_vendor_code_is_inconclusive(self):
|
||||
call = adapters.build_provider_call(
|
||||
config(provider="jev", checkpoint_ref=JEV_CHECKPOINT),
|
||||
jev_profile(task_validations=[validation(usage="offline")]),
|
||||
"extraction_grounding", "offline", fixture=vendor_fixture())
|
||||
result = judge(config(provider="jev", checkpoint_ref=JEV_CHECKPOINT),
|
||||
"RUN-T08-4", question("Q-EX-03"), provider_call=call)
|
||||
self.assertEqual(result["answer"]["kind"], "unknown")
|
||||
|
||||
|
||||
class TaskRegistrationTests(unittest.TestCase):
|
||||
"""AC-38/41：逐任务逐用途准入，连通或他任务通过都不能顶替。"""
|
||||
|
||||
def test_unregistered_task_is_refused_even_when_service_is_ready(self):
|
||||
with self.assertRaises(adapters.TaskNotValidated) as ctx:
|
||||
adapters.build_provider_call(
|
||||
config(), profile(), "trajectory_quality", "runtime",
|
||||
fixture=local_fixture())
|
||||
self.assertIn("未在用途 runtime 通过任务 trajectory_quality", str(ctx.exception))
|
||||
|
||||
def test_offline_pass_does_not_grant_runtime(self):
|
||||
offline_only = profile(task_validations=[validation(usage="offline")])
|
||||
with self.assertRaises(adapters.TaskNotValidated):
|
||||
adapters.build_provider_call(config(), offline_only,
|
||||
"extraction_grounding", "runtime",
|
||||
fixture=local_fixture())
|
||||
|
||||
def test_failed_validation_is_refused(self):
|
||||
failed = profile(task_validations=[validation(decision="failed")])
|
||||
with self.assertRaises(adapters.TaskNotValidated):
|
||||
adapters.build_provider_call(config(), failed, "extraction_grounding",
|
||||
"runtime", fixture=local_fixture())
|
||||
|
||||
def test_runtime_requires_independent_evaluation(self):
|
||||
"""同模型自评不能当独立效果证据（AC-41）。"""
|
||||
self_eval = profile(task_validations=[validation(usage="offline", independent=False),
|
||||
validation(usage="runtime", independent=False)])
|
||||
with self.assertRaises(adapters.TaskNotValidated) as ctx:
|
||||
adapters.build_provider_call(config(), self_eval, "extraction_grounding",
|
||||
"runtime", fixture=local_fixture())
|
||||
self.assertIn("独立评测", str(ctx.exception))
|
||||
# offline 用途只证明结构，非独立结果仍可用于离线质检
|
||||
adapters.build_provider_call(config(), self_eval, "extraction_grounding",
|
||||
"offline", fixture=local_fixture())
|
||||
|
||||
def test_unready_profile_is_refused(self):
|
||||
with self.assertRaises(adapters.TaskNotValidated):
|
||||
adapters.build_provider_call(config(), profile(readiness="unready"),
|
||||
"extraction_grounding", "runtime",
|
||||
fixture=local_fixture())
|
||||
|
||||
def test_refusal_is_recorded_as_incomplete_not_as_a_conclusion(self):
|
||||
def refusing(_config, _question):
|
||||
raise adapters.TaskNotValidated("未获准任务")
|
||||
|
||||
result = judge(config(), "RUN-T08-5", question("Q-EX-01"), provider_call=refusing)
|
||||
self.assertEqual(result["status"], "skipped")
|
||||
self.assertEqual(result["skip_reason"], "unsupported")
|
||||
self.assertIsNone(result["answer"], "未获准任务不得产出任何结论")
|
||||
|
||||
|
||||
class CheckpointTests(unittest.TestCase):
|
||||
"""AC-39/20：权重修订不可隐式更换，冻结值优先。"""
|
||||
|
||||
def test_unresolved_checkpoint_is_refused(self):
|
||||
with self.assertRaises(adapters.CheckpointMismatch):
|
||||
adapters.build_provider_call(config(checkpoint_ref="unresolved"),
|
||||
profile(checkpoint_ref="unresolved"),
|
||||
"extraction_grounding", "runtime",
|
||||
fixture=local_fixture())
|
||||
|
||||
def test_registry_drift_is_refused(self):
|
||||
with self.assertRaises(adapters.CheckpointMismatch):
|
||||
adapters.build_provider_call(
|
||||
config(), profile(checkpoint_ref="laya-1.1-zh@sha256:99be07"),
|
||||
"extraction_grounding", "runtime", fixture=local_fixture())
|
||||
|
||||
def test_provider_weight_mismatch_fails_the_question(self):
|
||||
"""提供方实际权重与冻结值不符：拒绝并记失败，不静默采用新结果。"""
|
||||
call = adapters.build_provider_call(
|
||||
config(), profile(), "extraction_grounding", "runtime",
|
||||
fixture=lambda item: local_answer("Q-EX-06"))
|
||||
result = judge(config(), "RUN-T08-6", question("Q-EX-01"), provider_call=call)
|
||||
self.assertEqual(result["status"], "failed")
|
||||
self.assertEqual(result["error_code"], ErrorCode.JUDGMENT_OUTPUT_INVALID)
|
||||
self.assertIsNone(result["answer"])
|
||||
self.assertEqual(result["evidence_ids"], ["EV-EX-01"], "失败也要保留原证据范围")
|
||||
|
||||
def test_frozen_config_wins_over_later_registry_change(self):
|
||||
"""排队后改登记（含换权重）不得改变既有运行（AC-20）。"""
|
||||
frozen = config()
|
||||
log: list[dict] = []
|
||||
call = adapters.build_provider_call(frozen, profile(), "extraction_grounding",
|
||||
"runtime", fixture=local_fixture(),
|
||||
call_log=log)
|
||||
# 之后管理员把登记改成新修订并换了权重：新运行会被拒绝，旧运行不受影响
|
||||
newer = profile(revision=2, checkpoint_ref="laya-1.2-zh@sha256:beef00")
|
||||
with self.assertRaises(adapters.CheckpointMismatch):
|
||||
adapters.build_provider_call(frozen, newer, "extraction_grounding",
|
||||
"runtime", fixture=local_fixture())
|
||||
|
||||
result = judge(frozen, "RUN-T08-7", question("Q-EX-01"), provider_call=call)
|
||||
|
||||
import json as _json
|
||||
payload = _json.loads(log[0]["payload"])
|
||||
self.assertEqual(payload["checkpoint_ref"], LOCAL_CHECKPOINT,
|
||||
"发送载荷必须使用运行冻结的权重修订")
|
||||
self.assertEqual(result["status"], "completed")
|
||||
|
||||
|
||||
class BudgetTests(unittest.TestCase):
|
||||
"""AC-40：超预算返回 E-25，保留原引用范围，绝不截断后发送。"""
|
||||
|
||||
def _padded(self) -> dict:
|
||||
item = question("Q-EX-05")
|
||||
item["evidence_text"] = item["evidence_text"] * (1 + item["evidence_repeat"])
|
||||
return item
|
||||
|
||||
def test_over_budget_fails_with_e25_and_sends_nothing(self):
|
||||
log: list[dict] = []
|
||||
call = adapters.build_provider_call(config(), profile(), "extraction_grounding",
|
||||
"runtime", fixture=local_fixture(),
|
||||
call_log=log)
|
||||
result = judge(config(), "RUN-T08-8", self._padded(), provider_call=call)
|
||||
|
||||
self.assertEqual(result["status"], "failed")
|
||||
self.assertEqual(result["error_code"], ErrorCode.JUDGMENT_CONTEXT_EXCEEDED)
|
||||
self.assertIsNone(result["answer"], "超预算不得给出任何结论")
|
||||
self.assertEqual(result["evidence_ids"], ["EV-EX-06"],
|
||||
"必须保留超出预算前的原证据范围")
|
||||
self.assertEqual(log, [], "超预算时不得发生任何发送")
|
||||
|
||||
def test_question_within_budget_still_runs(self):
|
||||
call = adapters.build_provider_call(config(), profile(), "extraction_grounding",
|
||||
"runtime", fixture=local_fixture())
|
||||
result = judge(config(), "RUN-T08-9", question("Q-EX-01"), provider_call=call)
|
||||
self.assertEqual(result["status"], "completed")
|
||||
|
||||
def test_budget_counts_options_and_prompt(self):
|
||||
call = adapters.build_provider_call(config(), profile(), "extraction_grounding",
|
||||
"runtime", fixture=local_fixture())
|
||||
item = question("Q-EX-01", options=["x" * 200])
|
||||
result = judge(config(), "RUN-T08-10", item, provider_call=call)
|
||||
self.assertEqual(result["error_code"], ErrorCode.JUDGMENT_CONTEXT_EXCEEDED,
|
||||
"选项预算也要计入，不能只看正文")
|
||||
|
||||
|
||||
class MappingAmbiguityTests(unittest.TestCase):
|
||||
"""AC-18：中文否定、条件例外、来源矛盾、未知映射一律不硬选正面结果。"""
|
||||
|
||||
def _run(self, qid: str) -> dict:
|
||||
call = adapters.build_provider_call(config(), profile(), "extraction_grounding",
|
||||
"runtime", fixture=local_fixture())
|
||||
return judge(config(), "RUN-T08-11", question(qid), provider_call=call)
|
||||
|
||||
def test_chinese_negation_is_not_mapped_to_a_positive_option(self):
|
||||
result = self._run("Q-EX-02")
|
||||
self.assertEqual(result["answer"]["kind"], "unknown")
|
||||
self.assertEqual(result["answer"]["reason_code"], "inconclusive")
|
||||
|
||||
def test_conditional_exception_becomes_unknown(self):
|
||||
self.assertEqual(self._run("Q-EX-03")["answer"]["kind"], "unknown")
|
||||
|
||||
def test_conflicting_sources_become_unknown(self):
|
||||
self.assertEqual(self._run("Q-EX-04")["answer"]["kind"], "unknown")
|
||||
|
||||
def test_self_reported_confidence_never_becomes_probability(self):
|
||||
result = self._run("Q-EX-01")
|
||||
metrics = result["native_metrics"]
|
||||
self.assertEqual([m["source"] for m in metrics], ["self_reported"])
|
||||
self.assertNotIn("probability", result)
|
||||
self.assertIsNone(result["answer"].get("probability", None),
|
||||
"自报置信度不得写入概率字段")
|
||||
|
||||
def test_missing_evidence_returns_unknown_without_calling_provider(self):
|
||||
def forbidden(_config, _question):
|
||||
raise AssertionError("缺证题目不应请求提供方")
|
||||
|
||||
item = question("Q-EX-01", evidence_ids=[])
|
||||
result = judge(config(), "RUN-T08-12", item, provider_call=forbidden)
|
||||
self.assertEqual(result["status"], "completed")
|
||||
self.assertEqual(result["answer"], {"kind": "unknown", "value": None,
|
||||
"reason_code": "missing_evidence"})
|
||||
|
||||
|
||||
class EgressTests(unittest.TestCase):
|
||||
"""AC-16/19：未批准出口不发送；无 live 传输层时明确不可用。"""
|
||||
|
||||
def test_jev_live_without_approval_is_not_sent(self):
|
||||
sent: list = []
|
||||
|
||||
def transport(target, payload):
|
||||
sent.append(target)
|
||||
return {}
|
||||
|
||||
call = adapters.build_provider_call(
|
||||
config(provider="jev", checkpoint_ref=JEV_CHECKPOINT), jev_profile(),
|
||||
"extraction_grounding", "runtime", transport=transport)
|
||||
result = judge(config(provider="jev", checkpoint_ref=JEV_CHECKPOINT),
|
||||
"RUN-T08-13", question("Q-EX-01"), provider_call=call)
|
||||
|
||||
self.assertEqual(result["status"], "skipped")
|
||||
self.assertEqual(result["skip_reason"], "not_authorized")
|
||||
self.assertEqual(sent, [], "未批准出口不得发生任何发送")
|
||||
|
||||
def test_local_live_without_transport_is_explicitly_unavailable(self):
|
||||
call = adapters.build_provider_call(config(), profile(), "extraction_grounding",
|
||||
"runtime")
|
||||
result = judge(config(), "RUN-T08-14", question("Q-EX-01"), provider_call=call)
|
||||
self.assertEqual(result["status"], "failed")
|
||||
self.assertEqual(result["error_code"], ErrorCode.JUDGMENT_UNAVAILABLE)
|
||||
self.assertIsNone(result["answer"])
|
||||
|
||||
def test_approved_jev_live_dispatches_through_transport(self):
|
||||
sent: list = []
|
||||
|
||||
def transport(target, payload):
|
||||
sent.append((target, payload))
|
||||
return vendor_answer("Q-EX-02")
|
||||
|
||||
approved = jev_profile(egress_approved=True)
|
||||
call = adapters.build_provider_call(
|
||||
config(provider="jev", checkpoint_ref=JEV_CHECKPOINT), approved,
|
||||
"extraction_grounding", "runtime", transport=transport)
|
||||
result = judge(config(provider="jev", checkpoint_ref=JEV_CHECKPOINT),
|
||||
"RUN-T08-15", question("Q-EX-02"), provider_call=call)
|
||||
|
||||
self.assertEqual([item[0] for item in sent], ["jev"])
|
||||
self.assertEqual(result["answer"]["value"], "unsupported")
|
||||
|
||||
|
||||
class ProfileContractTests(unittest.TestCase):
|
||||
"""配置本身的自洽性：本地不许外发、jev 必须外部、同任务同用途不重复登记。"""
|
||||
|
||||
def test_local_profile_cannot_use_external_egress(self):
|
||||
with self.assertRaises(ValueError):
|
||||
profile(deployment_mode="external_http")
|
||||
|
||||
def test_jev_profile_must_be_external(self):
|
||||
with self.assertRaises(ValueError):
|
||||
jev_profile(deployment_mode="internal_http")
|
||||
|
||||
def test_duplicate_task_usage_registration_is_rejected(self):
|
||||
with self.assertRaises(ValueError):
|
||||
profile(task_validations=[validation(), validation()])
|
||||
|
||||
def test_estimate_tokens_counts_chinese_by_character(self):
|
||||
self.assertEqual(adapters.estimate_tokens("中文四字"), 4)
|
||||
self.assertLess(adapters.estimate_tokens("abcdefgh"), 3)
|
||||
|
||||
|
||||
class ContractModuleGuardTests(unittest.TestCase):
|
||||
"""新契约模块同样适用"契约层不得出现凭据字段"的静态底线（E-10 / BR-16）。
|
||||
|
||||
profile 依规格必须带 tokenizer 与预算字段，因此这里按凭据词本身匹配，
|
||||
而不是像 prescreen.py 那样禁用整个 "token" 子串；这是收紧而非放宽。
|
||||
"""
|
||||
|
||||
def test_no_credential_fields_in_judgment_contract(self):
|
||||
import contracts.judgment as module
|
||||
|
||||
with open(module.__file__, encoding="utf-8") as handle:
|
||||
source = handle.read().lower()
|
||||
for banned in ("api_key", "apikey", "password", "secret",
|
||||
"access_token", "auth_token", "bearer"):
|
||||
self.assertNotIn(banned, source)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user