feat(v2): T08 专项离线质检——统一适配 fixture、逐任务准入与预算守卫
覆盖 AC-16/17/18/19/20/38/39/40/41（BR-17/18/19/20/37/38/39、E-10/13/14/15/25）。 新增： - backend/contracts/judgment.py：JudgeProfile/TaskValidation 契约（逐任务逐用途 验收登记 + 冻结 checkpoint）。放在独立契约模块，因为规格要求的 tokenizer/预算 字段会撞上 prescreen.py 既有的"契约层不得出现凭据字样"静态守卫；不放宽该守卫。 - backend/services/judgment_adapters.py：统一适配入口。准入（未登记任务/用途拒绝、 runtime 须独立评测）、冻结权重（unresolved 与不一致均拒绝）、上下文预算 （正文与选项分开计，超限返回 E-25 且零发送）、中文否定/条件例外/来源矛盾/未登记 映射一律落 unknown(inconclusive)，不硬选正面结果。 - backend/fixtures/judgment/：中文提炼依据题集、本地模型原始输出、Jev 厂商响应 fixture（虚构内容，离线契约用）。 - backend/test_judgment_adapters.py：30 项测试。 修改（最小面）： - contracts/prescreen.py：新增 E-25 运行级错误码 JUDGMENT_CONTEXT_EXCEEDED； Judgment 增 actual_checkpoint_ref、JudgmentConfig 增 checkpoint_ref。 - services/judgment_provider.py：映射适配层的策略拒绝异常；归一化保留实际权重。 未做（受授权限制）：没有 Jev 凭据与 Laya 权重，因此只交付离线 fixture 路径， live 传输层仅留注入点，未接入硬件与端口探测。fixture 通过不证明中文效果合格。 验证：backend 全量 unittest 440 项 OK（原 410 项无回归）。 Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
Claude Code
@@ -0,0 +1,251 @@
|
||||
"""T08 专项离线质检：统一适配 fixture、逐任务/用途准入、冻结 checkpoint 与上下文预算。
|
||||
|
||||
依据：docs/v2/11-judgment-providers.md §8、docs/v2/06-validation.md §4/§6；
|
||||
BR-17/18/19/20/37/38/39、E-10/13/14/15/25、AC-16/17/18/19/20/38/39/40/41。
|
||||
|
||||
本批**只交付离线 fixture 路径**。没有 Jev 凭据、没有 Laya 权重与授权，
|
||||
因此：
|
||||
- live 传输层只留注入点，未注入时明确 `LiveTransportUnavailable`；
|
||||
- fixture 通过只证明适配代码结构成立，**不证明**中文专项效果合格
|
||||
（06-validation.md §6："fixture 通过仅证明适配代码结构"）。
|
||||
|
||||
四条不可让渡的规则：
|
||||
1. **逐任务逐用途准入**：`task_validations` 里没有 (task, usage) 的 passed 记录，
|
||||
就不执行；服务能连通、别的任务通过、装好了权重，都不算准入（AC-38）。
|
||||
2. **权重不可隐式换**：运行冻结 `checkpoint_ref`；提供方实际权重不一致即拒绝，
|
||||
不按语言或负载偷偷路由到别的模型（AC-39）。
|
||||
3. **超预算不外发**：超出已验证上下文预算时返回 E-25 并保留原证据范围，
|
||||
绝不截断后半段照样发送（AC-40 / BR-39）。
|
||||
4. **歧义不硬选**：映射未知、中文否定、条件例外、来源矛盾都落 unknown/人工复核，
|
||||
不挑一个"看起来正面"的选项（AC-18）。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from contracts.judgment import JudgmentProfile
|
||||
from contracts.prescreen import JudgmentConfig
|
||||
|
||||
FIXTURE_ROOT = Path(__file__).resolve().parent.parent / "fixtures" / "judgment"
|
||||
|
||||
# 离线预算估算：中文按 1 字≈1 token、ASCII 按 4 字符≈1 token 的保守上界。
|
||||
# 真实预算必须按选定模型的 tokenizer 实测填写（11 §8.1），这里的估算只用于
|
||||
# 离线 fixture 与"宁可拒绝"的方向，不作为生产阈值。
|
||||
_CJK_FLOOR = 0x2E80
|
||||
|
||||
|
||||
class ContextBudgetExceeded(RuntimeError):
|
||||
"""超出已验证上下文预算（E-25）。携带原证据范围，供上层保留引用。"""
|
||||
|
||||
def __init__(self, message: str, evidence_ids=()):
|
||||
super().__init__(message)
|
||||
self.evidence_ids = list(evidence_ids)
|
||||
|
||||
|
||||
class TaskNotValidated(ValueError):
|
||||
"""该任务/用途没有 passed 的验收登记（AC-38）。"""
|
||||
|
||||
|
||||
class CheckpointMismatch(ValueError):
|
||||
"""实际权重修订与运行冻结值不一致（AC-39）。"""
|
||||
|
||||
|
||||
class EgressNotApproved(PermissionError):
|
||||
"""未获批准的外部出口：不发送（AC-16/19）。"""
|
||||
|
||||
|
||||
class LiveTransportUnavailable(RuntimeError):
|
||||
"""没有可用的 live 传输层（本批未接入硬件与授权）。"""
|
||||
|
||||
|
||||
def estimate_tokens(text: str) -> int:
|
||||
"""保守估算 token 数：CJK 每字 1，其它每 4 字符 1，向上取整。"""
|
||||
if not text:
|
||||
return 0
|
||||
cjk = sum(1 for char in text if ord(char) >= _CJK_FLOOR)
|
||||
other = len(text) - cjk
|
||||
return cjk + (other + 3) // 4
|
||||
|
||||
|
||||
def load_fixture(name: str) -> dict:
|
||||
return json.loads((FIXTURE_ROOT / name).read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ 准入
|
||||
def validation_for(profile: JudgmentProfile, task: str, usage: str):
|
||||
"""取出该任务与用途的登记；没有登记返回 None，不做"默认通过"。"""
|
||||
for item in profile.task_validations:
|
||||
if item.task == task and item.allowed_usage == usage:
|
||||
return item
|
||||
return None
|
||||
|
||||
|
||||
def assert_task_allowed(profile: JudgmentProfile, task: str, usage: str) -> None:
|
||||
"""逐任务逐用途准入（AC-38 / AC-41）。
|
||||
|
||||
offline 与 runtime 分开登记：只有 offline 通过的配置不能进 runtime；
|
||||
runtime 还要求验收是独立评测——同模型自评或其生成标签不算证据。
|
||||
"""
|
||||
if profile.readiness != "ready":
|
||||
raise TaskNotValidated("该专项配置尚未就绪，不能执行任何任务")
|
||||
record = validation_for(profile, task, usage)
|
||||
if record is None or record.decision != "passed":
|
||||
raise TaskNotValidated(
|
||||
f"该配置未在用途 {usage} 通过任务 {task} 的验收；"
|
||||
"连通、安装或其它任务通过都不能替代该任务准入")
|
||||
if usage == "runtime" and not record.independent:
|
||||
raise TaskNotValidated("runtime 准入必须是独立评测结果，同模型自评不能作为效果证据")
|
||||
|
||||
|
||||
def assert_checkpoint_frozen(config: JudgmentConfig, profile: JudgmentProfile) -> None:
|
||||
"""运行冻结值优先：配置登记被改动后，旧运行仍按原值执行（AC-20/39）。"""
|
||||
if profile.checkpoint_ref == "unresolved":
|
||||
raise CheckpointMismatch("权重修订不可复现（unresolved），不得用于任何执行")
|
||||
if config.checkpoint_ref and config.checkpoint_ref != profile.checkpoint_ref:
|
||||
raise CheckpointMismatch(
|
||||
"登记配置的权重修订与本次运行冻结值不一致；请新建运行而不是就地替换")
|
||||
|
||||
|
||||
def assert_within_budget(question: dict, profile: JudgmentProfile) -> None:
|
||||
"""调用前算预算；超限即拒绝，不截断、不外发（AC-40）。
|
||||
|
||||
正文（含提示词）与选项分开计：选项预算不能被正文的余量顶掉，反之亦然。
|
||||
"""
|
||||
body = "".join(str(item) for item in question.get("evidence_text") or ())
|
||||
prompt = str(question.get("prompt") or "")
|
||||
options = " ".join(str(item) for item in question.get("options") or ())
|
||||
evidence_ids = question.get("evidence_ids") or ()
|
||||
|
||||
body_tokens = estimate_tokens(body) + estimate_tokens(prompt)
|
||||
if body_tokens > profile.input_token_limit:
|
||||
raise ContextBudgetExceeded(
|
||||
f"正文约 {body_tokens} token，超出该模型已验证输入预算 "
|
||||
f"{profile.input_token_limit}；请缩小问题或转人工复核",
|
||||
evidence_ids=evidence_ids)
|
||||
|
||||
option_tokens = estimate_tokens(options)
|
||||
if option_tokens > profile.option_token_limit:
|
||||
raise ContextBudgetExceeded(
|
||||
f"选项约 {option_tokens} token，超出该模型已验证选项预算 "
|
||||
f"{profile.option_token_limit}；请缩减候选项或转人工复核",
|
||||
evidence_ids=evidence_ids)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ 映射
|
||||
def _unknown(reason: str) -> dict:
|
||||
return {"kind": "unknown", "reason_code": reason}
|
||||
|
||||
|
||||
def map_local_answer(raw: dict, question: dict, profile: JudgmentProfile) -> dict:
|
||||
"""本地模型原始输出 → 统一契约形状。
|
||||
|
||||
歧义一律落 unknown/inconclusive：映射未知、中文否定、条件例外、来源矛盾
|
||||
都不许"挑一个正面结果"（AC-18）。`confidence` 只作为 self_reported 指标
|
||||
保留原语义，**不**换算成任何概率。
|
||||
"""
|
||||
mapping = (raw.get("mapping") or {})
|
||||
if raw.get("conflicts"):
|
||||
return _unknown("inconclusive")
|
||||
if raw.get("exception_applies"):
|
||||
return _unknown("inconclusive")
|
||||
label = raw.get("label")
|
||||
if not label:
|
||||
return _unknown("missing_evidence" if not raw.get("evidence_ids") else "inconclusive")
|
||||
option = mapping.get(label)
|
||||
if option is None:
|
||||
# 例如中文否定表述（"不符合"）未登记映射：不得当成正面选项使用
|
||||
return _unknown("inconclusive")
|
||||
metrics = []
|
||||
if raw.get("confidence") is not None:
|
||||
metrics.append({"name": "self_confidence", "value": float(raw["confidence"]),
|
||||
"semantics": "模型自报置信度，未经校准，与审批概率无关",
|
||||
"source": "self_reported"})
|
||||
return {"kind": question["type"], "value": option, "native_metrics": metrics,
|
||||
"actual_checkpoint_ref": raw.get("actual_checkpoint_ref")}
|
||||
|
||||
|
||||
# jev 厂商字段名与本地完全不同：映射表按 mapping_version 固定，主审不读厂商字段
|
||||
# （AC-17）。新增厂商版本要新增一行映射并单独登记验收，不改这一层的行为。
|
||||
# 本行只登记 choice 题型的映射：其它题型没有登记就落 unknown，而不是借这一行
|
||||
# 顺带支持（06-validation.md §6："阈值和映射按提供方/模型/题型独立登记"）。
|
||||
_DEV_VENDOR_CHOICES = {
|
||||
"APPROVE": "supported",
|
||||
"REJECT": "unsupported",
|
||||
# COND_APPROVE（附条件）刻意**不**映射为通过：附条件不等于通过（BR-21）
|
||||
}
|
||||
|
||||
|
||||
def map_jev_answer(raw: dict, question: dict, profile: JudgmentProfile) -> dict:
|
||||
"""Jev 厂商响应 → 统一契约形状（fixture 验证用，不发送任何请求）。"""
|
||||
if question["type"] != "choice":
|
||||
return _unknown("inconclusive")
|
||||
envelope = raw.get("result") or {}
|
||||
code = envelope.get("decision_code")
|
||||
value = _DEV_VENDOR_CHOICES.get(code)
|
||||
if value is None:
|
||||
return _unknown("inconclusive")
|
||||
metrics = []
|
||||
for name, item in (raw.get("metrics") or {}).items():
|
||||
# 厂商原生指标保留语义与来源；不跨提供方套阈值，也不补齐缺失概率（AC-18）
|
||||
metrics.append({"name": str(name), "value": float(item),
|
||||
"semantics": "厂商原生指标，定义与兴邦校准口径不一致",
|
||||
"source": "provider_native"})
|
||||
return {"kind": question["type"], "value": value, "native_metrics": metrics,
|
||||
"actual_checkpoint_ref": raw.get("checkpoint")}
|
||||
|
||||
|
||||
MAPPERS = {"local": map_local_answer, "jev": map_jev_answer}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ 适配入口
|
||||
def build_provider_call(config: JudgmentConfig, profile: JudgmentProfile, task: str,
|
||||
usage: str, *, fixture=None, transport=None, call_log=None):
|
||||
"""构造 `judge()` 用的 provider_call：先过准入与冻结，再决定是否真的发送。
|
||||
|
||||
`fixture(question) -> raw` 是离线路径（不联网）；`transport(target, payload)`
|
||||
是 live 路径。两者不能互相冒充：call_log 会记录 mode，测试据此断言
|
||||
"local 运行期间没有任何 jev 请求"（AC-17）。
|
||||
"""
|
||||
assert_task_allowed(profile, task, usage)
|
||||
assert_checkpoint_frozen(config, profile)
|
||||
mapper = MAPPERS[profile.provider]
|
||||
|
||||
def call(config: JudgmentConfig, question: dict) -> dict:
|
||||
# 预算在发送之前判定：超限即结束，调用记录里不会出现任何发送
|
||||
assert_within_budget(question, profile)
|
||||
payload = json.dumps({
|
||||
"question_id": question["id"], "version": question["version"],
|
||||
"type": question["type"], "prompt": question.get("prompt", ""),
|
||||
"options": list(question.get("options") or ()),
|
||||
"evidence_ids": list(question.get("evidence_ids") or ()),
|
||||
"checkpoint_ref": config.checkpoint_ref,
|
||||
"prompt_version": config.prompt_version,
|
||||
"mapping_version": config.mapping_version,
|
||||
}, ensure_ascii=False)
|
||||
|
||||
if fixture is not None:
|
||||
raw = fixture(question)
|
||||
mode = "fixture"
|
||||
else:
|
||||
if config.provider == "jev" and not profile.egress_approved:
|
||||
raise EgressNotApproved("该外部出口未获批准，已拒绝发送")
|
||||
if transport is None:
|
||||
raise LiveTransportUnavailable(
|
||||
"本批未接入 live 传输层（缺硬件与授权），只能走离线 fixture")
|
||||
raw = transport(profile.provider, payload)
|
||||
mode = "live"
|
||||
|
||||
if call_log is not None:
|
||||
call_log.append({"target": config.provider, "mode": mode,
|
||||
"question_id": question["id"], "payload": payload})
|
||||
|
||||
adapted = mapper(raw, question, profile)
|
||||
actual = adapted.get("actual_checkpoint_ref")
|
||||
if actual and actual != config.checkpoint_ref:
|
||||
raise CheckpointMismatch(
|
||||
f"提供方实际权重 {actual} 与运行冻结值 {config.checkpoint_ref} 不一致")
|
||||
return adapted
|
||||
|
||||
return call
|
||||
@@ -26,6 +26,14 @@ from contracts.prescreen import (
|
||||
)
|
||||
from services import assistant_store as store
|
||||
from services.component_registry import TOOLS
|
||||
# T08：适配层的策略拒绝用异常表达，在这里统一映射成 Judgment 记录。
|
||||
# judgment_adapters 只依赖 contracts，不反向依赖本模块，因此没有循环导入。
|
||||
from services.judgment_adapters import (
|
||||
CheckpointMismatch,
|
||||
ContextBudgetExceeded,
|
||||
EgressNotApproved,
|
||||
TaskNotValidated,
|
||||
)
|
||||
|
||||
DISABLED_REASON_DEFAULT = "本次运行未启用专项判断；主审仍按规则、工具与经验形成意见"
|
||||
|
||||
@@ -69,7 +77,9 @@ def resolve_config(enabled: bool, profile_id: str | None = None,
|
||||
model_version=profile.get("model_version") or "unresolved",
|
||||
adapter_version=profile.get("adapter_version") or "not_applicable",
|
||||
prompt_version=profile.get("prompt_version") or "not_applicable",
|
||||
mapping_version=profile.get("mapping_version") or "not_applicable")
|
||||
mapping_version=profile.get("mapping_version") or "not_applicable",
|
||||
# T08：权重修订随配置一起冻结；缺失记 unresolved 而不是留空猜测（AC-39）
|
||||
checkpoint_ref=profile.get("checkpoint_ref") or "unresolved")
|
||||
|
||||
|
||||
def allowed_tools(skill_body: dict, registry_tools=None) -> set[str]:
|
||||
@@ -127,6 +137,20 @@ def judge(config: JudgmentConfig, run_id: str, question: dict,
|
||||
raw = provider_call(config, question)
|
||||
except TimeoutError:
|
||||
return _failed(run_id, question, config, ErrorCode.JUDGMENT_UNAVAILABLE)
|
||||
except ContextBudgetExceeded:
|
||||
# E-25：超预算不外发、不静默截断；evidence_ids 仍保留原引用范围（AC-40）
|
||||
return _failed(run_id, question, config, ErrorCode.JUDGMENT_CONTEXT_EXCEEDED)
|
||||
except TaskNotValidated:
|
||||
# 未获准的任务不给"未完成"以外的结论（AC-38）
|
||||
return {**_failed(run_id, question, config, ErrorCode.JUDGMENT_UNAVAILABLE),
|
||||
"status": "skipped", "error_code": None, "skip_reason": "unsupported"}
|
||||
except EgressNotApproved:
|
||||
# 未批准出口：不发送，记录策略拒绝（AC-16/19）
|
||||
return {**_failed(run_id, question, config, ErrorCode.JUDGMENT_UNAVAILABLE),
|
||||
"status": "skipped", "error_code": None, "skip_reason": "not_authorized"}
|
||||
except CheckpointMismatch:
|
||||
# 实际权重与冻结修订不符：拒绝并记失败，不得静默换模型（AC-39）
|
||||
return _failed(run_id, question, config, ErrorCode.JUDGMENT_OUTPUT_INVALID)
|
||||
except Exception:
|
||||
return _failed(run_id, question, config, ErrorCode.JUDGMENT_UNAVAILABLE)
|
||||
|
||||
@@ -179,7 +203,8 @@ def _normalise(run_id: str, question: dict, config: JudgmentConfig, raw: dict) -
|
||||
"provider": config.provider, "profile_id": config.profile_id,
|
||||
"profile_revision": config.profile_revision,
|
||||
"model_version": config.model_version, "native_metrics": metrics,
|
||||
"error_code": None, "skip_reason": None}
|
||||
"error_code": None, "skip_reason": None,
|
||||
"actual_checkpoint_ref": raw.get("actual_checkpoint_ref")}
|
||||
return Judgment(**body).model_dump(mode="json")
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user