feat(v2): T11 改进治理与演示交付——对照门禁、泄漏作废、三模块演示脚本
覆盖 AC-15/16/34/37/41（BR-15/16/40、E-11/24）。
后端：
- contracts/improvement.py：ImprovementCandidate + ContrastReport/ContrastMetrics。
零分母不得给比率或覆盖率（样本不足不是 0%）；非独立评测不得给改进/退化结论；
已发布必须引用既有不可变版本；没有对照证据不得进入待审核或已发布。
- services/improvements.py：
· evaluate_contrast 在服务端判定泄漏——预测锁定前读到标签即整次评测作废（E-11），
保留集来源进开发集的样本作废并单列，指标按分子/可观察分母/覆盖率/未观察分开报；
· transition 状态机：draft→evaluating→ready→in_review→approved→released，
自批拒绝、退化与结论不明拒绝发布、发布幂等且只引用既有版本、撤回只影响新运行。
- api/prescreen.py：候选列表、提出与状态机三个端点；评测请求体提交样本，
泄漏与指标一律由服务端计算，不接受客户端交来的"成品报告"。
- contracts/prescreen.py：补 E-11/E-24 错误码与 409 映射。
前端一致性修正：
- src/lib/prescreen-run.ts 的呈现判定改用后端既有词汇
（empty/list/grouped/radar/faceted，与 services/review_plan.presentation 一致），
避免前后端各一套判定导致界面画出后端判定为不该画雷达的图。
演示交付：
- scripts/xingbang_prescreen_demo.py：按 08 §7 八步演示，只驱动已实现入口，
未实现的步骤（批量清单、历史提炼、卡片治理）如实标注原因；
工程演示与模型效果分列，本地专项/Jev/真实概率一律标未验证或未接通。
- backend/test_demo_script.py：冒烟断言八步状态、未实现原因与分列口径。
验证：backend 全量 unittest 519 项 OK；前端 npm run build 通过。
Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
Claude Code
@@ -0,0 +1,147 @@
|
||||
"""T11 改进候选契约：对照证据、独立审核与发布。
|
||||
|
||||
Owner：本文件（docs/v2/03-detailed-design.md §9.4 ImprovementCandidate 与状态机、
|
||||
docs/v2/14-experience-loop.md §8、06-validation.md §5）。
|
||||
|
||||
三条不可让渡的规则：
|
||||
1. **没有对照证据不能发布**：候选必须带一次冻结版本的旧/新对照，且不是退化
|
||||
（AC-34 / 06 §5"未评测、自批、退化均不能发布"）。
|
||||
2. **自批不算审核**：提交人与审核人必须是不同的人；同模型自评也不能当独立评测
|
||||
（AC-41）。
|
||||
3. **发布是引用不可变版本**：发布只把候选指向一个既有的、不可变的发布结果，
|
||||
不允许"标成已发布却没有对应版本"（03 §9.4 末）。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import Field, model_validator
|
||||
|
||||
from contracts.prescreen import (
|
||||
DESC_MAX,
|
||||
ID_MAX,
|
||||
AwareDatetime,
|
||||
Contract,
|
||||
DataDomain,
|
||||
ErrorCode,
|
||||
Ref,
|
||||
)
|
||||
|
||||
CANDIDATE_STATUSES = ("draft", "evaluating", "ready", "in_review", "approved",
|
||||
"rejected", "released", "withdrawn")
|
||||
TARGETS = ("knowledge", "orchestration", "question_profile")
|
||||
|
||||
|
||||
class ContrastMetrics(Contract):
|
||||
"""高预测组指标口径（AC-37）：分子、可观察分母、覆盖率与未观察数分开报。
|
||||
|
||||
零分母时必须留空而不是填 0%——"没有样本"与"全都没通过"是两件事。
|
||||
"""
|
||||
|
||||
label: str = Field(min_length=1, max_length=DESC_MAX)
|
||||
numerator: int = Field(ge=0)
|
||||
observable_denominator: int = Field(ge=0)
|
||||
unobserved: int = Field(ge=0)
|
||||
coverage: float | None = Field(default=None, ge=0, le=1)
|
||||
rate: float | None = Field(default=None, ge=0, le=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _check_metrics(self) -> ContrastMetrics:
|
||||
if self.numerator > self.observable_denominator:
|
||||
raise ValueError("分子不得大于可观察分母")
|
||||
if self.observable_denominator == 0:
|
||||
if self.rate is not None or self.coverage is not None:
|
||||
raise ValueError("零分母时不得给出比率或覆盖率；样本不足不是 0%")
|
||||
elif self.rate is None:
|
||||
raise ValueError("有可观察样本时必须给出比率，避免用空白代替结果")
|
||||
return self
|
||||
|
||||
|
||||
class ContrastReport(Contract):
|
||||
"""一次旧/新对照的结果。versions 冻结后才可比较（06 §5）。"""
|
||||
|
||||
id: str = Field(min_length=1, max_length=ID_MAX)
|
||||
baseline_ref: Ref
|
||||
candidate_ref: Ref
|
||||
dataset_ref: str = Field(min_length=1, max_length=DESC_MAX)
|
||||
split_ref: str = Field(min_length=1, max_length=DESC_MAX)
|
||||
baseline_metrics: list[ContrastMetrics] = Field(default_factory=list, max_length=20)
|
||||
candidate_metrics: list[ContrastMetrics] = Field(default_factory=list, max_length=20)
|
||||
sample_size: int = Field(ge=0)
|
||||
contaminated: list[str] = Field(default_factory=list, max_length=200)
|
||||
independent: bool = False
|
||||
verdict: Literal["improved", "unchanged", "regressed", "inconclusive"]
|
||||
reviewer_id: str | None = Field(default=None, max_length=ID_MAX)
|
||||
created_at: AwareDatetime
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _check_report(self) -> ContrastReport:
|
||||
if self.verdict != "inconclusive" and not self.independent:
|
||||
raise ValueError("非独立评测不得给出改进/退化结论，只能记 inconclusive")
|
||||
if self.sample_size == 0 and self.verdict == "improved":
|
||||
raise ValueError("没有样本不得声称改进")
|
||||
return self
|
||||
|
||||
|
||||
class ImprovementCandidate(Contract):
|
||||
"""改进候选。发布引用既有不可变版本；撤回只影响新运行。"""
|
||||
|
||||
id: str = Field(min_length=1, max_length=ID_MAX)
|
||||
project_id: str = Field(min_length=1, max_length=ID_MAX)
|
||||
data_domain: DataDomain
|
||||
revision: int = Field(ge=1)
|
||||
target: Literal["knowledge", "orchestration", "question_profile"]
|
||||
target_ref: Ref
|
||||
source_run_ids: list[str] = Field(default_factory=list, max_length=200)
|
||||
outcome_ids: list[str] = Field(default_factory=list, max_length=200)
|
||||
change_summary: str = Field(min_length=1, max_length=DESC_MAX)
|
||||
proposed_revision_ref: Ref
|
||||
status: Literal["draft", "evaluating", "ready", "in_review", "approved",
|
||||
"rejected", "released", "withdrawn"]
|
||||
evaluation_ref: Ref | None = None
|
||||
contrast: ContrastReport | None = None
|
||||
reviewer_id: str | None = Field(default=None, max_length=ID_MAX)
|
||||
released_ref: Ref | None = None
|
||||
proposed_by: str = Field(min_length=1, max_length=ID_MAX)
|
||||
rejection_reason: str | None = Field(default=None, max_length=DESC_MAX)
|
||||
created_at: AwareDatetime
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _check_candidate(self) -> ImprovementCandidate:
|
||||
if self.status in ("approved", "released") and self.reviewer_id is None:
|
||||
raise ValueError("通过审核的候选必须记录审核人")
|
||||
if self.status == "released" and self.released_ref is None:
|
||||
raise ValueError("已发布必须引用一个既有的不可变发布版本，不得只改状态")
|
||||
if self.status == "rejected" and not self.rejection_reason:
|
||||
raise ValueError("拒绝必须留下原因")
|
||||
if self.status in ("ready", "in_review", "approved", "released") \
|
||||
and self.contrast is None:
|
||||
raise ValueError("没有对照证据的候选不得进入待审核或已发布状态")
|
||||
return self
|
||||
|
||||
def must_not_self_approve(self, actor: str) -> None:
|
||||
if actor == self.proposed_by:
|
||||
raise PermissionError("改进候选不能由提出者自己审核（自批不算独立审核）")
|
||||
|
||||
def release_blockers(self) -> list[str]:
|
||||
"""发布前必须为空的阻塞项列表；可读的失败原因，便于界面展示。"""
|
||||
blockers = []
|
||||
if self.contrast is None:
|
||||
blockers.append("缺少旧/新对照证据")
|
||||
else:
|
||||
if self.contrast.contaminated:
|
||||
blockers.append(f"对照样本存在泄漏：{self.contrast.contaminated}")
|
||||
if not self.contrast.independent:
|
||||
blockers.append("对照不是独立评测（同模型自评不算证据）")
|
||||
if self.contrast.verdict == "regressed":
|
||||
blockers.append("对照显示退化，不得发布")
|
||||
if self.contrast.verdict == "inconclusive":
|
||||
blockers.append("对照结论不明确，不得发布")
|
||||
if self.status != "approved":
|
||||
blockers.append(f"当前状态为 {self.status}，未通过审核")
|
||||
return blockers
|
||||
|
||||
|
||||
CONTAMINATION_CODE = ErrorCode.EVALUATION_CONTAMINATED
|
||||
NOT_VALIDATED_CODE = ErrorCode.IMPROVEMENT_NOT_VALIDATED
|
||||
@@ -109,6 +109,9 @@ class ErrorCode(StrEnum):
|
||||
DIMENSION_SCHEMA_CONFLICT = "DIMENSION_SCHEMA_CONFLICT"
|
||||
DEMO_ESTIMATE_FORBIDDEN = "DEMO_ESTIMATE_FORBIDDEN"
|
||||
SCENARIO_ASSUMPTION_CONFLICT = "SCENARIO_ASSUMPTION_CONFLICT"
|
||||
# T11 改进治理：泄漏与未验证都是"先改数据再重试"的冲突，走 409。
|
||||
EVALUATION_CONTAMINATED = "EVALUATION_CONTAMINATED"
|
||||
IMPROVEMENT_NOT_VALIDATED = "IMPROVEMENT_NOT_VALIDATED"
|
||||
# 运行级编码：记录在运行体（degradations / judgment）里，**不构成 HTTP 错误**，
|
||||
# 因此刻意不进 S1_ERROR_HTTP；见下方 RUN_LEVEL_CODES。
|
||||
JUDGMENT_UNAVAILABLE = "JUDGMENT_UNAVAILABLE"
|
||||
@@ -140,6 +143,8 @@ S1_ERROR_HTTP: dict[ErrorCode, int] = {
|
||||
ErrorCode.DIMENSION_SCHEMA_CONFLICT: 422,
|
||||
ErrorCode.DEMO_ESTIMATE_FORBIDDEN: 422,
|
||||
ErrorCode.SCENARIO_ASSUMPTION_CONFLICT: 422,
|
||||
ErrorCode.EVALUATION_CONTAMINATED: 409,
|
||||
ErrorCode.IMPROVEMENT_NOT_VALIDATED: 409,
|
||||
}
|
||||
|
||||
# 这些编码只出现在运行结果里（"HTTP 200 仅表示结果可读取"），永远不会作为
|
||||
|
||||
Reference in New Issue
Block a user