Files
mini-openclaw-v2/backend/test_prescreen_api.py
T
qiaoqichaoandClaude Code ff7894ae1f feat(v2): S1c′ 既有上传入口的批量摄取——S1 首次端到端闭环
T00/04 §4.2 要求的"既有材料上传入口扩展":**不建立平行上传 API**。
POST /documents 增加可选 batch_id / item_id 表单字段;不传时行为完全不变。

- api/prescreen.ingest_into_batch:清单核对 + 状态回写,仍走既有
  ProjectAssistantService.upload(),故解析、同哈希复用原件、pending_candidates、
  data_revision、mark_stale 等既有行为一字未改
- 服务端**重算** sha256 与大小,不信任客户端清单;不符则条目 failed + 422
- 返回体保持 **document 形状**(既有前端读 body.id / body.candidates),
  批次状态作为附加字段 import_batch 给出
- 拒绝重复摄入已解析条目(409)、已取消批次(409)、不在清单中的文件(404)

闭环已成立:登记批次 → 上传 → 解析定位 → 冲突检测 → 人工裁定 → 进事实。

测试:test_prescreen_api.py 新增 BatchIngestTests 10 项(含闭环与既有路径回归)。
全量 235 项 OK(225 → 235);scripts/verify.py 9/9 PASS。
真实 app 冒烟 claude_use/T02/smoke_s1.py 15/15 PASS。

**测试隔离缺陷已修**:BatchIngestTests 初版只 patch 了 prescreen 的 service 工厂,
未 patch api.project_assistant.service,导致不传 batch_id 的上传落到**真实 v2 数据库**。
已补 patch,并清空 v2 数据根从干净状态重跑验证。

Co-Authored-By: Claude Code <noreply@anthropic.com>
2026-09-29 17:43:57 +08:00

477 lines
25 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""S1 接口层测试:材料集合导入与事实确认的 HTTP 契约。
只挂 prescreen 路由到独立 app,数据库指向临时文件——不触碰真实库。
"""
import hashlib
import os
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from fastapi import FastAPI
from fastapi.exceptions import RequestValidationError
from fastapi.testclient import TestClient
from api import prescreen as api
from services import assistant_store as store
from services import fact_conflict
from services.project_assistant import ProjectAssistantService
HEADERS = {"X-Xingbang-Demo-Role": "manager"}
# 演示身份可访问的项目:DEMO-A/B/C、LAB-001、XB-2026-001
PROJECT = "DEMO-B"
FOREIGN_PROJECT = "OTHER-9" # 不在演示身份的项目列表里
def manifest(name="a.csv", sha="a" * 64, size=100):
return {"relative_path": name, "sha256": sha, "size_bytes": size, "mime_type": "text/csv"}
class PrescreenApiTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory(ignore_cleanup_errors=True)
self.path = Path(self.temp.name)
self.service = ProjectAssistantService(self.path / "assistant.sqlite3")
self.patches = [
patch.object(api, "_service", lambda: self.service),
patch.dict(os.environ, {"XINGBANG_AUTH_USERS": "", "XINGBANG_REQUIRE_AUTH": "0",
"XINGBANG_LOCAL_MODEL_URL": ""}),
]
for p in self.patches:
p.start()
app = FastAPI()
# 必须与 app.py 的注册保持一致:否则本测试的 app 不忠实于生产接线,
# 校验错误会走 FastAPI 默认形状而测不到信封。
app.add_exception_handler(RequestValidationError, api.validation_handler)
app.include_router(api.router)
self.client = TestClient(app)
self.base = "/api/xingbang/risk-agent/project-assistant/prescreen"
def tearDown(self):
self.client.close()
for p in reversed(self.patches):
p.stop()
self.temp.cleanup()
# ------------------------------------------------------------ 工具
def post(self, path, body=None, headers=None):
return self.client.post(self.base + path, json=body if body is not None else {},
headers={**HEADERS, **(headers or {})})
def get(self, path, headers=None):
return self.client.get(self.base + path, headers={**HEADERS, **(headers or {})})
def error(self, response):
body = response.json()
self.assertIn("error", body, body)
return body["error"]
def seed_conflict(self):
"""直接经服务层造一个未解决冲突(本文件测的是 HTTP 层,不重复测存储层)。
注意:演示种子项目 DEMO-B 可能**已有**已确认的 facility.amount,
所以不能假设"首次确认一定无冲突"。这里按现状自适应。
传 unit=None 使候选值不做万元→元换算,与已确认值同口径可比。
"""
def cand(cid, value, source, locator):
return {"id": cid, "field": "facility.amount", "value": value,
"unit": None, "source": source, "locator": locator}
with self.service.connect() as db:
db.execute("BEGIN IMMEDIATE")
project = store.get(db, "project", PROJECT)
existing = project.get("values", {}).get("facility.amount")
if existing is None:
payload = fact_conflict.detect(project, cand("c0", "1000", "a.csv", "第2行"),
"demo-manager", "初始确认")
store.put(db, "project", project)
self.assertIsNone(payload, "首次确认不应产生冲突")
project = store.get(db, "project", PROJECT)
existing = project["values"]["facility.amount"]
different = next(v for v in ("777", "888", "999") if str(v) != str(existing))
payload = fact_conflict.detect(project, cand("c1", different, "b.csv", "第9行"),
"demo-manager", "第二来源")
store.put(db, "project", project)
self.assertIsNotNone(payload, "与已确认值不同,应产生冲突")
return fact_conflict.create(db, PROJECT, payload, "demo-manager")
# ------------------------------------------------------------ 导入批次
def test_create_batch_returns_202_queued(self):
response = self.post(f"/projects/{PROJECT}/import-batches", {"manifest": [manifest()]})
self.assertEqual(response.status_code, 202, response.text)
batch = response.json()["batch"]
self.assertEqual(batch["status"], "queued")
self.assertEqual(batch["data_domain"], "demo")
self.assertEqual(batch["revision"], 1)
self.assertTrue(batch["items"][0]["id"].startswith("item-"))
self.assertEqual(batch["items"][0]["status"], "queued")
self.assertIsNone(batch["items"][0]["observed_at"])
def test_client_supplied_server_fields_rejected(self):
bad = dict(manifest(), status="parsed")
response = self.post(f"/projects/{PROJECT}/import-batches", {"manifest": [bad]})
self.assertEqual(response.status_code, 422, response.text)
self.assertEqual(self.error(response)["code"], "IMPORT_BATCH_INVALID")
def test_absolute_path_rejected_with_envelope(self):
response = self.post(f"/projects/{PROJECT}/import-batches",
{"manifest": [manifest(name="C:/x.csv")]})
self.assertEqual(response.status_code, 422)
error = self.error(response)
self.assertEqual(error["code"], "IMPORT_BATCH_INVALID")
self.assertFalse(error["retryable"])
self.assertTrue(error["request_id"])
def test_unknown_body_field_rejected_with_envelope(self):
"""校验层在进入端点前抛错,也必须走统一信封(否则同命名空间两种形状)。"""
response = self.post(f"/projects/{PROJECT}/import-batches",
{"manifest": [manifest()], "data_domain": "production"})
self.assertEqual(response.status_code, 422, response.text)
error = self.error(response)
self.assertEqual(error["code"], "INVALID_REQUEST")
self.assertTrue(error["request_id"])
def test_invalid_query_param_rejected_with_envelope(self):
response = self.get(f"/projects/{PROJECT}/import-batches?limit=notanumber")
self.assertEqual(response.status_code, 422, response.text)
self.assertEqual(self.error(response)["code"], "INVALID_REQUEST")
def test_empty_manifest_rejected(self):
response = self.post(f"/projects/{PROJECT}/import-batches", {"manifest": []})
self.assertEqual(response.status_code, 422, response.text)
def test_foreign_project_is_404_not_403(self):
# 03 §2:不存在与无访问权统一 404,不泄露对象是否存在
response = self.post(f"/projects/{FOREIGN_PROJECT}/import-batches",
{"manifest": [manifest()]})
self.assertEqual(response.status_code, 404, response.text)
self.assertEqual(self.error(response)["code"], "NOT_FOUND")
def test_get_batch_by_id(self):
created = self.post(f"/projects/{PROJECT}/import-batches", {"manifest": [manifest()]}).json()["batch"]
response = self.get(f"/import-batches/{created['id']}")
self.assertEqual(response.status_code, 200, response.text)
self.assertEqual(response.json()["id"], created["id"])
def test_get_unknown_batch_is_404_envelope(self):
response = self.get("/import-batches/import-does-not-exist")
self.assertEqual(response.status_code, 404)
error = self.error(response)
self.assertEqual(error["code"], "NOT_FOUND")
# 文案不暴露对象是否存在
self.assertEqual(error["message"], "对象不可访问或不存在")
def test_idempotent_replay_returns_same_batch(self):
body = {"manifest": [manifest()]}
first = self.post(f"/projects/{PROJECT}/import-batches", body,
headers={"Idempotency-Key": "k1"}).json()["batch"]
again = self.post(f"/projects/{PROJECT}/import-batches", body,
headers={"Idempotency-Key": "k1"}).json()["batch"]
self.assertEqual(first["id"], again["id"])
def test_idempotency_conflict_is_409(self):
self.post(f"/projects/{PROJECT}/import-batches", {"manifest": [manifest()]},
headers={"Idempotency-Key": "k2"})
response = self.post(f"/projects/{PROJECT}/import-batches",
{"manifest": [manifest(sha="c" * 64)]},
headers={"Idempotency-Key": "k2"})
self.assertEqual(response.status_code, 409, response.text)
self.assertEqual(self.error(response)["code"], "IDEMPOTENCY_CONFLICT")
def test_overlong_idempotency_key_rejected(self):
response = self.post(f"/projects/{PROJECT}/import-batches", {"manifest": [manifest()]},
headers={"Idempotency-Key": "x" * 200})
self.assertEqual(response.status_code, 422, response.text)
def test_list_batches_for_project(self):
self.post(f"/projects/{PROJECT}/import-batches", {"manifest": [manifest()]})
response = self.get(f"/projects/{PROJECT}/import-batches")
self.assertEqual(response.status_code, 200, response.text)
self.assertEqual(len(response.json()["items"]), 1)
def test_retry_only_failed_items(self):
batch = self.post(f"/projects/{PROJECT}/import-batches",
{"manifest": [manifest(name="ok.csv"),
manifest(name="bad.csv", sha="b" * 64)]}).json()["batch"]
ids = [item["id"] for item in batch["items"]]
with self.service.connect() as db:
from services import material_import
material_import.mark_item(db, batch["id"], ids[0], "parsed", document_id="doc-1")
batch = material_import.mark_item(db, batch["id"], ids[1], "failed",
error_code="MATERIAL_PARSE_FAILED")
response = self.post(f"/import-batches/{batch['id']}/retry",
{"item_ids": [ids[1]], "expected_revision": batch["revision"]})
# 202 表示已接受重试;但该请求是同步执行的,直接看返回体
self.assertIn(response.status_code, (200, 202), response.text)
items = {item["id"]: item["status"] for item in response.json()["items"]}
self.assertEqual(items[ids[1]], "queued")
def test_retry_with_stale_revision_is_409(self):
batch = self.post(f"/projects/{PROJECT}/import-batches", {"manifest": [manifest()]}).json()["batch"]
with self.service.connect() as db:
from services import material_import
batch = material_import.mark_item(db, batch["id"], batch["items"][0]["id"], "failed",
error_code="MATERIAL_PARSE_FAILED")
response = self.post(f"/import-batches/{batch['id']}/retry",
{"item_ids": [batch["items"][0]["id"]], "expected_revision": 1})
self.assertEqual(response.status_code, 409, response.text)
self.assertEqual(self.error(response)["code"], "REVISION_CONFLICT")
def test_cancel_then_retry_refused(self):
batch = self.post(f"/projects/{PROJECT}/import-batches", {"manifest": [manifest()]}).json()["batch"]
cancelled = self.post(f"/import-batches/{batch['id']}/cancel",
{"expected_revision": batch["revision"]})
self.assertEqual(cancelled.status_code, 200, cancelled.text)
self.assertEqual(cancelled.json()["status"], "cancelled")
again = self.post(f"/import-batches/{batch['id']}/cancel",
{"expected_revision": cancelled.json()["revision"]})
self.assertEqual(again.status_code, 409, again.text)
# ------------------------------------------------------------ 事实冲突
def test_list_conflicts_and_filter(self):
conflict = self.seed_conflict()
response = self.get(f"/projects/{PROJECT}/fact-conflicts?status=unresolved")
self.assertEqual(response.status_code, 200, response.text)
self.assertEqual([row["id"] for row in response.json()["items"]], [conflict["id"]])
empty = self.get(f"/projects/{PROJECT}/fact-conflicts?status=resolved")
self.assertEqual(empty.json()["items"], [])
def test_invalid_status_filter_rejected(self):
response = self.get(f"/projects/{PROJECT}/fact-conflicts?status=whatever")
self.assertEqual(response.status_code, 422, response.text)
self.assertEqual(self.error(response)["code"], "INVALID_REQUEST")
def test_get_conflict_by_id(self):
conflict = self.seed_conflict()
response = self.get(f"/fact-conflicts/{conflict['id']}")
self.assertEqual(response.status_code, 200, response.text)
self.assertEqual(response.json()["status"], "unresolved")
def test_resolve_happy_path_adopts_value_and_persists(self):
conflict = self.seed_conflict()
chosen = conflict["candidates"][1]["evidence_id"]
response = self.post(f"/fact-conflicts/{conflict['id']}/resolve",
{"selected_evidence_ids": [chosen],
"resolution_note": "以合同金额为准",
"expected_revision": conflict["revision"]})
self.assertEqual(response.status_code, 200, response.text)
self.assertEqual(response.json()["status"], "resolved")
# 值必须真的落到项目事实里(resolve 就地改 project,接口层负责保存)
with self.service.connect() as db:
project = store.get(db, "project", PROJECT)
self.assertEqual(str(project["values"]["facility.amount"]),
str(conflict["candidates"][1]["value"]))
self.assertEqual(project["selected_sources"]["facility.amount"], chosen)
self.assertEqual(project["facts"]["facility_amount"],
project["values"]["facility.amount"])
def test_resolve_with_foreign_evidence_is_422(self):
conflict = self.seed_conflict()
response = self.post(f"/fact-conflicts/{conflict['id']}/resolve",
{"selected_evidence_ids": ["ev-nope"],
"resolution_note": "说明",
"expected_revision": conflict["revision"]})
self.assertEqual(response.status_code, 422, response.text)
self.assertEqual(self.error(response)["code"], "INVALID_REQUEST")
def test_resolve_twice_is_409(self):
conflict = self.seed_conflict()
chosen = conflict["candidates"][1]["evidence_id"]
self.post(f"/fact-conflicts/{conflict['id']}/resolve",
{"selected_evidence_ids": [chosen], "resolution_note": "说明",
"expected_revision": conflict["revision"]})
again = self.post(f"/fact-conflicts/{conflict['id']}/resolve",
{"selected_evidence_ids": [chosen], "resolution_note": "再来",
"expected_revision": conflict["revision"]})
self.assertEqual(again.status_code, 409, again.text)
def test_resolve_unknown_conflict_is_404(self):
response = self.post("/fact-conflicts/fc-nope/resolve",
{"selected_evidence_ids": ["ev-1"], "resolution_note": "说明",
"expected_revision": 1})
self.assertEqual(response.status_code, 404, response.text)
# ------------------------------------------------------------ 边界
def test_envelope_shape_is_stable(self):
response = self.get("/import-batches/nope")
error = self.error(response)
self.assertEqual(set(error), {"code", "message", "retryable", "request_id"})
def test_existing_endpoints_still_use_detail_shape(self):
"""D2 决策的证据:本模块以外不产出 error 信封。"""
import api.project_assistant as legacy
self.assertFalse(hasattr(legacy, "_envelope"))
self.assertFalse(hasattr(legacy, "S1_ERROR_HTTP"))
def test_router_is_mounted_under_project_assistant_prefix(self):
# 命名空间定案 P1:零鉴权改动的关键前提是前缀在 project-assistant 之下
self.assertEqual(api.router.prefix,
"/api/xingbang/risk-agent/project-assistant/prescreen")
def test_storage_layer_untouched_by_api_layer(self):
"""接口层不直接写 SQL,只经服务层——避免出现第二份业务规则。"""
import api.prescreen as module
with open(module.__file__, encoding="utf-8") as handle:
source = handle.read()
for banned in ("INSERT INTO", "UPDATE ", "DELETE FROM", "SELECT "):
self.assertNotIn(banned, source)
class BatchIngestTests(unittest.TestCase):
"""S1c′:既有上传入口的批量摄取(T00/04 §4.2)。
文档端点属于 project_assistant 路由,故这里同时挂两个路由,与 app.py 一致。
"""
def setUp(self):
self.temp = tempfile.TemporaryDirectory(ignore_cleanup_errors=True)
self.path = Path(self.temp.name)
self.service = ProjectAssistantService(self.path / "assistant.sqlite3")
self.patches = [
patch.object(api, "_service", lambda: self.service),
patch.dict(os.environ, {"XINGBANG_AUTH_USERS": "", "XINGBANG_REQUIRE_AUTH": "0",
"XINGBANG_LOCAL_MODEL_URL": ""}),
]
for p in self.patches:
p.start()
from fastapi.exceptions import RequestValidationError as RVE
from api import project_assistant as legacy
# 两个路由各自的 service 工厂都要指向临时库——否则 POST /documents
# 会落到真实的 backend/xingbang_data 库上(测试隔离泄漏)。
self.patches.append(patch.object(legacy, "service", lambda: self.service))
for p in self.patches:
p.start()
app = FastAPI()
app.add_exception_handler(RVE, api.validation_handler)
app.include_router(legacy.router)
app.include_router(api.router)
self.client = TestClient(app)
self.base = "/api/xingbang/risk-agent/project-assistant"
self.pre = self.base + "/prescreen"
def tearDown(self):
self.client.close()
for p in reversed(self.patches):
p.stop()
self.temp.cleanup()
def make_batch(self, payload: bytes, name="流水.csv"):
entry = {"relative_path": name, "sha256": hashlib.sha256(payload).hexdigest(),
"size_bytes": len(payload), "mime_type": "text/csv"}
response = self.client.post(f"{self.pre}/projects/{PROJECT}/import-batches",
json={"manifest": [entry]}, headers=HEADERS)
self.assertEqual(response.status_code, 202, response.text)
return response.json()["batch"]
def send(self, batch, payload: bytes, name="流水.csv", item_id=None, batch_id=None):
data = {"batch_id": batch["id"] if batch_id is None else batch_id}
data["item_id"] = item_id if item_id is not None else batch["items"][0]["id"]
return self.client.post(f"{self.base}/projects/{PROJECT}/documents",
files={"file": (name, payload, "text/csv")},
data=data, headers=HEADERS)
def test_ingest_marks_item_parsed_and_links_document(self):
payload = "field,value\nfacility.amount,100\n".encode("utf-8-sig")
batch = self.make_batch(payload)
response = self.send(batch, payload)
self.assertEqual(response.status_code, 201, response.text)
document_id = response.json()["id"]
after = self.client.get(f"{self.pre}/import-batches/{batch['id']}", headers=HEADERS).json()
self.assertEqual(after["status"], "ready")
self.assertEqual(after["items"][0]["status"], "parsed")
self.assertEqual(after["items"][0]["document_id"], document_id)
def test_ingest_by_filename_when_item_id_absent(self):
payload = b"field,value\nfacility.amount,100\n"
batch = self.make_batch(payload)
response = self.client.post(f"{self.base}/projects/{PROJECT}/documents",
files={"file": ("流水.csv", payload, "text/csv")},
data={"batch_id": batch["id"]}, headers=HEADERS)
self.assertEqual(response.status_code, 201, response.text)
after = self.client.get(f"{self.pre}/import-batches/{batch['id']}", headers=HEADERS).json()
self.assertEqual(after["items"][0]["status"], "parsed")
def test_hash_mismatch_marks_failed_and_rejects(self):
declared = b"field,value\nfacility.amount,100\n"
batch = self.make_batch(declared)
tampered = b"field,value\nfacility.amount,999\n"
response = self.send(batch, tampered)
self.assertEqual(response.status_code, 422, response.text)
after = self.client.get(f"{self.pre}/import-batches/{batch['id']}", headers=HEADERS).json()
self.assertEqual(after["items"][0]["status"], "failed")
self.assertEqual(after["items"][0]["error_code"], "INVALID_EVIDENCE")
self.assertIsNone(after["items"][0]["document_id"])
def test_size_mismatch_marks_failed(self):
payload = b"field,value\nfacility.amount,100\n"
entry = {"relative_path": "x.csv", "sha256": hashlib.sha256(payload).hexdigest(),
"size_bytes": len(payload) + 5, "mime_type": "text/csv"}
batch = self.client.post(f"{self.pre}/projects/{PROJECT}/import-batches",
json={"manifest": [entry]}, headers=HEADERS).json()["batch"]
response = self.send(batch, payload, name="x.csv")
self.assertEqual(response.status_code, 422, response.text)
after = self.client.get(f"{self.pre}/import-batches/{batch['id']}", headers=HEADERS).json()
self.assertEqual(after["items"][0]["status"], "failed")
def test_item_not_in_batch_rejected(self):
payload = b"field,value\nfacility.amount,100\n"
batch = self.make_batch(payload)
response = self.send(batch, payload, item_id="item-not-mine")
self.assertEqual(response.status_code, 404, response.text)
def test_filename_not_in_manifest_rejected(self):
payload = b"field,value\nfacility.amount,100\n"
batch = self.make_batch(payload, name="declared.csv")
response = self.client.post(f"{self.base}/projects/{PROJECT}/documents",
files={"file": ("other.csv", payload, "text/csv")},
data={"batch_id": batch["id"]}, headers=HEADERS)
self.assertEqual(response.status_code, 404, response.text)
def test_double_ingest_rejected(self):
payload = b"field,value\nfacility.amount,100\n"
batch = self.make_batch(payload)
self.assertEqual(self.send(batch, payload).status_code, 201)
again = self.send(batch, payload)
self.assertEqual(again.status_code, 409, again.text)
def test_cancelled_batch_refuses_ingest(self):
payload = b"field,value\nfacility.amount,100\n"
batch = self.make_batch(payload)
self.client.post(f"{self.pre}/import-batches/{batch['id']}/cancel",
json={"expected_revision": batch["revision"]}, headers=HEADERS)
response = self.send(batch, payload)
self.assertEqual(response.status_code, 409, response.text)
def test_plain_upload_without_batch_id_is_unchanged(self):
"""既有行为回归:不传 batch_id 时仍走原路径,返回解析候选。"""
payload = b"field,value\nfacility.amount,100\n"
response = self.client.post(f"{self.base}/projects/{PROJECT}/documents",
files={"file": ("plain.csv", payload, "text/csv")},
headers=HEADERS)
self.assertEqual(response.status_code, 201, response.text)
body = response.json()
self.assertIn("candidates", body)
self.assertNotIn("raw_base64", body)
self.assertTrue(body["id"].startswith("doc-"))
def test_end_to_end_batch_closes_the_loop(self):
"""S1 首次闭环:登记批次 → 上传 → 解析定位 → 可确认。"""
payload = "field,value,currency,unit,period,scope\nfacility.amount,100,CNY,万元,2025,合并\n".encode("utf-8-sig")
batch = self.make_batch(payload)
self.assertEqual(self.send(batch, payload).status_code, 201)
final = self.client.get(f"{self.pre}/import-batches/{batch['id']}", headers=HEADERS).json()
self.assertEqual(final["status"], "ready")
document_id = final["items"][0]["document_id"]
detail = self.client.get(f"{self.base}/projects/{PROJECT}/documents/{document_id}",
headers=HEADERS)
self.assertEqual(detail.status_code, 200, detail.text)
self.assertTrue(detail.json()["candidates"], "解析应产出候选字段")
if __name__ == "__main__":
unittest.main()