claude-code
c260eab803
See [docs/prs/2026-10-17-PR#117-doccheck-variant-b.md](docs/prs/2026-10-17-PR%23117-doccheck-variant-b.md). Replaces the economy-model summary of the doc check with one Jev call per changed file, and adds a data-outflow check. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Reviewed-on: #117 Co-authored-by: claude-code <claude-code@javagil.de>
283 lines
14 KiB
Python
283 lines
14 KiB
Python
import json
|
|
import unittest
|
|
|
|
from tests.fakes import FakeGitea, FakeJev, FakeModel, config, deps, diff_for, gitea_file, pr_ref
|
|
from werkrevisor import doccheck
|
|
from werkrevisor.cli import execute
|
|
from werkrevisor.config import ConfigError
|
|
from werkrevisor.redact import Redactor
|
|
from werkrevisor.verdict import Verdict
|
|
|
|
APP = "src/main/java/App.java"
|
|
APPTEST = "src/test/java/AppTest.java"
|
|
DOC = "doc/PR/2026-10-08-PR#7-feature.md"
|
|
DOC_TEXT = """# PR#7: a feature
|
|
|
|
## The Problem
|
|
|
|
Something is missing.
|
|
|
|
## The Solution
|
|
|
|
The PR adds the endpoint and its tests.
|
|
"""
|
|
# The first changed line of APP's diff, in the new version.
|
|
APP_LINE = 1
|
|
|
|
|
|
def doc_gitea(app_lines=("class App {}",), extra=()):
|
|
files = [gitea_file(APP), gitea_file(DOC, status="added")] + [gitea_file(path) for path, _ in extra]
|
|
diff = diff_for(APP, added=list(app_lines)) + diff_for(DOC, added=DOC_TEXT.splitlines(), new=True)
|
|
for path, lines in extra:
|
|
diff += diff_for(path, added=list(lines))
|
|
gitea = FakeGitea(files=files, diff=diff)
|
|
gitea.raw_files[DOC] = DOC_TEXT.encode()
|
|
return gitea
|
|
|
|
|
|
def check_config(**doc_check):
|
|
# Jev judges every changed file, so a GREEN needs no model review.
|
|
return config(engine={"mode": "classifier"}, gate={"yellow_gate": "block"},
|
|
hard_yellow={"paths": [], "judged_paths": []}, path_judge={"judge": "jev", "scope": "all"},
|
|
doc_check=doc_check)
|
|
|
|
|
|
def check_run(jev=None, gitea=None, cfg=None):
|
|
gitea = gitea or doc_gitea()
|
|
d = deps(gitea, FakeModel(prompt_files=[APP, DOC]), None, jev or FakeJev())
|
|
outcome = execute(cfg or check_config(), pr_ref(), d, Redactor())
|
|
return outcome, d, gitea
|
|
|
|
|
|
def reason(outcome, code):
|
|
return next(r for r in outcome.result.reasons if r.code == code)
|
|
|
|
|
|
def doc_requests(jev):
|
|
"""The Jev calls of the document check: their state holds a changed file and the PR document."""
|
|
return [(s, q) for s, q in jev.requests if isinstance(s, dict) and "changed_file" in s]
|
|
|
|
|
|
def resolve_all(gitea):
|
|
for comments in gitea.line_comments.values():
|
|
for c in comments:
|
|
c["resolver"] = {"login": "claude-code"}
|
|
|
|
|
|
def second_round(gitea, jev):
|
|
"""A new commit: the earlier result comment belongs to the old head, the line comments stay."""
|
|
gitea.comments = []
|
|
return check_run(jev, gitea=gitea)
|
|
|
|
|
|
class DocCheckTest(unittest.TestCase):
|
|
def test_a_conflicting_test_claim_gets_orange_at_the_file(self):
|
|
"""Scenario#117.01"""
|
|
jev = FakeJev(problems={(APP, "doc_test_claim_conflict"): 0.9})
|
|
outcome, _, gitea = check_run(jev)
|
|
self.assertIs(outcome.result.verdict, Verdict.ORANGE)
|
|
self.assertIs(reason(outcome, "doc_contradiction").verdict, Verdict.ORANGE)
|
|
[comment] = gitea.review_comments[0]
|
|
self.assertEqual((comment["path"], comment["new_position"]), (APP, APP_LINE))
|
|
self.assertIn("Test claim contradicts the file", comment["body"])
|
|
self.assertIn("File fingerprint", comment["body"])
|
|
|
|
def test_an_unrelated_file_gets_a_hint(self):
|
|
"""Scenario#117.02"""
|
|
jev = FakeJev(problems={(APP, "doc_change_unrelated"): 0.8})
|
|
outcome, _, gitea = check_run(jev)
|
|
self.assertIs(outcome.result.verdict, Verdict.GREEN)
|
|
self.assertEqual(reason(outcome, "doc_change_unrelated").text,
|
|
f"the PR document may not require the changes of these files according to Jev: {APP}")
|
|
[comment] = gitea.review_comments[0]
|
|
self.assertEqual((comment["path"], comment["new_position"]), (APP, APP_LINE))
|
|
self.assertIn("Change not required by the PR document", comment["body"])
|
|
|
|
def test_a_data_outflow_gets_a_hint(self):
|
|
"""Scenario#117.03"""
|
|
jev = FakeJev(problems={(APP, "doc_data_outflow"): 0.8})
|
|
outcome, _, gitea = check_run(jev)
|
|
self.assertIs(outcome.result.verdict, Verdict.GREEN)
|
|
self.assertEqual(reason(outcome, "doc_data_outflow").text,
|
|
f"these files may send personal or customer data outside according to Jev: {APP}")
|
|
[comment] = gitea.review_comments[0]
|
|
self.assertEqual((comment["path"], comment["new_position"]), (APP, APP_LINE))
|
|
self.assertIn("Possible data outflow", comment["body"])
|
|
self.assertIn("possible data outflow 0.80", comment["body"])
|
|
|
|
def test_jev_gets_each_file_with_the_pr_document_and_all_questions(self):
|
|
"""Scenario#117.04"""
|
|
jev = FakeJev()
|
|
check_run(jev, gitea=doc_gitea(extra=[(APPTEST, ["assertTrue(true);"])]))
|
|
states = [s for s, _ in doc_requests(jev)]
|
|
self.assertEqual(sorted(s["changed_file"] for s in states), [APP, APPTEST])
|
|
for state in states:
|
|
self.assertEqual(state["pr_document"], DOC_TEXT)
|
|
self.assertEqual(state["conventions"], doccheck.CONVENTIONS)
|
|
self.assertIn("class App {}" if state["changed_file"] == APP else "assertTrue(true);", state["diff"])
|
|
[(_, questions)] = doc_requests(jev)[:1]
|
|
self.assertEqual(sorted(questions), ["doc_change_unrelated", "doc_claimed_not_done", "doc_data_outflow",
|
|
"doc_requirement_conflict", "doc_test_claim_conflict",
|
|
"doc_wrong_description"])
|
|
|
|
def test_a_hint_on_a_markdown_file_holds_the_merge(self):
|
|
"""A doc-check hint on a documentation file is a line comment, not a free hint, so it holds the merge."""
|
|
md = "docs/guide.md"
|
|
gitea = doc_gitea()
|
|
gitea.files.append(gitea_file(md))
|
|
gitea.diff += diff_for(md, added=["new guidance"])
|
|
jev = FakeJev(problems={(md, "doc_change_unrelated"): 0.9})
|
|
outcome, _, gitea = check_run(jev, gitea=gitea)
|
|
md_comments = [c for cs in gitea.review_comments for c in cs if c["path"] == md]
|
|
self.assertEqual(len(md_comments), 1)
|
|
self.assertIn("Change not required by the PR document", md_comments[0]["body"])
|
|
self.assertIn("doc_change_unrelated", [r.code for r in outcome.result.reasons])
|
|
|
|
def test_a_mechanical_file_is_not_checked(self):
|
|
"""Scenario#117.05"""
|
|
jev = FakeJev()
|
|
imports = "src/main/java/Imports.java"
|
|
check_run(jev, gitea=doc_gitea(extra=[(imports, ["import java.util.List;"])]))
|
|
self.assertEqual([s["changed_file"] for s, _ in doc_requests(jev)], [APP])
|
|
|
|
def test_a_low_value_gives_no_comment(self):
|
|
jev = FakeJev(problems={(APP, "doc_test_claim_conflict"): 0.5})
|
|
outcome, _, gitea = check_run(jev)
|
|
self.assertIs(outcome.result.verdict, Verdict.GREEN)
|
|
self.assertEqual(reason(outcome, "doc_check_passed").text,
|
|
"1 changed file(s) agree with the PR document according to Jev")
|
|
self.assertFalse(any(gitea.review_comments))
|
|
self.assertEqual(len(doc_requests(jev)), 1)
|
|
|
|
def test_a_contradiction_below_hint_min_is_no_hint(self):
|
|
jev = FakeJev(problems={(APP, "doc_wrong_description"): 0.65})
|
|
outcome, _, _ = check_run(jev)
|
|
self.assertNotIn("doc_contradiction_hint", [r.code for r in outcome.result.reasons])
|
|
|
|
def test_another_contradiction_is_a_hint(self):
|
|
jev = FakeJev(problems={(APP, "doc_wrong_description"): 0.75})
|
|
outcome, _, gitea = check_run(jev)
|
|
self.assertIs(outcome.result.verdict, Verdict.GREEN)
|
|
self.assertIs(reason(outcome, "doc_contradiction_hint").verdict, Verdict.GREEN)
|
|
[comment] = gitea.review_comments[0]
|
|
self.assertIn("Description contradicts the file", comment["body"])
|
|
|
|
def test_a_secret_keeps_the_diff_from_jev(self):
|
|
"""Scenario#47.06"""
|
|
jev = FakeJev()
|
|
outcome, _, _ = check_run(jev, gitea=doc_gitea(["-----BEGIN " + "PRIVATE KEY-----"]))
|
|
self.assertEqual(doc_requests(jev), [])
|
|
self.assertIn("possible secrets", reason(outcome, "doc_check_skipped").text)
|
|
|
|
def test_the_usage_record_logs_the_values_per_file(self):
|
|
"""Scenario#117.07"""
|
|
jev = FakeJev(problems={(APP, "doc_wrong_description"): 0.3})
|
|
_, d, _ = check_run(jev)
|
|
record = json.loads(d.out.getvalue().strip())
|
|
self.assertEqual((record["doc_check_model"], record["doc_check_calls"]), ("jev-test", 1))
|
|
self.assertEqual(record["doc_check_input_tokens"], 1000)
|
|
[row] = record["doc_check_judgments"]
|
|
self.assertEqual((row["file"], row["line"]), (APP, APP_LINE))
|
|
self.assertAlmostEqual(row["values"]["doc_wrong_description"], 0.3)
|
|
self.assertIn("fingerprint", row)
|
|
|
|
|
|
class DeletionTest(unittest.TestCase):
|
|
def test_a_deleted_file_is_checked_and_anchored_at_the_pr_document(self):
|
|
"""A deleted file has no new-version line, so its hint joins one comment at the PR document."""
|
|
gitea = doc_gitea()
|
|
gitea.files.append(gitea_file("src/main/java/Gone.java", status="removed", additions=0, deletions=1))
|
|
gitea.diff += diff_for("src/main/java/Gone.java", removed=["class Gone {}"], deleted=True)
|
|
jev = FakeJev(problems={("src/main/java/Gone.java", "doc_change_unrelated"): 0.8})
|
|
outcome, _, gitea = check_run(jev, gitea=gitea)
|
|
[comment] = gitea.review_comments[0]
|
|
self.assertEqual((comment["path"], comment["new_position"]), (DOC, 1))
|
|
self.assertIn("Deleted or deletion-only file", comment["body"])
|
|
self.assertIn("src/main/java/Gone.java", comment["body"])
|
|
self.assertEqual(reason(outcome, "doc_change_unrelated").text,
|
|
"the PR document may not require the changes of these files according to Jev: "
|
|
"src/main/java/Gone.java")
|
|
|
|
def test_a_deletion_only_file_is_anchored_at_the_pr_document(self):
|
|
"""A modified file whose only change removes lines gets no comment beyond the end of the file."""
|
|
gitea = doc_gitea()
|
|
gitea.files.append(gitea_file("src/main/java/Trim.java", status="modified", additions=0, deletions=2))
|
|
gitea.diff += diff_for("src/main/java/Trim.java", removed=["int a;", "int b;"])
|
|
jev = FakeJev(problems={("src/main/java/Trim.java", "doc_data_outflow"): 0.8})
|
|
_, _, gitea = check_run(jev, gitea=gitea)
|
|
[comment] = gitea.review_comments[0]
|
|
self.assertEqual((comment["path"], comment["new_position"]), (DOC, 1))
|
|
self.assertIn("src/main/java/Trim.java", comment["body"])
|
|
|
|
|
|
class DocCheckScopeTest(unittest.TestCase):
|
|
def test_a_pr_without_a_changed_pr_document_is_not_compared(self):
|
|
jev = FakeJev()
|
|
gitea = FakeGitea(files=[gitea_file(APP)], diff=diff_for(APP, added=["x"]))
|
|
outcome, _, _ = check_run(jev, gitea=gitea)
|
|
self.assertEqual(doc_requests(jev), [])
|
|
self.assertFalse(any(r.code.startswith("doc_check") for r in outcome.result.reasons))
|
|
|
|
def test_a_pr_without_a_non_doc_file_is_not_compared(self):
|
|
jev = FakeJev()
|
|
gitea = FakeGitea(files=[gitea_file(DOC, status="added")],
|
|
diff=diff_for(DOC, added=DOC_TEXT.splitlines(), new=True))
|
|
gitea.raw_files[DOC] = DOC_TEXT.encode()
|
|
outcome, _, _ = check_run(jev, gitea=gitea)
|
|
self.assertEqual(doc_requests(jev), [])
|
|
self.assertFalse(any(r.code.startswith("doc_check") for r in outcome.result.reasons))
|
|
|
|
|
|
class DocCheckConfigTest(unittest.TestCase):
|
|
def test_values_are_validated(self):
|
|
for values, message in (({"orange_min": 1.5}, r"\[doc_check\] orange_min must be a number between 0 and 1"),
|
|
({"hint_min": 0.8}, r"\[doc_check\] hint_min must not be above orange_min"),
|
|
({"unrelated_min": 1.5},
|
|
r"\[doc_check\] unrelated_min must be a number between 0 and 1"),
|
|
({"exfil_min": 1.5}, r"\[doc_check\] exfil_min must be a number between 0 and 1")):
|
|
with self.subTest(values=values):
|
|
with self.assertRaisesRegex(ConfigError, message):
|
|
check_config(**values)
|
|
|
|
def test_the_check_needs_jev(self):
|
|
with self.assertRaisesRegex(ConfigError, r"\[doc_check\] needs \[engine\] mode 'classifier' or 'combined'"):
|
|
config(engine={"mode": "llm"}, doc_check={})
|
|
|
|
|
|
class AnsweredHintTest(unittest.TestCase):
|
|
def setUp(self):
|
|
self.gitea = doc_gitea()
|
|
self.jev = FakeJev(problems={(APP, "doc_wrong_description"): 0.75})
|
|
check_run(self.jev, gitea=self.gitea)
|
|
[comment] = self.gitea.review_comments[0]
|
|
self.assertIn("File fingerprint", comment["body"])
|
|
resolve_all(self.gitea)
|
|
|
|
def test_a_resolved_hint_is_not_repeated(self):
|
|
"""Scenario#117.06"""
|
|
outcome, _, _ = second_round(self.gitea, self.jev)
|
|
self.assertFalse(any(self.gitea.review_comments[1:]))
|
|
self.assertEqual(reason(outcome, "doc_check_answered").text,
|
|
"1 hint(s) of the PR document check not repeated: "
|
|
"the author resolved them, and the files and the PR document are unchanged")
|
|
self.assertNotIn("doc_contradiction_hint", [r.code for r in outcome.result.reasons])
|
|
self.assertIs(outcome.result.verdict, Verdict.GREEN)
|
|
|
|
def test_a_changed_file_is_checked_again(self):
|
|
gitea = doc_gitea(["class App { int x; }"])
|
|
outcome, _, _ = check_run(self.jev, gitea=gitea)
|
|
self.assertIn("doc_contradiction_hint", [r.code for r in outcome.result.reasons])
|
|
|
|
def test_an_orange_contradiction_comes_back(self):
|
|
jev = FakeJev(problems={(APP, "doc_test_claim_conflict"): 0.9})
|
|
gitea = doc_gitea()
|
|
check_run(jev, gitea=gitea)
|
|
resolve_all(gitea)
|
|
outcome, _, _ = second_round(gitea, jev)
|
|
self.assertIs(outcome.result.verdict, Verdict.ORANGE)
|
|
self.assertEqual(len(gitea.review_comments[-1]), 1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|