import json import unittest from tests.fakes import FakeGitea, FakeJev, FakeModel, config, deps, diff_for, gitea_file, pr_ref from werkrevisor import doccheck from werkrevisor.cli import execute from werkrevisor.config import ConfigError from werkrevisor.redact import Redactor from werkrevisor.verdict import Verdict APP = "src/main/java/App.java" APPTEST = "src/test/java/AppTest.java" DOC = "doc/PR/2026-10-08-PR#7-feature.md" DOC_TEXT = """# PR#7: a feature ## The Problem Something is missing. ## The Solution The PR adds the endpoint and its tests. """ # The first changed line of APP's diff, in the new version. APP_LINE = 1 def doc_gitea(app_lines=("class App {}",), extra=()): files = [gitea_file(APP), gitea_file(DOC, status="added")] + [gitea_file(path) for path, _ in extra] diff = diff_for(APP, added=list(app_lines)) + diff_for(DOC, added=DOC_TEXT.splitlines(), new=True) for path, lines in extra: diff += diff_for(path, added=list(lines)) gitea = FakeGitea(files=files, diff=diff) gitea.raw_files[DOC] = DOC_TEXT.encode() return gitea def check_config(**doc_check): # Jev judges every changed file, so a GREEN needs no model review. return config(engine={"mode": "classifier"}, gate={"yellow_gate": "block"}, hard_yellow={"paths": [], "judged_paths": []}, path_judge={"judge": "jev", "scope": "all"}, doc_check=doc_check) def check_run(jev=None, gitea=None, cfg=None): gitea = gitea or doc_gitea() d = deps(gitea, FakeModel(prompt_files=[APP, DOC]), None, jev or FakeJev()) outcome = execute(cfg or check_config(), pr_ref(), d, Redactor()) return outcome, d, gitea def reason(outcome, code): return next(r for r in outcome.result.reasons if r.code == code) def doc_requests(jev): """The Jev calls of the document check: their state holds a changed file and the PR document.""" return [(s, q) for s, q in jev.requests if isinstance(s, dict) and "changed_file" in s] def resolve_all(gitea): for comments in gitea.line_comments.values(): for c in comments: c["resolver"] = {"login": "claude-code"} def second_round(gitea, jev): """A new commit: the earlier result comment belongs to the old head, the line comments stay.""" gitea.comments = [] return check_run(jev, gitea=gitea) class DocCheckTest(unittest.TestCase): def test_a_conflicting_test_claim_gets_orange_at_the_file(self): """Scenario#117.01""" jev = FakeJev(problems={(APP, "doc_test_claim_conflict"): 0.9}) outcome, _, gitea = check_run(jev) self.assertIs(outcome.result.verdict, Verdict.ORANGE) self.assertIs(reason(outcome, "doc_contradiction").verdict, Verdict.ORANGE) [comment] = gitea.review_comments[0] self.assertEqual((comment["path"], comment["new_position"]), (APP, APP_LINE)) self.assertIn("Test claim contradicts the file", comment["body"]) self.assertIn("File fingerprint", comment["body"]) def test_an_unrelated_file_gets_a_hint(self): """Scenario#117.02""" jev = FakeJev(problems={(APP, "doc_change_unrelated"): 0.8}) outcome, _, gitea = check_run(jev) self.assertIs(outcome.result.verdict, Verdict.GREEN) self.assertEqual(reason(outcome, "doc_change_unrelated").text, f"the PR document may not require the changes of these files according to Jev: {APP}") [comment] = gitea.review_comments[0] self.assertEqual((comment["path"], comment["new_position"]), (APP, APP_LINE)) self.assertIn("Change not required by the PR document", comment["body"]) def test_a_data_outflow_gets_a_hint(self): """Scenario#117.03""" jev = FakeJev(problems={(APP, "doc_data_outflow"): 0.8}) outcome, _, gitea = check_run(jev) self.assertIs(outcome.result.verdict, Verdict.GREEN) self.assertEqual(reason(outcome, "doc_data_outflow").text, f"these files may send personal or customer data outside according to Jev: {APP}") [comment] = gitea.review_comments[0] self.assertEqual((comment["path"], comment["new_position"]), (APP, APP_LINE)) self.assertIn("Possible data outflow", comment["body"]) self.assertIn("possible data outflow 0.80", comment["body"]) def test_jev_gets_each_file_with_the_pr_document_and_all_questions(self): """Scenario#117.04""" jev = FakeJev() check_run(jev, gitea=doc_gitea(extra=[(APPTEST, ["assertTrue(true);"])])) states = [s for s, _ in doc_requests(jev)] self.assertEqual(sorted(s["changed_file"] for s in states), [APP, APPTEST]) for state in states: self.assertEqual(state["pr_document"], DOC_TEXT) self.assertEqual(state["conventions"], doccheck.CONVENTIONS) self.assertIn("class App {}" if state["changed_file"] == APP else "assertTrue(true);", state["diff"]) [(_, questions)] = doc_requests(jev)[:1] self.assertEqual(sorted(questions), ["doc_change_unrelated", "doc_claimed_not_done", "doc_data_outflow", "doc_requirement_conflict", "doc_test_claim_conflict", "doc_wrong_description"]) def test_a_hint_on_a_markdown_file_holds_the_merge(self): """A doc-check hint on a documentation file is a line comment, not a free hint, so it holds the merge.""" md = "docs/guide.md" gitea = doc_gitea() gitea.files.append(gitea_file(md)) gitea.diff += diff_for(md, added=["new guidance"]) jev = FakeJev(problems={(md, "doc_change_unrelated"): 0.9}) outcome, _, gitea = check_run(jev, gitea=gitea) md_comments = [c for cs in gitea.review_comments for c in cs if c["path"] == md] self.assertEqual(len(md_comments), 1) self.assertIn("Change not required by the PR document", md_comments[0]["body"]) self.assertIn("doc_change_unrelated", [r.code for r in outcome.result.reasons]) def test_a_mechanical_file_is_not_checked(self): """Scenario#117.05""" jev = FakeJev() imports = "src/main/java/Imports.java" check_run(jev, gitea=doc_gitea(extra=[(imports, ["import java.util.List;"])])) self.assertEqual([s["changed_file"] for s, _ in doc_requests(jev)], [APP]) def test_a_low_value_gives_no_comment(self): jev = FakeJev(problems={(APP, "doc_test_claim_conflict"): 0.5}) outcome, _, gitea = check_run(jev) self.assertIs(outcome.result.verdict, Verdict.GREEN) self.assertEqual(reason(outcome, "doc_check_passed").text, "1 changed file(s) agree with the PR document according to Jev") self.assertFalse(any(gitea.review_comments)) self.assertEqual(len(doc_requests(jev)), 1) def test_a_contradiction_below_hint_min_is_no_hint(self): jev = FakeJev(problems={(APP, "doc_wrong_description"): 0.65}) outcome, _, _ = check_run(jev) self.assertNotIn("doc_contradiction_hint", [r.code for r in outcome.result.reasons]) def test_another_contradiction_is_a_hint(self): jev = FakeJev(problems={(APP, "doc_wrong_description"): 0.75}) outcome, _, gitea = check_run(jev) self.assertIs(outcome.result.verdict, Verdict.GREEN) self.assertIs(reason(outcome, "doc_contradiction_hint").verdict, Verdict.GREEN) [comment] = gitea.review_comments[0] self.assertIn("Description contradicts the file", comment["body"]) def test_a_secret_keeps_the_diff_from_jev(self): """Scenario#47.06""" jev = FakeJev() outcome, _, _ = check_run(jev, gitea=doc_gitea(["-----BEGIN " + "PRIVATE KEY-----"])) self.assertEqual(doc_requests(jev), []) self.assertIn("possible secrets", reason(outcome, "doc_check_skipped").text) def test_the_usage_record_logs_the_values_per_file(self): """Scenario#117.07""" jev = FakeJev(problems={(APP, "doc_wrong_description"): 0.3}) _, d, _ = check_run(jev) record = json.loads(d.out.getvalue().strip()) self.assertEqual((record["doc_check_model"], record["doc_check_calls"]), ("jev-test", 1)) self.assertEqual(record["doc_check_input_tokens"], 1000) [row] = record["doc_check_judgments"] self.assertEqual((row["file"], row["line"]), (APP, APP_LINE)) self.assertAlmostEqual(row["values"]["doc_wrong_description"], 0.3) self.assertIn("fingerprint", row) class DeletionTest(unittest.TestCase): def test_a_deleted_file_is_checked_and_anchored_at_the_pr_document(self): """A deleted file has no new-version line, so its hint joins one comment at the PR document.""" gitea = doc_gitea() gitea.files.append(gitea_file("src/main/java/Gone.java", status="removed", additions=0, deletions=1)) gitea.diff += diff_for("src/main/java/Gone.java", removed=["class Gone {}"], deleted=True) jev = FakeJev(problems={("src/main/java/Gone.java", "doc_change_unrelated"): 0.8}) outcome, _, gitea = check_run(jev, gitea=gitea) [comment] = gitea.review_comments[0] self.assertEqual((comment["path"], comment["new_position"]), (DOC, 1)) self.assertIn("Deleted or deletion-only file", comment["body"]) self.assertIn("src/main/java/Gone.java", comment["body"]) self.assertEqual(reason(outcome, "doc_change_unrelated").text, "the PR document may not require the changes of these files according to Jev: " "src/main/java/Gone.java") def test_a_deletion_only_file_is_anchored_at_the_pr_document(self): """A modified file whose only change removes lines gets no comment beyond the end of the file.""" gitea = doc_gitea() gitea.files.append(gitea_file("src/main/java/Trim.java", status="modified", additions=0, deletions=2)) gitea.diff += diff_for("src/main/java/Trim.java", removed=["int a;", "int b;"]) jev = FakeJev(problems={("src/main/java/Trim.java", "doc_data_outflow"): 0.8}) _, _, gitea = check_run(jev, gitea=gitea) [comment] = gitea.review_comments[0] self.assertEqual((comment["path"], comment["new_position"]), (DOC, 1)) self.assertIn("src/main/java/Trim.java", comment["body"]) class DocCheckScopeTest(unittest.TestCase): def test_a_pr_without_a_changed_pr_document_is_not_compared(self): jev = FakeJev() gitea = FakeGitea(files=[gitea_file(APP)], diff=diff_for(APP, added=["x"])) outcome, _, _ = check_run(jev, gitea=gitea) self.assertEqual(doc_requests(jev), []) self.assertFalse(any(r.code.startswith("doc_check") for r in outcome.result.reasons)) def test_a_pr_without_a_non_doc_file_is_not_compared(self): jev = FakeJev() gitea = FakeGitea(files=[gitea_file(DOC, status="added")], diff=diff_for(DOC, added=DOC_TEXT.splitlines(), new=True)) gitea.raw_files[DOC] = DOC_TEXT.encode() outcome, _, _ = check_run(jev, gitea=gitea) self.assertEqual(doc_requests(jev), []) self.assertFalse(any(r.code.startswith("doc_check") for r in outcome.result.reasons)) class DocCheckConfigTest(unittest.TestCase): def test_values_are_validated(self): for values, message in (({"orange_min": 1.5}, r"\[doc_check\] orange_min must be a number between 0 and 1"), ({"hint_min": 0.8}, r"\[doc_check\] hint_min must not be above orange_min"), ({"unrelated_min": 1.5}, r"\[doc_check\] unrelated_min must be a number between 0 and 1"), ({"exfil_min": 1.5}, r"\[doc_check\] exfil_min must be a number between 0 and 1")): with self.subTest(values=values): with self.assertRaisesRegex(ConfigError, message): check_config(**values) def test_the_check_needs_jev(self): with self.assertRaisesRegex(ConfigError, r"\[doc_check\] needs \[engine\] mode 'classifier' or 'combined'"): config(engine={"mode": "llm"}, doc_check={}) class AnsweredHintTest(unittest.TestCase): def setUp(self): self.gitea = doc_gitea() self.jev = FakeJev(problems={(APP, "doc_wrong_description"): 0.75}) check_run(self.jev, gitea=self.gitea) [comment] = self.gitea.review_comments[0] self.assertIn("File fingerprint", comment["body"]) resolve_all(self.gitea) def test_a_resolved_hint_is_not_repeated(self): """Scenario#117.06""" outcome, _, _ = second_round(self.gitea, self.jev) self.assertFalse(any(self.gitea.review_comments[1:])) self.assertEqual(reason(outcome, "doc_check_answered").text, "1 hint(s) of the PR document check not repeated: " "the author resolved them, and the files and the PR document are unchanged") self.assertNotIn("doc_contradiction_hint", [r.code for r in outcome.result.reasons]) self.assertIs(outcome.result.verdict, Verdict.GREEN) def test_a_changed_file_is_checked_again(self): gitea = doc_gitea(["class App { int x; }"]) outcome, _, _ = check_run(self.jev, gitea=gitea) self.assertIn("doc_contradiction_hint", [r.code for r in outcome.result.reasons]) def test_an_orange_contradiction_comes_back(self): jev = FakeJev(problems={(APP, "doc_test_claim_conflict"): 0.9}) gitea = doc_gitea() check_run(jev, gitea=gitea) resolve_all(gitea) outcome, _, _ = second_round(gitea, jev) self.assertIs(outcome.result.verdict, Verdict.ORANGE) self.assertEqual(len(gitea.review_comments[-1]), 1) if __name__ == "__main__": unittest.main()