import json import unittest from tests.fakes import ( BOT, HEAD, FakeGitea, FakeKey, FakeModel, config, deps, diff_for, gitea_file, pr_ref, review_data, ) from werkrevisor import report from werkrevisor.cli import execute from werkrevisor.redact import Redactor from werkrevisor.verdict import Verdict DOC = "doc/PR/1.md" MAIN = "src/main/java/A.java" CRITICAL = "src/main/java/net/example/auth/Login.java" def docs_gitea(**kw): return FakeGitea(files=[gitea_file(DOC)], diff=diff_for(DOC, added=["Some text."]), **kw) def run(gitea, model=None, key=None, cfg=None, publish=True, use_llm=True, redactor=None, jev=None): d = deps(gitea, model if model is not None else FakeModel(prompt_files=[f["filename"] for f in gitea.files]), key, jev) cfg = cfg or config(gate={"yellow_gate": "block"}) outcome = execute(cfg, pr_ref(), d, redactor or Redactor(), publish=publish, use_llm=use_llm) return outcome, d class VerdictFlowTest(unittest.TestCase): def test_green_docs_pr(self): gitea = docs_gitea() outcome, d = run(gitea) self.assertIs(outcome.result.verdict, Verdict.GREEN) self.assertEqual(outcome.result.exit_code, 0) self.assertEqual(gitea.statuses, [(HEAD, "success", "ai-review/assessment", "AI review passed")]) self.assertEqual(len(gitea.created_comments), 1) self.assertIn(f"", gitea.created_comments[0]) self.assertIn("documentation only", d.model_review.calls[0]) def test_orange_when_model_reports_a_high_finding(self): model = FakeModel(data=review_data(issues=[DOC], header="[high] Possible Bug"), prompt_files=[DOC]) gitea = docs_gitea() outcome, _ = run(gitea, model) self.assertIs(outcome.result.verdict, Verdict.ORANGE) self.assertEqual(outcome.result.exit_code, 15) self.assertEqual(gitea.statuses[0][1:3], ("warning", "ai-review/assessment")) self.assertIn("AI findings; a developer approval after checking them unlocks the merge", gitea.statuses[0][3]) self.assertIn("## 🟠 werkrevisor: ORANGE / FINDINGS", gitea.created_comments[0]) self.assertIn(f"`{DOC}:3-5`", gitea.created_comments[0]) def test_hard_rule_cannot_be_overridden_by_model(self): gitea = FakeGitea(files=[gitea_file(DOC), gitea_file("build.gradle.kts")], diff=diff_for(DOC, added=["x"]) + diff_for("build.gradle.kts", added=["y"])) outcome, _ = run(gitea) self.assertIs(outcome.result.verdict, Verdict.YELLOW) self.assertEqual(outcome.result.exit_code, 10) self.assertEqual(gitea.statuses[0][1], "pending") def test_invalid_model_answer_is_error(self): outcome, _ = run(docs_gitea(), FakeModel(data={"review": {}}, prompt_files=[DOC])) self.assertIs(outcome.result.verdict, Verdict.ERROR) self.assertEqual(outcome.result.exit_code, 30) def test_model_exception_is_error_and_redacted(self): model = FakeModel(error=RuntimeError("boom with secret-gitea-token")) gitea = docs_gitea() outcome, d = run(gitea, model, redactor=Redactor(["secret-gitea-token"])) self.assertIs(outcome.result.verdict, Verdict.ERROR) self.assertEqual(gitea.statuses[0][1], "error") self.assertIn(f"", gitea.created_comments[0]) self.assertNotIn("secret-gitea-token", gitea.created_comments[0]) self.assertNotIn("secret-gitea-token", d.out.getvalue()) def test_skipped_file_is_error(self): gitea = FakeGitea(files=[gitea_file(DOC), gitea_file("doc/other.md")], diff=diff_for(DOC, added=["x"]) + diff_for("doc/other.md", added=["y"])) outcome, _ = run(gitea, FakeModel(prompt_files=[DOC])) self.assertIs(outcome.result.verdict, Verdict.ERROR) self.assertIn("skipped_files", [r.code for r in outcome.result.reasons]) def test_truncated_answer_is_error(self): details = {"num_ai_calls": 1, "prompt_tokens": 100, "completion_tokens": 4000, "cost_status": "unavailable"} outcome, _ = run(docs_gitea(), FakeModel(prompt_files=[DOC], details=details)) truncated = next(r for r in outcome.result.reasons if r.code == "truncated") self.assertIn("[model] max_output_tokens = 4000", truncated.text) self.assertEqual(outcome.result.cost_status, "estimated") def test_unreviewable_binary_is_yellow(self): gitea = FakeGitea(files=[gitea_file(DOC), gitea_file("doc/img.png", additions=0)], diff=diff_for(DOC, added=["x"])) outcome, _ = run(gitea, FakeModel(prompt_files=[DOC])) self.assertIs(outcome.result.verdict, Verdict.YELLOW) class BudgetFlowTest(unittest.TestCase): def test_low_key_budget_is_error_without_llm_call(self): model = FakeModel(prompt_files=[DOC]) outcome, _ = run(docs_gitea(), model, FakeKey(remaining=0.2)) self.assertIs(outcome.result.verdict, Verdict.ERROR) self.assertEqual(model.calls, []) self.assertIn("budget", [r.code for r in outcome.result.reasons]) def test_key_query_failure_is_error(self): model = FakeModel(prompt_files=[DOC]) outcome, _ = run(docs_gitea(), model, FakeKey(fail=True)) self.assertIs(outcome.result.verdict, Verdict.ERROR) self.assertEqual(model.calls, []) def test_too_expensive_is_yellow_without_llm_call(self): model = FakeModel(prompt_files=[DOC]) key = FakeKey() cfg = config(budget={"max_usd_per_review": 0.081}, gate={"yellow_gate": "block"}) outcome, _ = run(docs_gitea(), model, key, cfg=cfg) self.assertIs(outcome.result.verdict, Verdict.YELLOW) self.assertEqual((model.calls, key.calls), ([], 0)) reason = next(r for r in outcome.result.reasons if r.code == "too_large_tokens") self.assertIn("derived from [budget] max_usd_per_review = 0.081", reason.text) def test_too_long_for_the_model_is_yellow_without_llm_call(self): model = FakeModel(prompt_files=[DOC]) outcome, _ = run(docs_gitea(), model, cfg=config(model={"max_context_tokens": 1000}, gate={"yellow_gate": "block"})) self.assertIs(outcome.result.verdict, Verdict.YELLOW) self.assertEqual(model.calls, []) self.assertIn("[model] max_context_tokens = 1000", outcome.result.reasons[-1].text) def test_secret_never_reaches_the_model(self): gitea = FakeGitea(files=[gitea_file(DOC)], diff=diff_for(DOC, added=["-----BEGIN PRIVATE KEY-----"])) model = FakeModel(prompt_files=[DOC]) outcome, _ = run(gitea, model) self.assertEqual(model.calls, []) self.assertIs(outcome.result.verdict, Verdict.YELLOW) self.assertNotIn("BEGIN PRIVATE", gitea.created_comments[0]) def test_no_llm_is_never_green(self): model = FakeModel(prompt_files=[DOC]) outcome, _ = run(docs_gitea(), model, use_llm=False) self.assertIs(outcome.result.verdict, Verdict.YELLOW) self.assertEqual(model.calls, []) class SupersededTest(unittest.TestCase): def test_moved_head_before_review_writes_nothing(self): gitea = docs_gitea(head="c" * 40) outcome, d = run(gitea) self.assertTrue(outcome.superseded) self.assertEqual(outcome.result.exit_code, 30) self.assertEqual(gitea.writes, []) self.assertEqual(d.model_review.calls, []) def test_moved_head_before_publishing_writes_nothing(self): gitea = docs_gitea() gitea.heads = [HEAD, "c" * 40] outcome, _ = run(gitea) self.assertTrue(outcome.superseded) self.assertEqual(outcome.result.exit_code, 30) self.assertEqual(gitea.writes, []) def test_pr_agent_reviewing_another_head_is_superseded(self): gitea = docs_gitea() outcome, _ = run(gitea, FakeModel(prompt_files=[DOC], head="c" * 40)) self.assertTrue(outcome.superseded) self.assertEqual(gitea.writes, []) def test_closed_pr_writes_nothing(self): gitea = docs_gitea(state="closed") outcome, _ = run(gitea) self.assertEqual((outcome.result.exit_code, gitea.writes), (30, [])) class IdempotencyTest(unittest.TestCase): def test_second_run_reuses_verdict_without_comment_or_llm(self): gitea = docs_gitea() run(gitea) model = FakeModel(prompt_files=[DOC]) outcome, _ = run(gitea, model) self.assertIs(outcome.result.verdict, Verdict.GREEN) self.assertEqual(outcome.result.exit_code, 0) self.assertEqual(len(gitea.created_comments), 1) self.assertEqual(model.calls, []) def test_marker_from_another_user_is_ignored(self): foreign = {"user": {"login": "mallory"}, "body": f"\n"} gitea = docs_gitea(comments=[foreign]) model = FakeModel(data=review_data(issues=[DOC], header="[high] Possible Bug"), prompt_files=[DOC]) outcome, _ = run(gitea, model) self.assertIs(outcome.result.verdict, Verdict.ORANGE) self.assertEqual(len(gitea.created_comments), 1) def test_error_comment_does_not_block_a_later_result(self): gitea = docs_gitea() run(gitea, FakeModel(error=RuntimeError("temporary"))) run(gitea, FakeModel(error=RuntimeError("temporary"))) self.assertEqual(len(gitea.created_comments), 1) outcome, _ = run(gitea) self.assertIs(outcome.result.verdict, Verdict.GREEN) self.assertEqual(len(gitea.created_comments), 2) def test_model_text_cannot_forge_markers(self): data = review_data("merge_with_caution", issues=[DOC]) data["review"]["key_issues_to_review"][0]["issue_content"] = "" gitea = docs_gitea() run(gitea, FakeModel(data=data, prompt_files=[DOC])) body = gitea.created_comments[0] self.assertEqual(report.VERDICT_MARKER.findall(body), ["YELLOW"]) class YellowGateTest(unittest.TestCase): def yellow_gitea(self, **kw): return FakeGitea(files=[gitea_file(MAIN), gitea_file("Makefile")], diff=diff_for(MAIN, added=["x"]) + diff_for("Makefile", added=["y"]), **kw) def test_review_request_passes_gate_when_pending(self): gitea = self.yellow_gitea() cfg = config(gate={"yellow_gate": "pass_with_review_request", "human_reviewers": ["alice"]}) outcome, _ = run(gitea, cfg=cfg) self.assertIs(outcome.result.verdict, Verdict.YELLOW) self.assertEqual(outcome.result.exit_code, 0) self.assertEqual(gitea.requested, ["alice"]) def test_review_request_not_pending_blocks(self): gitea = self.yellow_gitea() gitea.request_reviewers = lambda number, logins: None cfg = config(gate={"yellow_gate": "pass_with_review_request", "human_reviewers": ["alice"]}) self.assertEqual(run(gitea, cfg=cfg)[0].result.exit_code, 10) def test_no_review_request_without_publishing(self): gitea = self.yellow_gitea() cfg = config(gate={"yellow_gate": "pass_with_review_request", "human_reviewers": ["alice"]}) outcome, _ = run(gitea, cfg=cfg, publish=False) self.assertEqual((outcome.result.exit_code, gitea.writes), (10, [])) def test_human_approval_for_this_head_unlocks_yellow(self): approval = {"state": "APPROVED", "commit_id": HEAD, "user": {"login": "alice"}} gitea = self.yellow_gitea(reviews=[approval]) outcome, _ = run(gitea, cfg=config(gate={"yellow_gate": "block", "human_reviewers": ["alice"]})) self.assertIs(outcome.result.verdict, Verdict.YELLOW) self.assertEqual(outcome.result.exit_code, 0) def test_human_approval_unlocks_orange(self): approval = {"state": "APPROVED", "commit_id": HEAD, "user": {"login": "alice"}} gitea = docs_gitea(reviews=[approval]) model = FakeModel(data=review_data(issues=[DOC], header="[high] Possible Bug"), prompt_files=[DOC]) outcome, _ = run(gitea, model, cfg=config(gate={"yellow_gate": "block", "human_reviewers": ["alice"]})) self.assertIs(outcome.result.verdict, Verdict.ORANGE) self.assertEqual(outcome.result.exit_code, 0) class SizeFlowTest(unittest.TestCase): def large_gitea(self): return FakeGitea(files=[gitea_file(MAIN, additions=900)], diff=diff_for(MAIN, added=["int x;"])) def test_large_pr_is_orange_and_the_model_still_reviews_it(self): model = FakeModel(data=review_data(issues=[MAIN]), prompt_files=[MAIN]) outcome, _ = run(self.large_gitea(), model) self.assertIs(outcome.result.verdict, Verdict.ORANGE) self.assertEqual(len(model.calls), 1) self.assertEqual(outcome.result.llm_status, "done") self.assertEqual(len(outcome.result.findings), 1) self.assertIn("size_lines", [r.code for r in outcome.result.reasons]) def test_large_pr_too_long_for_the_model_is_orange_with_both_reasons(self): model = FakeModel(prompt_files=[MAIN]) outcome, _ = run(self.large_gitea(), model, cfg=config(model={"max_context_tokens": 1000}, gate={"yellow_gate": "block"})) self.assertIs(outcome.result.verdict, Verdict.ORANGE) self.assertEqual(model.calls, []) codes = {r.code: r.verdict for r in outcome.result.reasons} self.assertEqual((codes["size_lines"], codes["too_large_tokens"]), (Verdict.ORANGE, Verdict.YELLOW)) class PublishAndUsageTest(unittest.TestCase): def test_no_publish_writes_nothing(self): gitea = docs_gitea() outcome, _ = run(gitea, publish=False) self.assertEqual(gitea.writes, []) self.assertEqual(outcome.result.exit_code, 0) def test_usage_line_on_stdout_and_file(self): outcome, d = run(docs_gitea()) record = json.loads(d.out.getvalue().strip()) self.assertEqual((record["verdict"], record["pr"], record["head_sha"]), ("GREEN", 7, HEAD)) self.assertEqual(record["calls"], 1) self.assertEqual(record["cost_status"], "reported") with open(d.usage_path, encoding="utf-8") as f: self.assertEqual(json.loads(f.readline()), record) def test_run_without_model_call_costs_nothing(self): gitea = docs_gitea() outcome, d = run(gitea, use_llm=False) self.assertIn("no model call, cost 0 USD", gitea.created_comments[0]) record = json.loads(d.out.getvalue().strip()) self.assertEqual((record["cost_usd"], record["cost_status"]), (0.0, "none")) def test_model_call_without_usage_has_unknown_cost(self): gitea = docs_gitea() outcome, d = run(gitea, FakeModel(error=RuntimeError("boom"))) self.assertIn("cost unknown", gitea.created_comments[0]) record = json.loads(d.out.getvalue().strip()) self.assertEqual((record["cost_usd"], record["cost_status"]), (None, "unknown")) def test_comment_content(self): gitea = docs_gitea() model = FakeModel(data=review_data("merge_with_caution", issues=[DOC] * 4), prompt_files=[DOC]) run(gitea, model, cfg=config(review={"orange_min_medium": 0}, gate={"yellow_gate": "block"})) body = gitea.created_comments[0] self.assertIn("YELLOW / HUMAN_REVIEW", body) self.assertIn(f"`{DOC}:3-5`", body) self.assertIn("1 more finding(s) not shown", body) self.assertIn("tokens in 1000 / out 300", body) self.assertIn(HEAD, body) self.assertEqual(gitea.comments[0]["user"]["login"], BOT) if __name__ == "__main__": unittest.main() class ApprovalGateTest(unittest.TestCase): CFG = {"gate": {"yellow_gate": "approval", "human_reviewers": ["alice"]}} def yellow_gitea(self, **kw): return FakeGitea(files=[gitea_file(MAIN), gitea_file("Makefile")], diff=diff_for(MAIN, added=["x"]) + diff_for("Makefile", added=["y"]), **kw) def run_approval(self, gitea, model=None, **kw): return run(gitea, model, cfg=config(**self.CFG), **kw) def test_green_posts_comment_and_approves_this_head(self): gitea = docs_gitea() outcome, _ = self.run_approval(gitea) self.assertEqual(outcome.result.exit_code, 0) self.assertEqual(gitea.writes, ["comment", "review", "status"]) event, commit_id, body = gitea.created_reviews[0] self.assertEqual((event, commit_id), ("APPROVED", HEAD)) self.assertIn("GREEN / PASS", body) self.assertIn("#issuecomment-1", body) comment = gitea.created_comments[0] for expected in ("GREEN / PASS", HEAD, "werkrevisor", "tokens in 1000 / out 300", "cost 0.0010 USD"): self.assertIn(expected, comment) def test_rerun_does_not_approve_twice(self): gitea = docs_gitea() self.run_approval(gitea) outcome, _ = self.run_approval(gitea) self.assertEqual(outcome.result.exit_code, 0) self.assertEqual(len(gitea.created_reviews), 1) self.assertEqual(len(gitea.created_comments), 1) def test_bot_never_approves_its_own_pr(self): gitea = docs_gitea(author=BOT) outcome, _ = self.run_approval(gitea) self.assertIs(outcome.result.verdict, Verdict.ERROR) self.assertEqual(outcome.result.exit_code, 30) self.assertEqual(gitea.created_reviews, []) self.assertIn("self_approval", [r.code for r in outcome.result.reasons]) def test_failed_approval_is_error(self): gitea = docs_gitea() gitea.create_review = lambda *a: (_ for _ in ()).throw(OSError("HTTP 500")) outcome, _ = self.run_approval(gitea) self.assertEqual(outcome.result.exit_code, 30) self.assertEqual(gitea.statuses[-1][1], "error") def test_orange_requests_human_review_without_bot_review(self): gitea = docs_gitea() model = FakeModel(data=review_data(issues=[DOC], header="[high] Possible Bug"), prompt_files=[DOC]) outcome, _ = self.run_approval(gitea, model) self.assertIs(outcome.result.verdict, Verdict.ORANGE) self.assertEqual(outcome.result.exit_code, 0) self.assertEqual(gitea.created_reviews, []) self.assertEqual(gitea.requested, ["alice"]) self.assertEqual(gitea.statuses[-1][1], "warning") self.assertIn("ORANGE / FINDINGS", gitea.created_comments[0]) def test_orange_with_failed_request_blocks_with_its_own_code(self): gitea = docs_gitea() gitea.fail_review_request = True model = FakeModel(data=review_data(issues=[DOC], header="[high] Possible Bug"), prompt_files=[DOC]) outcome, _ = self.run_approval(gitea, model) self.assertEqual(outcome.result.exit_code, 15) def test_critical_path_requests_human_review_without_bot_review(self): gitea = FakeGitea(files=[gitea_file(CRITICAL)], diff=diff_for(CRITICAL, added=["x"])) outcome, _ = self.run_approval(gitea) self.assertIs(outcome.result.verdict, Verdict.ORANGE) self.assertEqual(outcome.result.exit_code, 0) self.assertEqual(gitea.created_reviews, []) self.assertEqual(gitea.requested, ["alice"]) self.assertEqual(gitea.statuses[-1][1], "warning") def test_critical_paths_reason_comes_before_the_yellow_reasons(self): gitea = FakeGitea(files=[gitea_file(CRITICAL), gitea_file("build.gradle.kts")], diff=diff_for(CRITICAL, added=["x"]) + diff_for("build.gradle.kts", added=["y"])) self.run_approval(gitea) listed = [line for line in gitea.created_comments[0].splitlines() if line.startswith("- **")] self.assertTrue(listed[0].startswith("- **ORANGE** critical paths changed, review them deliberately: " f"{CRITICAL} ([hard_orange] **/auth/**)"), listed) self.assertIn("- **YELLOW** risky paths changed: build.gradle.kts (build.gradle*)", listed[1:]) def test_yellow_requests_human_review_and_passes(self): gitea = self.yellow_gitea() outcome, _ = self.run_approval(gitea) self.assertIs(outcome.result.verdict, Verdict.YELLOW) self.assertEqual(outcome.result.exit_code, 0) self.assertEqual(gitea.requested, ["alice"]) self.assertEqual(gitea.created_reviews, []) def test_yellow_keeps_a_pending_request(self): gitea = self.yellow_gitea() gitea.requested = ["alice"] outcome, _ = self.run_approval(gitea) self.assertEqual(outcome.result.exit_code, 0) self.assertNotIn("request", gitea.writes) def test_yellow_with_human_approval_requests_nothing(self): approval = {"state": "APPROVED", "commit_id": HEAD, "user": {"login": "alice"}} gitea = self.yellow_gitea(reviews=[approval]) outcome, _ = self.run_approval(gitea) self.assertEqual(outcome.result.exit_code, 0) self.assertNotIn("request", gitea.writes) def test_yellow_with_failed_request_blocks(self): gitea = self.yellow_gitea() gitea.fail_review_request = True outcome, _ = self.run_approval(gitea) self.assertEqual(outcome.result.exit_code, 10) def test_error_submits_no_review(self): gitea = docs_gitea() outcome, _ = self.run_approval(gitea, FakeModel(error=RuntimeError("down"))) self.assertEqual(outcome.result.exit_code, 30) self.assertEqual(gitea.created_reviews, []) def test_superseded_writes_nothing(self): gitea = docs_gitea() gitea.heads = [HEAD, "c" * 40] outcome, _ = self.run_approval(gitea) self.assertEqual((outcome.result.exit_code, gitea.writes), (30, []))