import json import os import tomllib import unittest from tests.fakes import ROOT, FakeGitea, FakeJev, FakeModel, config, diff_for, gitea_file, review_data from tests.test_flow import run from werkrevisor import findings from werkrevisor.config import ConfigError, build_config from werkrevisor.verdict import Finding, Verdict MAIN = "src/main/java/A.java" LOCATION = f"{MAIN}:3-5" def combined(**review): return config(engine={"mode": "combined"}, gate={"yellow_gate": "block"}, review=review) def main_gitea(): gitea = FakeGitea(files=[gitea_file(MAIN)], diff=diff_for(MAIN, added=["int x;"])) gitea.raw_files[MAIN] = "".join(f"line {i}\n" for i in range(1, 101)).encode() return gitea def judged_run(header, real, severity, jev=None, cfg=None): """severity is Jev's expected level from 0 to 3, as in the measurement.""" gitea = main_gitea() jev = jev or FakeJev(problems={"real_defect": real, "severity": severity / findings.SEVERITY_TOP}) model = FakeModel(data=review_data(issues=[MAIN], header=header), prompt_files=[MAIN]) outcome, d = run(gitea, model, cfg=cfg or combined(), jev=jev) return outcome, gitea, json.loads(d.out.getvalue().strip()), jev class FindingJudgmentTest(unittest.TestCase): def test_jev_answers_decide_the_verdict(self): for header, real, severity, verdict in (("[medium] Bug", 0.72, 2.6, Verdict.ORANGE), ("[high] Bug", 0.40, 2.8, Verdict.YELLOW), ("[high] Bug", 0.90, 2.0, Verdict.GREEN), ("[low] Bug", 0.95, 2.9, Verdict.GREEN)): with self.subTest(header=header, real=real, severity=severity): outcome = judged_run(header, real, severity)[0] self.assertIs(outcome.result.verdict, verdict) def test_low_findings_cost_no_call(self): jev = judged_run("[low] Naming", 0.9, 2.9)[3] self.assertEqual([q for _, q in jev.requests if "real_defect" in q], []) def test_comment_shows_the_jev_values(self): gitea = judged_run("[medium] Bug", 0.72, 2.6)[1] body = gitea.comments[-1]["body"] self.assertIn(f"**[medium] Bug** at `{LOCATION}`, ORANGE, as a line comment", body) self.assertIn("ORANGE; Jev: real 0.72, severity 2.6 of 3", gitea.review_comments[0][0]["body"]) self.assertIn(f"1 finding(s) that Jev rates as probably real and severe: {LOCATION}", body) self.assertIn("finding judge `jev-test` calls 1", body) def test_hint_is_shown_as_hint(self): gitea = judged_run("[high] Bug", 0.9, 2.0)[1] body = gitea.comments[-1]["body"] self.assertIn("hint; Jev: real 0.90, severity 2.0 of 3", gitea.review_comments[0][0]["body"]) self.assertIn(f"- **hint** 1 finding(s) of low severity or rated minor by Jev: {LOCATION}", body) def test_failed_judgment_gives_yellow(self): outcome, gitea, record, _ = judged_run("[high] Bug", 0, 0, jev=FakeJev(error=OSError("down"))) self.assertIs(outcome.result.verdict, Verdict.YELLOW) reason = next(r for r in outcome.result.reasons if r.code == "finding_not_judged") self.assertEqual(reason.text, f"1 finding(s) not judged by Jev (OSError: down): {LOCATION}") self.assertIn("YELLOW; Jev: not judged", gitea.review_comments[0][0]["body"]) def test_invalid_answer_counts_one_call(self): class InvalidJev(FakeJev): def ask(self, state, questions): answer = super().ask(state, questions) answer.answers["real_defect"]["noul"] = 2.0 return answer outcome, _, record, _ = judged_run("[high] Bug", 0, 0, jev=InvalidJev()) self.assertIs(outcome.result.verdict, Verdict.YELLOW) self.assertEqual((record["finding_judge_calls"], record["finding_judge_input_tokens"]), (1, 1000)) def test_usage_record_holds_the_jev_values(self): gitea = main_gitea() jev = FakeJev(problems={"real_defect": 0.72, "severity": 0.9}) data = review_data(issues=[MAIN, MAIN], header="[medium] Bug") data["review"]["key_issues_to_review"][1].update(start_line=40, end_line=41, issue_header="[low] Style") _, d = run(gitea, FakeModel(data=data, prompt_files=[MAIN]), cfg=combined(), jev=jev) record = json.loads(d.out.getvalue().strip()) self.assertEqual(record["finding_judgments"], [ {"file": MAIN, "start_line": 3, "end_line": 5, "model_severity": "medium", "jev_real": 0.72, "jev_severity": 2.7, "jev_error": None, "jev_category": None, "jev_category_confidence": None, "verdict": "ORANGE"}, {"file": MAIN, "start_line": 40, "end_line": 41, "model_severity": "low", "jev_real": None, "jev_severity": None, "jev_error": None, "jev_category": None, "jev_category_confidence": None, "verdict": "GREEN"}, ]) self.assertEqual((record["finding_judge_calls"], record["finding_judge_input_tokens"]), (1, 1000)) def test_llm_mode_judges_without_jev(self): jev = FakeJev() outcome, _, record, _ = judged_run("[high] Bug", 0, 0, jev=jev, cfg=config(gate={"yellow_gate": "block"})) self.assertIs(outcome.result.verdict, Verdict.ORANGE) self.assertEqual(jev.requests, []) self.assertEqual(record["finding_judge_calls"], 0) def test_thresholds_come_from_the_config(self): outcome = judged_run("[medium] Bug", 0.72, 2.6, cfg=combined(finding_real_min=0.8))[0] self.assertIs(outcome.result.verdict, Verdict.YELLOW) outcome = judged_run("[medium] Bug", 0.72, 2.6, cfg=combined(finding_severity_min=2.8))[0] self.assertIs(outcome.result.verdict, Verdict.GREEN) class FindingStateTest(unittest.TestCase): def test_state_holds_the_finding_and_the_code_excerpt(self): text = "".join(f"code {i}\n" for i in range(1, 101)) finding = Finding(file="app.py", start_line=40, end_line=41, header="Off by one", content="Loops once more.") state = findings.state(finding, text) self.assertTrue(state.startswith("Finding: Off by one\nLocation: app.py:40-41\n\nLoops once more.\n\n" "Code excerpt:\n")) excerpt = state.split("Code excerpt:\n")[1].splitlines() self.assertEqual((excerpt[0], excerpt[-1], len(excerpt)), (" 15 | code 15", " 66 | code 66", 52)) def test_excerpt_without_code_or_line(self): self.assertEqual(findings.excerpt(None, 3, 5), "(code not available)") self.assertEqual(findings.excerpt("a\n", 0, 0), "(the finding names no line)") self.assertEqual(findings.excerpt("a\nb\n", 1, 1), " 1 | a\n 2 | b") class FindingConfigTest(unittest.TestCase): def test_thresholds_must_be_in_range(self): for name, value in (("finding_real_min", 1.1), ("finding_real_min", -0.1), ("finding_real_min", True), ("finding_severity_min", 3.5), ("finding_severity_min", "2")): with self.subTest(name=name, value=value): with self.assertRaisesRegex(ConfigError, rf"\[review\] {name} must be a number between 0 and"): config(review={name: value}) self.assertEqual((config().review.finding_real_min, config().review.finding_severity_min), (0.6, 2.5)) def test_thresholds_are_host_only(self): with open(os.path.join(ROOT, "config", "werkprobe.toml"), "rb") as f: host = tomllib.load(f) with self.assertRaisesRegex(ConfigError, r"must not set \[review\] finding_real_min: only the host config"): build_config(host, {"review": {"finding_real_min": 0.1}}) if __name__ == "__main__": unittest.main()