145 lines
7.6 KiB
Python
145 lines
7.6 KiB
Python
import json
|
|
import os
|
|
import tomllib
|
|
import unittest
|
|
|
|
from tests.fakes import ROOT, FakeGitea, FakeJev, FakeModel, config, diff_for, gitea_file, review_data
|
|
from tests.test_flow import run
|
|
from werkrevisor import findings
|
|
from werkrevisor.config import ConfigError, build_config
|
|
from werkrevisor.verdict import Finding, Verdict
|
|
|
|
MAIN = "src/main/java/A.java"
|
|
LOCATION = f"{MAIN}:3-5"
|
|
|
|
|
|
def combined(**review):
|
|
return config(engine={"mode": "combined"}, gate={"yellow_gate": "block"}, review=review)
|
|
|
|
|
|
def main_gitea():
|
|
gitea = FakeGitea(files=[gitea_file(MAIN)], diff=diff_for(MAIN, added=["int x;"]))
|
|
gitea.raw_files[MAIN] = "".join(f"line {i}\n" for i in range(1, 101)).encode()
|
|
return gitea
|
|
|
|
|
|
def judged_run(header, real, severity, jev=None, cfg=None):
|
|
"""severity is Jev's expected level from 0 to 3, as in the measurement."""
|
|
gitea = main_gitea()
|
|
jev = jev or FakeJev(problems={"real_defect": real, "severity": severity / findings.SEVERITY_TOP})
|
|
model = FakeModel(data=review_data(issues=[MAIN], header=header), prompt_files=[MAIN])
|
|
outcome, d = run(gitea, model, cfg=cfg or combined(), jev=jev)
|
|
return outcome, gitea, json.loads(d.out.getvalue().strip()), jev
|
|
|
|
|
|
class FindingJudgmentTest(unittest.TestCase):
|
|
def test_jev_answers_decide_the_verdict(self):
|
|
for header, real, severity, verdict in (("[medium] Bug", 0.72, 2.6, Verdict.ORANGE),
|
|
("[high] Bug", 0.40, 2.8, Verdict.YELLOW),
|
|
("[high] Bug", 0.90, 2.0, Verdict.GREEN),
|
|
("[low] Bug", 0.95, 2.9, Verdict.GREEN)):
|
|
with self.subTest(header=header, real=real, severity=severity):
|
|
outcome = judged_run(header, real, severity)[0]
|
|
self.assertIs(outcome.result.verdict, verdict)
|
|
|
|
def test_low_findings_cost_no_call(self):
|
|
jev = judged_run("[low] Naming", 0.9, 2.9)[3]
|
|
self.assertEqual([q for _, q in jev.requests if "real_defect" in q], [])
|
|
|
|
def test_comment_shows_the_jev_values(self):
|
|
gitea = judged_run("[medium] Bug", 0.72, 2.6)[1]
|
|
body = gitea.comments[-1]["body"]
|
|
self.assertIn(f"**[medium] Bug** at `{LOCATION}`, ORANGE, as a line comment", body)
|
|
self.assertIn("ORANGE; Jev: real 0.72, severity 2.6 of 3", gitea.review_comments[0][0]["body"])
|
|
self.assertIn(f"1 finding(s) that Jev rates as probably real and severe: {LOCATION}", body)
|
|
self.assertIn("finding judge `jev-test` calls 1", body)
|
|
|
|
def test_hint_is_shown_as_hint(self):
|
|
gitea = judged_run("[high] Bug", 0.9, 2.0)[1]
|
|
body = gitea.comments[-1]["body"]
|
|
self.assertIn("hint; Jev: real 0.90, severity 2.0 of 3", gitea.review_comments[0][0]["body"])
|
|
self.assertIn(f"- **hint** 1 finding(s) of low severity or rated minor by Jev: {LOCATION}", body)
|
|
|
|
def test_failed_judgment_gives_yellow(self):
|
|
outcome, gitea, record, _ = judged_run("[high] Bug", 0, 0, jev=FakeJev(error=OSError("down")))
|
|
self.assertIs(outcome.result.verdict, Verdict.YELLOW)
|
|
reason = next(r for r in outcome.result.reasons if r.code == "finding_not_judged")
|
|
self.assertEqual(reason.text, f"1 finding(s) not judged by Jev (OSError: down): {LOCATION}")
|
|
self.assertIn("YELLOW; Jev: not judged", gitea.review_comments[0][0]["body"])
|
|
|
|
def test_invalid_answer_counts_one_call(self):
|
|
class InvalidJev(FakeJev):
|
|
def ask(self, state, questions):
|
|
answer = super().ask(state, questions)
|
|
answer.answers["real_defect"]["noul"] = 2.0
|
|
return answer
|
|
outcome, _, record, _ = judged_run("[high] Bug", 0, 0, jev=InvalidJev())
|
|
self.assertIs(outcome.result.verdict, Verdict.YELLOW)
|
|
self.assertEqual((record["finding_judge_calls"], record["finding_judge_input_tokens"]), (1, 1000))
|
|
|
|
def test_usage_record_holds_the_jev_values(self):
|
|
gitea = main_gitea()
|
|
jev = FakeJev(problems={"real_defect": 0.72, "severity": 0.9})
|
|
data = review_data(issues=[MAIN, MAIN], header="[medium] Bug")
|
|
data["review"]["key_issues_to_review"][1].update(start_line=40, end_line=41, issue_header="[low] Style")
|
|
_, d = run(gitea, FakeModel(data=data, prompt_files=[MAIN]), cfg=combined(), jev=jev)
|
|
record = json.loads(d.out.getvalue().strip())
|
|
self.assertEqual(record["finding_judgments"], [
|
|
{"file": MAIN, "start_line": 3, "end_line": 5, "model_severity": "medium", "jev_real": 0.72,
|
|
"jev_severity": 2.7, "jev_error": None, "jev_category": None, "jev_category_confidence": None,
|
|
"verdict": "ORANGE"},
|
|
{"file": MAIN, "start_line": 40, "end_line": 41, "model_severity": "low", "jev_real": None,
|
|
"jev_severity": None, "jev_error": None, "jev_category": None, "jev_category_confidence": None,
|
|
"verdict": "GREEN"},
|
|
])
|
|
self.assertEqual((record["finding_judge_calls"], record["finding_judge_input_tokens"]), (1, 1000))
|
|
|
|
def test_llm_mode_judges_without_jev(self):
|
|
jev = FakeJev()
|
|
outcome, _, record, _ = judged_run("[high] Bug", 0, 0, jev=jev, cfg=config(gate={"yellow_gate": "block"}))
|
|
self.assertIs(outcome.result.verdict, Verdict.ORANGE)
|
|
self.assertEqual(jev.requests, [])
|
|
self.assertEqual(record["finding_judge_calls"], 0)
|
|
|
|
def test_thresholds_come_from_the_config(self):
|
|
outcome = judged_run("[medium] Bug", 0.72, 2.6, cfg=combined(finding_real_min=0.8))[0]
|
|
self.assertIs(outcome.result.verdict, Verdict.YELLOW)
|
|
outcome = judged_run("[medium] Bug", 0.72, 2.6, cfg=combined(finding_severity_min=2.8))[0]
|
|
self.assertIs(outcome.result.verdict, Verdict.GREEN)
|
|
|
|
|
|
class FindingStateTest(unittest.TestCase):
|
|
def test_state_holds_the_finding_and_the_code_excerpt(self):
|
|
text = "".join(f"code {i}\n" for i in range(1, 101))
|
|
finding = Finding(file="app.py", start_line=40, end_line=41, header="Off by one", content="Loops once more.")
|
|
state = findings.state(finding, text)
|
|
self.assertTrue(state.startswith("Finding: Off by one\nLocation: app.py:40-41\n\nLoops once more.\n\n"
|
|
"Code excerpt:\n"))
|
|
excerpt = state.split("Code excerpt:\n")[1].splitlines()
|
|
self.assertEqual((excerpt[0], excerpt[-1], len(excerpt)), (" 15 | code 15", " 66 | code 66", 52))
|
|
|
|
def test_excerpt_without_code_or_line(self):
|
|
self.assertEqual(findings.excerpt(None, 3, 5), "(code not available)")
|
|
self.assertEqual(findings.excerpt("a\n", 0, 0), "(the finding names no line)")
|
|
self.assertEqual(findings.excerpt("a\nb\n", 1, 1), " 1 | a\n 2 | b")
|
|
|
|
|
|
class FindingConfigTest(unittest.TestCase):
|
|
def test_thresholds_must_be_in_range(self):
|
|
for name, value in (("finding_real_min", 1.1), ("finding_real_min", -0.1), ("finding_real_min", True),
|
|
("finding_severity_min", 3.5), ("finding_severity_min", "2")):
|
|
with self.subTest(name=name, value=value):
|
|
with self.assertRaisesRegex(ConfigError, rf"\[review\] {name} must be a number between 0 and"):
|
|
config(review={name: value})
|
|
self.assertEqual((config().review.finding_real_min, config().review.finding_severity_min), (0.6, 2.5))
|
|
|
|
def test_thresholds_are_host_only(self):
|
|
with open(os.path.join(ROOT, "config", "werkprobe.toml"), "rb") as f:
|
|
host = tomllib.load(f)
|
|
with self.assertRaisesRegex(ConfigError, r"must not set \[review\] finding_real_min: only the host config"):
|
|
build_config(host, {"review": {"finding_real_min": 0.1}})
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|