claude-code
e8b256ef92
werkator build succeeded after 00:22
werkrevisor 0.14.0: the severity of the model findings decides between ORANGE and YELLOW. - The model marks every finding as `[high]`, `[medium]` or `[low]`; werkrevisor reads and strips the mark. - ORANGE needs one high finding or `[review] orange_min_medium` (default 2) medium ones; other findings give YELLOW. - `[review] orange_min_medium` replaces `red_min_findings`. Details: the PR document under `docs/prs/`. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Reviewed-on: #14 Co-authored-by: claude-code <claude-code@javagil.de>
188 lines
9.2 KiB
Python
188 lines
9.2 KiB
Python
import unittest
|
|
|
|
from tests.fakes import HEAD, review_data
|
|
from werkrevisor.verdict import (
|
|
InvalidModelAnswer,
|
|
Verdict,
|
|
assess_model,
|
|
gate_decision,
|
|
human_approval,
|
|
parse_model_review,
|
|
worst,
|
|
)
|
|
|
|
|
|
def assess(data, orange_min_medium=2):
|
|
return assess_model(parse_model_review(data), orange_min_medium)
|
|
|
|
|
|
def with_headers(*headers, recommendation="safe_to_merge"):
|
|
data = review_data(recommendation, issues=["a.py"] * len(headers))
|
|
for issue, header in zip(data["review"]["key_issues_to_review"], headers):
|
|
issue["issue_header"] = header
|
|
return data
|
|
|
|
|
|
class ExitCodeTest(unittest.TestCase):
|
|
def test_exit_codes(self):
|
|
self.assertEqual([v.exit_code for v in Verdict], [0, 10, 15, 20, 30])
|
|
|
|
def test_ordering(self):
|
|
self.assertEqual([v.name for v in sorted(Verdict, key=lambda v: v.rank)],
|
|
["GREEN", "YELLOW", "ORANGE", "RED", "ERROR"])
|
|
|
|
def test_worst(self):
|
|
self.assertIs(worst(Verdict.GREEN, Verdict.RED, Verdict.YELLOW), Verdict.RED)
|
|
self.assertIs(worst(Verdict.YELLOW, Verdict.ORANGE), Verdict.ORANGE)
|
|
self.assertIs(worst(Verdict.ORANGE, Verdict.RED), Verdict.RED)
|
|
self.assertIs(worst(Verdict.RED, Verdict.ERROR), Verdict.ERROR)
|
|
|
|
|
|
class ModelMappingTest(unittest.TestCase):
|
|
def test_safe_without_issues_is_green(self):
|
|
self.assertIs(assess(review_data())[0], Verdict.GREEN)
|
|
|
|
def test_caution_is_yellow(self):
|
|
self.assertIs(assess(review_data("merge_with_caution"))[0], Verdict.YELLOW)
|
|
|
|
def test_high_risk_is_yellow(self):
|
|
self.assertIs(assess(review_data(risk="high"))[0], Verdict.YELLOW)
|
|
|
|
def test_security_concern_is_yellow(self):
|
|
self.assertIs(assess(review_data(security="SQL injection: ..."))[0], Verdict.YELLOW)
|
|
|
|
def test_threshold_reason_names_the_config_key(self):
|
|
reasons = assess(with_headers("[medium] A", "[medium] B"))[1]
|
|
self.assertIn("model reports 2 findings of medium severity ([review] orange_min_medium = 2)",
|
|
[r.text for r in reasons])
|
|
|
|
def test_invalid_answers(self):
|
|
for data in (None, {}, {"review": "x"},
|
|
{"review": {"risk_level": "low", "key_issues_to_review": []}},
|
|
{"review": {"merge_recommendation": "ship it", "risk_level": "low"}},
|
|
{"review": {"merge_recommendation": "safe_to_merge"}},
|
|
{"review": {"merge_recommendation": "safe_to_merge", "risk_level": "low",
|
|
"key_issues_to_review": "none"}}):
|
|
with self.subTest(data=data):
|
|
with self.assertRaises(InvalidModelAnswer):
|
|
parse_model_review(data)
|
|
|
|
def test_review_markers_are_stripped(self):
|
|
for content in ("FIXME.review: x\nBreaks X.", "// FIXME.review: x\nBreaks X.", "# FIXME.review: x\nBreaks X.",
|
|
"-- FIXME.review: x\nBreaks X.", "<!-- FIXME.review: x -->\nBreaks X.", "Breaks X."):
|
|
with self.subTest(content=content):
|
|
data = review_data(issues=["a.py"])
|
|
data["review"]["key_issues_to_review"][0]["issue_content"] = content
|
|
self.assertEqual(parse_model_review(data).findings[0].content, "Breaks X.")
|
|
|
|
def test_lone_review_marker_keeps_its_note(self):
|
|
data = review_data(issues=["a.py"])
|
|
data["review"]["key_issues_to_review"][0]["issue_content"] = "// FIXME.review: missing null check"
|
|
self.assertEqual(parse_model_review(data).findings[0].content, "missing null check")
|
|
|
|
def test_finding_location(self):
|
|
finding = parse_model_review(review_data(issues=["a.py"])).findings[0]
|
|
self.assertEqual(finding.location, "a.py:3-5")
|
|
|
|
|
|
class SeverityParsingTest(unittest.TestCase):
|
|
def test_known_prefixes_are_parsed_and_stripped(self):
|
|
for header, severity, title in (("[high] Possible Bug", "high", "Possible Bug"),
|
|
("[Medium] Missing Test", "medium", "Missing Test"),
|
|
(" [ LOW ] Log Noise", "low", "Log Noise"),
|
|
("[HIGH]Possible Bug", "high", "Possible Bug")):
|
|
with self.subTest(header=header):
|
|
finding = parse_model_review(with_headers(header)).findings[0]
|
|
self.assertEqual((finding.severity, finding.severity_known, finding.header), (severity, True, title))
|
|
|
|
def test_missing_or_unknown_prefix_counts_as_medium(self):
|
|
for header in ("Possible Bug", "[critical] Possible Bug", "Possible Bug [high]", ""):
|
|
with self.subTest(header=header):
|
|
finding = parse_model_review(with_headers(header)).findings[0]
|
|
self.assertEqual((finding.severity, finding.severity_known, finding.header),
|
|
("medium", False, header))
|
|
|
|
|
|
class SeverityVerdictTest(unittest.TestCase):
|
|
def test_verdict_follows_the_severities(self):
|
|
for headers, verdict in ((["[high] A"], Verdict.ORANGE),
|
|
(["[medium] A", "[medium] B"], Verdict.ORANGE),
|
|
(["[medium] A", "B"], Verdict.ORANGE),
|
|
(["[medium] A"], Verdict.YELLOW),
|
|
(["[medium] A", "[low] B", "[low] C", "[low] D"], Verdict.YELLOW),
|
|
(["[low] A", "[low] B", "[low] C"], Verdict.YELLOW),
|
|
([], Verdict.GREEN)):
|
|
with self.subTest(headers=headers):
|
|
verdict_, reasons = assess(with_headers(*headers))
|
|
self.assertIs(verdict_, verdict)
|
|
if not headers:
|
|
self.assertEqual(reasons, [])
|
|
|
|
def test_minor_findings_reason_counts_the_severities(self):
|
|
reasons = assess(with_headers("[low] A", "[medium] B", "[low] C"))[1]
|
|
self.assertEqual([(r.code, r.text, r.verdict) for r in reasons],
|
|
[("model_minor_findings", "model reports minor findings: 1 medium, 2 low", Verdict.YELLOW)])
|
|
|
|
def test_high_reason(self):
|
|
reasons = assess(with_headers("[high] A", "[low] B"))[1]
|
|
self.assertEqual([(r.text, r.verdict) for r in reasons],
|
|
[("model reports 1 finding(s) of high severity", Verdict.ORANGE)])
|
|
|
|
def test_changes_required_gives_orange_only_with_severe_findings(self):
|
|
for headers, verdict in ((["[low] A"], Verdict.YELLOW), ([], Verdict.YELLOW), (["[high] A"], Verdict.ORANGE),
|
|
(["[medium] A", "[medium] B"], Verdict.ORANGE)):
|
|
with self.subTest(headers=headers):
|
|
verdict_, reasons = assess(with_headers(*headers, recommendation="changes_required"))
|
|
self.assertIs(verdict_, verdict)
|
|
self.assertIn(("model_changes_required", Verdict.YELLOW), [(r.code, r.verdict) for r in reasons])
|
|
|
|
def test_orange_min_medium_zero_disables_the_medium_rule(self):
|
|
self.assertIs(assess(with_headers("[medium] A", "[medium] B", "C"), orange_min_medium=0)[0], Verdict.YELLOW)
|
|
self.assertIs(assess(with_headers("[high] A"), orange_min_medium=0)[0], Verdict.ORANGE)
|
|
|
|
|
|
class GateTest(unittest.TestCase):
|
|
def test_non_yellow_maps_directly(self):
|
|
for v in (Verdict.GREEN, Verdict.RED, Verdict.ERROR):
|
|
self.assertEqual(gate_decision(v, "pass_with_review_request", "alice", True).exit_code, v.exit_code)
|
|
|
|
def test_yellow_blocks_by_default(self):
|
|
self.assertEqual(gate_decision(Verdict.YELLOW, "block").exit_code, 10)
|
|
|
|
def test_yellow_with_pending_request(self):
|
|
self.assertEqual(gate_decision(Verdict.YELLOW, "pass_with_review_request", None, True).exit_code, 0)
|
|
self.assertEqual(gate_decision(Verdict.YELLOW, "pass_with_review_request", None, False).exit_code, 10)
|
|
self.assertEqual(gate_decision(Verdict.YELLOW, "block", None, True).exit_code, 10)
|
|
|
|
def test_yellow_unlocked_by_approval(self):
|
|
self.assertEqual(gate_decision(Verdict.YELLOW, "block", "alice").exit_code, 0)
|
|
|
|
def test_orange_gates_like_yellow_with_its_own_exit_code(self):
|
|
self.assertEqual(gate_decision(Verdict.ORANGE, "block").exit_code, 15)
|
|
self.assertEqual(gate_decision(Verdict.ORANGE, "block", "alice").exit_code, 0)
|
|
self.assertEqual(gate_decision(Verdict.ORANGE, "pass_with_review_request", None, True).exit_code, 0)
|
|
self.assertEqual(gate_decision(Verdict.ORANGE, "pass_with_review_request", None, False).exit_code, 15)
|
|
|
|
def test_red_is_not_unlocked_by_approval(self):
|
|
self.assertEqual(gate_decision(Verdict.RED, "block", "alice", True).exit_code, 20)
|
|
|
|
|
|
class ApprovalTest(unittest.TestCase):
|
|
def review(self, **kw):
|
|
r = {"state": "APPROVED", "commit_id": HEAD, "user": {"login": "alice"}, "dismissed": False, "stale": False}
|
|
r.update(kw)
|
|
return r
|
|
|
|
def test_valid_approval(self):
|
|
self.assertEqual(human_approval([self.review()], HEAD, ["Alice"]), "alice")
|
|
|
|
def test_rejected_approvals(self):
|
|
for kw in ({"commit_id": "c" * 40}, {"state": "COMMENT"}, {"user": {"login": "mallory"}},
|
|
{"dismissed": True}, {"stale": True}):
|
|
with self.subTest(kw=kw):
|
|
self.assertIsNone(human_approval([self.review(**kw)], HEAD, ["alice"]))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|