import unittest from tests.fakes import HEAD, review_data from werkrevisor.verdict import ( InvalidModelAnswer, Verdict, assess_model, gate_decision, human_approval, parse_model_review, worst, ) def assess(data, orange_min_medium=2): return assess_model(parse_model_review(data), orange_min_medium) def with_headers(*headers, recommendation="safe_to_merge"): data = review_data(recommendation, issues=["a.py"] * len(headers)) for issue, header in zip(data["review"]["key_issues_to_review"], headers): issue["issue_header"] = header return data class ExitCodeTest(unittest.TestCase): def test_exit_codes(self): self.assertEqual([v.exit_code for v in Verdict], [0, 10, 15, 20, 30]) def test_ordering(self): self.assertEqual([v.name for v in sorted(Verdict, key=lambda v: v.rank)], ["GREEN", "YELLOW", "ORANGE", "RED", "ERROR"]) def test_worst(self): self.assertIs(worst(Verdict.GREEN, Verdict.RED, Verdict.YELLOW), Verdict.RED) self.assertIs(worst(Verdict.YELLOW, Verdict.ORANGE), Verdict.ORANGE) self.assertIs(worst(Verdict.ORANGE, Verdict.RED), Verdict.RED) self.assertIs(worst(Verdict.RED, Verdict.ERROR), Verdict.ERROR) class ModelMappingTest(unittest.TestCase): def test_safe_without_issues_is_green(self): self.assertIs(assess(review_data())[0], Verdict.GREEN) def test_caution_is_yellow(self): self.assertIs(assess(review_data("merge_with_caution"))[0], Verdict.YELLOW) def test_high_risk_is_yellow(self): self.assertIs(assess(review_data(risk="high"))[0], Verdict.YELLOW) def test_security_concern_is_yellow(self): self.assertIs(assess(review_data(security="SQL injection: ..."))[0], Verdict.YELLOW) def test_threshold_reason_names_the_config_key(self): reasons = assess(with_headers("[medium] A", "[medium] B"))[1] self.assertIn("model reports 2 findings of medium severity ([review] orange_min_medium = 2)", [r.text for r in reasons]) def test_invalid_answers(self): for data in (None, {}, {"review": "x"}, {"review": {"risk_level": "low", "key_issues_to_review": []}}, {"review": {"merge_recommendation": "ship it", "risk_level": "low"}}, {"review": {"merge_recommendation": "safe_to_merge"}}, {"review": {"merge_recommendation": "safe_to_merge", "risk_level": "low", "key_issues_to_review": "none"}}): with self.subTest(data=data): with self.assertRaises(InvalidModelAnswer): parse_model_review(data) def test_review_markers_are_stripped(self): for content in ("FIXME.review: x\nBreaks X.", "// FIXME.review: x\nBreaks X.", "# FIXME.review: x\nBreaks X.", "-- FIXME.review: x\nBreaks X.", "\nBreaks X.", "Breaks X."): with self.subTest(content=content): data = review_data(issues=["a.py"]) data["review"]["key_issues_to_review"][0]["issue_content"] = content self.assertEqual(parse_model_review(data).findings[0].content, "Breaks X.") def test_lone_review_marker_keeps_its_note(self): data = review_data(issues=["a.py"]) data["review"]["key_issues_to_review"][0]["issue_content"] = "// FIXME.review: missing null check" self.assertEqual(parse_model_review(data).findings[0].content, "missing null check") def test_finding_location(self): finding = parse_model_review(review_data(issues=["a.py"])).findings[0] self.assertEqual(finding.location, "a.py:3-5") class SeverityParsingTest(unittest.TestCase): def test_known_prefixes_are_parsed_and_stripped(self): for header, severity, title in (("[high] Possible Bug", "high", "Possible Bug"), ("[Medium] Missing Test", "medium", "Missing Test"), (" [ LOW ] Log Noise", "low", "Log Noise"), ("[HIGH]Possible Bug", "high", "Possible Bug")): with self.subTest(header=header): finding = parse_model_review(with_headers(header)).findings[0] self.assertEqual((finding.severity, finding.severity_known, finding.header), (severity, True, title)) def test_missing_or_unknown_prefix_counts_as_medium(self): for header in ("Possible Bug", "[critical] Possible Bug", "Possible Bug [high]", ""): with self.subTest(header=header): finding = parse_model_review(with_headers(header)).findings[0] self.assertEqual((finding.severity, finding.severity_known, finding.header), ("medium", False, header)) class SeverityVerdictTest(unittest.TestCase): def test_verdict_follows_the_severities(self): for headers, verdict in ((["[high] A"], Verdict.ORANGE), (["[medium] A", "[medium] B"], Verdict.ORANGE), (["[medium] A", "B"], Verdict.ORANGE), (["[medium] A"], Verdict.YELLOW), (["[medium] A", "[low] B", "[low] C", "[low] D"], Verdict.YELLOW), (["[low] A", "[low] B", "[low] C"], Verdict.YELLOW), ([], Verdict.GREEN)): with self.subTest(headers=headers): verdict_, reasons = assess(with_headers(*headers)) self.assertIs(verdict_, verdict) if not headers: self.assertEqual(reasons, []) def test_minor_findings_reason_counts_the_severities(self): reasons = assess(with_headers("[low] A", "[medium] B", "[low] C"))[1] self.assertEqual([(r.code, r.text, r.verdict) for r in reasons], [("model_minor_findings", "model reports minor findings: 1 medium, 2 low", Verdict.YELLOW)]) def test_high_reason(self): reasons = assess(with_headers("[high] A", "[low] B"))[1] self.assertEqual([(r.text, r.verdict) for r in reasons], [("model reports 1 finding(s) of high severity", Verdict.ORANGE)]) def test_changes_required_gives_orange_only_with_severe_findings(self): for headers, verdict in ((["[low] A"], Verdict.YELLOW), ([], Verdict.YELLOW), (["[high] A"], Verdict.ORANGE), (["[medium] A", "[medium] B"], Verdict.ORANGE)): with self.subTest(headers=headers): verdict_, reasons = assess(with_headers(*headers, recommendation="changes_required")) self.assertIs(verdict_, verdict) self.assertIn(("model_changes_required", Verdict.YELLOW), [(r.code, r.verdict) for r in reasons]) def test_orange_min_medium_zero_disables_the_medium_rule(self): self.assertIs(assess(with_headers("[medium] A", "[medium] B", "C"), orange_min_medium=0)[0], Verdict.YELLOW) self.assertIs(assess(with_headers("[high] A"), orange_min_medium=0)[0], Verdict.ORANGE) class GateTest(unittest.TestCase): def test_non_yellow_maps_directly(self): for v in (Verdict.GREEN, Verdict.RED, Verdict.ERROR): self.assertEqual(gate_decision(v, "pass_with_review_request", "alice", True).exit_code, v.exit_code) def test_yellow_blocks_by_default(self): self.assertEqual(gate_decision(Verdict.YELLOW, "block").exit_code, 10) def test_yellow_with_pending_request(self): self.assertEqual(gate_decision(Verdict.YELLOW, "pass_with_review_request", None, True).exit_code, 0) self.assertEqual(gate_decision(Verdict.YELLOW, "pass_with_review_request", None, False).exit_code, 10) self.assertEqual(gate_decision(Verdict.YELLOW, "block", None, True).exit_code, 10) def test_yellow_unlocked_by_approval(self): self.assertEqual(gate_decision(Verdict.YELLOW, "block", "alice").exit_code, 0) def test_orange_gates_like_yellow_with_its_own_exit_code(self): self.assertEqual(gate_decision(Verdict.ORANGE, "block").exit_code, 15) self.assertEqual(gate_decision(Verdict.ORANGE, "block", "alice").exit_code, 0) self.assertEqual(gate_decision(Verdict.ORANGE, "pass_with_review_request", None, True).exit_code, 0) self.assertEqual(gate_decision(Verdict.ORANGE, "pass_with_review_request", None, False).exit_code, 15) def test_red_is_not_unlocked_by_approval(self): self.assertEqual(gate_decision(Verdict.RED, "block", "alice", True).exit_code, 20) class ApprovalTest(unittest.TestCase): def review(self, **kw): r = {"state": "APPROVED", "commit_id": HEAD, "user": {"login": "alice"}, "dismissed": False, "stale": False} r.update(kw) return r def test_valid_approval(self): self.assertEqual(human_approval([self.review()], HEAD, ["Alice"]), "alice") def test_rejected_approvals(self): for kw in ({"commit_id": "c" * 40}, {"state": "COMMENT"}, {"user": {"login": "mallory"}}, {"dismissed": True}, {"stale": True}): with self.subTest(kw=kw): self.assertIsNone(human_approval([self.review(**kw)], HEAD, ["alice"])) if __name__ == "__main__": unittest.main()