import unittest from tests.fakes import HEAD, config, review_data from werkrevisor.verdict import ( InvalidModelAnswer, Verdict, assess_model, gate_decision, human_approval, parse_model_review, worst, ) def assess(data, orange_min_medium=2): return assess_model(parse_model_review(data), config(review={"orange_min_medium": orange_min_medium}).review) def with_headers(*headers, recommendation="safe_to_merge"): data = review_data(recommendation, issues=["a.py"] * len(headers)) for issue, header in zip(data["review"]["key_issues_to_review"], headers): issue["issue_header"] = header return data class ExitCodeTest(unittest.TestCase): def test_exit_codes(self): self.assertEqual([v.exit_code for v in Verdict], [0, 10, 15, 20, 30]) def test_ordering(self): self.assertEqual([v.name for v in sorted(Verdict, key=lambda v: v.rank)], ["GREEN", "YELLOW", "ORANGE", "RED", "ERROR"]) def test_worst(self): self.assertIs(worst(Verdict.GREEN, Verdict.RED, Verdict.YELLOW), Verdict.RED) self.assertIs(worst(Verdict.YELLOW, Verdict.ORANGE), Verdict.ORANGE) self.assertIs(worst(Verdict.ORANGE, Verdict.RED), Verdict.RED) self.assertIs(worst(Verdict.RED, Verdict.ERROR), Verdict.ERROR) class ModelMappingTest(unittest.TestCase): def test_safe_without_issues_is_green(self): self.assertIs(assess(review_data())[0], Verdict.GREEN) def test_caution_is_a_hint(self): verdict, reasons = assess(review_data("merge_with_caution")) self.assertIs(verdict, Verdict.GREEN) self.assertEqual([(r.code, r.verdict) for r in reasons], [("model_caution", Verdict.GREEN)]) def test_high_risk_is_yellow(self): self.assertIs(assess(review_data(risk="high"))[0], Verdict.YELLOW) def test_security_concern_is_yellow_and_quoted(self): """Scenario#67.01""" verdict, reasons = assess(review_data(security="SQL injection: the query concatenates the user name.")) self.assertIs(verdict, Verdict.YELLOW) self.assertIn("model reports a security concern: SQL injection: the query concatenates the user name.", [r.text for r in reasons]) def test_a_long_security_concern_is_one_shortened_line(self): """Scenario#67.02""" concern = "Token leak:\n the token " + "goes into the log " * 30 text = next(r.text for r in assess(review_data(security=concern))[1] if r.code == "model_security") quoted = text.removeprefix("model reports a security concern: ") self.assertNotIn("\n", quoted) self.assertTrue(quoted.startswith("Token leak: the token goes into the log")) self.assertEqual((len(quoted), quoted[-1]), (401, "…")) def test_no_security_concern_in_any_spelling_gives_no_reason(self): """Scenario#71.01""" for answer in (False, "False", "false", "no", "No.", "None", None, ""): with self.subTest(answer=answer): self.assertEqual(assess(review_data(security=answer)), (Verdict.GREEN, [])) def test_true_stays_a_security_concern(self): verdict, reasons = assess(review_data(security=True)) self.assertIs(verdict, Verdict.YELLOW) self.assertIn("model reports a security concern: True", [r.text for r in reasons]) def test_threshold_reason_names_the_config_key(self): reasons = assess(with_headers("[medium] A", "[medium] B"))[1] self.assertIn("2 finding(s) that the model rates as high or medium ([review] orange_min_medium = 2): " "a.py:3-5, a.py:3-5", [r.text for r in reasons]) def test_invalid_answers(self): for data in (None, {}, {"review": "x"}, {"review": {"risk_level": "low", "key_issues_to_review": []}}, {"review": {"merge_recommendation": "ship it", "risk_level": "low"}}, {"review": {"merge_recommendation": "safe_to_merge"}}, {"review": {"merge_recommendation": "safe_to_merge", "risk_level": "low", "key_issues_to_review": "none"}}): with self.subTest(data=data): with self.assertRaises(InvalidModelAnswer): parse_model_review(data) def test_review_markers_are_stripped(self): for content in ("FIXME.review: x\nBreaks X.", "// FIXME.review: x\nBreaks X.", "# FIXME.review: x\nBreaks X.", "-- FIXME.review: x\nBreaks X.", "\nBreaks X.", "Breaks X."): with self.subTest(content=content): data = review_data(issues=["a.py"]) data["review"]["key_issues_to_review"][0]["issue_content"] = content self.assertEqual(parse_model_review(data).findings[0].content, "Breaks X.") def test_lone_review_marker_keeps_its_note(self): data = review_data(issues=["a.py"]) data["review"]["key_issues_to_review"][0]["issue_content"] = "// FIXME.review: missing null check" self.assertEqual(parse_model_review(data).findings[0].content, "missing null check") def test_finding_location(self): finding = parse_model_review(review_data(issues=["a.py"])).findings[0] self.assertEqual(finding.location, "a.py:3-5") class SeverityParsingTest(unittest.TestCase): def test_known_prefixes_are_parsed_and_stripped(self): for header, severity, title in (("[high] Possible Bug", "high", "Possible Bug"), ("[Medium] Missing Test", "medium", "Missing Test"), (" [ LOW ] Log Noise", "low", "Log Noise"), ("[HIGH]Possible Bug", "high", "Possible Bug")): with self.subTest(header=header): finding = parse_model_review(with_headers(header)).findings[0] self.assertEqual((finding.severity, finding.severity_known, finding.header), (severity, True, title)) def test_missing_or_unknown_prefix_counts_as_medium(self): for header in ("Possible Bug", "[critical] Possible Bug", "Possible Bug [high]", ""): with self.subTest(header=header): finding = parse_model_review(with_headers(header)).findings[0] self.assertEqual((finding.severity, finding.severity_known, finding.header), ("medium", False, header)) class SeverityVerdictTest(unittest.TestCase): def test_verdict_follows_the_severities(self): for headers, verdict in ((["[high] A"], Verdict.ORANGE), (["[medium] A", "[medium] B"], Verdict.ORANGE), (["[medium] A", "B"], Verdict.ORANGE), (["[medium] A"], Verdict.YELLOW), (["[medium] A", "[low] B", "[low] C", "[low] D"], Verdict.YELLOW), (["[low] A", "[low] B", "[low] C"], Verdict.GREEN), ([], Verdict.GREEN)): with self.subTest(headers=headers): verdict_, reasons = assess(with_headers(*headers)) self.assertIs(verdict_, verdict) if not headers: self.assertEqual(reasons, []) def test_low_findings_are_hints(self): reasons = assess(with_headers("[low] A", "[medium] B", "[low] C"))[1] self.assertEqual([(r.code, r.text, r.verdict) for r in reasons], [("finding_unconfirmed", "1 finding(s) of medium severity: a.py:3-5", Verdict.YELLOW), ("model_minor_findings", "2 finding(s) of low severity: a.py:3-5, a.py:3-5", Verdict.GREEN)]) def test_high_reason(self): reasons = assess(with_headers("[high] A", "[low] B"))[1] self.assertEqual([(r.text, r.verdict) for r in reasons], [("1 finding(s) that the model rates as high: a.py:3-5", Verdict.ORANGE), ("1 finding(s) of low severity: a.py:3-5", Verdict.GREEN)]) def test_recommendation_alone_is_a_hint(self): for data, verdict in ((with_headers("[low] A", recommendation="merge_with_caution"), Verdict.GREEN), (with_headers(recommendation="changes_required"), Verdict.GREEN), (with_headers("[high] A", recommendation="changes_required"), Verdict.ORANGE), (review_data(risk="high"), Verdict.YELLOW), (review_data(security="SQL injection: ..."), Verdict.YELLOW)): with self.subTest(data=data): verdict_, reasons = assess(data) self.assertIs(verdict_, verdict) reasons = assess(with_headers(recommendation="changes_required"))[1] self.assertEqual([(r.code, r.verdict) for r in reasons], [("model_changes_required", Verdict.GREEN)]) def test_orange_min_medium_zero_disables_the_medium_rule(self): self.assertIs(assess(with_headers("[medium] A", "[medium] B", "C"), orange_min_medium=0)[0], Verdict.YELLOW) self.assertIs(assess(with_headers("[high] A"), orange_min_medium=0)[0], Verdict.ORANGE) class GateTest(unittest.TestCase): def test_non_yellow_maps_directly(self): for v in (Verdict.GREEN, Verdict.RED, Verdict.ERROR): self.assertEqual(gate_decision(v, "pass_with_review_request", "alice", True).exit_code, v.exit_code) def test_yellow_blocks_by_default(self): self.assertEqual(gate_decision(Verdict.YELLOW, "block").exit_code, 10) def test_yellow_with_pending_request(self): self.assertEqual(gate_decision(Verdict.YELLOW, "pass_with_review_request", None, True).exit_code, 0) self.assertEqual(gate_decision(Verdict.YELLOW, "pass_with_review_request", None, False).exit_code, 10) self.assertEqual(gate_decision(Verdict.YELLOW, "block", None, True).exit_code, 10) def test_yellow_unlocked_by_approval(self): self.assertEqual(gate_decision(Verdict.YELLOW, "block", "alice").exit_code, 0) def test_orange_gates_like_yellow_with_its_own_exit_code(self): self.assertEqual(gate_decision(Verdict.ORANGE, "block").exit_code, 15) self.assertEqual(gate_decision(Verdict.ORANGE, "block", "alice").exit_code, 0) self.assertEqual(gate_decision(Verdict.ORANGE, "pass_with_review_request", None, True).exit_code, 0) self.assertEqual(gate_decision(Verdict.ORANGE, "pass_with_review_request", None, False).exit_code, 15) def test_red_is_not_unlocked_by_approval(self): self.assertEqual(gate_decision(Verdict.RED, "block", "alice", True).exit_code, 20) class ApprovalTest(unittest.TestCase): def review(self, **kw): r = {"state": "APPROVED", "commit_id": HEAD, "user": {"login": "alice"}, "dismissed": False, "stale": False} r.update(kw) return r def test_valid_approval(self): self.assertEqual(human_approval([self.review()], HEAD, ["Alice"]), "alice") def test_rejected_approvals(self): for kw in ({"commit_id": "c" * 40}, {"state": "COMMENT"}, {"user": {"login": "mallory"}}, {"dismissed": True}, {"stale": True}): with self.subTest(kw=kw): self.assertIsNone(human_approval([self.review(**kw)], HEAD, ["alice"])) if __name__ == "__main__": unittest.main()