221 lines
11 KiB
Python
221 lines
11 KiB
Python
import unittest
|
|
|
|
from tests.fakes import HEAD, config, review_data
|
|
from werkrevisor.verdict import (
|
|
InvalidModelAnswer,
|
|
Verdict,
|
|
assess_model,
|
|
gate_decision,
|
|
human_approval,
|
|
parse_model_review,
|
|
worst,
|
|
)
|
|
|
|
|
|
def assess(data, orange_min_medium=2):
|
|
return assess_model(parse_model_review(data), config(review={"orange_min_medium": orange_min_medium}).review)
|
|
|
|
|
|
def with_headers(*headers, recommendation="safe_to_merge"):
|
|
data = review_data(recommendation, issues=["a.py"] * len(headers))
|
|
for issue, header in zip(data["review"]["key_issues_to_review"], headers):
|
|
issue["issue_header"] = header
|
|
return data
|
|
|
|
|
|
class ExitCodeTest(unittest.TestCase):
|
|
def test_exit_codes(self):
|
|
self.assertEqual([v.exit_code for v in Verdict], [0, 10, 15, 20, 30])
|
|
|
|
def test_ordering(self):
|
|
self.assertEqual([v.name for v in sorted(Verdict, key=lambda v: v.rank)],
|
|
["GREEN", "YELLOW", "ORANGE", "RED", "ERROR"])
|
|
|
|
def test_worst(self):
|
|
self.assertIs(worst(Verdict.GREEN, Verdict.RED, Verdict.YELLOW), Verdict.RED)
|
|
self.assertIs(worst(Verdict.YELLOW, Verdict.ORANGE), Verdict.ORANGE)
|
|
self.assertIs(worst(Verdict.ORANGE, Verdict.RED), Verdict.RED)
|
|
self.assertIs(worst(Verdict.RED, Verdict.ERROR), Verdict.ERROR)
|
|
|
|
|
|
class ModelMappingTest(unittest.TestCase):
|
|
def test_safe_without_issues_is_green(self):
|
|
self.assertIs(assess(review_data())[0], Verdict.GREEN)
|
|
|
|
def test_caution_is_a_hint(self):
|
|
verdict, reasons = assess(review_data("merge_with_caution"))
|
|
self.assertIs(verdict, Verdict.GREEN)
|
|
self.assertEqual([(r.code, r.verdict) for r in reasons], [("model_caution", Verdict.GREEN)])
|
|
|
|
def test_high_risk_is_yellow(self):
|
|
self.assertIs(assess(review_data(risk="high"))[0], Verdict.YELLOW)
|
|
|
|
def test_security_concern_is_yellow_and_quoted(self):
|
|
"""Scenario#67.01"""
|
|
verdict, reasons = assess(review_data(security="SQL injection: the query concatenates the user name."))
|
|
self.assertIs(verdict, Verdict.YELLOW)
|
|
self.assertIn("model reports a security concern: SQL injection: the query concatenates the user name.",
|
|
[r.text for r in reasons])
|
|
|
|
def test_a_long_security_concern_is_one_shortened_line(self):
|
|
"""Scenario#67.02"""
|
|
concern = "Token leak:\n the token " + "goes into the log " * 30
|
|
text = next(r.text for r in assess(review_data(security=concern))[1] if r.code == "model_security")
|
|
quoted = text.removeprefix("model reports a security concern: ")
|
|
self.assertNotIn("\n", quoted)
|
|
self.assertTrue(quoted.startswith("Token leak: the token goes into the log"))
|
|
self.assertEqual((len(quoted), quoted[-1]), (401, "…"))
|
|
|
|
def test_no_security_concern_in_any_spelling_gives_no_reason(self):
|
|
"""Scenario#71.01"""
|
|
for answer in (False, "False", "false", "no", "No.", "None", None, ""):
|
|
with self.subTest(answer=answer):
|
|
self.assertEqual(assess(review_data(security=answer)), (Verdict.GREEN, []))
|
|
|
|
def test_true_stays_a_security_concern(self):
|
|
verdict, reasons = assess(review_data(security=True))
|
|
self.assertIs(verdict, Verdict.YELLOW)
|
|
self.assertIn("model reports a security concern: True", [r.text for r in reasons])
|
|
|
|
def test_threshold_reason_names_the_config_key(self):
|
|
reasons = assess(with_headers("[medium] A", "[medium] B"))[1]
|
|
self.assertIn("2 finding(s) that the model rates as high or medium ([review] orange_min_medium = 2): "
|
|
"a.py:3-5, a.py:3-5", [r.text for r in reasons])
|
|
|
|
def test_invalid_answers(self):
|
|
for data in (None, {}, {"review": "x"},
|
|
{"review": {"risk_level": "low", "key_issues_to_review": []}},
|
|
{"review": {"merge_recommendation": "ship it", "risk_level": "low"}},
|
|
{"review": {"merge_recommendation": "safe_to_merge"}},
|
|
{"review": {"merge_recommendation": "safe_to_merge", "risk_level": "low",
|
|
"key_issues_to_review": "none"}}):
|
|
with self.subTest(data=data):
|
|
with self.assertRaises(InvalidModelAnswer):
|
|
parse_model_review(data)
|
|
|
|
def test_review_markers_are_stripped(self):
|
|
for content in ("FIXME.review: x\nBreaks X.", "// FIXME.review: x\nBreaks X.", "# FIXME.review: x\nBreaks X.",
|
|
"-- FIXME.review: x\nBreaks X.", "<!-- FIXME.review: x -->\nBreaks X.", "Breaks X."):
|
|
with self.subTest(content=content):
|
|
data = review_data(issues=["a.py"])
|
|
data["review"]["key_issues_to_review"][0]["issue_content"] = content
|
|
self.assertEqual(parse_model_review(data).findings[0].content, "Breaks X.")
|
|
|
|
def test_lone_review_marker_keeps_its_note(self):
|
|
data = review_data(issues=["a.py"])
|
|
data["review"]["key_issues_to_review"][0]["issue_content"] = "// FIXME.review: missing null check"
|
|
self.assertEqual(parse_model_review(data).findings[0].content, "missing null check")
|
|
|
|
def test_finding_location(self):
|
|
finding = parse_model_review(review_data(issues=["a.py"])).findings[0]
|
|
self.assertEqual(finding.location, "a.py:3-5")
|
|
|
|
|
|
class SeverityParsingTest(unittest.TestCase):
|
|
def test_known_prefixes_are_parsed_and_stripped(self):
|
|
for header, severity, title in (("[high] Possible Bug", "high", "Possible Bug"),
|
|
("[Medium] Missing Test", "medium", "Missing Test"),
|
|
(" [ LOW ] Log Noise", "low", "Log Noise"),
|
|
("[HIGH]Possible Bug", "high", "Possible Bug")):
|
|
with self.subTest(header=header):
|
|
finding = parse_model_review(with_headers(header)).findings[0]
|
|
self.assertEqual((finding.severity, finding.severity_known, finding.header), (severity, True, title))
|
|
|
|
def test_missing_or_unknown_prefix_counts_as_medium(self):
|
|
for header in ("Possible Bug", "[critical] Possible Bug", "Possible Bug [high]", ""):
|
|
with self.subTest(header=header):
|
|
finding = parse_model_review(with_headers(header)).findings[0]
|
|
self.assertEqual((finding.severity, finding.severity_known, finding.header),
|
|
("medium", False, header))
|
|
|
|
|
|
class SeverityVerdictTest(unittest.TestCase):
|
|
def test_verdict_follows_the_severities(self):
|
|
for headers, verdict in ((["[high] A"], Verdict.ORANGE),
|
|
(["[medium] A", "[medium] B"], Verdict.ORANGE),
|
|
(["[medium] A", "B"], Verdict.ORANGE),
|
|
(["[medium] A"], Verdict.YELLOW),
|
|
(["[medium] A", "[low] B", "[low] C", "[low] D"], Verdict.YELLOW),
|
|
(["[low] A", "[low] B", "[low] C"], Verdict.GREEN),
|
|
([], Verdict.GREEN)):
|
|
with self.subTest(headers=headers):
|
|
verdict_, reasons = assess(with_headers(*headers))
|
|
self.assertIs(verdict_, verdict)
|
|
if not headers:
|
|
self.assertEqual(reasons, [])
|
|
|
|
def test_low_findings_are_hints(self):
|
|
reasons = assess(with_headers("[low] A", "[medium] B", "[low] C"))[1]
|
|
self.assertEqual([(r.code, r.text, r.verdict) for r in reasons],
|
|
[("finding_unconfirmed", "1 finding(s) of medium severity: a.py:3-5", Verdict.YELLOW),
|
|
("model_minor_findings", "2 finding(s) of low severity: a.py:3-5, a.py:3-5",
|
|
Verdict.GREEN)])
|
|
|
|
def test_high_reason(self):
|
|
reasons = assess(with_headers("[high] A", "[low] B"))[1]
|
|
self.assertEqual([(r.text, r.verdict) for r in reasons],
|
|
[("1 finding(s) that the model rates as high: a.py:3-5", Verdict.ORANGE),
|
|
("1 finding(s) of low severity: a.py:3-5", Verdict.GREEN)])
|
|
|
|
def test_recommendation_alone_is_a_hint(self):
|
|
for data, verdict in ((with_headers("[low] A", recommendation="merge_with_caution"), Verdict.GREEN),
|
|
(with_headers(recommendation="changes_required"), Verdict.GREEN),
|
|
(with_headers("[high] A", recommendation="changes_required"), Verdict.ORANGE),
|
|
(review_data(risk="high"), Verdict.YELLOW),
|
|
(review_data(security="SQL injection: ..."), Verdict.YELLOW)):
|
|
with self.subTest(data=data):
|
|
verdict_, reasons = assess(data)
|
|
self.assertIs(verdict_, verdict)
|
|
reasons = assess(with_headers(recommendation="changes_required"))[1]
|
|
self.assertEqual([(r.code, r.verdict) for r in reasons], [("model_changes_required", Verdict.GREEN)])
|
|
|
|
def test_orange_min_medium_zero_disables_the_medium_rule(self):
|
|
self.assertIs(assess(with_headers("[medium] A", "[medium] B", "C"), orange_min_medium=0)[0], Verdict.YELLOW)
|
|
self.assertIs(assess(with_headers("[high] A"), orange_min_medium=0)[0], Verdict.ORANGE)
|
|
|
|
|
|
class GateTest(unittest.TestCase):
|
|
def test_non_yellow_maps_directly(self):
|
|
for v in (Verdict.GREEN, Verdict.RED, Verdict.ERROR):
|
|
self.assertEqual(gate_decision(v, "pass_with_review_request", "alice", True).exit_code, v.exit_code)
|
|
|
|
def test_yellow_blocks_by_default(self):
|
|
self.assertEqual(gate_decision(Verdict.YELLOW, "block").exit_code, 10)
|
|
|
|
def test_yellow_with_pending_request(self):
|
|
self.assertEqual(gate_decision(Verdict.YELLOW, "pass_with_review_request", None, True).exit_code, 0)
|
|
self.assertEqual(gate_decision(Verdict.YELLOW, "pass_with_review_request", None, False).exit_code, 10)
|
|
self.assertEqual(gate_decision(Verdict.YELLOW, "block", None, True).exit_code, 10)
|
|
|
|
def test_yellow_unlocked_by_approval(self):
|
|
self.assertEqual(gate_decision(Verdict.YELLOW, "block", "alice").exit_code, 0)
|
|
|
|
def test_orange_gates_like_yellow_with_its_own_exit_code(self):
|
|
self.assertEqual(gate_decision(Verdict.ORANGE, "block").exit_code, 15)
|
|
self.assertEqual(gate_decision(Verdict.ORANGE, "block", "alice").exit_code, 0)
|
|
self.assertEqual(gate_decision(Verdict.ORANGE, "pass_with_review_request", None, True).exit_code, 0)
|
|
self.assertEqual(gate_decision(Verdict.ORANGE, "pass_with_review_request", None, False).exit_code, 15)
|
|
|
|
def test_red_is_not_unlocked_by_approval(self):
|
|
self.assertEqual(gate_decision(Verdict.RED, "block", "alice", True).exit_code, 20)
|
|
|
|
|
|
class ApprovalTest(unittest.TestCase):
|
|
def review(self, **kw):
|
|
r = {"state": "APPROVED", "commit_id": HEAD, "user": {"login": "alice"}, "dismissed": False, "stale": False}
|
|
r.update(kw)
|
|
return r
|
|
|
|
def test_valid_approval(self):
|
|
self.assertEqual(human_approval([self.review()], HEAD, ["Alice"]), "alice")
|
|
|
|
def test_rejected_approvals(self):
|
|
for kw in ({"commit_id": "c" * 40}, {"state": "COMMENT"}, {"user": {"login": "mallory"}},
|
|
{"dismissed": True}, {"stale": True}):
|
|
with self.subTest(kw=kw):
|
|
self.assertIsNone(human_approval([self.review(**kw)], HEAD, ["alice"]))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|