180 lines
9.7 KiB
Python
180 lines
9.7 KiB
Python
import dataclasses
|
|
import json
|
|
import unittest
|
|
|
|
from tests.fakes import FakeJev, FakeModel, config, example_config
|
|
from tests.test_flow import run
|
|
from tests.test_pathjudge import APP, APP_TEST, BUILD, all_gitea, reason_codes
|
|
from werkrevisor import risk
|
|
from werkrevisor.config import ConfigError
|
|
from werkrevisor.verdict import Verdict
|
|
|
|
TEMPLATE = "jvm-spring-backend"
|
|
# A measured question may end with one sentence that delimits it, but never without a question mark.
|
|
QUESTION_FORMAT = r"\?( [^?.]+\.)?$"
|
|
|
|
|
|
def risk_config(template=TEMPLATE):
|
|
path_judge = {"judge": "jev", "scope": "all"}
|
|
if template is not None:
|
|
path_judge["risk_template"] = template
|
|
return config(engine={"mode": "combined", "model_role": "advises"}, gate={"yellow_gate": "block"},
|
|
hard_yellow={"paths": [], "judged_paths": ["server/*.gradle.kts"]}, path_judge=path_judge)
|
|
|
|
|
|
def risk_run(jev, paths=(APP, APP_TEST), template=TEMPLATE):
|
|
return run(all_gitea(*paths), FakeModel(prompt_files=list(paths)), cfg=risk_config(template), jev=jev)
|
|
|
|
|
|
def questions_for(jev, path):
|
|
return next(q for s, q in jev.requests if s.startswith(f"Changed file: {path}"))
|
|
|
|
|
|
def reason(outcome, code, verdict):
|
|
return next(r for r in outcome.result.reasons if r.code == code and r.verdict is verdict)
|
|
|
|
|
|
class RiskCategoriesTest(unittest.TestCase):
|
|
def test_a_likely_risk_gives_yellow_and_names_the_category(self):
|
|
"""Scenario#46.01"""
|
|
outcome, _ = risk_run(FakeJev(problems={(APP, "object_level_authorization"): 0.96}))
|
|
self.assertIs(outcome.result.verdict, Verdict.YELLOW)
|
|
self.assertEqual(reason(outcome, "risk_object_level_authorization", Verdict.YELLOW).text,
|
|
f"Object-level authorization according to Jev: {APP} (0.96); "
|
|
"a human checks it, unless you revert the change because the task does not need it")
|
|
|
|
def test_a_quality_category_gives_only_a_hint(self):
|
|
"""Scenario#46.02"""
|
|
outcome, _ = risk_run(FakeJev(problems={(APP, "performance"): 0.97}))
|
|
self.assertIs(outcome.result.verdict, Verdict.GREEN)
|
|
self.assertEqual(reason(outcome, "risk_performance", Verdict.GREEN).text,
|
|
f"Database and I/O performance according to Jev, a hint: {APP} (0.97)")
|
|
|
|
def test_a_very_likely_risk_gives_orange(self):
|
|
"""Scenario#46.03"""
|
|
outcome, _ = risk_run(FakeJev(problems={(APP, "transaction_integrity"): 0.997}))
|
|
self.assertIs(outcome.result.verdict, Verdict.ORANGE)
|
|
self.assertEqual(reason(outcome, "risk_transaction_integrity", Verdict.ORANGE).text,
|
|
f"Transaction integrity, very likely according to Jev: {APP} (1.00); "
|
|
"lower the risk, or revert the change because the task does not need it")
|
|
|
|
def test_werkrevisor_holds_a_weakened_gate_for_the_human(self):
|
|
"""Scenario#107.01, Scenario#107.02"""
|
|
own = example_config("werkrevisor.toml")
|
|
[gate] = [c for c in risk.resolve(own.path_judge.risk_template, own.risk.categories) if c.name == "gate_weakened"]
|
|
self.assertEqual((gate.kinds, gate.yellow_min, gate.action, gate.orange_min), (("main",), 0.75, "yellow", None))
|
|
cfg = risk_config(template=None)
|
|
cfg = dataclasses.replace(cfg, risk=dataclasses.replace(cfg.risk, categories=own.risk.categories))
|
|
for value, verdict in ((0.84, Verdict.YELLOW), (0.75, Verdict.YELLOW), (0.74, Verdict.GREEN)):
|
|
with self.subTest(value=value):
|
|
outcome, _ = run(all_gitea(APP), FakeModel(prompt_files=[APP]), cfg=cfg,
|
|
jev=FakeJev(problems={(APP, "gate_weakened"): value}))
|
|
self.assertIs(outcome.result.verdict, verdict)
|
|
self.assertEqual(reason_codes(outcome).get("risk_gate_weakened") is Verdict.YELLOW,
|
|
verdict is Verdict.YELLOW)
|
|
|
|
def test_a_test_file_gets_only_the_categories_for_tests(self):
|
|
"""Scenario#46.04"""
|
|
jev = FakeJev(problems={(APP_TEST, "authorization"): 0.99})
|
|
outcome, _ = risk_run(jev)
|
|
self.assertEqual(set(questions_for(jev, APP_TEST)), {"weakens_checks"})
|
|
self.assertEqual(set(questions_for(jev, APP)),
|
|
{"weakens_checks", "security", *(c.name for c in risk.load(TEMPLATE))})
|
|
self.assertIs(outcome.result.verdict, Verdict.GREEN)
|
|
|
|
def test_the_security_question_stays_besides_the_categories(self):
|
|
"""Scenario#46.05"""
|
|
outcome, _ = risk_run(FakeJev(problems={(APP, "security"): 0.97}))
|
|
self.assertIs(reason_codes(outcome)["files_touch_security"], Verdict.YELLOW)
|
|
|
|
def test_without_a_template_the_two_measured_questions_stay(self):
|
|
"""Scenario#46.06"""
|
|
jev = FakeJev()
|
|
risk_run(jev, template=None)
|
|
self.assertEqual(set(questions_for(jev, APP)), {"weakens_checks", "security"})
|
|
|
|
def test_an_unknown_template_is_a_config_error(self):
|
|
"""Scenario#46.07"""
|
|
with self.assertRaisesRegex(ConfigError, r"\[path_judge\] risk_template: unknown risk template "
|
|
r"'no-such-template'; known: jvm-spring-backend, typescript-frontend"):
|
|
risk_config("no-such-template")
|
|
|
|
def test_the_usage_record_logs_each_category_value(self):
|
|
"""Scenario#46.08"""
|
|
_, d = risk_run(FakeJev(problems={(APP, "authentication"): 0.4}))
|
|
record = json.loads(d.out.getvalue().strip())
|
|
judgment = next(j for j in record["path_judgments"] if j["file"] == APP)
|
|
self.assertEqual(set(judgment["risks"]), {c.name for c in risk.load(TEMPLATE)})
|
|
self.assertAlmostEqual(judgment["risks"]["authentication"], 0.4)
|
|
self.assertEqual(judgment["verdict"], "GREEN")
|
|
_, d = risk_run(FakeJev(problems={(APP, "authentication"): 0.96}))
|
|
judgment = next(j for j in json.loads(d.out.getvalue().strip())["path_judgments"] if j["file"] == APP)
|
|
self.assertEqual(judgment["verdict"], "YELLOW")
|
|
|
|
def test_a_harmless_file_names_the_template_in_the_hint(self):
|
|
outcome, _ = risk_run(FakeJev())
|
|
self.assertEqual(reason(outcome, "files_judged", Verdict.GREEN).text,
|
|
"2 changed file(s) that neither weaken the checks, touch security "
|
|
f"nor show a risk category of {TEMPLATE} according to Jev")
|
|
self.assertNotIn("files_touch_security", reason_codes(outcome))
|
|
|
|
def test_a_judged_build_file_keeps_its_questions(self):
|
|
jev = FakeJev()
|
|
risk_run(jev, paths=(BUILD, APP))
|
|
build = next(q for s, q in jev.requests if s.startswith(f"Build file: {BUILD}"))
|
|
self.assertEqual(set(build), {"weakens_checks", "build_risk", "security"})
|
|
|
|
def test_the_template_needs_scope_all_and_is_host_only(self):
|
|
with self.assertRaisesRegex(ConfigError, r"risk_template needs scope = 'all'"):
|
|
config(path_judge={"judge": "jev", "risk_template": TEMPLATE})
|
|
|
|
|
|
class RiskTemplatesTest(unittest.TestCase):
|
|
def test_every_template_loads(self):
|
|
for name in risk.names():
|
|
with self.subTest(name):
|
|
categories = risk.load(name)
|
|
self.assertEqual(len(categories), {"jvm-spring-backend": 16, "typescript-frontend": 15}[name])
|
|
for c in categories:
|
|
self.assertRegex(c.question, QUESTION_FORMAT, c.name)
|
|
|
|
def test_the_question_format_needs_a_question_mark(self):
|
|
for text in ("Does the change alter X?", "Does the change alter X? Tests cannot."):
|
|
self.assertRegex(text, QUESTION_FORMAT)
|
|
for text in ("The change alters X.", "Does the change alter X", "Does it? Tests cannot. Docs neither."):
|
|
self.assertNotRegex(text, QUESTION_FORMAT)
|
|
|
|
def test_quality_categories_start_as_hints(self):
|
|
hints = {name: sorted(c.name for c in risk.load(name) if c.action == "hint") for name in risk.names()}
|
|
self.assertEqual(hints, {"jvm-spring-backend": ["api_compatibility", "architecture", "performance"],
|
|
"typescript-frontend": ["accessibility", "api_contract", "performance"]})
|
|
|
|
def test_invalid_categories_are_refused(self):
|
|
base = {"name": "x", "title": "X", "question": "Q?", "yellow_min": 0.9, "action": "yellow"}
|
|
for change, message in (({"name": "security"}, r"name must be an identifier other than"),
|
|
({"action": "maybe"}, r"action must be one of yellow, hint"),
|
|
({"kinds": ["docs"]}, r"kinds must be a non-empty list of main, test"),
|
|
({"yellow_min": 2}, r"yellow_min must be a number between 0 and 1"),
|
|
({"action": "hint", "orange_min": 0.99}, r"orange_min needs action = 'yellow'"),
|
|
({"orange_min": 0.5}, r"orange_min must not be below yellow_min"),
|
|
({"color": "red"}, r"unknown keys color")):
|
|
with self.subTest(change=change):
|
|
with self.assertRaisesRegex(risk.TemplateError, message):
|
|
risk._category("t", {**base, **change})
|
|
|
|
|
|
|
|
class DataCorrectionTest(unittest.TestCase):
|
|
def test_a_change_that_very_likely_writes_persisted_data_gives_yellow(self):
|
|
"""Scenario#54.01"""
|
|
outcome, _ = risk_run(FakeJev(problems={(APP, "data_correction"): 0.99}))
|
|
self.assertIs(outcome.result.verdict, Verdict.YELLOW)
|
|
self.assertTrue(reason(outcome, "risk_data_correction", Verdict.YELLOW).text.startswith(
|
|
f"Data that would need corrections according to Jev: {APP} (0.99); "))
|
|
outcome, _ = risk_run(FakeJev(problems={(APP, "data_correction"): 0.84}))
|
|
self.assertIs(outcome.result.verdict, Verdict.GREEN)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|