import dataclasses import json import unittest from tests.fakes import FakeJev, FakeModel, config, example_config from tests.test_flow import run from tests.test_pathjudge import APP, APP_TEST, BUILD, all_gitea, reason_codes from werkrevisor import risk from werkrevisor.config import ConfigError from werkrevisor.verdict import Verdict TEMPLATE = "jvm-spring-backend" # A measured question may end with one sentence that delimits it, but never without a question mark. QUESTION_FORMAT = r"\?( [^?.]+\.)?$" def risk_config(template=TEMPLATE): path_judge = {"judge": "jev", "scope": "all"} if template is not None: path_judge["risk_template"] = template return config(engine={"mode": "combined", "model_role": "advises"}, gate={"yellow_gate": "block"}, hard_yellow={"paths": [], "judged_paths": ["server/*.gradle.kts"]}, path_judge=path_judge) def risk_run(jev, paths=(APP, APP_TEST), template=TEMPLATE): return run(all_gitea(*paths), FakeModel(prompt_files=list(paths)), cfg=risk_config(template), jev=jev) def questions_for(jev, path): return next(q for s, q in jev.requests if s.startswith(f"Changed file: {path}")) def reason(outcome, code, verdict): return next(r for r in outcome.result.reasons if r.code == code and r.verdict is verdict) class RiskCategoriesTest(unittest.TestCase): def test_a_likely_risk_gives_yellow_and_names_the_category(self): """Scenario#46.01""" outcome, _ = risk_run(FakeJev(problems={(APP, "object_level_authorization"): 0.96})) self.assertIs(outcome.result.verdict, Verdict.YELLOW) self.assertEqual(reason(outcome, "risk_object_level_authorization", Verdict.YELLOW).text, f"Object-level authorization according to Jev: {APP} (0.96); " "a human checks it, unless you revert the change because the task does not need it") def test_a_quality_category_gives_only_a_hint(self): """Scenario#46.02""" outcome, _ = risk_run(FakeJev(problems={(APP, "performance"): 0.97})) self.assertIs(outcome.result.verdict, Verdict.GREEN) self.assertEqual(reason(outcome, "risk_performance", Verdict.GREEN).text, f"Database and I/O performance according to Jev, a hint: {APP} (0.97)") def test_a_very_likely_risk_gives_orange(self): """Scenario#46.03""" outcome, _ = risk_run(FakeJev(problems={(APP, "transaction_integrity"): 0.997})) self.assertIs(outcome.result.verdict, Verdict.ORANGE) self.assertEqual(reason(outcome, "risk_transaction_integrity", Verdict.ORANGE).text, f"Transaction integrity, very likely according to Jev: {APP} (1.00); " "lower the risk, or revert the change because the task does not need it") def test_werkrevisor_holds_a_weakened_gate_for_the_human(self): """Scenario#107.01, Scenario#107.02""" own = example_config("werkrevisor.toml") [gate] = [c for c in risk.resolve(own.path_judge.risk_template, own.risk.categories) if c.name == "gate_weakened"] self.assertEqual((gate.kinds, gate.yellow_min, gate.action, gate.orange_min), (("main",), 0.75, "yellow", None)) cfg = risk_config(template=None) cfg = dataclasses.replace(cfg, risk=dataclasses.replace(cfg.risk, categories=own.risk.categories)) for value, verdict in ((0.84, Verdict.YELLOW), (0.75, Verdict.YELLOW), (0.74, Verdict.GREEN)): with self.subTest(value=value): outcome, _ = run(all_gitea(APP), FakeModel(prompt_files=[APP]), cfg=cfg, jev=FakeJev(problems={(APP, "gate_weakened"): value})) self.assertIs(outcome.result.verdict, verdict) self.assertEqual(reason_codes(outcome).get("risk_gate_weakened") is Verdict.YELLOW, verdict is Verdict.YELLOW) def test_a_test_file_gets_only_the_categories_for_tests(self): """Scenario#46.04""" jev = FakeJev(problems={(APP_TEST, "authorization"): 0.99}) outcome, _ = risk_run(jev) self.assertEqual(set(questions_for(jev, APP_TEST)), {"weakens_checks"}) self.assertEqual(set(questions_for(jev, APP)), {"weakens_checks", "security", *(c.name for c in risk.load(TEMPLATE))}) self.assertIs(outcome.result.verdict, Verdict.GREEN) def test_the_security_question_stays_besides_the_categories(self): """Scenario#46.05""" outcome, _ = risk_run(FakeJev(problems={(APP, "security"): 0.97})) self.assertIs(reason_codes(outcome)["files_touch_security"], Verdict.YELLOW) def test_without_a_template_the_two_measured_questions_stay(self): """Scenario#46.06""" jev = FakeJev() risk_run(jev, template=None) self.assertEqual(set(questions_for(jev, APP)), {"weakens_checks", "security"}) def test_an_unknown_template_is_a_config_error(self): """Scenario#46.07""" with self.assertRaisesRegex(ConfigError, r"\[path_judge\] risk_template: unknown risk template " r"'no-such-template'; known: jvm-spring-backend, typescript-frontend"): risk_config("no-such-template") def test_the_usage_record_logs_each_category_value(self): """Scenario#46.08""" _, d = risk_run(FakeJev(problems={(APP, "authentication"): 0.4})) record = json.loads(d.out.getvalue().strip()) judgment = next(j for j in record["path_judgments"] if j["file"] == APP) self.assertEqual(set(judgment["risks"]), {c.name for c in risk.load(TEMPLATE)}) self.assertAlmostEqual(judgment["risks"]["authentication"], 0.4) self.assertEqual(judgment["verdict"], "GREEN") _, d = risk_run(FakeJev(problems={(APP, "authentication"): 0.96})) judgment = next(j for j in json.loads(d.out.getvalue().strip())["path_judgments"] if j["file"] == APP) self.assertEqual(judgment["verdict"], "YELLOW") def test_a_harmless_file_names_the_template_in_the_hint(self): outcome, _ = risk_run(FakeJev()) self.assertEqual(reason(outcome, "files_judged", Verdict.GREEN).text, "2 changed file(s) that neither weaken the checks, touch security " f"nor show a risk category of {TEMPLATE} according to Jev") self.assertNotIn("files_touch_security", reason_codes(outcome)) def test_a_judged_build_file_keeps_its_questions(self): jev = FakeJev() risk_run(jev, paths=(BUILD, APP)) build = next(q for s, q in jev.requests if s.startswith(f"Build file: {BUILD}")) self.assertEqual(set(build), {"weakens_checks", "build_risk", "security"}) def test_the_template_needs_scope_all_and_is_host_only(self): with self.assertRaisesRegex(ConfigError, r"risk_template needs scope = 'all'"): config(path_judge={"judge": "jev", "risk_template": TEMPLATE}) class RiskTemplatesTest(unittest.TestCase): def test_every_template_loads(self): for name in risk.names(): with self.subTest(name): categories = risk.load(name) self.assertEqual(len(categories), {"jvm-spring-backend": 16, "typescript-frontend": 15}[name]) for c in categories: self.assertRegex(c.question, QUESTION_FORMAT, c.name) def test_the_question_format_needs_a_question_mark(self): for text in ("Does the change alter X?", "Does the change alter X? Tests cannot."): self.assertRegex(text, QUESTION_FORMAT) for text in ("The change alters X.", "Does the change alter X", "Does it? Tests cannot. Docs neither."): self.assertNotRegex(text, QUESTION_FORMAT) def test_quality_categories_start_as_hints(self): hints = {name: sorted(c.name for c in risk.load(name) if c.action == "hint") for name in risk.names()} self.assertEqual(hints, {"jvm-spring-backend": ["api_compatibility", "architecture", "performance"], "typescript-frontend": ["accessibility", "api_contract", "performance"]}) def test_invalid_categories_are_refused(self): base = {"name": "x", "title": "X", "question": "Q?", "yellow_min": 0.9, "action": "yellow"} for change, message in (({"name": "security"}, r"name must be an identifier other than"), ({"action": "maybe"}, r"action must be one of yellow, hint"), ({"kinds": ["docs"]}, r"kinds must be a non-empty list of main, test"), ({"yellow_min": 2}, r"yellow_min must be a number between 0 and 1"), ({"action": "hint", "orange_min": 0.99}, r"orange_min needs action = 'yellow'"), ({"orange_min": 0.5}, r"orange_min must not be below yellow_min"), ({"color": "red"}, r"unknown keys color")): with self.subTest(change=change): with self.assertRaisesRegex(risk.TemplateError, message): risk._category("t", {**base, **change}) class DataCorrectionTest(unittest.TestCase): def test_a_change_that_very_likely_writes_persisted_data_gives_yellow(self): """Scenario#54.01""" outcome, _ = risk_run(FakeJev(problems={(APP, "data_correction"): 0.99})) self.assertIs(outcome.result.verdict, Verdict.YELLOW) self.assertTrue(reason(outcome, "risk_data_correction", Verdict.YELLOW).text.startswith( f"Data that would need corrections according to Jev: {APP} (0.99); ")) outcome, _ = risk_run(FakeJev(problems={(APP, "data_correction"): 0.84})) self.assertIs(outcome.result.verdict, Verdict.GREEN) if __name__ == "__main__": unittest.main()