import json import unittest from tests.fakes import FakeJev, FakeModel, review_data from tests.test_flow import run from tests.test_pathjudge import APP, APP_TEST, all_gitea from tests.test_risk import risk_config from werkrevisor.verdict import Verdict OTHER = "src/main/java/Other.java" FINDING_UNIT = f"{APP}:3-5" def gitea_with_texts(*paths): gitea = all_gitea(*paths) for p in paths: gitea.raw_files[p] = "".join(f"line {i}\n" for i in range(1, 11)).encode() return gitea def confirmed_finding(path=APP): """A finding of the model in path that Jev rates as probably real and severe.""" model = FakeModel(data=review_data(issues=[path], header="[high] Lost update"), prompt_files=[APP, OTHER, APP_TEST]) problems = {(f"{path}:3-5", "real_defect"): 0.9, (f"{path}:3-5", "severity"): 0.9} return model, problems def risk_run(problems, model, paths=(APP, OTHER, APP_TEST)): return run(gitea_with_texts(*paths), model, cfg=risk_config(), jev=FakeJev(problems=problems)) def reason(outcome, code): return next(r for r in outcome.result.reasons if r.code == code) class RiskWithIssueTest(unittest.TestCase): def test_a_very_likely_risk_and_a_confirmed_finding_in_the_same_file_give_red(self): """Scenario#51.01""" model, problems = confirmed_finding(APP) outcome, _ = risk_run({**problems, (APP, "concurrency"): 0.995}, model) self.assertIs(outcome.result.verdict, Verdict.RED) self.assertEqual(reason(outcome, "risk_with_issue").text, "a very likely risk and a finding that Jev confirms in the same file: " f"{APP} (Concurrency and idempotency; Lost update at {APP}:3-5); " "only a new commit that lowers the risk or fixes the finding lifts it") def test_a_risk_and_a_finding_in_different_files_stay_orange(self): """Scenario#51.02""" model, problems = confirmed_finding(OTHER) outcome, _ = risk_run({**problems, (APP, "concurrency"): 0.995}, model) self.assertIs(outcome.result.verdict, Verdict.ORANGE) self.assertNotIn("risk_with_issue", [r.code for r in outcome.result.reasons]) def test_a_finding_that_jev_doubts_gives_no_red(self): model, problems = confirmed_finding(APP) problems[(FINDING_UNIT, "real_defect")] = 0.2 outcome, _ = risk_run({**problems, (APP, "concurrency"): 0.995}, model) self.assertNotIn("risk_with_issue", [r.code for r in outcome.result.reasons]) class PhaseTimingTest(unittest.TestCase): def test_the_usage_record_and_the_comment_show_the_duration_of_each_phase(self): """Scenario#51.03""" model, problems = confirmed_finding(APP) gitea = gitea_with_texts(APP, OTHER, APP_TEST) outcome, d = run(gitea, model, cfg=risk_config(), jev=FakeJev(problems=problems)) record = json.loads(d.out.getvalue().strip()) self.assertLessEqual({"path_judge", "model_review", "finding_judge"}, set(record["phase_seconds"])) self.assertTrue(all(isinstance(v, float) and v >= 0 for v in record["phase_seconds"].values())) self.assertIn("phases: path_judge ", gitea.created_comments[0]) # No PR document changed, so the classifier called Jev for nothing. self.assertNotIn("classifier", record["phase_seconds"]) SERVICE = "src/main/java/Service.java" def category_run(category, confidence=0.9, finding_in=APP): """OTHER is very likely risky for concurrency; the confirmed finding lies in finding_in.""" model, problems = confirmed_finding(finding_in) problems = {**problems, (OTHER, "concurrency"): 0.995, (f"{finding_in}:3-5", "category"): category} jev = FakeJev(problems=problems, confidences={(f"{finding_in}:3-5", "category"): confidence}) return run(gitea_with_texts(APP, OTHER, APP_TEST), model, cfg=risk_config(), jev=jev) class RiskCategoryTest(unittest.TestCase): def test_a_very_likely_risk_and_a_confirmed_finding_of_the_same_category_give_red(self): """Scenario#53.01""" outcome, d = category_run("concurrency") self.assertIs(outcome.result.verdict, Verdict.RED) self.assertEqual(reason(outcome, "risk_with_issue").text, "a very likely risk and a finding that Jev confirms in the same category: " f"Concurrency and idempotency in {OTHER}; Lost update at {APP}:3-5; " "only a new commit that lowers the risk or fixes the finding lifts it") record = json.loads(d.out.getvalue().strip()) judgment = next(j for j in record["finding_judgments"] if j["file"] == APP) self.assertEqual((judgment["jev_category"], judgment["jev_category_confidence"]), ("concurrency", 0.9)) def test_an_uncertain_category_gives_no_red(self): """Scenario#53.02""" outcome, _ = category_run("concurrency", confidence=0.6) self.assertIs(outcome.result.verdict, Verdict.ORANGE) def test_a_finding_in_test_code_gives_no_red_by_its_category(self): """Scenario#53.03""" outcome, _ = category_run("concurrency", finding_in=APP_TEST) self.assertIs(outcome.result.verdict, Verdict.ORANGE) def test_another_category_gives_no_red(self): outcome, _ = category_run("error_handling") self.assertIs(outcome.result.verdict, Verdict.ORANGE) def test_without_a_template_the_finding_judge_asks_no_category(self): model, problems = confirmed_finding(APP) jev = FakeJev(problems=problems) from tests.test_pathjudge import all_config run(gitea_with_texts(APP, OTHER, APP_TEST), model, cfg=all_config(), jev=jev) finding_questions = next(q for s, q in jev.requests if isinstance(s, str) and s.startswith("Finding:")) self.assertEqual(set(finding_questions), {"real_defect", "severity"}) if __name__ == "__main__": unittest.main()