125 lines
5.8 KiB
Python
125 lines
5.8 KiB
Python
import json
|
|
import unittest
|
|
|
|
from tests.fakes import FakeJev, FakeModel, review_data
|
|
from tests.test_flow import run
|
|
from tests.test_pathjudge import APP, APP_TEST, all_gitea
|
|
from tests.test_risk import risk_config
|
|
from werkrevisor.verdict import Verdict
|
|
|
|
OTHER = "src/main/java/Other.java"
|
|
FINDING_UNIT = f"{APP}:3-5"
|
|
|
|
|
|
def gitea_with_texts(*paths):
|
|
gitea = all_gitea(*paths)
|
|
for p in paths:
|
|
gitea.raw_files[p] = "".join(f"line {i}\n" for i in range(1, 11)).encode()
|
|
return gitea
|
|
|
|
|
|
def confirmed_finding(path=APP):
|
|
"""A finding of the model in path that Jev rates as probably real and severe."""
|
|
model = FakeModel(data=review_data(issues=[path], header="[high] Lost update"), prompt_files=[APP, OTHER, APP_TEST])
|
|
problems = {(f"{path}:3-5", "real_defect"): 0.9, (f"{path}:3-5", "severity"): 0.9}
|
|
return model, problems
|
|
|
|
|
|
def risk_run(problems, model, paths=(APP, OTHER, APP_TEST)):
|
|
return run(gitea_with_texts(*paths), model, cfg=risk_config(), jev=FakeJev(problems=problems))
|
|
|
|
|
|
def reason(outcome, code):
|
|
return next(r for r in outcome.result.reasons if r.code == code)
|
|
|
|
|
|
class RiskWithIssueTest(unittest.TestCase):
|
|
def test_a_very_likely_risk_and_a_confirmed_finding_in_the_same_file_give_red(self):
|
|
"""Scenario#51.01"""
|
|
model, problems = confirmed_finding(APP)
|
|
outcome, _ = risk_run({**problems, (APP, "concurrency"): 0.995}, model)
|
|
self.assertIs(outcome.result.verdict, Verdict.RED)
|
|
self.assertEqual(reason(outcome, "risk_with_issue").text,
|
|
"a very likely risk and a finding that Jev confirms in the same file: "
|
|
f"{APP} (Concurrency and idempotency; Lost update at {APP}:3-5); "
|
|
"only a new commit that lowers the risk or fixes the finding lifts it")
|
|
|
|
def test_a_risk_and_a_finding_in_different_files_stay_orange(self):
|
|
"""Scenario#51.02"""
|
|
model, problems = confirmed_finding(OTHER)
|
|
outcome, _ = risk_run({**problems, (APP, "concurrency"): 0.995}, model)
|
|
self.assertIs(outcome.result.verdict, Verdict.ORANGE)
|
|
self.assertNotIn("risk_with_issue", [r.code for r in outcome.result.reasons])
|
|
|
|
def test_a_finding_that_jev_doubts_gives_no_red(self):
|
|
model, problems = confirmed_finding(APP)
|
|
problems[(FINDING_UNIT, "real_defect")] = 0.2
|
|
outcome, _ = risk_run({**problems, (APP, "concurrency"): 0.995}, model)
|
|
self.assertNotIn("risk_with_issue", [r.code for r in outcome.result.reasons])
|
|
|
|
|
|
class PhaseTimingTest(unittest.TestCase):
|
|
def test_the_usage_record_and_the_comment_show_the_duration_of_each_phase(self):
|
|
"""Scenario#51.03"""
|
|
model, problems = confirmed_finding(APP)
|
|
gitea = gitea_with_texts(APP, OTHER, APP_TEST)
|
|
outcome, d = run(gitea, model, cfg=risk_config(), jev=FakeJev(problems=problems))
|
|
record = json.loads(d.out.getvalue().strip())
|
|
self.assertLessEqual({"path_judge", "model_review", "finding_judge"}, set(record["phase_seconds"]))
|
|
self.assertTrue(all(isinstance(v, float) and v >= 0 for v in record["phase_seconds"].values()))
|
|
self.assertIn("phases: path_judge ", gitea.created_comments[0])
|
|
# No PR document changed, so the classifier called Jev for nothing.
|
|
self.assertNotIn("classifier", record["phase_seconds"])
|
|
|
|
|
|
|
|
SERVICE = "src/main/java/Service.java"
|
|
|
|
|
|
def category_run(category, confidence=0.9, finding_in=APP):
|
|
"""OTHER is very likely risky for concurrency; the confirmed finding lies in finding_in."""
|
|
model, problems = confirmed_finding(finding_in)
|
|
problems = {**problems, (OTHER, "concurrency"): 0.995, (f"{finding_in}:3-5", "category"): category}
|
|
jev = FakeJev(problems=problems, confidences={(f"{finding_in}:3-5", "category"): confidence})
|
|
return run(gitea_with_texts(APP, OTHER, APP_TEST), model, cfg=risk_config(), jev=jev)
|
|
|
|
|
|
class RiskCategoryTest(unittest.TestCase):
|
|
def test_a_very_likely_risk_and_a_confirmed_finding_of_the_same_category_give_red(self):
|
|
"""Scenario#53.01"""
|
|
outcome, d = category_run("concurrency")
|
|
self.assertIs(outcome.result.verdict, Verdict.RED)
|
|
self.assertEqual(reason(outcome, "risk_with_issue").text,
|
|
"a very likely risk and a finding that Jev confirms in the same category: "
|
|
f"Concurrency and idempotency in {OTHER}; Lost update at {APP}:3-5; "
|
|
"only a new commit that lowers the risk or fixes the finding lifts it")
|
|
record = json.loads(d.out.getvalue().strip())
|
|
judgment = next(j for j in record["finding_judgments"] if j["file"] == APP)
|
|
self.assertEqual((judgment["jev_category"], judgment["jev_category_confidence"]), ("concurrency", 0.9))
|
|
|
|
def test_an_uncertain_category_gives_no_red(self):
|
|
"""Scenario#53.02"""
|
|
outcome, _ = category_run("concurrency", confidence=0.6)
|
|
self.assertIs(outcome.result.verdict, Verdict.ORANGE)
|
|
|
|
def test_a_finding_in_test_code_gives_no_red_by_its_category(self):
|
|
"""Scenario#53.03"""
|
|
outcome, _ = category_run("concurrency", finding_in=APP_TEST)
|
|
self.assertIs(outcome.result.verdict, Verdict.ORANGE)
|
|
|
|
def test_another_category_gives_no_red(self):
|
|
outcome, _ = category_run("error_handling")
|
|
self.assertIs(outcome.result.verdict, Verdict.ORANGE)
|
|
|
|
def test_without_a_template_the_finding_judge_asks_no_category(self):
|
|
model, problems = confirmed_finding(APP)
|
|
jev = FakeJev(problems=problems)
|
|
from tests.test_pathjudge import all_config
|
|
run(gitea_with_texts(APP, OTHER, APP_TEST), model, cfg=all_config(), jev=jev)
|
|
finding_questions = next(q for s, q in jev.requests if isinstance(s, str) and s.startswith("Finding:"))
|
|
self.assertEqual(set(finding_questions), {"real_defect", "severity"})
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|