255 lines
15 KiB
Python
255 lines
15 KiB
Python
import os
|
|
import tomllib
|
|
import unittest
|
|
|
|
from tests.fakes import TEST_CONFIG, FakeGitea, FakeModel, config, deps, diff_for, gitea_file, pr_ref
|
|
from werkrevisor.cli import execute
|
|
from werkrevisor.config import build_config, parse_repository_config
|
|
from werkrevisor.diff import parse_unified_diff
|
|
from werkrevisor.redact import Redactor
|
|
from werkrevisor.rules import change_kind_instruction, evaluate_rules, files_from_gitea
|
|
from werkrevisor.verdict import Verdict
|
|
|
|
MAIN = "src/main/java/A.java"
|
|
TEST = "src/test/java/ATest.java"
|
|
DOC = "doc/PR/1.md"
|
|
API = "src/main/resources/api-definition/api.yaml"
|
|
SCHEMA = "src/main/resources/db/changelog/1-create.sql"
|
|
|
|
|
|
def evaluate(files, diff="", **overrides):
|
|
return evaluate_rules(files_from_gitea(files), parse_unified_diff(diff), config(**overrides))
|
|
|
|
|
|
def main_and_test(test_added=(), test_removed=(), **overrides):
|
|
files = [gitea_file(MAIN), gitea_file(TEST, deletions=len(test_removed))]
|
|
diff = diff_for(MAIN, added=["int x;"]) + diff_for(TEST, added=test_added, removed=test_removed)
|
|
return evaluate(files, diff, **overrides)
|
|
|
|
|
|
def weakening(outcome):
|
|
return next((r.text for r in outcome.reasons if r.code == "tests_weakened"), None)
|
|
|
|
|
|
class ChangeKindTest(unittest.TestCase):
|
|
def test_change_kind_matrix(self):
|
|
added = ["@Test void y() { assertThat(1).isOne(); }"]
|
|
changed = {"added": ["var a = new A(2);"], "removed": ["var a = new A(1);"]}
|
|
cases = (
|
|
("docs", [DOC], diff_for(DOC, added=["x"]), "docs_only", True),
|
|
("added tests", [TEST], diff_for(TEST, added=added), "tests_only_added", True),
|
|
("docs and added tests", [DOC, TEST], diff_for(TEST, added=added), "docs_and_tests", True),
|
|
("main code", [MAIN], diff_for(MAIN, added=["x"]), "refactoring", True),
|
|
("main code and docs", [MAIN, DOC], diff_for(MAIN, added=["x"]), "refactoring", True),
|
|
("main code and added tests", [MAIN, TEST], diff_for(TEST, added=added), "feature_with_tests", True),
|
|
("main code, added tests and docs", [MAIN, TEST, DOC], diff_for(TEST, added=added),
|
|
"feature_with_tests", True),
|
|
("tests with a changed line", [TEST], diff_for(TEST, **changed), "tests_only_added", True),
|
|
("main code and tests with a changed line", [MAIN, TEST], diff_for(TEST, **changed),
|
|
"feature_with_tests", True),
|
|
("tests with a lost assertion", [TEST], diff_for(TEST, removed=["assertThat(1).isOne();"]),
|
|
"tests_modified", False),
|
|
("main code and tests with a lost assertion", [MAIN, TEST], diff_for(TEST, removed=["assertThat(1).isOne();"]),
|
|
"feature_with_modified_tests", False),
|
|
("main code and an API spec", [MAIN, API], diff_for(MAIN, added=["x"]), "mixed", True),
|
|
("main code, tests and an other file", [MAIN, TEST, "Makefile"], diff_for(TEST, added=added),
|
|
"mixed", True),
|
|
)
|
|
for name, paths, diff, kind, approvable in cases:
|
|
with self.subTest(name):
|
|
outcome = evaluate([gitea_file(p) for p in paths], diff)
|
|
self.assertEqual(outcome.change_kind, kind)
|
|
self.assertIs(outcome.auto_approvable, approvable)
|
|
self.assertIs(outcome.verdict, Verdict.GREEN if approvable else Verdict.YELLOW)
|
|
|
|
def test_not_approvable_reason_names_the_feature_kind(self):
|
|
outcome = main_and_test(test_added=["@Test void y() {}"], categories={"auto_approvable": ["docs_only"]})
|
|
reason = next(r for r in outcome.reasons if r.code == "category")
|
|
self.assertEqual(reason.text, "change kind 'feature_with_tests' (main code, tests); "
|
|
"AI-approvable are only: docs only")
|
|
|
|
|
|
class FeatureWithTestsFlowTest(unittest.TestCase):
|
|
def test_feature_with_additive_tests_is_green(self):
|
|
gitea = FakeGitea(files=[gitea_file(MAIN), gitea_file(TEST)],
|
|
diff=diff_for(MAIN, added=["int x;"])
|
|
+ diff_for(TEST, added=[" @Test void y() {", " assertThat(a.y()).isEqualTo(2);",
|
|
" }"]))
|
|
model = FakeModel(prompt_files=[MAIN, TEST])
|
|
outcome = execute(config(gate={"yellow_gate": "block"}), pr_ref(), deps(gitea, model), Redactor())
|
|
self.assertIs(outcome.result.verdict, Verdict.GREEN)
|
|
self.assertEqual(outcome.result.change_kind, "feature_with_tests")
|
|
self.assertIn("Check that the added tests cover the changed behavior", model.calls[0])
|
|
|
|
|
|
class TestWeakeningTest(unittest.TestCase):
|
|
def test_new_assert_line_in_an_existing_method_is_additive(self):
|
|
outcome = main_and_test(test_added=[" assertThat(a.y()).isEqualTo(2);"])
|
|
self.assertEqual(outcome.change_kind, "feature_with_tests")
|
|
self.assertIsNone(weakening(outcome))
|
|
self.assertEqual(outcome.changed_tests, [])
|
|
|
|
def test_only_lost_or_disabled_tests_give_yellow(self):
|
|
cases = (
|
|
("changed assertion", [" assertEquals(2, x);"], [" assertEquals(1, x);"], None),
|
|
("changed setup line", [" var a = new A(2);"], [" var a = new A(1);"], None),
|
|
("removed assertion", [], [" assertEquals(1, x);"],
|
|
f"{TEST}:1 removes 1 test(s) or assertion(s) without a replacement"),
|
|
("removed test method", [], [" @Test", " void x() { assertEquals(1, x()); }"],
|
|
f"{TEST}:1 removes 2 test(s) or assertion(s) without a replacement"),
|
|
("added skip marker", [" @Disabled"], [], f"{TEST}:1 disables a test"),
|
|
("removed assertion, added test method", [" @Test void y() {}"], [" assertEquals(1, x);"],
|
|
f"{TEST}:1 removes 1 test(s) or assertion(s) without a replacement"),
|
|
)
|
|
for name, added, removed, finding in cases:
|
|
with self.subTest(name):
|
|
outcome = main_and_test(test_added=added, test_removed=removed)
|
|
if finding is None:
|
|
self.assertIsNone(weakening(outcome))
|
|
self.assertIs(outcome.verdict, Verdict.GREEN)
|
|
self.assertEqual(outcome.change_kind, "feature_with_tests")
|
|
else:
|
|
self.assertIs(outcome.verdict, Verdict.YELLOW)
|
|
self.assertEqual(outcome.change_kind, "feature_with_modified_tests")
|
|
self.assertEqual(weakening(outcome), f"tests lost or disabled: {finding}")
|
|
outcome = evaluate([gitea_file(MAIN), gitea_file(TEST, status="deleted", deletions=1)],
|
|
diff_for(MAIN, added=["x"]) + diff_for(TEST, removed=["class ATest {}"], deleted=True))
|
|
self.assertEqual(weakening(outcome), f"tests lost or disabled: {TEST} deleted")
|
|
|
|
def test_changed_test_lines_go_to_the_model(self):
|
|
outcome = main_and_test(test_added=[" assertEquals(2, x);"], test_removed=[" assertEquals(1, x);"])
|
|
self.assertEqual(outcome.changed_tests, [f"{TEST}:1"])
|
|
self.assertIn(f"Existing test lines changed at {TEST}:1. Check that each change keeps the test "
|
|
"at least as strict as before; report a weakened test as a [high] issue.",
|
|
change_kind_instruction(outcome))
|
|
gitea = FakeGitea(files=[gitea_file(MAIN), gitea_file(TEST, deletions=1)],
|
|
diff=diff_for(MAIN, added=["int x;"])
|
|
+ diff_for(TEST, added=[" assertEquals(2, x);"], removed=[" assertEquals(1, x);"]))
|
|
model = FakeModel(prompt_files=[MAIN, TEST])
|
|
outcome = execute(config(gate={"yellow_gate": "block"}), pr_ref(), deps(gitea, model), Redactor())
|
|
self.assertIs(outcome.result.verdict, Verdict.GREEN)
|
|
self.assertIn(f"Existing test lines changed at {TEST}:1.", model.calls[0])
|
|
|
|
def test_re_added_structural_line_is_not_a_change(self):
|
|
outcome = main_and_test(test_added=[" @Test void y() {}", "}"], test_removed=["}"])
|
|
self.assertIsNone(weakening(outcome))
|
|
self.assertEqual(outcome.changed_tests, [])
|
|
|
|
def test_a_moved_line_with_content_goes_to_the_model(self):
|
|
line = " var a = new A(1);"
|
|
outcome = main_and_test(test_added=[" @Test void y() {", line, " }"], test_removed=[line])
|
|
self.assertIsNone(weakening(outcome))
|
|
self.assertEqual(outcome.changed_tests, [f"{TEST}:1"])
|
|
|
|
def test_skip_markers_inside_string_literals_are_test_data(self):
|
|
for line in (' ("@Disabled", "@Ignore"),', """ ('xit("a")', '.skip(')"""):
|
|
with self.subTest(line=line):
|
|
self.assertIsNone(weakening(main_and_test(test_added=[line])))
|
|
|
|
def test_blank_line_removal_is_additive(self):
|
|
outcome = main_and_test(test_added=[" @Test void y() {}"], test_removed=[""])
|
|
self.assertIsNone(weakening(outcome))
|
|
self.assertEqual(outcome.changed_tests, [])
|
|
|
|
def test_disabled_test_is_yellow(self):
|
|
for line in ("@Disabled", "@Ignore", "@pytest.mark.skip", '@unittest.skip("flaky")', 'xit("works", () => {})',
|
|
'describe.skip("all", () => {})', 'it.only("this one", () => {})', 'self.skipTest("later")',
|
|
"@pytest.mark.xfail"):
|
|
with self.subTest(line=line):
|
|
outcome = main_and_test(test_added=[line])
|
|
self.assertIs(outcome.verdict, Verdict.YELLOW)
|
|
self.assertEqual(weakening(outcome), f"tests lost or disabled: {TEST}:1 disables a test")
|
|
|
|
|
|
class MixedTest(unittest.TestCase):
|
|
def test_mixed_is_approvable(self):
|
|
gitea = FakeGitea(files=[gitea_file(MAIN), gitea_file("Makefile")],
|
|
diff=diff_for(MAIN, added=["int x;"]) + diff_for("Makefile", added=["all: build"]))
|
|
model = FakeModel(prompt_files=[MAIN, "Makefile"])
|
|
outcome = execute(config(gate={"yellow_gate": "block"}), pr_ref(), deps(gitea, model), Redactor())
|
|
self.assertEqual(outcome.result.change_kind, "mixed")
|
|
self.assertIs(outcome.result.verdict, Verdict.GREEN)
|
|
|
|
|
|
class HarmlessTest(unittest.TestCase):
|
|
def test_harmless_file_does_not_spoil_the_change_kind(self):
|
|
harmless = {"categories": {"harmless": [".editorconfig"]}}
|
|
for other, kind in ((".editorconfig", "refactoring"), ("Makefile", "mixed")):
|
|
with self.subTest(other=other):
|
|
outcome = evaluate([gitea_file(MAIN), gitea_file(other)], diff_for(MAIN, added=["x"]), **harmless)
|
|
self.assertEqual(outcome.change_kind, kind)
|
|
outcome = evaluate([gitea_file(DOC), gitea_file(".editorconfig")], **harmless)
|
|
self.assertEqual(outcome.change_kind, "docs_only")
|
|
outcome = evaluate([gitea_file(MAIN), gitea_file(TEST), gitea_file(".editorconfig")],
|
|
diff_for(TEST, added=["@Test void y() {}"]), **harmless)
|
|
self.assertEqual(outcome.change_kind, "feature_with_tests")
|
|
|
|
def test_harmless_only_is_approvable(self):
|
|
outcome = evaluate([gitea_file(".editorconfig")], categories={"harmless": [".editorconfig"]})
|
|
self.assertEqual(outcome.change_kind, "harmless_only")
|
|
self.assertTrue(outcome.auto_approvable)
|
|
self.assertIs(outcome.verdict, Verdict.GREEN)
|
|
|
|
def test_harmless_never_hides_risky_paths_or_main_code(self):
|
|
outcome = evaluate([gitea_file(".gitea/workflows/ci.yaml")], categories={"harmless": ["**"]})
|
|
self.assertIs(outcome.verdict, Verdict.YELLOW)
|
|
self.assertIn("risky paths changed: .gitea/workflows/ci.yaml (.gitea/**)", [r.text for r in outcome.reasons])
|
|
outcome = evaluate([gitea_file(MAIN), gitea_file(API)], categories={"harmless": ["**"]})
|
|
self.assertEqual(outcome.categories, {MAIN: "main", API: "api_spec"})
|
|
self.assertEqual(outcome.change_kind, "mixed")
|
|
|
|
def test_harmless_comes_from_host_or_repository_and_defaults_to_empty(self):
|
|
with open(TEST_CONFIG, "rb") as f:
|
|
host = tomllib.load(f)
|
|
self.assertEqual(build_config(host).categories.harmless, ())
|
|
repo = parse_repository_config(b'[categories]\nharmless = [".editorconfig"]\n')
|
|
self.assertEqual(build_config(host, repo).categories.harmless, (".editorconfig",))
|
|
host["categories"]["harmless"] = [".gitignore"]
|
|
self.assertEqual(build_config(host).categories.harmless, (".gitignore",))
|
|
|
|
|
|
class DbSchemaTest(unittest.TestCase):
|
|
PATHS = (SCHEMA, "app/migrations/0002_add_field.py", "src/main/resources/db/migration/V2__x.java",
|
|
"src/main/resources/liquibase/master.xml", "src/main/resources/flyway/V3.sql", "db/schema.changelog.yaml",
|
|
"src/main/resources/db.changelog-master.yaml")
|
|
|
|
def test_db_schema_change_is_yellow(self):
|
|
for path in self.PATHS:
|
|
with self.subTest(path=path):
|
|
outcome = evaluate([gitea_file(path)], hard_yellow={"paths": []})
|
|
self.assertIs(outcome.verdict, Verdict.YELLOW)
|
|
self.assertIn(f"DB schema changed: {path} (built in)", [r.text for r in outcome.reasons])
|
|
|
|
def test_no_layer_removes_the_db_schema_paths(self):
|
|
weakest = {"hard_yellow": {"paths": []}, "categories": {"docs": ["**"], "harmless": ["**"]}}
|
|
outcome = evaluate([gitea_file(SCHEMA)], **weakest)
|
|
self.assertIn(f"DB schema changed: {SCHEMA} (built in)", [r.text for r in outcome.reasons])
|
|
self.assertIs(outcome.verdict, Verdict.YELLOW)
|
|
with open(TEST_CONFIG, "rb") as f:
|
|
host = tomllib.load(f)
|
|
host["hard_yellow"]["paths"] = []
|
|
repo = parse_repository_config(b'[hard_yellow]\npaths = []\n[categories]\ndocs = ["**"]\nharmless = ["**"]\n')
|
|
outcome = evaluate_rules(files_from_gitea([gitea_file(SCHEMA)]), {}, build_config(host, repo))
|
|
self.assertIn(f"DB schema changed: {SCHEMA} (built in)", [r.text for r in outcome.reasons])
|
|
outcome = evaluate([gitea_file("app/migrations/README.md")], **weakest)
|
|
self.assertNotIn("db_schema", [r.code for r in outcome.reasons])
|
|
|
|
def test_test_files_are_no_db_schema(self):
|
|
for path in ("src/test/java/net/x/migration/LiquibaseCompatibilityIntegrationTest.java",
|
|
"src/test/resources/migration/released-prod-schema.sql", "tests/migrations/test_0002.py"):
|
|
with self.subTest(path=path):
|
|
outcome = evaluate([gitea_file(path)], hard_yellow={"paths": []})
|
|
self.assertNotIn("db_schema", [r.code for r in outcome.reasons])
|
|
outcome = evaluate([gitea_file(SCHEMA)], hard_yellow={"paths": []}, categories={"tests": ["**"]})
|
|
self.assertIn("db_schema", [r.code for r in outcome.reasons])
|
|
|
|
def test_renamed_schema_file_counts_with_its_old_path(self):
|
|
f = gitea_file("archive/1-create.txt", status="renamed")
|
|
f["previous_filename"] = SCHEMA
|
|
outcome = evaluate([f], hard_yellow={"paths": []})
|
|
self.assertIn(f"DB schema changed: {SCHEMA} (built in)", [r.text for r in outcome.reasons])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|