import os import tomllib import unittest from tests.fakes import TEST_CONFIG, FakeGitea, FakeModel, config, deps, diff_for, gitea_file, pr_ref from werkrevisor.cli import execute from werkrevisor.config import build_config, parse_repository_config from werkrevisor.diff import parse_unified_diff from werkrevisor.redact import Redactor from werkrevisor.rules import change_kind_instruction, evaluate_rules, files_from_gitea from werkrevisor.verdict import Verdict MAIN = "src/main/java/A.java" TEST = "src/test/java/ATest.java" DOC = "doc/PR/1.md" API = "src/main/resources/api-definition/api.yaml" SCHEMA = "src/main/resources/db/changelog/1-create.sql" def evaluate(files, diff="", **overrides): return evaluate_rules(files_from_gitea(files), parse_unified_diff(diff), config(**overrides)) def main_and_test(test_added=(), test_removed=(), **overrides): files = [gitea_file(MAIN), gitea_file(TEST, deletions=len(test_removed))] diff = diff_for(MAIN, added=["int x;"]) + diff_for(TEST, added=test_added, removed=test_removed) return evaluate(files, diff, **overrides) def weakening(outcome): return next((r.text for r in outcome.reasons if r.code == "tests_weakened"), None) class ChangeKindTest(unittest.TestCase): def test_change_kind_matrix(self): added = ["@Test void y() { assertThat(1).isOne(); }"] changed = {"added": ["var a = new A(2);"], "removed": ["var a = new A(1);"]} cases = ( ("docs", [DOC], diff_for(DOC, added=["x"]), "docs_only", True), ("added tests", [TEST], diff_for(TEST, added=added), "tests_only_added", True), ("docs and added tests", [DOC, TEST], diff_for(TEST, added=added), "docs_and_tests", True), ("main code", [MAIN], diff_for(MAIN, added=["x"]), "refactoring", True), ("main code and docs", [MAIN, DOC], diff_for(MAIN, added=["x"]), "refactoring", True), ("main code and added tests", [MAIN, TEST], diff_for(TEST, added=added), "feature_with_tests", True), ("main code, added tests and docs", [MAIN, TEST, DOC], diff_for(TEST, added=added), "feature_with_tests", True), ("tests with a changed line", [TEST], diff_for(TEST, **changed), "tests_only_added", True), ("main code and tests with a changed line", [MAIN, TEST], diff_for(TEST, **changed), "feature_with_tests", True), ("tests with a lost assertion", [TEST], diff_for(TEST, removed=["assertThat(1).isOne();"]), "tests_modified", False), ("main code and tests with a lost assertion", [MAIN, TEST], diff_for(TEST, removed=["assertThat(1).isOne();"]), "feature_with_modified_tests", False), ("main code and an API spec", [MAIN, API], diff_for(MAIN, added=["x"]), "mixed", True), ("main code, tests and an other file", [MAIN, TEST, "Makefile"], diff_for(TEST, added=added), "mixed", True), ) for name, paths, diff, kind, approvable in cases: with self.subTest(name): outcome = evaluate([gitea_file(p) for p in paths], diff) self.assertEqual(outcome.change_kind, kind) self.assertIs(outcome.auto_approvable, approvable) self.assertIs(outcome.verdict, Verdict.GREEN if approvable else Verdict.YELLOW) def test_not_approvable_reason_names_the_feature_kind(self): outcome = main_and_test(test_added=["@Test void y() {}"], categories={"auto_approvable": ["docs_only"]}) reason = next(r for r in outcome.reasons if r.code == "category") self.assertEqual(reason.text, "change kind 'feature_with_tests' (main code, tests); " "AI-approvable are only: docs only") class FeatureWithTestsFlowTest(unittest.TestCase): def test_feature_with_additive_tests_is_green(self): gitea = FakeGitea(files=[gitea_file(MAIN), gitea_file(TEST)], diff=diff_for(MAIN, added=["int x;"]) + diff_for(TEST, added=[" @Test void y() {", " assertThat(a.y()).isEqualTo(2);", " }"])) model = FakeModel(prompt_files=[MAIN, TEST]) outcome = execute(config(gate={"yellow_gate": "block"}), pr_ref(), deps(gitea, model), Redactor()) self.assertIs(outcome.result.verdict, Verdict.GREEN) self.assertEqual(outcome.result.change_kind, "feature_with_tests") self.assertIn("Check that the added tests cover the changed behavior", model.calls[0]) class TestWeakeningTest(unittest.TestCase): def test_new_assert_line_in_an_existing_method_is_additive(self): outcome = main_and_test(test_added=[" assertThat(a.y()).isEqualTo(2);"]) self.assertEqual(outcome.change_kind, "feature_with_tests") self.assertIsNone(weakening(outcome)) self.assertEqual(outcome.changed_tests, []) def test_only_lost_or_disabled_tests_give_yellow(self): cases = ( ("changed assertion", [" assertEquals(2, x);"], [" assertEquals(1, x);"], None), ("changed setup line", [" var a = new A(2);"], [" var a = new A(1);"], None), ("removed assertion", [], [" assertEquals(1, x);"], f"{TEST}:1 removes 1 test(s) or assertion(s) without a replacement"), ("removed test method", [], [" @Test", " void x() { assertEquals(1, x()); }"], f"{TEST}:1 removes 2 test(s) or assertion(s) without a replacement"), ("added skip marker", [" @Disabled"], [], f"{TEST}:1 disables a test"), ("removed assertion, added test method", [" @Test void y() {}"], [" assertEquals(1, x);"], f"{TEST}:1 removes 1 test(s) or assertion(s) without a replacement"), ) for name, added, removed, finding in cases: with self.subTest(name): outcome = main_and_test(test_added=added, test_removed=removed) if finding is None: self.assertIsNone(weakening(outcome)) self.assertIs(outcome.verdict, Verdict.GREEN) self.assertEqual(outcome.change_kind, "feature_with_tests") else: self.assertIs(outcome.verdict, Verdict.YELLOW) self.assertEqual(outcome.change_kind, "feature_with_modified_tests") self.assertEqual(weakening(outcome), f"tests lost or disabled: {finding}") outcome = evaluate([gitea_file(MAIN), gitea_file(TEST, status="deleted", deletions=1)], diff_for(MAIN, added=["x"]) + diff_for(TEST, removed=["class ATest {}"], deleted=True)) self.assertEqual(weakening(outcome), f"tests lost or disabled: {TEST} deleted") def test_changed_test_lines_go_to_the_model(self): outcome = main_and_test(test_added=[" assertEquals(2, x);"], test_removed=[" assertEquals(1, x);"]) self.assertEqual(outcome.changed_tests, [f"{TEST}:1"]) self.assertIn(f"Existing test lines changed at {TEST}:1. Check that each change keeps the test " "at least as strict as before; report a weakened test as a [high] issue.", change_kind_instruction(outcome)) gitea = FakeGitea(files=[gitea_file(MAIN), gitea_file(TEST, deletions=1)], diff=diff_for(MAIN, added=["int x;"]) + diff_for(TEST, added=[" assertEquals(2, x);"], removed=[" assertEquals(1, x);"])) model = FakeModel(prompt_files=[MAIN, TEST]) outcome = execute(config(gate={"yellow_gate": "block"}), pr_ref(), deps(gitea, model), Redactor()) self.assertIs(outcome.result.verdict, Verdict.GREEN) self.assertIn(f"Existing test lines changed at {TEST}:1.", model.calls[0]) def test_re_added_structural_line_is_not_a_change(self): outcome = main_and_test(test_added=[" @Test void y() {}", "}"], test_removed=["}"]) self.assertIsNone(weakening(outcome)) self.assertEqual(outcome.changed_tests, []) def test_a_moved_line_with_content_goes_to_the_model(self): line = " var a = new A(1);" outcome = main_and_test(test_added=[" @Test void y() {", line, " }"], test_removed=[line]) self.assertIsNone(weakening(outcome)) self.assertEqual(outcome.changed_tests, [f"{TEST}:1"]) def test_skip_markers_inside_string_literals_are_test_data(self): for line in (' ("@Disabled", "@Ignore"),', """ ('xit("a")', '.skip(')"""): with self.subTest(line=line): self.assertIsNone(weakening(main_and_test(test_added=[line]))) def test_blank_line_removal_is_additive(self): outcome = main_and_test(test_added=[" @Test void y() {}"], test_removed=[""]) self.assertIsNone(weakening(outcome)) self.assertEqual(outcome.changed_tests, []) def test_disabled_test_is_yellow(self): for line in ("@Disabled", "@Ignore", "@pytest.mark.skip", '@unittest.skip("flaky")', 'xit("works", () => {})', 'describe.skip("all", () => {})', 'it.only("this one", () => {})', 'self.skipTest("later")', "@pytest.mark.xfail"): with self.subTest(line=line): outcome = main_and_test(test_added=[line]) self.assertIs(outcome.verdict, Verdict.YELLOW) self.assertEqual(weakening(outcome), f"tests lost or disabled: {TEST}:1 disables a test") class MixedTest(unittest.TestCase): def test_mixed_is_approvable(self): gitea = FakeGitea(files=[gitea_file(MAIN), gitea_file("Makefile")], diff=diff_for(MAIN, added=["int x;"]) + diff_for("Makefile", added=["all: build"])) model = FakeModel(prompt_files=[MAIN, "Makefile"]) outcome = execute(config(gate={"yellow_gate": "block"}), pr_ref(), deps(gitea, model), Redactor()) self.assertEqual(outcome.result.change_kind, "mixed") self.assertIs(outcome.result.verdict, Verdict.GREEN) class HarmlessTest(unittest.TestCase): def test_harmless_file_does_not_spoil_the_change_kind(self): harmless = {"categories": {"harmless": [".editorconfig"]}} for other, kind in ((".editorconfig", "refactoring"), ("Makefile", "mixed")): with self.subTest(other=other): outcome = evaluate([gitea_file(MAIN), gitea_file(other)], diff_for(MAIN, added=["x"]), **harmless) self.assertEqual(outcome.change_kind, kind) outcome = evaluate([gitea_file(DOC), gitea_file(".editorconfig")], **harmless) self.assertEqual(outcome.change_kind, "docs_only") outcome = evaluate([gitea_file(MAIN), gitea_file(TEST), gitea_file(".editorconfig")], diff_for(TEST, added=["@Test void y() {}"]), **harmless) self.assertEqual(outcome.change_kind, "feature_with_tests") def test_harmless_only_is_approvable(self): outcome = evaluate([gitea_file(".editorconfig")], categories={"harmless": [".editorconfig"]}) self.assertEqual(outcome.change_kind, "harmless_only") self.assertTrue(outcome.auto_approvable) self.assertIs(outcome.verdict, Verdict.GREEN) def test_harmless_never_hides_risky_paths_or_main_code(self): outcome = evaluate([gitea_file(".gitea/workflows/ci.yaml")], categories={"harmless": ["**"]}) self.assertIs(outcome.verdict, Verdict.YELLOW) self.assertIn("risky paths changed: .gitea/workflows/ci.yaml (.gitea/**)", [r.text for r in outcome.reasons]) outcome = evaluate([gitea_file(MAIN), gitea_file(API)], categories={"harmless": ["**"]}) self.assertEqual(outcome.categories, {MAIN: "main", API: "api_spec"}) self.assertEqual(outcome.change_kind, "mixed") def test_harmless_comes_from_host_or_repository_and_defaults_to_empty(self): with open(TEST_CONFIG, "rb") as f: host = tomllib.load(f) self.assertEqual(build_config(host).categories.harmless, ()) repo = parse_repository_config(b'[categories]\nharmless = [".editorconfig"]\n') self.assertEqual(build_config(host, repo).categories.harmless, (".editorconfig",)) host["categories"]["harmless"] = [".gitignore"] self.assertEqual(build_config(host).categories.harmless, (".gitignore",)) class DbSchemaTest(unittest.TestCase): PATHS = (SCHEMA, "app/migrations/0002_add_field.py", "src/main/resources/db/migration/V2__x.java", "src/main/resources/liquibase/master.xml", "src/main/resources/flyway/V3.sql", "db/schema.changelog.yaml", "src/main/resources/db.changelog-master.yaml") def test_db_schema_change_is_yellow(self): for path in self.PATHS: with self.subTest(path=path): outcome = evaluate([gitea_file(path)], hard_yellow={"paths": []}) self.assertIs(outcome.verdict, Verdict.YELLOW) self.assertIn(f"DB schema changed: {path} (built in)", [r.text for r in outcome.reasons]) def test_no_layer_removes_the_db_schema_paths(self): weakest = {"hard_yellow": {"paths": []}, "categories": {"docs": ["**"], "harmless": ["**"]}} outcome = evaluate([gitea_file(SCHEMA)], **weakest) self.assertIn(f"DB schema changed: {SCHEMA} (built in)", [r.text for r in outcome.reasons]) self.assertIs(outcome.verdict, Verdict.YELLOW) with open(TEST_CONFIG, "rb") as f: host = tomllib.load(f) host["hard_yellow"]["paths"] = [] repo = parse_repository_config(b'[hard_yellow]\npaths = []\n[categories]\ndocs = ["**"]\nharmless = ["**"]\n') outcome = evaluate_rules(files_from_gitea([gitea_file(SCHEMA)]), {}, build_config(host, repo)) self.assertIn(f"DB schema changed: {SCHEMA} (built in)", [r.text for r in outcome.reasons]) outcome = evaluate([gitea_file("app/migrations/README.md")], **weakest) self.assertNotIn("db_schema", [r.code for r in outcome.reasons]) def test_test_files_are_no_db_schema(self): for path in ("src/test/java/net/x/migration/LiquibaseCompatibilityIntegrationTest.java", "src/test/resources/migration/released-prod-schema.sql", "tests/migrations/test_0002.py"): with self.subTest(path=path): outcome = evaluate([gitea_file(path)], hard_yellow={"paths": []}) self.assertNotIn("db_schema", [r.code for r in outcome.reasons]) outcome = evaluate([gitea_file(SCHEMA)], hard_yellow={"paths": []}, categories={"tests": ["**"]}) self.assertIn("db_schema", [r.code for r in outcome.reasons]) def test_renamed_schema_file_counts_with_its_old_path(self): f = gitea_file("archive/1-create.txt", status="renamed") f["previous_filename"] = SCHEMA outcome = evaluate([f], hard_yellow={"paths": []}) self.assertIn(f"DB schema changed: {SCHEMA} (built in)", [r.text for r in outcome.reasons]) if __name__ == "__main__": unittest.main()