claude-code
c50ad05877
werkator build succeeded after 01:05
See docs/prs/2026-10-05-PR#16-colors-by-certainty.md. Replaces PR#15: dangerous paths give YELLOW, not ORANGE. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Reviewed-on: #16 Co-authored-by: claude-code <claude-code@javagil.de>
234 lines
12 KiB
Python
234 lines
12 KiB
Python
import unittest
|
|
|
|
from tests.fakes import config, diff_for, gitea_file
|
|
from werkrevisor.diff import parse_unified_diff
|
|
from werkrevisor.globs import matches
|
|
from werkrevisor.rules import categorize, evaluate_rules, files_from_gitea, scan_secrets
|
|
from werkrevisor.verdict import Verdict
|
|
|
|
|
|
def evaluate(files, diff="", **overrides):
|
|
return evaluate_rules(files_from_gitea(files), parse_unified_diff(diff), config(**overrides))
|
|
|
|
|
|
class GlobTest(unittest.TestCase):
|
|
def test_double_star_matches_any_depth(self):
|
|
self.assertTrue(matches("README.md", "**/*.md"))
|
|
self.assertTrue(matches("doc/PR/302.md", "**/*.md"))
|
|
self.assertTrue(matches("src/test/java/A.java", "src/test/**"))
|
|
self.assertFalse(matches("src/main/java/A.java", "src/test/**"))
|
|
|
|
def test_pattern_without_slash_matches_file_name_anywhere(self):
|
|
self.assertTrue(matches("build.gradle.kts", "build.gradle*"))
|
|
self.assertTrue(matches("sub/build.gradle", "build.gradle*"))
|
|
self.assertFalse(matches("src/a.java", "*.md"))
|
|
|
|
def test_single_star_stays_in_one_directory(self):
|
|
self.assertFalse(matches("doc/a/b.md", "doc/*.md"))
|
|
self.assertTrue(matches("doc/b.md", "doc/*.md"))
|
|
|
|
|
|
class DiffParserTest(unittest.TestCase):
|
|
def test_line_numbers_and_files(self):
|
|
text = (diff_for("a.txt", added=["x", "y"], removed=["old"])
|
|
+ diff_for("gone.txt", removed=["z"], deleted=True)
|
|
+ diff_for("new.txt", added=["n"], new=True))
|
|
files = parse_unified_diff(text)
|
|
self.assertEqual(set(files), {"a.txt", "gone.txt", "new.txt"})
|
|
self.assertEqual(files["a.txt"].added, [(1, "x"), (2, "y")])
|
|
self.assertEqual(files["a.txt"].removed, [(1, "old")])
|
|
self.assertTrue(files["gone.txt"].deleted)
|
|
self.assertTrue(files["new.txt"].new)
|
|
|
|
def test_added_line_that_looks_like_a_header_stays_content(self):
|
|
text = "diff --git a/f b/f\n--- a/f\n+++ b/f\n@@ -1,0 +1,1 @@\n++++ not a header\n"
|
|
self.assertEqual(parse_unified_diff(text)["f"].added, [(1, "+++ not a header")])
|
|
|
|
|
|
class CategoryTest(unittest.TestCase):
|
|
def test_categories(self):
|
|
c = config().categories
|
|
self.assertEqual(categorize("doc/PR/1.md", c), "docs")
|
|
self.assertEqual(categorize("src/test/java/ATest.java", c), "tests")
|
|
self.assertEqual(categorize("src/main/java/A.java", c), "main")
|
|
self.assertEqual(categorize("src/main/resources/api-definition/x.yaml", c), "api_spec")
|
|
self.assertEqual(categorize("Makefile", c), "other")
|
|
|
|
def test_docs_only_is_approvable(self):
|
|
outcome = evaluate([gitea_file("README.md")], diff_for("README.md", added=["text"]))
|
|
self.assertEqual(outcome.change_kind, "docs_only")
|
|
self.assertTrue(outcome.auto_approvable)
|
|
self.assertEqual(outcome.verdict, Verdict.GREEN)
|
|
|
|
def test_added_tests_with_docs(self):
|
|
diff = diff_for("src/test/java/ATest.java", added=["@Test void x() { assertThat(1).isOne(); }"])
|
|
outcome = evaluate([gitea_file("src/test/java/ATest.java"), gitea_file("doc/x.md")], diff)
|
|
self.assertEqual(outcome.change_kind, "docs_and_tests")
|
|
self.assertEqual(outcome.verdict, Verdict.GREEN)
|
|
|
|
def test_removed_assertion_is_not_tests_only_added(self):
|
|
diff = diff_for("src/test/java/ATest.java", removed=[" assertThat(x).isEqualTo(1);"])
|
|
outcome = evaluate([gitea_file("src/test/java/ATest.java", deletions=1)], diff)
|
|
self.assertEqual(outcome.change_kind, "tests_modified")
|
|
self.assertEqual(outcome.verdict, Verdict.YELLOW)
|
|
self.assertIn("tests_weakened", [r.code for r in outcome.reasons])
|
|
|
|
def test_disabled_test_is_not_tests_only_added(self):
|
|
diff = diff_for("src/test/java/ATest.java", added=[" @Disabled"])
|
|
outcome = evaluate([gitea_file("src/test/java/ATest.java")], diff)
|
|
self.assertEqual(outcome.change_kind, "tests_modified")
|
|
|
|
def test_deleted_test_file_is_not_tests_only_added(self):
|
|
outcome = evaluate([gitea_file("src/test/java/ATest.java", status="deleted", deletions=3)],
|
|
diff_for("src/test/java/ATest.java", removed=["class ATest {}"], deleted=True))
|
|
self.assertEqual(outcome.change_kind, "tests_modified")
|
|
|
|
def test_refactoring_allows_pr_docs(self):
|
|
outcome = evaluate([gitea_file("src/main/java/A.java"), gitea_file("doc/PR/1.md")],
|
|
diff_for("src/main/java/A.java", added=["int x;"]))
|
|
self.assertEqual(outcome.change_kind, "refactoring")
|
|
self.assertTrue(outcome.auto_approvable)
|
|
|
|
def test_main_with_other_files_is_mixed(self):
|
|
outcome = evaluate([gitea_file("src/main/java/A.java"), gitea_file("Makefile")])
|
|
self.assertEqual(outcome.change_kind, "mixed")
|
|
self.assertEqual(outcome.verdict, Verdict.GREEN)
|
|
|
|
def test_mixed_reason_names_the_file_kinds_and_the_approvable_change_kinds(self):
|
|
outcome = evaluate([gitea_file("src/main/java/A.java"), gitea_file("Makefile"), gitea_file("doc/PR/1.md")],
|
|
categories={"auto_approvable": ["docs_only", "refactoring"]})
|
|
reason = next(r for r in outcome.reasons if r.code == "category")
|
|
self.assertEqual(reason.text, "change kind 'mixed' (main code, other files, docs); AI-approvable are only: "
|
|
"docs only, refactoring with unchanged tests")
|
|
|
|
def test_mixed_reason_names_api_spec_and_tests(self):
|
|
outcome = evaluate([gitea_file("src/main/resources/api-definition/api.yaml"),
|
|
gitea_file("src/test/java/ATest.java")],
|
|
categories={"auto_approvable": ["docs_only"]})
|
|
reason = next(r for r in outcome.reasons if r.code == "category")
|
|
self.assertEqual(reason.text, "change kind 'mixed' (API spec, tests); AI-approvable are only: docs only")
|
|
|
|
def test_api_spec_change_is_not_refactoring(self):
|
|
outcome = evaluate([gitea_file("src/main/java/A.java"),
|
|
gitea_file("src/main/resources/api-definition/api.yaml")])
|
|
self.assertEqual(outcome.change_kind, "mixed")
|
|
|
|
def test_kind_outside_auto_approvable_is_yellow(self):
|
|
outcome = evaluate([gitea_file("src/main/java/A.java")],
|
|
categories={"auto_approvable": ["docs_only"]})
|
|
self.assertFalse(outcome.auto_approvable)
|
|
self.assertIn("category", [r.code for r in outcome.reasons])
|
|
|
|
def test_no_files_is_error(self):
|
|
self.assertEqual(evaluate([]).verdict, Verdict.ERROR)
|
|
|
|
|
|
class HardYellowTest(unittest.TestCase):
|
|
def test_risky_path_is_yellow_even_for_docs(self):
|
|
outcome = evaluate([gitea_file("README.md"), gitea_file("build.gradle.kts")])
|
|
self.assertEqual(outcome.verdict, Verdict.YELLOW)
|
|
self.assertIn("risky_paths", [r.code for r in outcome.reasons])
|
|
|
|
def test_renamed_from_risky_path_counts(self):
|
|
f = gitea_file("doc/x.md", status="renamed")
|
|
f["previous_filename"] = ".gitea/workflows/ci.yaml"
|
|
self.assertIn("risky_paths", [r.code for r in evaluate([f]).reasons])
|
|
|
|
def test_documentation_never_matches_risky_paths(self):
|
|
for path in ("doc/PR/2026-08-17-PR#288-rbac-generator-hardening.md", "docs/auth/README.md"):
|
|
with self.subTest(path=path):
|
|
self.assertNotIn("risky_paths", [r.code for r in evaluate([gitea_file(path)]).reasons])
|
|
self.assertIn("risky_paths", [r.code for r in evaluate([gitea_file("src/main/java/rbac/A.java")]).reasons])
|
|
|
|
def test_repository_config_is_always_a_risky_path(self):
|
|
for overrides in ({}, {"hard_yellow": {"paths": []}, "categories": {"docs": ["**"]}}):
|
|
with self.subTest(overrides=overrides):
|
|
outcome = evaluate([gitea_file(".werkrevisor.toml")], **overrides)
|
|
self.assertIn("risky paths changed: .werkrevisor.toml (built in)", [r.text for r in outcome.reasons])
|
|
self.assertIs(outcome.verdict, Verdict.YELLOW)
|
|
|
|
def test_long_lists_show_three_examples(self):
|
|
outcome = evaluate([gitea_file(f".gitea/w{i}.yml") for i in range(5)])
|
|
self.assertIn("risky paths changed: .gitea/w0.yml (.gitea/**), .gitea/w1.yml (.gitea/**), "
|
|
".gitea/w2.yml (.gitea/**) and 2 more", [r.text for r in outcome.reasons])
|
|
|
|
def test_size_limits_give_yellow(self):
|
|
outcome = evaluate([gitea_file("src/main/java/A.java", additions=801)])
|
|
reason = next(r for r in outcome.reasons if r.code == "size_lines")
|
|
self.assertIs(reason.verdict, Verdict.YELLOW)
|
|
self.assertEqual(reason.text, "PR too large for an AI-only approval: 801 changed lines "
|
|
"(docs and mechanical files not counted) "
|
|
"exceed [size] max_changed_lines = 800; please review it yourself")
|
|
self.assertEqual(outcome.skip_llm_reason, "")
|
|
outcome = evaluate([gitea_file(f"src/main/java/A{i}.java") for i in range(31)])
|
|
reason = next(r for r in outcome.reasons if r.code == "size_files")
|
|
self.assertIs(reason.verdict, Verdict.YELLOW)
|
|
self.assertIn("31 changed files (docs and mechanical files not counted) exceed [size] max_files = 30",
|
|
reason.text)
|
|
|
|
def test_docs_do_not_count_for_the_size_limits(self):
|
|
files = ([gitea_file("doc/PR/1.md", additions=2000)] + [gitea_file(f"doc/{i}.md") for i in range(40)]
|
|
+ [gitea_file("src/main/java/A.java", additions=500, deletions=300)])
|
|
outcome = evaluate(files)
|
|
self.assertEqual(outcome.changed_lines, 800)
|
|
self.assertFalse({"size_lines", "size_files"} & {r.code for r in outcome.reasons})
|
|
|
|
|
|
class SecretScanTest(unittest.TestCase):
|
|
def scan(self, line):
|
|
return scan_secrets(parse_unified_diff(diff_for("a.txt", added=[line])))
|
|
|
|
def test_detects_secrets(self):
|
|
for line in ("-----BEGIN RSA PRIVATE KEY-----",
|
|
"-----BEGIN CERTIFICATE-----",
|
|
"aws = AKIAABCDEFGHIJKLMNOP",
|
|
'password = "correct-horse-battery"',
|
|
'api_key: "0123456789abcdef"',
|
|
"key hsak_admin." + "0" * 64,
|
|
"OPENROUTER=sk-or-v1-" + "f" * 40):
|
|
with self.subTest(line=line):
|
|
self.assertEqual(len(self.scan(line)), 1)
|
|
|
|
def test_ignores_harmless_lines(self):
|
|
for line in ("format `hsak_<subject-name>.<64 hex>`", 'password = ""', "token = getToken()",
|
|
"the sk-or- prefix marks OpenRouter keys"):
|
|
with self.subTest(line=line):
|
|
self.assertEqual(self.scan(line), [])
|
|
|
|
def test_hit_names_location_but_not_the_value(self):
|
|
hits = self.scan('secret = "supersecretvalue123"')
|
|
self.assertEqual(hits, ["a.txt:1 (generic_secret)"])
|
|
|
|
def test_secret_skips_the_model(self):
|
|
outcome = evaluate([gitea_file("README.md")], diff_for("README.md", added=["-----BEGIN PRIVATE KEY-----"]))
|
|
self.assertEqual(outcome.skip_llm_reason, "possible secrets in the diff")
|
|
self.assertEqual(outcome.verdict, Verdict.YELLOW)
|
|
|
|
def test_excluded_test_data_keeps_only_real_key_formats(self):
|
|
path = "src/test/java/net/hostsharing/BillingUseCase.java"
|
|
exclude = config().hard_yellow.secret_scan_exclude
|
|
self.assertEqual(exclude, ("src/test/**",))
|
|
|
|
def scan(line, at=path):
|
|
return scan_secrets(parse_unified_diff(diff_for(at, added=[line])), exclude)
|
|
|
|
for line in ('password = "correct-horse-battery"', 'api_key: "0123456789abcdef"', "-----BEGIN CERTIFICATE-----",
|
|
"key hsak_admin." + "0" * 16, "OPENROUTER=sk-or-v1-" + "f" * 40):
|
|
with self.subTest(line=line):
|
|
self.assertEqual(scan(line), [])
|
|
for line, name in (("-----BEGIN OPENSSH PRIVATE KEY-----", "private_key"),
|
|
("aws = AKIAABCDEFGHIJKLMNOP", "aws_access_key"),
|
|
("key hsak_admin." + "0" * 64, "hostsharing_api_key"),
|
|
("OPENROUTER=sk-or-v1-" + "f" * 64, "openrouter_key")):
|
|
with self.subTest(line=line):
|
|
self.assertEqual(scan(line), [f"{path}:1 ({name})"])
|
|
self.assertEqual(scan('password = "correct-horse-battery"', "src/main/java/A.java"),
|
|
["src/main/java/A.java:1 (generic_secret)"])
|
|
|
|
def test_removed_lines_are_not_scanned(self):
|
|
self.assertEqual(scan_secrets(parse_unified_diff(diff_for("a", removed=["-----BEGIN X"]))), [])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|