[engine] mode chooses the review: "llm" (default, unchanged), "classifier" (Jev alone, no LLM) or "combined" (Jev, then the LLM with Jev's flagged units as hints in the user prompt). werkrevisor splits each changed PR document into ##/### sections and #### Scenario units, asks Jev once per unit with the operator's questions, and maps every answer to a problem value: each question asks for a problem, a score runs from "no problem" to "worst problem". A clear spec problem is RED, a doubtful one or an oversized unit YELLOW, a Jev error ERROR. The hints carry file, line, scenario id, question and value, but no PR text. tools/jev-eval measures the questions offline on local PR documents. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
124 lines
5.7 KiB
Python
124 lines
5.7 KiB
Python
import json
|
|
import os
|
|
import unittest
|
|
|
|
from tests.fakes import ROOT
|
|
from werkrevisor.specdocs import extract_units, is_pr_doc, parse_steps
|
|
|
|
FIXTURES = os.path.join(ROOT, "tests", "fixtures")
|
|
DE = "doc/PR/2026-10-01-PR#900-mandate-notice.md"
|
|
EN = "docs/prs/2026-10-02-PR#901-build-status-badge.md"
|
|
|
|
|
|
def units_of(path):
|
|
with open(os.path.join(FIXTURES, os.path.basename(path)), encoding="utf-8") as f:
|
|
return extract_units(f.read(), path)
|
|
|
|
|
|
class PrDocSelectionTest(unittest.TestCase):
|
|
def test_globs_and_exclusions(self):
|
|
globs = ("doc/PR/*.md", "docs/prs/*.md")
|
|
self.assertTrue(is_pr_doc(DE, globs))
|
|
self.assertTrue(is_pr_doc(EN, globs))
|
|
for path in ("doc/PR/TEMPLATE.md", "doc/PR/TEMPLATE-de.md", "doc/PR/README.md", "README.md",
|
|
"doc/other/x.md", "doc/PR/sub/x.md"):
|
|
with self.subTest(path=path):
|
|
self.assertFalse(is_pr_doc(path, globs))
|
|
|
|
|
|
class GermanDocTest(unittest.TestCase):
|
|
def setUp(self):
|
|
self.units = units_of(DE)
|
|
self.scenarios = [u for u in self.units if u.kind == "scenario"]
|
|
self.sections = {u.title: u for u in self.units if u.kind == "section"}
|
|
|
|
def test_units_in_document_order(self):
|
|
self.assertEqual([(u.kind, u.label) for u in self.units], [
|
|
("section", "section 'Relevante Links (Hostsharing-intern)'"),
|
|
("section", "section 'Das Problem'"),
|
|
("section", "section 'Feature: Auslaufende SEPA-Mandate anzeigen'"),
|
|
("scenario", "Scenario#900.01"),
|
|
("scenario", "Scenario 'Ein abgelaufenes Mandat erscheint nicht'"),
|
|
("section", "section 'Die Lösung'"),
|
|
])
|
|
self.assertEqual(self.scenarios[0].line, 20)
|
|
|
|
def test_scenario_with_goal_steps_and_context(self):
|
|
s = self.scenarios[0]
|
|
self.assertEqual((s.scenario_id, s.title), ("900.01", "Ein auslaufendes Mandat erscheint in der Liste"))
|
|
self.assertEqual(s.feature, "Auslaufende SEPA-Mandate anzeigen")
|
|
self.assertIn("läuft aus", s.background)
|
|
self.assertEqual(s.document_title, "PR#900: Hinweis auf auslaufende SEPA-Mandate")
|
|
self.assertTrue(s.text.startswith("… damit die Verwaltung"))
|
|
self.assertEqual(s.steps, (
|
|
"Given ein Mandat, dessen Gültigkeitsende in 10 Tagen liegt",
|
|
" And ein Mandat, das noch ein Jahr gilt",
|
|
"When die Verwaltung `GET /api/hs/office/sepamandates?expiring=true` aufruft",
|
|
"Then antwortet die API mit `200 OK` und die Liste enthält nur das erste Mandat",
|
|
))
|
|
|
|
def test_verified_by_is_not_part_of_any_unit(self):
|
|
for unit in self.units:
|
|
with self.subTest(unit=unit.label):
|
|
self.assertNotIn("Verified by", unit.text)
|
|
self.assertNotIn("HsOfficeScenarioTests", unit.text)
|
|
self.assertNotIn("gehört dem Generator", unit.text)
|
|
|
|
def test_scenario_without_id(self):
|
|
s = self.scenarios[1]
|
|
self.assertEqual((s.scenario_id, s.line), ("", 35))
|
|
self.assertEqual(s.hint_label, "scenario")
|
|
self.assertEqual(len(s.steps), 3)
|
|
|
|
def test_sections_hold_prose_and_background_but_no_scenarios(self):
|
|
feature = self.sections["Feature: Auslaufende SEPA-Mandate anzeigen"]
|
|
self.assertIn("läuft aus", feature.text)
|
|
self.assertNotIn("Given", feature.text)
|
|
self.assertNotIn("Die Szenarien", self.sections)
|
|
self.assertNotIn("Anhänge", self.sections)
|
|
|
|
def test_headings_in_code_blocks_do_not_count(self):
|
|
solution = self.sections["Die Lösung"]
|
|
self.assertIn("## Not a heading", solution.text)
|
|
self.assertIn("#### Scenario#900.99", solution.text)
|
|
self.assertIn("#### Hinweis zur Zeitzone", solution.text)
|
|
self.assertIn("Europe/Berlin", solution.text)
|
|
self.assertNotIn("900.99", [u.scenario_id for u in self.scenarios])
|
|
|
|
def test_state_is_json_with_named_fields(self):
|
|
state = self.scenarios[0].state()
|
|
self.assertEqual(set(state), {"pr_document", "document_title", "feature", "background", "scenario"})
|
|
self.assertEqual(set(state["scenario"]), {"id", "title", "text", "steps"})
|
|
self.assertEqual(set(self.units[0].state()["section"]), {"title", "level", "text"})
|
|
self.assertEqual(self.units[0].state_chars(), len(json.dumps(self.units[0].state(), ensure_ascii=False)))
|
|
|
|
|
|
class EnglishDocTest(unittest.TestCase):
|
|
def test_features_scope_their_scenarios(self):
|
|
scenarios = [u for u in units_of(EN) if u.kind == "scenario"]
|
|
self.assertEqual([(s.scenario_id, s.feature) for s in scenarios],
|
|
[("901.01", "status badge per branch"), ("901.02", "badges for unknown branches")])
|
|
self.assertEqual(scenarios[0].steps[-1], " And it is served with `no-store` cache headers.")
|
|
self.assertTrue(scenarios[0].text.startswith("So that a README"))
|
|
|
|
def test_tilde_fence_hides_headings(self):
|
|
sections = [u.title for u in units_of(EN) if u.kind == "section"]
|
|
self.assertEqual(sections, ["The Problem", "The Solution"])
|
|
|
|
def test_empty_feature_sections_are_skipped_but_scenarios_stay(self):
|
|
self.assertEqual(len([u for u in units_of(EN) if u.kind == "scenario"]), 2)
|
|
|
|
|
|
class ParseStepsTest(unittest.TestCase):
|
|
def test_keywords_case_and_other_bullets(self):
|
|
lines = ["- **given** x", "* **WHEN** y", "- plain bullet", "- **Then** z", " continued", "", "after"]
|
|
self.assertEqual(parse_steps(lines), ("Given x", "When y", "Then z continued"))
|
|
|
|
def test_german_keywords(self):
|
|
lines = ["- **Gegeben** x,", "- **wenn** y,", "- **dann** z,", "- **und** w."]
|
|
self.assertEqual(parse_steps(lines), ("Given x,", "When y,", "Then z,", "And w."))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|