Files
werkrevisor/tests/test_specdocs.py
mhoennigandClaude Opus 5.5 04461a274a werkrevisor 0.5.0: Jev judges the sections and Gherkin scenarios of the PR documents
[engine] mode chooses the review: "llm" (default, unchanged), "classifier" (Jev alone, no LLM)
or "combined" (Jev, then the LLM with Jev's flagged units as hints in the user prompt).
werkrevisor splits each changed PR document into ##/### sections and #### Scenario units,
asks Jev once per unit with the operator's questions, and maps every answer to a problem value:
each question asks for a problem, a score runs from "no problem" to "worst problem".
A clear spec problem is RED, a doubtful one or an oversized unit YELLOW, a Jev error ERROR.
The hints carry file, line, scenario id, question and value, but no PR text.
tools/jev-eval measures the questions offline on local PR documents.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-05 11:18:47 +02:00

124 lines
5.7 KiB
Python

import json
import os
import unittest
from tests.fakes import ROOT
from werkrevisor.specdocs import extract_units, is_pr_doc, parse_steps
FIXTURES = os.path.join(ROOT, "tests", "fixtures")
DE = "doc/PR/2026-10-01-PR#900-mandate-notice.md"
EN = "docs/prs/2026-10-02-PR#901-build-status-badge.md"
def units_of(path):
with open(os.path.join(FIXTURES, os.path.basename(path)), encoding="utf-8") as f:
return extract_units(f.read(), path)
class PrDocSelectionTest(unittest.TestCase):
def test_globs_and_exclusions(self):
globs = ("doc/PR/*.md", "docs/prs/*.md")
self.assertTrue(is_pr_doc(DE, globs))
self.assertTrue(is_pr_doc(EN, globs))
for path in ("doc/PR/TEMPLATE.md", "doc/PR/TEMPLATE-de.md", "doc/PR/README.md", "README.md",
"doc/other/x.md", "doc/PR/sub/x.md"):
with self.subTest(path=path):
self.assertFalse(is_pr_doc(path, globs))
class GermanDocTest(unittest.TestCase):
def setUp(self):
self.units = units_of(DE)
self.scenarios = [u for u in self.units if u.kind == "scenario"]
self.sections = {u.title: u for u in self.units if u.kind == "section"}
def test_units_in_document_order(self):
self.assertEqual([(u.kind, u.label) for u in self.units], [
("section", "section 'Relevante Links (Hostsharing-intern)'"),
("section", "section 'Das Problem'"),
("section", "section 'Feature: Auslaufende SEPA-Mandate anzeigen'"),
("scenario", "Scenario#900.01"),
("scenario", "Scenario 'Ein abgelaufenes Mandat erscheint nicht'"),
("section", "section 'Die Lösung'"),
])
self.assertEqual(self.scenarios[0].line, 20)
def test_scenario_with_goal_steps_and_context(self):
s = self.scenarios[0]
self.assertEqual((s.scenario_id, s.title), ("900.01", "Ein auslaufendes Mandat erscheint in der Liste"))
self.assertEqual(s.feature, "Auslaufende SEPA-Mandate anzeigen")
self.assertIn("läuft aus", s.background)
self.assertEqual(s.document_title, "PR#900: Hinweis auf auslaufende SEPA-Mandate")
self.assertTrue(s.text.startswith("&hellip; damit die Verwaltung"))
self.assertEqual(s.steps, (
"Given ein Mandat, dessen Gültigkeitsende in 10 Tagen liegt",
" And ein Mandat, das noch ein Jahr gilt",
"When die Verwaltung `GET /api/hs/office/sepamandates?expiring=true` aufruft",
"Then antwortet die API mit `200 OK` und die Liste enthält nur das erste Mandat",
))
def test_verified_by_is_not_part_of_any_unit(self):
for unit in self.units:
with self.subTest(unit=unit.label):
self.assertNotIn("Verified by", unit.text)
self.assertNotIn("HsOfficeScenarioTests", unit.text)
self.assertNotIn("gehört dem Generator", unit.text)
def test_scenario_without_id(self):
s = self.scenarios[1]
self.assertEqual((s.scenario_id, s.line), ("", 35))
self.assertEqual(s.hint_label, "scenario")
self.assertEqual(len(s.steps), 3)
def test_sections_hold_prose_and_background_but_no_scenarios(self):
feature = self.sections["Feature: Auslaufende SEPA-Mandate anzeigen"]
self.assertIn("läuft aus", feature.text)
self.assertNotIn("Given", feature.text)
self.assertNotIn("Die Szenarien", self.sections)
self.assertNotIn("Anhänge", self.sections)
def test_headings_in_code_blocks_do_not_count(self):
solution = self.sections["Die Lösung"]
self.assertIn("## Not a heading", solution.text)
self.assertIn("#### Scenario#900.99", solution.text)
self.assertIn("#### Hinweis zur Zeitzone", solution.text)
self.assertIn("Europe/Berlin", solution.text)
self.assertNotIn("900.99", [u.scenario_id for u in self.scenarios])
def test_state_is_json_with_named_fields(self):
state = self.scenarios[0].state()
self.assertEqual(set(state), {"pr_document", "document_title", "feature", "background", "scenario"})
self.assertEqual(set(state["scenario"]), {"id", "title", "text", "steps"})
self.assertEqual(set(self.units[0].state()["section"]), {"title", "level", "text"})
self.assertEqual(self.units[0].state_chars(), len(json.dumps(self.units[0].state(), ensure_ascii=False)))
class EnglishDocTest(unittest.TestCase):
def test_features_scope_their_scenarios(self):
scenarios = [u for u in units_of(EN) if u.kind == "scenario"]
self.assertEqual([(s.scenario_id, s.feature) for s in scenarios],
[("901.01", "status badge per branch"), ("901.02", "badges for unknown branches")])
self.assertEqual(scenarios[0].steps[-1], " And it is served with `no-store` cache headers.")
self.assertTrue(scenarios[0].text.startswith("So that a README"))
def test_tilde_fence_hides_headings(self):
sections = [u.title for u in units_of(EN) if u.kind == "section"]
self.assertEqual(sections, ["The Problem", "The Solution"])
def test_empty_feature_sections_are_skipped_but_scenarios_stay(self):
self.assertEqual(len([u for u in units_of(EN) if u.kind == "scenario"]), 2)
class ParseStepsTest(unittest.TestCase):
def test_keywords_case_and_other_bullets(self):
lines = ["- **given** x", "* **WHEN** y", "- plain bullet", "- **Then** z", " continued", "", "after"]
self.assertEqual(parse_steps(lines), ("Given x", "When y", "Then z continued"))
def test_german_keywords(self):
lines = ["- **Gegeben** x,", "- **wenn** y,", "- **dann** z,", "- **und** w."]
self.assertEqual(parse_steps(lines), ("Given x,", "When y,", "Then z,", "And w."))
if __name__ == "__main__":
unittest.main()