import copy import os import tempfile import unittest from tests.fakes import FakeGitea from werkrevisor import codex, reviewreport HEAD = "6d1842aa2111a50cbb0614aa840ed1cb8a75985c" DRY_RUN = f""" ## 🟡 werkrevisor: YELLOW / HUMAN_REVIEW Change kind: `mixed`. ### Reasons - **YELLOW** risky paths changed: build.gradle.kts Usage: model `m`, cost 0.17 USD (reported).
Reviewed head `{HEAD}`.
{{"verdict": "YELLOW"}} """ FINDING = {"severity": "medium", "file": "tools/x", "line": 148, "title": "Claims collapse", "explanation": "Why."} def record(answer, exit_code=0): return {"model": "gpt-6-astra", "effort": "high", "exit_code": exit_code, "seconds": 72.2, "usage": {"input_tokens": 311095, "cached_input_tokens": 256000, "output_tokens": 2143, "reasoning_output_tokens": 643}, "commands": [], "errors": [], "answer": answer} class ReviewReportTest(unittest.TestCase): def test_the_werkrevisor_part_has_no_markers(self): title, body, usage = reviewreport.werkrevisor_part(DRY_RUN) self.assertEqual(title, "🟡 YELLOW / HUMAN_REVIEW") self.assertIn("#### Reasons", body) self.assertNotIn("", comment["body"]) self.assertIn("![pr-lens-flow.svg](https://git.example.org/attachments/1-pr-lens-flow.svg)", comment["body"]) self.assertNotIn("Attach the SVGs", comment["body"]) self.assertEqual([a["name"] for a in comment["assets"]], ["pr-lens-flow.svg"]) def test_an_unchecked_draft_is_refused(self): """Scenario#130.02""" draft = reviewreport.compose(HEAD, reviewreport.werkrevisor_part(DRY_RUN), None, codex_error="x") with self.assertRaisesRegex(reviewreport.AttachError, "not checked"): reviewreport.attach(draft, self.dir, self.gitea, 7, HEAD) self.assertEqual(self.gitea.writes, []) def test_a_draft_of_another_head_is_refused(self): """Scenario#130.02""" with self.assertRaisesRegex(reviewreport.AttachError, "run hs-review again"): reviewreport.attach(self.checked, self.dir, self.gitea, 7, "f" * 40) self.assertEqual(self.gitea.writes, []) def test_a_second_post_for_the_same_head_replaces_the_report(self): """Scenario#139.02""" first = reviewreport.attach(self.checked, self.dir, self.gitea, 7, HEAD) second = reviewreport.attach(self.checked, self.dir, self.gitea, 7, HEAD, checked="Finding 2 does not hold.") self.assertEqual(second, first) self.assertEqual(len(self.gitea.comments), 1) comment = self.gitea.comments[0] self.assertTrue(comment["body"].startswith(f"\n")) self.assertIn("Finding 2 does not hold.", comment["body"]) self.assertEqual([a["id"] for a in comment["assets"]], [2]) self.assertIn("attachments/2-pr-lens-flow.svg", comment["body"]) self.assertNotIn("attachments/1-pr-lens-flow.svg", comment["body"]) reviewreport.attach(self.checked, self.dir, self.gitea, 7, HEAD, checked="A third reason.") self.assertEqual([a["id"] for a in self.gitea.comments[0]["assets"]], [3]) self.assertIn("attachments/3-pr-lens-flow.svg", self.gitea.comments[0]["body"]) def test_a_replacement_keeps_a_file_added_by_hand(self): """Scenario#139.04""" reviewreport.attach(self.checked, self.dir, self.gitea, 7, HEAD) self.gitea.create_comment_asset(1, "server.log", b"log", "text/plain") reviewreport.attach(self.checked, self.dir, self.gitea, 7, HEAD, checked="Another reason.") self.assertEqual(sorted(a["name"] for a in self.gitea.comments[0]["assets"]), ["pr-lens-flow.svg", "server.log"]) def test_a_replacement_keeps_an_svg_linked_by_hand(self): """Scenario#139.04""" reviewreport.attach(self.checked, self.dir, self.gitea, 7, HEAD) custom = self.gitea.create_comment_asset(1, "pr-lens-custom.svg", b"", "image/svg+xml") comment = self.gitea.comments[0] comment["body"] += f"\n![custom]({custom['browser_download_url']})" reviewreport.attach(self.checked, self.dir, self.gitea, 7, HEAD, checked="Another reason.") self.assertIn("pr-lens-custom.svg", [a["name"] for a in comment["assets"]]) def test_a_replacement_keeps_an_old_svg_that_the_reason_links(self): """Scenario#139.03""" reviewreport.attach(self.checked, self.dir, self.gitea, 7, HEAD) url = self.gitea.comments[0]["assets"][0]["browser_download_url"] reviewreport.attach(self.checked, self.dir, self.gitea, 7, HEAD, checked=f"See the [old diagram]({url}).") self.assertIn(url, [a["browser_download_url"] for a in self.gitea.comments[0]["assets"]]) reviewreport.attach(self.checked, self.dir, self.gitea, 7, HEAD, checked="No link.") self.assertNotIn(url, [a["browser_download_url"] for a in self.gitea.comments[0]["assets"]]) def test_an_edit_applied_despite_a_lost_response_keeps_its_svgs(self): """Scenario#139.03""" reviewreport.attach(self.checked, self.dir, self.gitea, 7, HEAD) edit = self.gitea.edit_issue_comment def lost(comment_id, body): edit(comment_id, body) raise OSError("connection reset") self.gitea.edit_issue_comment = lost with self.assertRaises(OSError): reviewreport.attach(self.checked, self.dir, self.gitea, 7, HEAD, checked="Another reason.") comment = self.gitea.comments[0] self.assertIn("attachments/2-pr-lens-flow.svg", comment["body"]) self.assertIn(2, [a["id"] for a in comment["assets"]]) def test_a_failed_replacement_keeps_the_report_and_its_svgs(self): """Scenario#139.03""" reviewreport.attach(self.checked, self.dir, self.gitea, 7, HEAD) before = copy.deepcopy(self.gitea.comments) def refused(*args): raise OSError("HTTP 500") self.gitea.edit_issue_comment = refused with self.assertRaises(OSError): reviewreport.attach(self.checked, self.dir, self.gitea, 7, HEAD, checked="Another reason.") self.assertEqual(self.gitea.comments, before) def test_a_report_of_someone_else_is_neither_replaced_nor_folded(self): """Scenario#139.04""" self.gitea.comments.append({"id": 1, "user": {"login": "other"}, "html_url": "u", "body": f"\n## AI review"}) reviewreport.attach(self.checked, self.dir, self.gitea, 7, HEAD) self.assertEqual(len(self.gitea.comments), 2) self.assertEqual(self.gitea.comments[0]["body"], f"\n## AI review") def test_a_report_of_a_new_head_folds_the_earlier_reports(self): """Scenario#139.05""" old = "a" * 40 earlier = self.checked.replace(f"`{HEAD[:8]}`", f"`{old[:8]}`", 1) reviewreport.attach(earlier, self.dir, self.gitea, 7, old) url = reviewreport.attach(self.checked, self.dir, self.gitea, 7, HEAD) first, second = self.gitea.comments self.assertEqual(second["html_url"], url) self.assertTrue(first["body"].startswith(f"\n\n
")) self.assertIn(f'superseded by the review of {HEAD[:8]}', first["body"]) self.assertTrue(first["body"].endswith("
")) self.assertNotIn("
", second["body"]) writes = list(self.gitea.writes) reviewreport.attach(self.checked, self.dir, self.gitea, 7, HEAD, checked="Reason.") self.assertEqual(self.gitea.comments[0]["body"].count("
"), 1) self.assertEqual(self.gitea.writes.count("edit") - writes.count("edit"), 1) def test_a_fold_keeps_an_edit_made_meanwhile(self): """Scenario#139.05""" old = "a" * 40 reviewreport.attach(self.checked.replace(f"`{HEAD[:8]}`", f"`{old[:8]}`", 1), self.dir, self.gitea, 7, old) listed = self.gitea.list_issue_comments self.gitea.list_issue_comments = lambda number: copy.deepcopy(listed(number)) create = self.gitea.create_issue_comment def edited_meanwhile(number, body): self.gitea.comments[0]["body"] += "\nAdded by hand." return create(number, body) self.gitea.create_issue_comment = edited_meanwhile reviewreport.attach(self.checked, self.dir, self.gitea, 7, HEAD) self.assertIn("
", self.gitea.comments[0]["body"]) self.assertIn("Added by hand.", self.gitea.comments[0]["body"]) def test_a_replaced_report_folds_the_reports_of_other_heads(self): """Scenario#139.05""" new = "b" * 40 later = self.checked.replace(f"`{HEAD[:8]}`", f"`{new[:8]}`", 1) first = reviewreport.attach(self.checked, self.dir, self.gitea, 7, HEAD) reviewreport.attach(later, self.dir, self.gitea, 7, new) reviewreport.attach(self.checked, self.dir, self.gitea, 7, HEAD, checked="Back at this commit.") replaced, other = self.gitea.comments self.assertNotIn("
", replaced["body"]) self.assertIn(f'superseded by the review of {HEAD[:8]}', other["body"]) def test_a_failed_upload_leaves_no_comment_and_allows_a_retry(self): """Scenario#130.01""" create = self.gitea.create_comment_asset def refused(*args): raise OSError("HTTP 413") self.gitea.create_comment_asset = refused with self.assertRaises(OSError): reviewreport.attach(self.checked, self.dir, self.gitea, 7, HEAD) self.assertEqual(self.gitea.comments, []) self.gitea.create_comment_asset = create reviewreport.attach(self.checked, self.dir, self.gitea, 7, HEAD) self.assertEqual(len(self.gitea.comments), 1) def test_a_finding_that_quotes_the_draft_line_is_no_refusal(self): """Scenario#130.02""" quoted = self.checked.replace("### Codex", "> **Draft:** quoted in a finding\n\n### Codex") reviewreport.attach(quoted, self.dir, self.gitea, 7, HEAD) self.assertIn("quoted in a finding", self.gitea.comments[-1]["body"]) def test_a_reason_follows_the_report_as_assessment(self): """Scenario#139.01""" draft = reviewreport.compose(HEAD, reviewreport.werkrevisor_part(DRY_RUN), None, codex_error="x", diagrams=["pr-lens-flow.svg"]) reason = "Finding 1 is fixed in a1b2c3d.\n\nFinding 2 does not hold: the check runs in the repository." reviewreport.attach(draft, self.dir, self.gitea, 7, HEAD, checked=reason) body = self.gitea.comments[-1]["body"] self.assertNotIn("> **Draft:**", body) self.assertIn("### Assessment of the findings\n\nFinding 1 is fixed in a1b2c3d.\n\n" "Finding 2 does not hold: the check runs in the repository.\n", body) self.assertLess(body.index("### Codex"), body.index("### Assessment of the findings")) self.assertLess(body.index("### Assessment of the findings"), body.index("")) self.assertTrue(body.rstrip().endswith("")) self.assertNotIn("\n\n\n", body) def test_a_reason_that_quotes_the_draft_line_stays_whole(self): """Scenario#139.01""" draft = reviewreport.compose(HEAD, reviewreport.werkrevisor_part(DRY_RUN), None, codex_error="x") reviewreport.attach(draft, self.dir, self.gitea, 7, HEAD, checked="> **Draft:** the remaining finding is already covered.\nSecond line.") body = self.gitea.comments[-1]["body"] self.assertIn("### Assessment of the findings\n\n> **Draft:** the remaining finding is already covered.\n" "Second line.\n", body) def test_an_empty_reason_is_refused_before_any_write(self): """Scenario#136.02""" draft = reviewreport.compose(HEAD, reviewreport.werkrevisor_part(DRY_RUN), None, codex_error="x") for reason in ("", " \n\t\n"): with self.subTest(reason=reason), self.assertRaisesRegex(reviewreport.AttachError, "needs a reason"): reviewreport.attach(draft, self.dir, self.gitea, 7, HEAD, checked=reason) self.assertEqual(self.gitea.writes, []) def test_a_reason_also_follows_a_draft_line_deleted_by_hand(self): """Scenario#139.01""" reviewreport.attach(self.checked, self.dir, self.gitea, 7, HEAD, checked="All points hold; fixed.") body = self.gitea.comments[-1]["body"] self.assertIn("### Assessment of the findings\n\nAll points hold; fixed.", body) self.assertLess(body.index("### Codex"), body.index("### Assessment of the findings")) def test_a_missing_svg_is_refused_before_any_write(self): os.remove(os.path.join(self.dir, "pr-lens-flow.svg")) with self.assertRaisesRegex(reviewreport.AttachError, "pr-lens-flow.svg"): reviewreport.attach(self.checked, self.dir, self.gitea, 7, HEAD) self.assertEqual(self.gitea.writes, []) class ErrorTextTest(unittest.TestCase): def test_the_cause_of_a_failure_comes_first(self): """Scenario#131.01""" stderr = ("werkrevisor: operator config from /werkrevisor/config/hsadmin-ng.toml\n" "werkrevisor: cannot read the PR: GET /repos/o/r/pulls/343 failed: The handshake operation timed out\n") text = reviewreport.error_text(stderr) report = reviewreport.compose(HEAD, None, None, werkrevisor_error=text, codex_error="x") self.assertTrue(text.startswith("werkrevisor: cannot read the PR")) self.assertIn("The review did not complete: werkrevisor: cannot read the PR", report) self.assertNotIn("operator config", report) def test_a_gitea_request_waits_90_seconds(self): """Scenario#131.02""" from werkrevisor.gitea import GiteaClient self.assertEqual(GiteaClient("https://g.example.org", "o", "r", "t")._timeout, 90) class TitleTest(unittest.TestCase): URL = "https://git.example.org/team/app/pulls/343" def test_the_title_links_the_pr_and_keeps_the_commit(self): """Scenario#134.02""" report = reviewreport.compose(HEAD, None, None, codex_error="x", pr=(343, self.URL)) self.assertTrue(report.startswith(f"## AI review of [PR#343]({self.URL}) (`6d1842aa`)\n")) def test_attach_reads_the_commit_from_the_new_and_the_old_title(self): """Scenario#134.02""" for pr in ((343, self.URL), None): with self.subTest(pr=pr): draft = reviewreport.compose(HEAD, None, None, codex_error="x", pr=pr) checked = "\n".join(line for line in draft.split("\n") if not line.startswith("> **Draft:**")) gitea = FakeGitea() reviewreport.attach(checked, tempfile.gettempdir(), gitea, 7, HEAD) self.assertEqual(len(gitea.comments), 1) with self.assertRaisesRegex(reviewreport.AttachError, "run hs-review again"): reviewreport.attach(checked, tempfile.gettempdir(), FakeGitea(), 7, "f" * 40)