import contextlib import importlib.machinery import importlib.util import io import json import os import tempfile import unittest import unittest.mock from tests.fakes import ROOT loader = importlib.machinery.SourceFileLoader("codex_eval_tool", os.path.join(ROOT, "tools", "codex-eval")) spec = importlib.util.spec_from_loader("codex_eval_tool", loader) tool = importlib.util.module_from_spec(spec) loader.exec_module(tool) HEAD, BASE = "h" * 40, "b" * 40 class FakeClient: def __init__(self, base_url, owner, repo, token, timeout=30): self.where = (base_url, owner, repo, token) def get_pr(self, number): return {"number": number, "title": "Scope", "body": "", "head": {"sha": HEAD}, "base": {"sha": "x" * 40}, "merge_base": BASE} class CodexEvalToolTest(unittest.TestCase): def test_help_and_missing_urls(self): out = io.StringIO() with contextlib.redirect_stdout(out): self.assertEqual(tool.main(["--help"]), 0) self.assertEqual(tool.main([]), 2) self.assertIn("Usage: tools/codex-eval", out.getvalue()) def test_a_run_writes_its_record_and_passes_model_and_effort(self): calls = [] def run(work, prompt, model, effort): calls.append((work, model, effort, "git diff " + BASE + "..HEAD" in prompt)) return {"model": model, "effort": effort, "exit_code": 0, "seconds": 1.0, "commands": [], "errors": [], "usage": {"input_tokens": 9, "cached_input_tokens": 4, "output_tokens": 2, "reasoning_output_tokens": 1}, "answer": {"critical": True, "critical_reason": "RBAC", "findings": []}} with tempfile.TemporaryDirectory() as out_dir, \ unittest.mock.patch.dict(os.environ, {"GITEA_TOKEN": "secret"}), \ unittest.mock.patch.object(tool, "GiteaClient", FakeClient), \ unittest.mock.patch.object(tool.codex, "checkout", lambda *args: "/work"), \ unittest.mock.patch.object(tool.codex, "run", run): printed = io.StringIO() with contextlib.redirect_stdout(printed): tool.main(["--effort", "high", "--model", "m", "--out", out_dir, "https://git.example.org/team/app/pulls/306"]) with open(os.path.join(out_dir, "app-306-high.json")) as f: record = json.load(f) self.assertEqual(calls, [("/work", "m", "high", True)]) self.assertEqual((record["head_sha"], record["merge_base"], record["answer"]["critical"]), (HEAD, BASE, True)) self.assertIn("app#306 high: exit 0, critical True", printed.getvalue()) self.assertNotIn("secret", printed.getvalue())