62 lines
2.7 KiB
Python
62 lines
2.7 KiB
Python
import contextlib
|
|
import importlib.machinery
|
|
import importlib.util
|
|
import io
|
|
import json
|
|
import os
|
|
import tempfile
|
|
import unittest
|
|
import unittest.mock
|
|
|
|
from tests.fakes import ROOT
|
|
|
|
loader = importlib.machinery.SourceFileLoader("codex_eval_tool", os.path.join(ROOT, "tools", "codex-eval"))
|
|
spec = importlib.util.spec_from_loader("codex_eval_tool", loader)
|
|
tool = importlib.util.module_from_spec(spec)
|
|
loader.exec_module(tool)
|
|
|
|
HEAD, BASE = "h" * 40, "b" * 40
|
|
|
|
|
|
class FakeClient:
|
|
def __init__(self, base_url, owner, repo, token, timeout=30):
|
|
self.where = (base_url, owner, repo, token)
|
|
|
|
def get_pr(self, number):
|
|
return {"number": number, "title": "Scope", "body": "", "head": {"sha": HEAD}, "base": {"sha": "x" * 40},
|
|
"merge_base": BASE}
|
|
|
|
|
|
class CodexEvalToolTest(unittest.TestCase):
|
|
def test_help_and_missing_urls(self):
|
|
out = io.StringIO()
|
|
with contextlib.redirect_stdout(out):
|
|
self.assertEqual(tool.main(["--help"]), 0)
|
|
self.assertEqual(tool.main([]), 2)
|
|
self.assertIn("Usage: tools/codex-eval", out.getvalue())
|
|
|
|
def test_a_run_writes_its_record_and_passes_model_and_effort(self):
|
|
calls = []
|
|
|
|
def run(work, prompt, model, effort):
|
|
calls.append((work, model, effort, "git diff " + BASE + "..HEAD" in prompt))
|
|
return {"model": model, "effort": effort, "exit_code": 0, "seconds": 1.0, "commands": [], "errors": [],
|
|
"usage": {"input_tokens": 9, "cached_input_tokens": 4, "output_tokens": 2,
|
|
"reasoning_output_tokens": 1},
|
|
"answer": {"critical": True, "critical_reason": "RBAC", "findings": []}}
|
|
with tempfile.TemporaryDirectory() as out_dir, \
|
|
unittest.mock.patch.dict(os.environ, {"GITEA_TOKEN": "secret"}), \
|
|
unittest.mock.patch.object(tool, "GiteaClient", FakeClient), \
|
|
unittest.mock.patch.object(tool.codex, "checkout", lambda *args: "/work"), \
|
|
unittest.mock.patch.object(tool.codex, "run", run):
|
|
printed = io.StringIO()
|
|
with contextlib.redirect_stdout(printed):
|
|
tool.main(["--effort", "high", "--model", "m", "--out", out_dir,
|
|
"https://git.example.org/team/app/pulls/306"])
|
|
with open(os.path.join(out_dir, "app-306-high.json")) as f:
|
|
record = json.load(f)
|
|
self.assertEqual(calls, [("/work", "m", "high", True)])
|
|
self.assertEqual((record["head_sha"], record["merge_base"], record["answer"]["critical"]), (HEAD, BASE, True))
|
|
self.assertIn("app#306 high: exit 0, critical True", printed.getvalue())
|
|
self.assertNotIn("secret", printed.getvalue())
|