67 lines
3.1 KiB
Python
Executable File
67 lines
3.1 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Usage: tools/codex-eval [--effort LEVEL] [--model MODEL] [--out DIR] <pr-url>...
|
|
|
|
Measures a review by the Codex CLI on real Gitea PRs, without writing to Gitea.
|
|
Per PR it clones the repository into <out>/repos, checks the PR head out as a worktree,
|
|
and runs `codex exec` read-only in it, with the PR title and description and the review question.
|
|
Codex gets neither a token nor network access; it may read the whole repository and run read-only commands.
|
|
Per run it writes <out>/<repo>-<n>-<effort>.json: the answer, the token use, the duration and the commands Codex ran,
|
|
and it prints one line per run.
|
|
Needs GITEA_TOKEN with read access in the environment and a Codex CLI logged in; it never prints the token.
|
|
The defaults are --effort medium, --model gpt-6-astra and --out build/codex-eval.
|
|
|
|
Example:
|
|
GITEA_TOKEN=… tools/codex-eval --effort high https://git.example.org/owner/repo/pulls/306
|
|
"""
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
|
|
from werkrevisor import codex # noqa: E402
|
|
from werkrevisor.gitea import GiteaClient # noqa: E402
|
|
|
|
|
|
def main(argv):
|
|
effort, model, out, urls = "medium", "gpt-6-astra", "build/codex-eval", []
|
|
args = list(argv)
|
|
while args:
|
|
arg = args.pop(0)
|
|
if arg in ("-h", "--help"):
|
|
print(__doc__.strip())
|
|
return 0
|
|
if arg in ("--effort", "--model", "--out"):
|
|
value = args.pop(0)
|
|
effort, model, out = (value if arg == "--effort" else effort, value if arg == "--model" else model,
|
|
value if arg == "--out" else out)
|
|
else:
|
|
urls.append(arg)
|
|
if not urls:
|
|
print(__doc__.strip())
|
|
return 2
|
|
token = os.environ["GITEA_TOKEN"]
|
|
os.makedirs(out, exist_ok=True)
|
|
for url in urls:
|
|
base_url, owner, repo, number = codex.parse_pr_url(url)
|
|
pr = GiteaClient(base_url, owner, repo, token, timeout=60).get_pr(number)
|
|
head, base = pr["head"]["sha"], pr.get("merge_base") or pr["base"]["sha"]
|
|
work = codex.checkout(base_url, owner, repo, number, head, base, token, out)
|
|
record = {"pr_url": url, "head_sha": head, "merge_base": base,
|
|
**codex.run(work, codex.prompt_for(pr, base), model, effort)}
|
|
result_path = os.path.join(out, f"{repo}-{number}-{effort}.json")
|
|
with open(result_path, "w", encoding="utf-8") as f:
|
|
json.dump(record, f, indent=2)
|
|
answer, usage = record["answer"] or {}, record["usage"]
|
|
findings = answer.get("findings") or []
|
|
counts = ", ".join(f"{sum(1 for x in findings if x.get('severity') == s)} {s}" for s in codex.SEVERITIES)
|
|
print(f"{repo}#{number} {effort}: exit {record['exit_code']}, critical {answer.get('critical')}, "
|
|
f"findings {counts}, tokens in {usage['input_tokens']} (cached {usage['cached_input_tokens']}) "
|
|
f"out {usage['output_tokens']} (reasoning {usage['reasoning_output_tokens']}), "
|
|
f"{len(record['commands'])} commands, {record['seconds']} s -> {result_path}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv[1:]))
|