mhoennigandClaude Opus 5.5
04461a274a
[engine] mode chooses the review: "llm" (default, unchanged), "classifier" (Jev alone, no LLM) or "combined" (Jev, then the LLM with Jev's flagged units as hints in the user prompt). werkrevisor splits each changed PR document into ##/### sections and #### Scenario units, asks Jev once per unit with the operator's questions, and maps every answer to a problem value: each question asks for a problem, a score runs from "no problem" to "worst problem". A clear spec problem is RED, a doubtful one or an oversized unit YELLOW, a Jev error ERROR. The hints carry file, line, scenario id, question and value, but no PR text. tools/jev-eval measures the questions offline on local PR documents. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
59 lines
2.4 KiB
Bash
Executable File
59 lines
2.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Usage: tools/jev-eval [-h] [--config FILE] [--units-only] <markdown files...>
|
|
#
|
|
# Evaluates the classifier questions on local PR documents, without Gitea and without an LLM.
|
|
# It extracts the sections and Gherkin scenarios, asks Jev per unit,
|
|
# and prints one line per unit and question plus the totals (calls, tokens, cost).
|
|
# With --units-only it prints the extracted units, calls nothing and runs without network.
|
|
# Needs TYPESAFE_API_KEY in the environment unless --units-only is given.
|
|
# The default config is config/hsadmin-ng.toml; it must lie inside this repository.
|
|
# A set WERKREVISOR_CONFIG_TOML wins over it, as in production.
|
|
# The files are mounted read-only.
|
|
#
|
|
# Examples:
|
|
# tools/jev-eval --units-only ../hsadmin-ng/doc/PR/*PR#302*.md
|
|
# tools/jev-eval --config config/hsadmin-ng.toml ../hsadmin-ng/doc/PR/*.md
|
|
set -euo pipefail
|
|
|
|
if [[ $# -lt 1 || "$1" =~ ^(-h|-\?|--help)$ ]]; then
|
|
awk 'NR > 1 && /^#/ { sub(/^# ?/, ""); print; next } NR > 1 { exit }' "$0"
|
|
exit 0
|
|
fi
|
|
|
|
image="pragent/pr-agent@sha256:7d98954c29289846a08faf00aaea0f1f1375b1676cd8a2ed303e0af07010af7a"
|
|
config="config/hsadmin-ng.toml"
|
|
units_only=()
|
|
files=()
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--config) config="$2"; shift 2 ;;
|
|
--units-only) units_only=(--units-only); shift ;;
|
|
*) files+=("$1"); shift ;;
|
|
esac
|
|
done
|
|
[[ ${#files[@]} -gt 0 ]] || { echo "jev-eval: no files given" >&2; exit 2; }
|
|
|
|
docker_args=(--network none)
|
|
if [[ ${#units_only[@]} -eq 0 ]]; then
|
|
: "${TYPESAFE_API_KEY:?TYPESAFE_API_KEY must be set unless --units-only is given}"
|
|
# Secrets are passed by name, so their values never appear in the process list.
|
|
docker_args=(-e TYPESAFE_API_KEY)
|
|
fi
|
|
[[ -n "${WERKREVISOR_CONFIG_TOML:-}" ]] && docker_args+=(-e WERKREVISOR_CONFIG_TOML)
|
|
|
|
# Each file gets its own directory, so equal base names from different directories do not collide.
|
|
container_files=()
|
|
for i in "${!files[@]}"; do
|
|
source=$(realpath "${files[$i]}")
|
|
target="/input/$i/$(basename "$source")"
|
|
docker_args+=(--mount "type=bind,source=$source,target=$target,readonly")
|
|
container_files+=("$target")
|
|
done
|
|
|
|
docker run --rm \
|
|
-v "$PWD":/werkrevisor:ro -w /werkrevisor \
|
|
-e PYTHONPATH=/werkrevisor:/app -e PYTHONDONTWRITEBYTECODE=1 -e PYTHONNOUSERSITE=1 \
|
|
"${docker_args[@]}" \
|
|
--entrypoint python "$image" \
|
|
-s -m werkrevisor.jeveval --config "/werkrevisor/$config" "${units_only[@]}" "${container_files[@]}"
|