75 lines
3.5 KiB
Bash
Executable File
75 lines
3.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Usage: tools/jev-eval [-h] [--config FILE] [--repo-config FILE | --no-repo-config] [--units-only] <markdown files...>
|
|
#
|
|
# Evaluates the classifier questions on local PR documents, without Gitea and without an LLM.
|
|
# It extracts the sections and Gherkin scenarios, asks Jev per unit,
|
|
# and prints one line per unit and question plus the totals (calls, tokens, cost).
|
|
# With --units-only it prints the extracted units, calls nothing and runs without network.
|
|
# Needs TYPESAFE_API_KEY in the environment unless --units-only is given.
|
|
# The default config is config/hsadmin-ng.toml; it must lie inside this repository.
|
|
# A set WERKREVISOR_CONFIG_TOML wins over it, as in production.
|
|
# --repo-config puts a repository config on top, with the rules of the review;
|
|
# without it, the .werkrevisor.toml of the current directory is used, unless --no-repo-config is given.
|
|
# To measure another repository, give its .werkrevisor.toml or --no-repo-config.
|
|
# The files are mounted read-only.
|
|
# It runs with docker, else podman; CONTAINER_ENGINE overrides the choice.
|
|
#
|
|
# Examples:
|
|
# tools/jev-eval --units-only ../hsadmin-ng/doc/PR/*PR#302*.md
|
|
# tools/jev-eval --config config/hsadmin-ng.toml --no-repo-config ../hsadmin-ng/doc/PR/*.md
|
|
# tools/jev-eval --config config/werkrevisor.toml docs/prs/*.md
|
|
set -euo pipefail
|
|
|
|
if [[ $# -lt 1 || "$1" =~ ^(-h|-\?|--help)$ ]]; then
|
|
awk 'NR > 1 && /^#/ { sub(/^# ?/, ""); print; next } NR > 1 { exit }' "$0"
|
|
exit 0
|
|
fi
|
|
|
|
image="docker.io/pragent/pr-agent@sha256:7d98954c29289846a08faf00aaea0f1f1375b1676cd8a2ed303e0af07010af7a"
|
|
container_engine="${CONTAINER_ENGINE:-$(command -v docker || command -v podman || true)}"
|
|
[[ -n "$container_engine" ]] || { echo "jev-eval: neither docker nor podman found; set CONTAINER_ENGINE" >&2; exit 2; }
|
|
config="config/hsadmin-ng.toml"
|
|
units_only=()
|
|
repo_config=()
|
|
repo_config_file=""
|
|
files=()
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--config) config="$2"; shift 2 ;;
|
|
--repo-config) repo_config_file="$2"; shift 2 ;;
|
|
--no-repo-config) repo_config=(--no-repo-config); shift ;;
|
|
--units-only) units_only=(--units-only); shift ;;
|
|
*) files+=("$1"); shift ;;
|
|
esac
|
|
done
|
|
[[ ${#files[@]} -gt 0 ]] || { echo "jev-eval: no files given" >&2; exit 2; }
|
|
|
|
docker_args=(--network none)
|
|
if [[ ${#units_only[@]} -eq 0 ]]; then
|
|
: "${TYPESAFE_API_KEY:?TYPESAFE_API_KEY must be set unless --units-only is given}"
|
|
# Secrets are passed by name, so their values never appear in the process list.
|
|
docker_args=(-e TYPESAFE_API_KEY)
|
|
fi
|
|
[[ -n "${WERKREVISOR_CONFIG_TOML:-}" ]] && docker_args+=(-e WERKREVISOR_CONFIG_TOML)
|
|
|
|
if [[ -n "$repo_config_file" ]]; then
|
|
docker_args+=(--mount "type=bind,source=$(realpath "$repo_config_file"),target=/input/repo-config.toml,readonly")
|
|
repo_config=(--repo-config /input/repo-config.toml)
|
|
fi
|
|
|
|
# Each file gets its own directory, so equal base names from different directories do not collide.
|
|
container_files=()
|
|
for i in "${!files[@]}"; do
|
|
source=$(realpath "${files[$i]}")
|
|
target="/input/$i/$(basename "$source")"
|
|
docker_args+=(--mount "type=bind,source=$source,target=$target,readonly")
|
|
container_files+=("$target")
|
|
done
|
|
|
|
"$container_engine" run --rm \
|
|
-v "$PWD":/werkrevisor:ro -w /werkrevisor \
|
|
-e PYTHONPATH=/werkrevisor:/app -e PYTHONDONTWRITEBYTECODE=1 -e PYTHONNOUSERSITE=1 \
|
|
"${docker_args[@]}" \
|
|
--entrypoint python "$image" \
|
|
-s -m werkrevisor.jeveval --config "/werkrevisor/$config" "${repo_config[@]}" "${units_only[@]}" "${container_files[@]}"
|