#!/bin/sh
# One model at a time, at an hour when nobody is learning.
#
# The first attempt at this comparison ran both models at once in the middle of a school day: 34 GB resident,
# two sets of 48 cores at 100%, load average 88, and every page on the platform slow for the students on it.
# So this script exists rather than a one-line command, and it holds three things the command did not:
# it runs each model in its own invocation and unloads it before the next, it refuses to start if the machine
# is already busy, and it stops in time for the 03:15 backup.
#
# Usage:  infra/bench/assist_bench.sh [runs] [course]     (from the stack directory, or with STACK set)
set -eu

STACK=${STACK:-/var/www/private/thinkera.academy/enterprise}
RUNS=${1:-2}
COURSE=${2:-2}
MODELS='qwen3:30b-a3b gpt-oss:20b'
LOG_DIR=${LOG_DIR:-$STACK/../backups}
LOG=$LOG_DIR/assist-bench-$(date +%Y%m%d-%H%M).log

cd "$STACK"

say() { printf '%s  %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*" >>"$LOG"; }

# A machine already working is a machine with somebody on it. 48 cores, so 8 is quiet by any reading.
load=$(cut -d' ' -f1 /proc/loadavg)
if [ "$(printf '%.0f' "$load")" -gt 8 ]; then
    say "not starting: load average is $load, which means something else is running"
    exit 0
fi

say "start — load $load, runs $RUNS, course $COURSE"
for model in $MODELS; do
    say "── $model"
    docker compose exec -T -u www-data -w /var/www/html moodle \
        php public/local/thinkera/cli/assist_bench.php \
        --course="$COURSE" --models="$model" --runs="$RUNS" >>"$LOG" 2>&1 || say "$model failed, see above"
    # unloaded before the next one is touched: this is the whole point of the script
    docker compose exec -T llm ollama stop "$model" >>"$LOG" 2>&1 || true
    say "$model unloaded"
done
say "done — load $(cut -d' ' -f1 /proc/loadavg)"
printf '%s\n' "$LOG"