ci: janitor cancels jobs no runner can ever take
windy-pro alone left ~4 jobs per run waiting forever (build-electron on macos/windows/ubuntu-latest, deploy if:false): Gitea evaluates job if: only at pick time, the labels do not exist here, and waiting jobs are invisible in /actions/tasks. 37 such jobs across 10 runs today. After 30 min they are cancelled and the run status recomputed. Runs each sync, non-fatal. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
6
scripts/cancel_unrunnable.sh
Executable file
6
scripts/cancel_unrunnable.sh
Executable file
@@ -0,0 +1,6 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Cancel jobs no runner can ever take (see cancel_unrunnable.sql). Run on Veron as root.
|
||||||
|
set -euo pipefail
|
||||||
|
n=$(docker exec -i windy-git-db-1 sh -c 'psql -U "$POSTGRES_USER" -d gitea -At -v ON_ERROR_STOP=1' \
|
||||||
|
< "$(dirname "$0")/cancel_unrunnable.sql" | grep -cE '^[0-9]+$' || true)
|
||||||
|
echo "[janitor] cancelled unrunnable jobs in ${n} run(s)"
|
||||||
29
scripts/cancel_unrunnable.sql
Normal file
29
scripts/cancel_unrunnable.sql
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
-- Cancel CI jobs that can never run (called by scripts/cancel_unrunnable.sh).
|
||||||
|
--
|
||||||
|
-- A job whose runs-on names a label no Windy Git runner offers (ubuntu-latest,
|
||||||
|
-- macos-latest, windows-latest …) waits forever: Gitea evaluates a job's `if:`
|
||||||
|
-- only when a runner picks it, so even `if: false` / tag-only jobs sit in the
|
||||||
|
-- queue, invisible to /actions/tasks, and keep their run "waiting" for good.
|
||||||
|
-- After 30 minutes they are cancelled here; the run's status is then recomputed
|
||||||
|
-- (failure > still-active > cancelled > success), the same precedence Gitea uses.
|
||||||
|
-- Keep RUNNER_LABELS in step with deploy/runner/config.yaml.
|
||||||
|
BEGIN;
|
||||||
|
WITH dead AS (
|
||||||
|
UPDATE action_run_job j
|
||||||
|
SET status = 3, stopped = extract(epoch from now())::bigint, updated = extract(epoch from now())::bigint
|
||||||
|
WHERE j.status IN (5, 7)
|
||||||
|
AND to_timestamp(j.created) < now() - interval '30 minutes'
|
||||||
|
AND EXISTS (SELECT 1 FROM jsonb_array_elements_text(j.runs_on::jsonb) l
|
||||||
|
WHERE l NOT IN ('veron-1', 'linux-x64', 'self-hosted', 'linux', 'x64'))
|
||||||
|
RETURNING j.run_id
|
||||||
|
)
|
||||||
|
UPDATE action_run r
|
||||||
|
SET status = CASE
|
||||||
|
WHEN EXISTS (SELECT 1 FROM action_run_job x WHERE x.run_id = r.id AND x.status = 2) THEN 2
|
||||||
|
WHEN EXISTS (SELECT 1 FROM action_run_job x WHERE x.run_id = r.id AND x.status IN (5, 6, 7)) THEN r.status
|
||||||
|
WHEN EXISTS (SELECT 1 FROM action_run_job x WHERE x.run_id = r.id AND x.status = 3) THEN 3
|
||||||
|
ELSE 1 END,
|
||||||
|
stopped = CASE WHEN r.stopped = 0 THEN extract(epoch from now())::bigint ELSE r.stopped END
|
||||||
|
WHERE r.id IN (SELECT DISTINCT run_id FROM dead)
|
||||||
|
RETURNING r.id;
|
||||||
|
COMMIT;
|
||||||
@@ -80,6 +80,10 @@ for r in $REPOS; do
|
|||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
|
# Jobs that name labels no runner has (ubuntu/macos/windows-latest) would wait
|
||||||
|
# forever and invisibly; cancel them after 30 min. Never fails the sync.
|
||||||
|
bash "$(dirname "$0")/cancel_unrunnable.sh" || log "janitor failed (non-fatal)"
|
||||||
|
|
||||||
# Private repos can't run GitHub Actions; mirror their open PRs here so CI
|
# Private repos can't run GitHub Actions; mirror their open PRs here so CI
|
||||||
# fires, and post the verdicts back to GitHub as commit statuses.
|
# fires, and post the verdicts back to GitHub as commit statuses.
|
||||||
if ! python3 "$(dirname "$0")/pr_status_bridge.py"; then
|
if ! python3 "$(dirname "$0")/pr_status_bridge.py"; then
|
||||||
|
|||||||
Reference in New Issue
Block a user