2 Commits

Author SHA1 Message Date
cd5967031b ci: janitor cancels jobs no runner can ever take
All checks were successful
check / gate (push) Successful in 20s
canary / probe (push) Successful in 6s
windy-pro alone left ~4 jobs per run waiting forever (build-electron on
macos/windows/ubuntu-latest, deploy if:false): Gitea evaluates job if:
only at pick time, the labels do not exist here, and waiting jobs are
invisible in /actions/tasks. 37 such jobs across 10 runs today. After
30 min they are cancelled and the run status recomputed. Runs each sync,
non-fatal.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-23 10:58:53 -04:00
f246417095 ci: windy-pro desktop/installer jobs are NON-BLOCKING (Grant, 09-23)
build-desktop, test-installer and reality-check still run on Windy Git
and stay visible there, but the bridge no longer posts them to GitHub, so
they cannot turn windy-pro's combined status red. Windy Git side only;
the desktop code is Grant's to fix. BRIDGE_NON_BLOCKING, per repo.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-23 10:58:07 -04:00
5 changed files with 78 additions and 0 deletions

View File

@@ -147,3 +147,27 @@ def test_image_build_jobs_are_not_posted(fake):
) )
bridge.post_statuses("r", SHA) bridge.post_statuses("r", SHA)
assert f.posted == [] assert f.posted == []
def test_non_blocking_jobs_are_not_posted_for_that_repo_only(fake, monkeypatch):
"""Grant ruled windy-pro's desktop/installer jobs non-blocking: they must not
reach GitHub for windy-pro, and the rule must not leak to other repos."""
monkeypatch.setattr(bridge, "NON_BLOCKING", {"windy-pro": {"ci/build-desktop"}})
runs = [_run(1, "ci.yml", "build-desktop", "failure"), _run(2, "ci.yml", "test", "success")]
f = fake(runs=runs)
bridge.post_statuses("windy-pro", SHA)
assert [p["context"] for p in f.posted] == ["windy-git/ci/test"]
f2 = fake(runs=runs)
bridge.post_statuses("windy-chat", SHA)
assert sorted(p["context"] for p in f2.posted) == [
"windy-git/ci/build-desktop",
"windy-git/ci/test",
]
def test_default_non_blocking_is_grants_ruling():
assert bridge.NON_BLOCKING.get("windy-pro") == {
"ci/build-desktop",
"ci/test-installer",
"ci/reality-check",
}

6
scripts/cancel_unrunnable.sh Executable file
View File

@@ -0,0 +1,6 @@
#!/usr/bin/env bash
# Cancel jobs no runner can ever take (see cancel_unrunnable.sql). Run on Veron as root.
set -euo pipefail
n=$(docker exec -i windy-git-db-1 sh -c 'psql -U "$POSTGRES_USER" -d gitea -At -v ON_ERROR_STOP=1' \
< "$(dirname "$0")/cancel_unrunnable.sql" | grep -cE '^[0-9]+$' || true)
echo "[janitor] cancelled unrunnable jobs in ${n} run(s)"

View File

@@ -0,0 +1,29 @@
-- Cancel CI jobs that can never run (called by scripts/cancel_unrunnable.sh).
--
-- A job whose runs-on names a label no Windy Git runner offers (ubuntu-latest,
-- macos-latest, windows-latest …) waits forever: Gitea evaluates a job's `if:`
-- only when a runner picks it, so even `if: false` / tag-only jobs sit in the
-- queue, invisible to /actions/tasks, and keep their run "waiting" for good.
-- After 30 minutes they are cancelled here; the run's status is then recomputed
-- (failure > still-active > cancelled > success), the same precedence Gitea uses.
-- Keep RUNNER_LABELS in step with deploy/runner/config.yaml.
BEGIN;
WITH dead AS (
UPDATE action_run_job j
SET status = 3, stopped = extract(epoch from now())::bigint, updated = extract(epoch from now())::bigint
WHERE j.status IN (5, 7)
AND to_timestamp(j.created) < now() - interval '30 minutes'
AND EXISTS (SELECT 1 FROM jsonb_array_elements_text(j.runs_on::jsonb) l
WHERE l NOT IN ('veron-1', 'linux-x64', 'self-hosted', 'linux', 'x64'))
RETURNING j.run_id
)
UPDATE action_run r
SET status = CASE
WHEN EXISTS (SELECT 1 FROM action_run_job x WHERE x.run_id = r.id AND x.status = 2) THEN 2
WHEN EXISTS (SELECT 1 FROM action_run_job x WHERE x.run_id = r.id AND x.status IN (5, 6, 7)) THEN r.status
WHEN EXISTS (SELECT 1 FROM action_run_job x WHERE x.run_id = r.id AND x.status = 3) THEN 3
ELSE 1 END,
stopped = CASE WHEN r.stopped = 0 THEN extract(epoch from now())::bigint ELSE r.stopped END
WHERE r.id IN (SELECT DISTINCT run_id FROM dead)
RETURNING r.id;
COMMIT;

View File

@@ -71,6 +71,19 @@ MIRROR_TAG = "[GH#"
# exists; that is a decision, recorded in docs/CUTOVER.md, not a failure. # exists; that is a decision, recorded in docs/CUTOVER.md, not a failure.
NO_DAEMON_JOB = re.compile(r"docker", re.IGNORECASE) NO_DAEMON_JOB = re.compile(r"docker", re.IGNORECASE)
# Jobs Grant ruled NON-BLOCKING (GRANT_DECISIONS_2026-09-23): still run on
# Windy Git and visible there, but not posted to GitHub, so they cannot turn a
# commit's combined status red. Format: "repo:workflow/job,workflow/job;repo2:..."
# windy-pro's desktop/installer jobs belong to Grant's desktop side (fixed from
# his Mac mini), not to any lane's merge gate.
NON_BLOCKING: dict[str, set[str]] = {}
for _entry in os.environ.get(
"BRIDGE_NON_BLOCKING", "windy-pro:ci/build-desktop,ci/test-installer,ci/reality-check"
).split(";"):
if ":" in _entry:
_repo, _jobs = _entry.split(":", 1)
NON_BLOCKING[_repo.strip()] = {j.strip() for j in _jobs.split(",") if j.strip()}
def _call(base: str, token_header: str, method: str, path: str, body=None): def _call(base: str, token_header: str, method: str, path: str, body=None):
req = urllib.request.Request( req = urllib.request.Request(
@@ -155,6 +168,8 @@ def post_statuses(repo: str, sha: str) -> None:
for r in runs: for r in runs:
if r["head_sha"] != sha or NO_DAEMON_JOB.search(r["name"]): if r["head_sha"] != sha or NO_DAEMON_JOB.search(r["name"]):
continue continue
if f"{r['workflow_id'].removesuffix('.yml')}/{r['name']}" in NON_BLOCKING.get(repo, ()):
continue
ctx = f"windy-git/{r['workflow_id'].removesuffix('.yml')}/{r['name']}" ctx = f"windy-git/{r['workflow_id'].removesuffix('.yml')}/{r['name']}"
if ctx not in latest or r["id"] > latest[ctx]["id"]: if ctx not in latest or r["id"] > latest[ctx]["id"]:
latest[ctx] = r latest[ctx] = r

View File

@@ -80,6 +80,10 @@ for r in $REPOS; do
fi fi
done done
# Jobs that name labels no runner has (ubuntu/macos/windows-latest) would wait
# forever and invisibly; cancel them after 30 min. Never fails the sync.
bash "$(dirname "$0")/cancel_unrunnable.sh" || log "janitor failed (non-fatal)"
# Private repos can't run GitHub Actions; mirror their open PRs here so CI # Private repos can't run GitHub Actions; mirror their open PRs here so CI
# fires, and post the verdicts back to GitHub as commit statuses. # fires, and post the verdicts back to GitHub as commit statuses.
if ! python3 "$(dirname "$0")/pr_status_bridge.py"; then if ! python3 "$(dirname "$0")/pr_status_bridge.py"; then