From 2d4fadb09066b641aa04fb26646e3a0265e380c4 Mon Sep 17 00:00:00 2001 From: Kit OC5 Date: Wed, 23 Sep 2026 12:54:34 -0400 Subject: [PATCH] sync: bound the janitor and telemetry steps (docker exec hangs in an IO stall) 09-23 16:43Z the Veron data2 SMR stall left runc exec in D state; the janitor's docker exec never returned, so the sync sat 'activating' and no repo mirrored or got a status for any lane. Both steps are non-fatal; now they time out (120 s / 180 s) and the run continues. Co-Authored-By: Claude Opus 5.5 --- scripts/sync_from_github.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/sync_from_github.sh b/scripts/sync_from_github.sh index e949876..2e1e709 100755 --- a/scripts/sync_from_github.sh +++ b/scripts/sync_from_github.sh @@ -82,7 +82,11 @@ done # Jobs that name labels no runner has (ubuntu/macos/windows-latest) would wait # forever and invisibly; cancel them after 30 min. Never fails the sync. -bash "$(dirname "$0")/cancel_unrunnable.sh" || log "janitor failed (non-fatal)" +# Both DB steps go through `docker exec`, which hangs outright while the host +# is in an IO stall (09-23: data2 SMR cliff wedged this sync for 10+ min and +# stopped mirroring + the bridge for every lane). They are optional; mirroring +# and the bridge are not. Bound them so a stuck exec costs one step, not the run. +timeout -k 10 120 bash "$(dirname "$0")/cancel_unrunnable.sh" || log "janitor failed or timed out (non-fatal)" # Private repos can't run GitHub Actions; mirror their open PRs here so CI # fires, and post the verdicts back to GitHub as commit statuses. @@ -92,7 +96,7 @@ fi # CI telemetry -> admin.windyword.ai (shapes declared with Windy Telemetry 40). # Sends nothing until WINDYGIT_TELEMETRY_TOKEN is set; never fails the sync. -python3 "$(dirname "$0")/telemetry_emit.py" || log "telemetry emit failed (non-fatal)" +timeout -k 10 180 python3 "$(dirname "$0")/telemetry_emit.py" || log "telemetry emit failed or timed out (non-fatal)" [[ "$FAILED" -ne 0 ]] && { log "COMPLETED WITH FAILURES"; exit 1; } log "all repos in step with GitHub"