ci: make Windy Git CI permanent for the private repos
- Four runners x capacity 1 instead of one x capacity 4. Concurrent jobs in one act_runner share /root/.cache/act; a refresh racing a copy killed 3 of windy-chat's ~20 jobs at setup-node (lstat ... no such file). Separate processes have separate caches. Same parallelism, same capped dind. - Behavioral tests for pr_status_bridge (latest verdict wins, no reposting, skipped never painted green, fork PRs never run, pagination, PR lifecycle). - import_from_github.py reads IMPORT_GITEA_URL, not GITEA_BASE_URL: sourcing the deploy .env pointed it at http://gitea:3000 and it died on DNS after the mirror it replaces had already been deleted. - CUTOVER.md: the private-repo CI path, onboarding steps, and the /actions/tasks-hides-queued-runs trap. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
124
api/tests/test_pr_status_bridge.py
Normal file
124
api/tests/test_pr_status_bridge.py
Normal file
@@ -0,0 +1,124 @@
|
|||||||
|
"""Behavioral tests for scripts/pr_status_bridge.py.
|
||||||
|
|
||||||
|
The bridge is the ONLY CI signal the private platform repos get on GitHub, so
|
||||||
|
these drive its real functions against fake Gitea/GitHub APIs rather than
|
||||||
|
grepping its source: a status painted green that nobody tested is worse than
|
||||||
|
no status at all.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
_spec = importlib.util.spec_from_file_location("pr_status_bridge", ROOT / "scripts" / "pr_status_bridge.py")
|
||||||
|
bridge = importlib.util.module_from_spec(_spec)
|
||||||
|
_spec.loader.exec_module(bridge)
|
||||||
|
|
||||||
|
SHA = "a" * 40
|
||||||
|
|
||||||
|
|
||||||
|
def _run(i, wf, job, status, sha=SHA, n=1):
|
||||||
|
return {"id": i, "workflow_id": wf, "name": job, "status": status, "head_sha": sha, "run_number": n}
|
||||||
|
|
||||||
|
|
||||||
|
class Fake:
|
||||||
|
def __init__(self, runs=(), statuses=(), gh_prs=(), wg_prs=()):
|
||||||
|
self.runs, self.statuses = list(runs), list(statuses)
|
||||||
|
self.gh_prs, self.wg_prs = list(gh_prs), list(wg_prs)
|
||||||
|
self.posted, self.opened, self.closed = [], [], []
|
||||||
|
|
||||||
|
def gitea(self, method, path, body=None):
|
||||||
|
if "/actions/tasks" in path:
|
||||||
|
page = int(path.rsplit("page=", 1)[1])
|
||||||
|
return 200, {"workflow_runs": self.runs[(page - 1) * 50: page * 50]}
|
||||||
|
if method == "GET" and path.endswith("/pulls?state=open&limit=50"):
|
||||||
|
return 200, self.wg_prs
|
||||||
|
if method == "POST" and path.endswith("/pulls"):
|
||||||
|
self.opened.append(body)
|
||||||
|
return 201, {}
|
||||||
|
if method == "PATCH":
|
||||||
|
self.closed.append(path)
|
||||||
|
return 201, {}
|
||||||
|
raise AssertionError(path)
|
||||||
|
|
||||||
|
def github(self, method, path, body=None):
|
||||||
|
if "/statuses" in path and method == "GET":
|
||||||
|
return 200, self.statuses
|
||||||
|
if "/statuses/" in path and method == "POST":
|
||||||
|
self.posted.append(body)
|
||||||
|
return 201, {}
|
||||||
|
if "/pulls?" in path:
|
||||||
|
return 200, self.gh_prs
|
||||||
|
raise AssertionError(path)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def fake(monkeypatch):
|
||||||
|
def make(**kw):
|
||||||
|
f = Fake(**kw)
|
||||||
|
monkeypatch.setattr(bridge, "gitea", f.gitea)
|
||||||
|
monkeypatch.setattr(bridge, "github", f.github)
|
||||||
|
return f
|
||||||
|
return make
|
||||||
|
|
||||||
|
|
||||||
|
def test_posts_latest_verdict_per_job(fake):
|
||||||
|
f = fake(runs=[_run(1, "ci.yml", "test", "failure"), _run(2, "ci.yml", "test", "success", n=2)])
|
||||||
|
bridge.post_statuses("r", SHA)
|
||||||
|
assert [(p["context"], p["state"]) for p in f.posted] == [("windy-git/ci/test", "success")]
|
||||||
|
assert f.posted[0]["target_url"].endswith("/actions/runs/2")
|
||||||
|
|
||||||
|
|
||||||
|
def test_unchanged_state_is_not_reposted(fake):
|
||||||
|
f = fake(runs=[_run(1, "ci.yml", "test", "success")],
|
||||||
|
statuses=[{"context": "windy-git/ci/test", "state": "success"}])
|
||||||
|
bridge.post_statuses("r", SHA)
|
||||||
|
assert f.posted == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_skipped_job_is_never_painted_green(fake):
|
||||||
|
f = fake(runs=[_run(1, "substrate-drift.yml", "check", "skipped")])
|
||||||
|
bridge.post_statuses("r", SHA)
|
||||||
|
assert f.posted == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_other_commits_runs_are_ignored(fake):
|
||||||
|
f = fake(runs=[_run(1, "ci.yml", "test", "failure", sha="b" * 40)])
|
||||||
|
bridge.post_statuses("r", SHA)
|
||||||
|
assert f.posted == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_runs_past_the_first_page_are_seen(fake):
|
||||||
|
noise = [_run(100 + i, "drift.yml", "x", "skipped", sha="c" * 40) for i in range(50)]
|
||||||
|
f = fake(runs=noise + [_run(1, "ci.yml", "test", "success")])
|
||||||
|
bridge.post_statuses("r", SHA)
|
||||||
|
assert [p["state"] for p in f.posted] == ["success"]
|
||||||
|
|
||||||
|
|
||||||
|
def _gh_pr(n, repo="sneakyfree/r"):
|
||||||
|
return {"number": n, "title": "t", "html_url": "u",
|
||||||
|
"head": {"ref": f"b{n}", "sha": SHA, "repo": {"full_name": repo} if repo else None},
|
||||||
|
"base": {"ref": "main"}}
|
||||||
|
|
||||||
|
|
||||||
|
def test_fork_prs_are_never_mirrored(fake, monkeypatch):
|
||||||
|
monkeypatch.setattr(bridge, "GH_OWNER", "sneakyfree")
|
||||||
|
f = fake(gh_prs=[_gh_pr(1, repo="stranger/r"), _gh_pr(2, repo=None)])
|
||||||
|
assert bridge.sync_prs("r") == []
|
||||||
|
assert f.opened == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_pr_mirror_opened_once_and_closed_when_github_closes(fake, monkeypatch):
|
||||||
|
monkeypatch.setattr(bridge, "GH_OWNER", "sneakyfree")
|
||||||
|
f = fake(gh_prs=[_gh_pr(7)], wg_prs=[{"number": 3, "title": "[GH#5] gone"}])
|
||||||
|
assert bridge.sync_prs("r") == [SHA]
|
||||||
|
assert [o["head"] for o in f.opened] == ["b7"]
|
||||||
|
assert f.closed == ["/repos/windyadmin/r/pulls/3"]
|
||||||
|
|
||||||
|
f2 = fake(gh_prs=[_gh_pr(7)], wg_prs=[{"number": 4, "title": "[GH#7] t"}])
|
||||||
|
bridge.sync_prs("r")
|
||||||
|
assert f2.opened == [] and f2.closed == []
|
||||||
@@ -11,7 +11,7 @@ log:
|
|||||||
|
|
||||||
runner:
|
runner:
|
||||||
file: /data/.runner
|
file: /data/.runner
|
||||||
capacity: 4 # concurrent jobs; Veron has 24 cores, dind is capped at 12
|
capacity: 1 # per runner; parallelism = number of runner services (4). See docker-compose.yml
|
||||||
timeout: 30m
|
timeout: 30m
|
||||||
shutdown_timeout: 3m
|
shutdown_timeout: 3m
|
||||||
insecure: false
|
insecure: false
|
||||||
|
|||||||
@@ -49,7 +49,19 @@ services:
|
|||||||
mem_limit: 64g
|
mem_limit: 64g
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|
||||||
runner:
|
# ── FOUR runners × capacity 1, not one runner × capacity 4 (2026-09-23) ──
|
||||||
|
#
|
||||||
|
# act caches every action repo at /root/.cache/act/<hash> INSIDE the runner
|
||||||
|
# process and re-fetches it at the start of each job. With capacity 4, four
|
||||||
|
# concurrent jobs share that one directory: one job's refresh rewrites it while
|
||||||
|
# another is tarring it into its job container, and the job dies with
|
||||||
|
# `lstat /root/.cache/act/<hash>/…: no such file or directory` on
|
||||||
|
# `actions/setup-node` / `setup-uv` — a failure that reads like a broken
|
||||||
|
# workflow. windy-chat (~20 jobs per push) hit it on 3 jobs in its first run.
|
||||||
|
# `rm -rf /root/.cache/act` only reset the clock. Separate processes get
|
||||||
|
# separate caches, so the race cannot occur. Same total parallelism, same
|
||||||
|
# single capped dind — the blast radius is unchanged.
|
||||||
|
runner: &runner
|
||||||
# 0.2.11 -> 0.6.1 on 2026-08-14. The bundled act in 0.2.11 only knows
|
# 0.2.11 -> 0.6.1 on 2026-08-14. The bundled act in 0.2.11 only knows
|
||||||
# `runs.using: node12|node16|node20`, so ANY repo pinning a current action
|
# `runs.using: node12|node16|node20`, so ANY repo pinning a current action
|
||||||
# major dies before its first step with "The runs.using key in action.yml
|
# major dies before its first step with "The runs.using key in action.yml
|
||||||
@@ -99,6 +111,30 @@ services:
|
|||||||
mem_limit: 4g
|
mem_limit: 4g
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|
||||||
|
# Each extra runner registers itself on first start (own name, own volume —
|
||||||
|
# the registration lives in /data/.runner, so volumes must never be shared).
|
||||||
|
runner-2:
|
||||||
|
<<: *runner
|
||||||
|
environment: &env2
|
||||||
|
DOCKER_HOST: tcp://dind:2375
|
||||||
|
GITEA_INSTANCE_URL: https://app.windygit.com
|
||||||
|
GITEA_RUNNER_REGISTRATION_TOKEN: ${RUNNER_TOKEN:?set RUNNER_TOKEN}
|
||||||
|
GITEA_RUNNER_NAME: veron-1-2
|
||||||
|
CONFIG_FILE: /config.yaml
|
||||||
|
volumes: [./config.yaml:/config.yaml:ro, runner-data-2:/data]
|
||||||
|
runner-3:
|
||||||
|
<<: *runner
|
||||||
|
environment:
|
||||||
|
<<: *env2
|
||||||
|
GITEA_RUNNER_NAME: veron-1-3
|
||||||
|
volumes: [./config.yaml:/config.yaml:ro, runner-data-3:/data]
|
||||||
|
runner-4:
|
||||||
|
<<: *runner
|
||||||
|
environment:
|
||||||
|
<<: *env2
|
||||||
|
GITEA_RUNNER_NAME: veron-1-4
|
||||||
|
volumes: [./config.yaml:/config.yaml:ro, runner-data-4:/data]
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
jobs:
|
jobs:
|
||||||
# Untrusted job containers live here. No route to the forge.
|
# Untrusted job containers live here. No route to the forge.
|
||||||
@@ -107,4 +143,7 @@ networks:
|
|||||||
volumes:
|
volumes:
|
||||||
dind-storage:
|
dind-storage:
|
||||||
runner-data:
|
runner-data:
|
||||||
|
runner-data-2:
|
||||||
|
runner-data-3:
|
||||||
|
runner-data-4:
|
||||||
|
|
||||||
|
|||||||
@@ -87,6 +87,46 @@ Per repo, deliberately, when that repo is quiet:
|
|||||||
4. later, when it flips to Windy-Git-first: remove it from `REPOS` *first*,
|
4. later, when it flips to Windy-Git-first: remove it from `REPOS` *first*,
|
||||||
repoint its sessions, add a push-mirror back to GitHub
|
repoint its sessions, add a push-mirror back to GitHub
|
||||||
|
|
||||||
|
## Private repos: Windy Git IS their CI (permanent, 2026-09-23)
|
||||||
|
|
||||||
|
The platform repos stay **private** on GitHub (Grant, 2026-09-23), and private
|
||||||
|
repos cannot run GitHub Actions on this account at all. Windy Git is therefore
|
||||||
|
their CI permanently, not a stopgap:
|
||||||
|
|
||||||
|
GitHub push ──sync (15 min)──▶ Windy Git ──runner──▶ Veron 1
|
||||||
|
▲ │
|
||||||
|
└──── commit status windy-git/<workflow>/<job> ◀───┘ scripts/pr_status_bridge.py
|
||||||
|
|
||||||
|
- `pr_status_bridge.py` runs at the end of every sync. It opens a `[GH#N]`
|
||||||
|
mirror PR in Windy Git for every open **same-repo** GitHub PR (so
|
||||||
|
`pull_request` workflows fire), closes it when the GitHub PR closes, and posts
|
||||||
|
each job's result back to GitHub on PR heads and the default-branch head.
|
||||||
|
**Never merge a `[GH#N]` PR here** — merge on GitHub.
|
||||||
|
- Fork PRs are never run: their branch is never synced, and untrusted code
|
||||||
|
beside the privileged dind is the open audit finding.
|
||||||
|
- Covered repos: `BRIDGE_REPOS` in the script. Public repos are left out on
|
||||||
|
purpose; they run real GitHub Actions and two verdicts per commit is noise.
|
||||||
|
- `skipped` jobs post nothing — no green for a job nobody ran.
|
||||||
|
|
||||||
|
**Onboarding another private repo** — the promotion steps below, then:
|
||||||
|
|
||||||
|
# on Veron 1, as root
|
||||||
|
set -a; . /srv/windygit/src/.env; set +a
|
||||||
|
python3 scripts/import_from_github.py <repo> # writable; aborts if the repo exists
|
||||||
|
# disable EVERY deploying workflow before anything is pushed:
|
||||||
|
curl -X PUT -H "Authorization: token $GITEA_ADMIN_TOKEN" \
|
||||||
|
http://localhost:3080/api/v1/repos/windyadmin/<repo>/actions/workflows/deploy.yml/disable
|
||||||
|
# add <repo> to REPOS in sync_from_github.sh AND BRIDGE_REPOS in pr_status_bridge.py
|
||||||
|
|
||||||
|
An import fires no push event, so `main` has no verdict until its next commit.
|
||||||
|
To get one now: force Windy Git's `main` back one commit, then
|
||||||
|
`systemctl start windygit-sync` — the sync pushes it forward and CI fires.
|
||||||
|
|
||||||
|
⚠️ **`/actions/tasks` lists only jobs a runner has PICKED UP.** Queued runs are
|
||||||
|
invisible there, so a repo can read "0 runs" while work is waiting. The truth is
|
||||||
|
`action_run` in the `gitea` database (status 1 success, 2 failure, 5 waiting,
|
||||||
|
6 running).
|
||||||
|
|
||||||
## ⚠️ Deploy workflows are DISABLED on Windy Git, deliberately
|
## ⚠️ Deploy workflows are DISABLED on Windy Git, deliberately
|
||||||
|
|
||||||
Six workflows fire on `push:` and deploy to production:
|
Six workflows fire on `push:` and deploy to production:
|
||||||
|
|||||||
@@ -43,7 +43,12 @@ import urllib.request
|
|||||||
#
|
#
|
||||||
# Bulk import belongs on the host anyway: no hairpin through the edge, no
|
# Bulk import belongs on the host anyway: no hairpin through the edge, no
|
||||||
# Cloudflare ~100s proxy ceiling (G4A.5) on a large clone. Run this on Veron 1.
|
# Cloudflare ~100s proxy ceiling (G4A.5) on a large clone. Run this on Veron 1.
|
||||||
GITEA = os.environ.get("GITEA_BASE_URL", "http://localhost:3080")
|
#
|
||||||
|
# 🔴 Deliberately NOT `GITEA_BASE_URL`: the deploy `.env` sets that to
|
||||||
|
# `http://gitea:3000` for the API container, and sourcing `.env` on the host
|
||||||
|
# made this script die on DNS *after* a caller had already deleted the mirror it
|
||||||
|
# was meant to replace (2026-09-23).
|
||||||
|
GITEA = os.environ.get("IMPORT_GITEA_URL", "http://localhost:3080")
|
||||||
GITEA_TOKEN = os.environ.get("GITEA_ADMIN_TOKEN", "")
|
GITEA_TOKEN = os.environ.get("GITEA_ADMIN_TOKEN", "")
|
||||||
GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN", "")
|
GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN", "")
|
||||||
GITHUB_OWNER = os.environ.get("GITHUB_OWNER", "sneakyfree")
|
GITHUB_OWNER = os.environ.get("GITHUB_OWNER", "sneakyfree")
|
||||||
|
|||||||
Reference in New Issue
Block a user