G7: CI runners with real isolation, and the gate as a workflow
Some checks failed
check / gate (push) Failing after 50s
Some checks failed
check / gate (push) Failing after 50s
I-5 says runners execute untrusted code and must be isolated by machine
boundary. act_runner needs a Docker daemon to start job containers, and the
tempting move — what every published example does — is to mount the host's
/var/run/docker.sock. That hands every workflow, including whatever a
transitive dependency's postinstall script feels like doing, the ability to
start a privileged container mounting / — root on Grant's workstation.
Instead the runner talks to its OWN dind daemon:
- runner (TRUSTED, the act_runner daemon) sits on the forge network only to
collect jobs from gitea:3000
- dind and every job container it spawns are UNTRUSTED, on a private network
with no route to the forge, its Postgres, or its .env
- jobs cannot bind-mount from the daemon host (valid_volumes: []) and are not
handed the runner's own socket (docker_host: -)
- separate compose project, cpu/memory bounded — Veron 1 is Grant's
workstation, not a dedicated build box
The gate itself now runs as a workflow, including the migration round-trip that
already caught two bugs review did not, and the I-12 check that a COMMIT_SHA
env override cannot change what /version reports.
Labels are explicit and pinned. A workflow naming a label nobody provides
queues forever and presents as a hung CI system rather than a typo — which is
what ubuntu-latest does on every windy-registry run today.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
73
.gitea/workflows/check.yml
Normal file
73
.gitea/workflows/check.yml
Normal file
@@ -0,0 +1,73 @@
|
|||||||
|
# The gate, running on our own hardware (G7.3).
|
||||||
|
#
|
||||||
|
# This is the dogfood: windy-git verifies itself before anything else migrates.
|
||||||
|
#
|
||||||
|
# `runs-on: veron-1` is a label this runner actually provides. NEVER
|
||||||
|
# `ubuntu-latest` (G7.5) — a self-hosted runner has no such label, so a workflow
|
||||||
|
# naming it queues forever and presents as a hung CI system rather than a typo.
|
||||||
|
|
||||||
|
name: check
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
pull_request:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
gate:
|
||||||
|
runs-on: veron-1
|
||||||
|
services:
|
||||||
|
postgres:
|
||||||
|
image: postgres:16-alpine
|
||||||
|
env:
|
||||||
|
POSTGRES_USER: windygit
|
||||||
|
POSTGRES_PASSWORD: windygit
|
||||||
|
POSTGRES_DB: windygit
|
||||||
|
options: >-
|
||||||
|
--health-cmd "pg_isready -U windygit"
|
||||||
|
--health-interval 5s
|
||||||
|
--health-retries 10
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: install
|
||||||
|
run: |
|
||||||
|
python3 -m venv .venv
|
||||||
|
.venv/bin/pip install -q -e ".[dev]"
|
||||||
|
|
||||||
|
- name: lint
|
||||||
|
run: .venv/bin/ruff check api scripts
|
||||||
|
|
||||||
|
- name: vocabulary audit (D-9)
|
||||||
|
run: python3 scripts/vocab_audit.py
|
||||||
|
|
||||||
|
- name: tests
|
||||||
|
run: .venv/bin/pytest -q
|
||||||
|
|
||||||
|
# G0.4 — a migration nobody has run is a migration nobody can trust. This
|
||||||
|
# is the step that caught two bugs review did not: SQLAlchemy Enum
|
||||||
|
# persisting .name instead of .value, and create_table re-emitting
|
||||||
|
# CREATE TYPE without checkfirst.
|
||||||
|
- name: migration round-trip (upgrade -> downgrade -> upgrade)
|
||||||
|
env:
|
||||||
|
DATABASE_URL: postgresql://windygit:windygit@postgres:5432/windygit
|
||||||
|
run: |
|
||||||
|
.venv/bin/alembic upgrade head
|
||||||
|
.venv/bin/alembic downgrade base
|
||||||
|
.venv/bin/alembic upgrade head
|
||||||
|
|
||||||
|
# I-12 — the honesty check. Nine sibling services cannot name the commit
|
||||||
|
# they are running; one reports another repo's commit entirely.
|
||||||
|
- name: /version must equal HEAD
|
||||||
|
run: |
|
||||||
|
HEAD_SHA=$(git rev-parse HEAD)
|
||||||
|
COMMIT_SHA=deadbeefdeadbeefdeadbeefdeadbeefdeadbeef \
|
||||||
|
.venv/bin/python -c "
|
||||||
|
import os, sys
|
||||||
|
sys.path.insert(0, '.')
|
||||||
|
from api.app.buildinfo import get_build_info
|
||||||
|
info = get_build_info()
|
||||||
|
expected = '$HEAD_SHA'
|
||||||
|
assert info.commit_sha == expected, f'{info.commit_sha} != {expected}'
|
||||||
|
print('I-12 holds: env override ignored, reported', info.commit_sha[:12])
|
||||||
|
"
|
||||||
@@ -393,3 +393,58 @@ def test_i04_never_synced_is_not_reported_as_merely_behind():
|
|||||||
src = (ROOT / "api" / "app" / "services" / "mirror.py").read_text()
|
src = (ROOT / "api" / "app" / "services" / "mirror.py").read_text()
|
||||||
assert "never_synced" in src
|
assert "never_synced" in src
|
||||||
assert '"pending"' in src
|
assert '"pending"' in src
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# G7 / I-5 — CI never shares a kernel with identity
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
def test_i05_runner_never_mounts_the_host_docker_socket():
|
||||||
|
"""The tempting move — and what every published act_runner example does —
|
||||||
|
is to mount /var/run/docker.sock. That hands every workflow, including a
|
||||||
|
transitive dependency's postinstall script, the ability to start a
|
||||||
|
privileged container mounting / — i.e. root on the host."""
|
||||||
|
compose = (ROOT / "deploy" / "runner" / "docker-compose.yml").read_text()
|
||||||
|
active = [ln for ln in compose.splitlines() if ln.strip() and not ln.strip().startswith("#")]
|
||||||
|
for ln in active:
|
||||||
|
assert "/var/run/docker.sock" not in ln, "I-5: never mount the host docker socket"
|
||||||
|
|
||||||
|
|
||||||
|
def test_i05_jobs_cannot_bind_mount_from_the_daemon_host():
|
||||||
|
cfg = (ROOT / "deploy" / "runner" / "config.yaml").read_text()
|
||||||
|
assert "valid_volumes: []" in cfg
|
||||||
|
assert 'docker_host: "-"' in cfg
|
||||||
|
|
||||||
|
|
||||||
|
def test_i05_runner_is_a_separate_compose_project_from_the_forge():
|
||||||
|
"""Runners restart, crash, get starved and get killed. None of that should
|
||||||
|
ever touch the thing serving repositories."""
|
||||||
|
runner = (ROOT / "deploy" / "runner" / "docker-compose.yml").read_text()
|
||||||
|
forge = (ROOT / "docker-compose.yml").read_text()
|
||||||
|
assert "name: windy-git-runner" in runner
|
||||||
|
assert "name: windy-git" in forge
|
||||||
|
|
||||||
|
|
||||||
|
def test_g15_runner_is_cpu_and_memory_bounded():
|
||||||
|
"""Veron 1 is Grant's workstation, not a dedicated build box."""
|
||||||
|
compose = (ROOT / "deploy" / "runner" / "docker-compose.yml").read_text()
|
||||||
|
assert "cpus:" in compose
|
||||||
|
assert "mem_limit:" in compose
|
||||||
|
|
||||||
|
|
||||||
|
def test_g75_workflows_use_a_label_this_runner_actually_provides():
|
||||||
|
"""A workflow naming a label nobody provides queues forever and presents as
|
||||||
|
a hung CI system rather than a typo."""
|
||||||
|
cfg = (ROOT / "deploy" / "runner" / "config.yaml").read_text()
|
||||||
|
provided = {
|
||||||
|
ln.split(":")[0].strip().strip('"- ')
|
||||||
|
for ln in cfg.splitlines()
|
||||||
|
if "docker://" in ln
|
||||||
|
}
|
||||||
|
assert provided, "runner declares no labels"
|
||||||
|
for wf in ROOT.rglob(".gitea/workflows/*.y*ml"):
|
||||||
|
for ln in wf.read_text().splitlines():
|
||||||
|
# Skip comments — a doc line explaining runs-on is not a runs-on.
|
||||||
|
if ln.strip().startswith("#") or "runs-on:" not in ln:
|
||||||
|
continue
|
||||||
|
label = ln.split("runs-on:")[1].strip()
|
||||||
|
assert label in provided, f"{wf.name}: '{label}' is not a provided label"
|
||||||
|
|||||||
38
deploy/runner/config.yaml
Normal file
38
deploy/runner/config.yaml
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
# act_runner configuration (G7.1).
|
||||||
|
#
|
||||||
|
# Labels are EXPLICIT and PINNED. `ubuntu-latest` is banned (G7.5): all four
|
||||||
|
# windy-registry workflows use it and every single run fails, because a
|
||||||
|
# self-hosted runner has no such label unless you invent one. A workflow that
|
||||||
|
# names a label nobody provides queues forever and looks like a hung CI system
|
||||||
|
# rather than a typo.
|
||||||
|
|
||||||
|
log:
|
||||||
|
level: info
|
||||||
|
|
||||||
|
runner:
|
||||||
|
file: /data/.runner
|
||||||
|
capacity: 4 # concurrent jobs; Veron has 24 cores, dind is capped at 12
|
||||||
|
timeout: 30m
|
||||||
|
shutdown_timeout: 3m
|
||||||
|
insecure: false
|
||||||
|
fetch_timeout: 5s
|
||||||
|
fetch_interval: 2s
|
||||||
|
labels:
|
||||||
|
- "veron-1:docker://catthehacker/ubuntu:act-22.04"
|
||||||
|
- "linux-x64:docker://catthehacker/ubuntu:act-22.04"
|
||||||
|
|
||||||
|
cache:
|
||||||
|
enabled: true
|
||||||
|
dir: /data/cache
|
||||||
|
|
||||||
|
container:
|
||||||
|
# Job containers join the dind daemon's own bridge. NOT the forge network:
|
||||||
|
# untrusted code must never be able to reach the forge's Postgres or its
|
||||||
|
# environment (I-5).
|
||||||
|
network: bridge
|
||||||
|
privileged: false
|
||||||
|
options:
|
||||||
|
workdir_parent: /workspace
|
||||||
|
valid_volumes: [] # a job cannot bind-mount anything from the daemon host
|
||||||
|
docker_host: "-" # do NOT expose the runner's own docker socket to jobs
|
||||||
|
force_pull: false
|
||||||
81
deploy/runner/docker-compose.yml
Normal file
81
deploy/runner/docker-compose.yml
Normal file
@@ -0,0 +1,81 @@
|
|||||||
|
# CI runners (strand G7) — a SEPARATE compose project from the forge.
|
||||||
|
#
|
||||||
|
# Separate on purpose: runners restart, crash, get starved and get killed. None
|
||||||
|
# of that should ever touch the thing serving repositories. This is the cell
|
||||||
|
# doctrine applied one level down.
|
||||||
|
#
|
||||||
|
# ── I-5, and why there is a dind sidecar ───────────────────────────────────
|
||||||
|
#
|
||||||
|
# "CI never shares a kernel with identity. Runners execute untrusted code and
|
||||||
|
# are isolated by machine boundary, not container boundary. No runner may hold
|
||||||
|
# a credential scoped beyond its own job."
|
||||||
|
#
|
||||||
|
# act_runner needs a Docker daemon to start job containers. The tempting move is
|
||||||
|
# to mount the host's `/var/run/docker.sock`. That would hand every workflow —
|
||||||
|
# including whatever a transitive dependency's postinstall script feels like
|
||||||
|
# doing — the ability to start a privileged container mounting `/`, which is
|
||||||
|
# root on Veron 1. Every published act_runner example does exactly this.
|
||||||
|
#
|
||||||
|
# Instead the runner talks to its OWN daemon (`dind`). Untrusted job code runs
|
||||||
|
# as a child of that daemon, on an isolated network, with no route to the host
|
||||||
|
# socket and no route to the forge's database.
|
||||||
|
#
|
||||||
|
# The split that makes this work:
|
||||||
|
# * `runner` is TRUSTED code (the act_runner daemon). It sits on the forge
|
||||||
|
# network only so it can reach gitea:3000 to collect jobs.
|
||||||
|
# * `dind` and every job container it spawns are UNTRUSTED. They are on a
|
||||||
|
# private network with no access to the forge, its database, or its .env.
|
||||||
|
#
|
||||||
|
# dind itself is privileged — that is the cost, and it is the reason a job
|
||||||
|
# escape lands in a disposable daemon rather than on Grant's workstation.
|
||||||
|
#
|
||||||
|
# ⚠️ Do NOT "simplify" this by mounting the host docker socket.
|
||||||
|
|
||||||
|
name: windy-git-runner
|
||||||
|
|
||||||
|
services:
|
||||||
|
dind:
|
||||||
|
image: docker.io/library/docker:27-dind
|
||||||
|
privileged: true
|
||||||
|
environment:
|
||||||
|
DOCKER_TLS_CERTDIR: "" # plain TCP on an isolated network, no host route
|
||||||
|
command: ["dockerd", "--host=tcp://0.0.0.0:2375", "--tls=false"]
|
||||||
|
networks: [jobs]
|
||||||
|
volumes:
|
||||||
|
- dind-storage:/var/lib/docker
|
||||||
|
# G1.5 — bounded so a fork-bomb workflow cannot starve Grant's interactive
|
||||||
|
# session. Veron 1 is his workstation, not a dedicated build box.
|
||||||
|
cpus: 12.0 # 12 of 24 cores
|
||||||
|
mem_limit: 64g
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
runner:
|
||||||
|
image: docker.io/gitea/act_runner:0.2.11
|
||||||
|
depends_on: [dind]
|
||||||
|
environment:
|
||||||
|
# The runner reaches its OWN daemon. Never the host's.
|
||||||
|
DOCKER_HOST: tcp://dind:2375
|
||||||
|
GITEA_INSTANCE_URL: http://gitea:3000
|
||||||
|
GITEA_RUNNER_REGISTRATION_TOKEN: ${RUNNER_TOKEN:?set RUNNER_TOKEN}
|
||||||
|
GITEA_RUNNER_NAME: veron-1
|
||||||
|
CONFIG_FILE: /config.yaml
|
||||||
|
volumes:
|
||||||
|
- ./config.yaml:/config.yaml:ro
|
||||||
|
- runner-data:/data
|
||||||
|
networks: [jobs, forge]
|
||||||
|
cpus: 2.0
|
||||||
|
mem_limit: 4g
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
networks:
|
||||||
|
jobs:
|
||||||
|
# Untrusted job containers live here. No route to the forge.
|
||||||
|
internal: false # jobs legitimately need to fetch dependencies
|
||||||
|
forge:
|
||||||
|
# Pre-existing network owned by the forge compose project.
|
||||||
|
external: true
|
||||||
|
name: windy-git_default
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
dind-storage:
|
||||||
|
runner-data:
|
||||||
Reference in New Issue
Block a user