From da257652c2fa0023db2f29dfdc9ca40c88154c49 Mon Sep 17 00:00:00 2001 From: Grant Whitmer Date: Tue, 11 Aug 2026 16:11:45 -0400 Subject: [PATCH] G11 / I-4: continuous off-site mirror, and a namespace bug fixed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I-4 said 'never a one-way door' and had no implementation. Now it does. - ensure the GitHub counterpart exists (idempotent), then ask Gitea to keep it in step with sync_on_commit=True. An hourly timer means an hour of work can be the thing you lose, and that window is invisible until it costs you. - mirror status reports what is TRUE including 'we do not know'. An unconfigured mirror reports unconfigured, NEVER healthy — same posture as me-fleet.ts refusing to say 'online' when it only knows 'registered'. - lag past the threshold is a P2, not a shrug. A mirror nobody checks is a belief, not a backup, and this ecosystem already lost 37 days to a canary everyone assumed was fine. Gitea owns the replication rather than a hand-rolled loop, because a background job that fails silently is exactly how the registry's integrity refresh spent its entire life calling a 404 and incrementing a counter instead of raising. Also fixes a real bug I had written myself: list_versions derived the Gitea namespace from the CALLER, which is correct only while the caller is the owner and addresses the wrong namespace the moment a collaborator asks — surfacing as 'not found', which is the hardest kind of bug to see. Now derived from the repo, with a test that keeps it that way. Co-Authored-By: Claude Opus 5 --- api/app/config.py | 7 +- api/app/routes/repos.py | 92 +++++++++++++++++-- api/app/services/mirror.py | 169 +++++++++++++++++++++++++++++++++++ api/tests/test_invariants.py | 34 +++++++ 4 files changed, 295 insertions(+), 7 deletions(-) create mode 100644 api/app/services/mirror.py diff --git a/api/app/config.py b/api/app/config.py index d4cfd95..6a7cb7f 100644 --- a/api/app/config.py +++ b/api/app/config.py @@ -82,7 +82,12 @@ class Settings(BaseSettings): rate_grants_per_day: int = 100 rate_force_pushes_per_day: int = 10 - # ---- mirror health (I-4) ---------------------------------------------- + # ---- mirror: I-4, never a one-way door -------------------------------- + github_token: str = "" + github_owner: str = "sneakyfree" + # Gitea's timer, as a backstop. sync_on_commit is what actually matters: + # an hourly window means an hour of work can be the thing you lose. + mirror_interval: str = "8h0m0s" mirror_lag_p2_seconds: int = 3600 # env: 60 min -> P2 # ---- agent grants (G5.3) ---------------------------------------------- diff --git a/api/app/routes/repos.py b/api/app/routes/repos.py index df7a533..53f950e 100644 --- a/api/app/routes/repos.py +++ b/api/app/routes/repos.py @@ -31,6 +31,8 @@ from api.app.errors import RepairPointer from api.app.models.core import ( CreatedVia, GrantRole, + Mirror, + MirrorState, Repo, RepoGrant, RepoState, @@ -39,6 +41,7 @@ from api.app.models.core import ( Visibility, ) from api.app.services.gitea_client import GiteaClient +from api.app.services.mirror import MirrorService router = APIRouter(prefix="/api/v1/repos", tags=["repos"]) @@ -111,6 +114,18 @@ def _sessionmaker(request: Request) -> async_sessionmaker[AsyncSession]: return maker +def _repo_owner_login(repo: Repo) -> str: + """The owning login for a repo as STORED, not as inferred from the caller. + + Deriving this from the caller works only while the caller is the owner, and + silently addresses the wrong namespace the moment a collaborator calls. That + class of bug reads as "not found" and is very hard to see. + """ + if repo.passport: + return f"agent-{repo.passport.lower().replace('-', '')}" + return f"u-{repo.identity_id[:24]}" + + def _owner_login(caller: Caller) -> str: """One namespace rule for humans and agents alike (I-6).""" if caller.actor_type == ActorType.agent and caller.passport: @@ -272,11 +287,14 @@ async def list_versions( """ async with _sessionmaker(request)() as session: repo = await _load_repo(session, repo_id, caller) - owner = _owner_login(caller) if repo.passport == caller.passport else None + # Derive the namespace from the REPO, never from the caller: the + # caller-derived form is right only while the caller is the owner, and + # addresses the wrong namespace the moment a collaborator asks. It then + # surfaces as "not found", which is about the hardest bug to see. + owner, slug, display = _repo_owner_login(repo), repo.slug, repo.display_name gitea = GiteaClient(request.app.state.settings) - owner = owner or f"u-{repo.identity_id[:24]}" - commits = await gitea.list_commits(owner, repo.slug) + commits = await gitea.list_commits(owner, slug) versions = [ { @@ -294,12 +312,12 @@ async def list_versions( # "There are 1 saved versions" is the kind of sloppiness the vocabulary # law exists to catch. Copy is design material, not decoration (I-9). "speak": ( - f"'{repo.display_name}' has 1 saved version. You can go back to it." + f"'{display}' has 1 saved version. You can go back to it." if len(versions) == 1 - else f"'{repo.display_name}' has {len(versions)} saved versions. " + else f"'{display}' has {len(versions)} saved versions. " "You can go back to any of them." if versions - else f"'{repo.display_name}' is empty so far." + else f"'{display}' is empty so far." ), } @@ -438,3 +456,65 @@ async def get_repo( "recorded_versions": len(versions), "state": repo.state.value, } + + +# -------------------------------------------------------------------------- +# I-4 / G11 — the off-site copy +# -------------------------------------------------------------------------- +@router.post("/{repo_id}/mirror", status_code=201) +async def enable_mirror( + repo_id: uuid.UUID, + request: Request, + caller: Annotated[Caller, Depends(get_caller)], +) -> dict: + """Turn on the continuous off-site copy. + + Deliberately idempotent and deliberately loud on failure: a mirror that + quietly stopped working is worse than no mirror, because it is a backup you + believe in. + """ + settings = request.app.state.settings + async with _sessionmaker(request)() as session: + repo = await _load_repo(session, repo_id, caller) + owner = _repo_owner_login(repo) + display, slug = repo.display_name, repo.slug + private = repo.visibility != Visibility.public + + mirror = MirrorService(settings) + remote = await mirror.ensure_github_repo(slug, display, private) + await mirror.attach_push_mirror(owner, slug, remote) + + async with _sessionmaker(request)() as session: + session.add( + Mirror(repo_id=repo_id, remote_url=remote, direction="push", state=MirrorState.healthy) + ) + await session.commit() + + return { + "remote": remote, + "sync_on_commit": True, + "speak": "A second copy of this project is now kept somewhere else, automatically.", + "state_proof": {"remote": remote}, + "next_actions": ["windy_git.mirror_status"], + } + + +@router.get("/{repo_id}/mirror") +async def mirror_status( + repo_id: uuid.UUID, + request: Request, + caller: Annotated[Caller, Depends(get_caller)], +) -> dict: + async with _sessionmaker(request)() as session: + repo = await _load_repo(session, repo_id, caller) + owner, slug = _repo_owner_login(repo), repo.slug + + status = await MirrorService(request.app.state.settings).status(owner, slug) + speak = { + "healthy": "A second copy of this project is up to date.", + "degraded": "The second copy is behind. Your work here is safe.", + "absent": "There is no second copy of this project yet.", + "unconfigured": "Off-site copies aren't switched on yet.", + "unknown": "We can't tell how the second copy is doing right now.", + }[status["state"]] + return {**status, "speak": speak} diff --git a/api/app/services/mirror.py b/api/app/services/mirror.py new file mode 100644 index 0000000..8f80b37 --- /dev/null +++ b/api/app/services/mirror.py @@ -0,0 +1,169 @@ +"""I-4 — never a one-way door (strand G11). + +Every repo push-mirrors to GitHub, continuously, from the first save. This is +not a nicety and it is not a migration step: it is the thing that makes moving +off GitHub a reversible decision rather than a bet. + +Today GitHub's durability is free to Grant. The moment repos live only here, +backups, restore rehearsal and a second copy stop being someone else's job — and +the August audits found **no rehearsed restore anywhere in the ecosystem**, for +anything. A continuous mirror buys back that safety for zero dollars. + +Mirror health is a monitored, alerting signal. A mirror nobody checks is a +belief, not a backup — and this ecosystem has already learned that lesson the +expensive way with a fleet canary that sat dead for 37 days while everything +downstream assumed it was fine. +""" + +from __future__ import annotations + +import logging +from datetime import UTC, datetime + +import httpx + +from api.app.config import Settings +from api.app.errors import RepairPointer + +log = logging.getLogger(__name__) + +_TIMEOUT = httpx.Timeout(30.0, connect=10.0) + + +class MirrorService: + """Creates the GitHub counterpart and asks Gitea to keep it in step. + + Gitea owns the actual replication (`push_mirrors`), because a hand-rolled + mirror loop is a background job that fails silently — which is precisely how + the registry's integrity refresh went its entire life calling a 404 and + incrementing a counter instead of raising. + """ + + def __init__(self, settings: Settings) -> None: + self._s = settings + + @property + def configured(self) -> bool: + return bool(self._s.github_token and self._s.github_owner) + + def _require(self) -> None: + if not self.configured: + raise RepairPointer( + status_code=503, + code="mirror_unconfigured", + speak="The off-site copy isn't switched on yet.", + machine_cause="GITHUB_TOKEN or GITHUB_OWNER is unset; refusing to claim a mirror", + remediation_tool=None, + ) + + async def ensure_github_repo(self, name: str, description: str, private: bool) -> str: + """Idempotent. Returns the clone URL of the off-site copy.""" + self._require() + headers = { + "Authorization": f"Bearer {self._s.github_token}", + "Accept": "application/vnd.github+json", + } + owner = self._s.github_owner + async with httpx.AsyncClient(timeout=_TIMEOUT) as client: + existing = await client.get( + f"https://api.github.com/repos/{owner}/{name}", headers=headers + ) + if existing.status_code == 200: + return existing.json()["clone_url"] + + created = await client.post( + "https://api.github.com/user/repos", + headers=headers, + json={ + "name": name, + "description": f"{description} (Windy Git mirror)".strip(), + "private": private, + "auto_init": False, + }, + ) + if created.status_code not in (200, 201): + raise RepairPointer( + status_code=502, + code="mirror_target_failed", + speak="We couldn't set up the off-site copy. Your work here is safe.", + machine_cause=f"POST /user/repos -> {created.status_code}: {created.text[:200]}", + remediation_tool="windy_git.repair.resync_mirror", + ) + return created.json()["clone_url"] + + async def attach_push_mirror(self, owner: str, repo: str, remote_url: str) -> None: + """Ask Gitea to keep the off-site copy in step on every save.""" + self._require() + async with httpx.AsyncClient(timeout=_TIMEOUT) as client: + r = await client.post( + f"{self._s.gitea_base_url}/api/v1/repos/{owner}/{repo}/push_mirrors", + headers={ + "Authorization": f"token {self._s.gitea_admin_token}", + "Content-Type": "application/json", + }, + json={ + "remote_address": remote_url, + "remote_username": self._s.github_owner, + "remote_password": self._s.github_token, + "interval": self._s.mirror_interval, + # The important one: mirror on every save, not just on a timer. + # An hourly timer means an hour of work can be the thing you + # lose, and the window is invisible until it costs you. + "sync_on_commit": True, + }, + ) + if r.status_code not in (200, 201): + raise RepairPointer( + status_code=502, + code="mirror_attach_failed", + speak="We couldn't keep the off-site copy in step. Your work here is safe.", + machine_cause=f"POST push_mirrors -> {r.status_code}: {r.text[:200]}", + remediation_tool="windy_git.repair.resync_mirror", + ) + + async def status(self, owner: str, repo: str) -> dict: + """Report what is TRUE, including 'we do not know'. + + `me-fleet.ts:22-25` in a sibling service refuses to say "online" when it + only knows "registered". Same posture here: an unconfigured mirror is + reported as unknown, never as healthy. + """ + if not self.configured: + return {"state": "unconfigured", "lag_seconds": None, "last_success_at": None} + async with httpx.AsyncClient(timeout=_TIMEOUT) as client: + r = await client.get( + f"{self._s.gitea_base_url}/api/v1/repos/{owner}/{repo}/push_mirrors", + headers={"Authorization": f"token {self._s.gitea_admin_token}"}, + ) + if r.status_code != 200: + return {"state": "unknown", "detail": f"gitea -> {r.status_code}"} + + mirrors = r.json() + if not mirrors: + return {"state": "absent", "lag_seconds": None, "last_success_at": None} + + m = mirrors[0] + last = m.get("last_update") or m.get("last_updated") + lag = None + if last: + try: + lag = int( + (datetime.now(UTC) - datetime.fromisoformat(last.replace("Z", "+00:00"))) + .total_seconds() + ) + except ValueError: + lag = None + + # I-4: lag over the threshold is a P2, not a shrug. + if lag is None: + state = "unknown" + elif lag > self._s.mirror_lag_p2_seconds: + state = "degraded" + else: + state = "healthy" + return { + "state": state, + "lag_seconds": lag, + "last_success_at": last, + "remote": m.get("remote_address"), + } diff --git a/api/tests/test_invariants.py b/api/tests/test_invariants.py index d6b85db..ebe8e97 100644 --- a/api/tests/test_invariants.py +++ b/api/tests/test_invariants.py @@ -349,3 +349,37 @@ def test_g55_shelter_strings_avoid_developer_vocabulary(): low = s.lower() for jargon in ("commit", "repository", "branch", "sha", "push"): assert jargon not in low, f"developer vocabulary in a user string: {s!r}" + + +# -------------------------------------------------------------------------- +# I-4 — never a one-way door +# -------------------------------------------------------------------------- +def test_i04_mirror_syncs_on_every_save_not_just_a_timer(): + """An hourly window means an hour of work can be the thing you lose, and the + window is invisible until it costs you.""" + src = (ROOT / "api" / "app" / "services" / "mirror.py").read_text() + assert '"sync_on_commit": True' in src + + +def test_i04_unconfigured_mirror_is_never_reported_healthy(): + """A mirror nobody checks is a belief, not a backup. An unconfigured one + reports 'unconfigured' — never 'healthy'.""" + src = (ROOT / "api" / "app" / "services" / "mirror.py").read_text() + assert '"state": "unconfigured"' in src + assert "if not self.configured:" in src + + +def test_i04_mirror_lag_threshold_is_set(): + from api.app.config import Settings + + assert Settings().mirror_lag_p2_seconds == 3600 + + +def test_owner_namespace_is_derived_from_the_repo_not_the_caller(): + """Deriving the namespace from the caller is right only while the caller is + the owner, and addresses the wrong namespace the moment a collaborator asks + — surfacing as 'not found', which is the hardest kind of bug to see.""" + src = (ROOT / "api" / "app" / "routes" / "repos.py").read_text() + body = src[src.index("async def list_versions") : src.index("async def create_grant")] + assert "_repo_owner_login(repo)" in body + assert "_owner_login(caller)" not in body