windyadmin/windy-git
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

G11 / I-4: continuous off-site mirror, and a namespace bug fixed

Browse Source 
I-4 said 'never a one-way door' and had no implementation. Now it does.

  - ensure the GitHub counterpart exists (idempotent), then ask Gitea to keep
    it in step with sync_on_commit=True. An hourly timer means an hour of work
    can be the thing you lose, and that window is invisible until it costs you.
  - mirror status reports what is TRUE including 'we do not know'. An
    unconfigured mirror reports unconfigured, NEVER healthy — same posture as
    me-fleet.ts refusing to say 'online' when it only knows 'registered'.
  - lag past the threshold is a P2, not a shrug. A mirror nobody checks is a
    belief, not a backup, and this ecosystem already lost 37 days to a canary
    everyone assumed was fine.

Gitea owns the replication rather than a hand-rolled loop, because a background
job that fails silently is exactly how the registry's integrity refresh spent
its entire life calling a 404 and incrementing a counter instead of raising.

Also fixes a real bug I had written myself: list_versions derived the Gitea
namespace from the CALLER, which is correct only while the caller is the owner
and addresses the wrong namespace the moment a collaborator asks — surfacing as
'not found', which is the hardest kind of bug to see. Now derived from the repo,
with a test that keeps it that way.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Grant Whitmer
2026-08-11 16:11:45 -04:00
parent 066de34489
commit da257652c2
4 changed files with 295 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
api/app/config.py
View File

@@ -82,7 +82,12 @@ class Settings(BaseSettings):
rate_grants_per_day: int = 100
rate_force_pushes_per_day: int = 10
# ---- mirror health (I-4) ----------------------------------------------
# ---- mirror: I-4, never a one-way door --------------------------------
github_token: str = ""
github_owner: str = "sneakyfree"
# Gitea's timer, as a backstop. sync_on_commit is what actually matters:
# an hourly window means an hour of work can be the thing you lose.
mirror_interval: str = "8h0m0s"
mirror_lag_p2_seconds: int = 3600 # env: 60 min -> P2
# ---- agent grants (G5.3) ----------------------------------------------

92
api/app/routes/repos.py
View File

@@ -31,6 +31,8 @@ from api.app.errors import RepairPointer
from api.app.models.core import (
CreatedVia,
GrantRole,
Mirror,
MirrorState,
Repo,
RepoGrant,
RepoState,
@@ -39,6 +41,7 @@ from api.app.models.core import (
Visibility,
)
from api.app.services.gitea_client import GiteaClient
from api.app.services.mirror import MirrorService
router = APIRouter(prefix="/api/v1/repos", tags=["repos"])
@@ -111,6 +114,18 @@ def _sessionmaker(request: Request) -> async_sessionmaker[AsyncSession]:
return maker
def _repo_owner_login(repo: Repo) -> str:
"""The owning login for a repo as STORED, not as inferred from the caller.
Deriving this from the caller works only while the caller is the owner, and
silently addresses the wrong namespace the moment a collaborator calls. That
class of bug reads as "not found" and is very hard to see.
"""
if repo.passport:
return f"agent-{repo.passport.lower().replace('-', '')}"
return f"u-{repo.identity_id[:24]}"
def _owner_login(caller: Caller) -> str:
"""One namespace rule for humans and agents alike (I-6)."""
if caller.actor_type == ActorType.agent and caller.passport:
@@ -272,11 +287,14 @@ async def list_versions(
"""
async with _sessionmaker(request)() as session:
repo = await _load_repo(session, repo_id, caller)
owner = _owner_login(caller) if repo.passport == caller.passport else None
# Derive the namespace from the REPO, never from the caller: the
# caller-derived form is right only while the caller is the owner, and
# addresses the wrong namespace the moment a collaborator asks. It then
# surfaces as "not found", which is about the hardest bug to see.
owner, slug, display = _repo_owner_login(repo), repo.slug, repo.display_name
gitea = GiteaClient(request.app.state.settings)
owner = owner or f"u-{repo.identity_id[:24]}"
commits = await gitea.list_commits(owner, repo.slug)
commits = await gitea.list_commits(owner, slug)
versions = [
{
@@ -294,12 +312,12 @@ async def list_versions(
# "There are 1 saved versions" is the kind of sloppiness the vocabulary
# law exists to catch. Copy is design material, not decoration (I-9).
"speak": (
f"'{repo.display_name}' has 1 saved version. You can go back to it."
f"'{display}' has 1 saved version. You can go back to it."
if len(versions) == 1
else f"'{repo.display_name}' has {len(versions)} saved versions. "
else f"'{display}' has {len(versions)} saved versions. "
"You can go back to any of them."
if versions
else f"'{repo.display_name}' is empty so far."
else f"'{display}' is empty so far."
),
}
@@ -438,3 +456,65 @@ async def get_repo(
"recorded_versions": len(versions),
"state": repo.state.value,
}
# --------------------------------------------------------------------------
# I-4 / G11 — the off-site copy
# --------------------------------------------------------------------------
@router.post("/{repo_id}/mirror", status_code=201)
async def enable_mirror(
repo_id: uuid.UUID,
request: Request,
caller: Annotated[Caller, Depends(get_caller)],
) -> dict:
"""Turn on the continuous off-site copy.
Deliberately idempotent and deliberately loud on failure: a mirror that
quietly stopped working is worse than no mirror, because it is a backup you
believe in.
"""
settings = request.app.state.settings
async with _sessionmaker(request)() as session:
repo = await _load_repo(session, repo_id, caller)
owner = _repo_owner_login(repo)
display, slug = repo.display_name, repo.slug
private = repo.visibility != Visibility.public
mirror = MirrorService(settings)
remote = await mirror.ensure_github_repo(slug, display, private)
await mirror.attach_push_mirror(owner, slug, remote)
async with _sessionmaker(request)() as session:
session.add(
Mirror(repo_id=repo_id, remote_url=remote, direction="push", state=MirrorState.healthy)
)
await session.commit()
return {
"remote": remote,
"sync_on_commit": True,
"speak": "A second copy of this project is now kept somewhere else, automatically.",
"state_proof": {"remote": remote},
"next_actions": ["windy_git.mirror_status"],
}
@router.get("/{repo_id}/mirror")
async def mirror_status(
repo_id: uuid.UUID,
request: Request,
caller: Annotated[Caller, Depends(get_caller)],
) -> dict:
async with _sessionmaker(request)() as session:
repo = await _load_repo(session, repo_id, caller)
owner, slug = _repo_owner_login(repo), repo.slug
status = await MirrorService(request.app.state.settings).status(owner, slug)
speak = {
"healthy": "A second copy of this project is up to date.",
"degraded": "The second copy is behind. Your work here is safe.",
"absent": "There is no second copy of this project yet.",
"unconfigured": "Off-site copies aren't switched on yet.",
"unknown": "We can't tell how the second copy is doing right now.",
}[status["state"]]
return {**status, "speak": speak}

169
api/app/services/mirror.py Normal file
View File

@@ -0,0 +1,169 @@
"""I-4 — never a one-way door (strand G11).
Every repo push-mirrors to GitHub, continuously, from the first save. This is
not a nicety and it is not a migration step: it is the thing that makes moving
off GitHub a reversible decision rather than a bet.
Today GitHub's durability is free to Grant. The moment repos live only here,
backups, restore rehearsal and a second copy stop being someone else's job — and
the August audits found **no rehearsed restore anywhere in the ecosystem**, for
anything. A continuous mirror buys back that safety for zero dollars.
Mirror health is a monitored, alerting signal. A mirror nobody checks is a
belief, not a backup — and this ecosystem has already learned that lesson the
expensive way with a fleet canary that sat dead for 37 days while everything
downstream assumed it was fine.
"""
from __future__ import annotations
import logging
from datetime import UTC, datetime
import httpx
from api.app.config import Settings
from api.app.errors import RepairPointer
log = logging.getLogger(__name__)
_TIMEOUT = httpx.Timeout(30.0, connect=10.0)
class MirrorService:
"""Creates the GitHub counterpart and asks Gitea to keep it in step.
Gitea owns the actual replication (`push_mirrors`), because a hand-rolled
mirror loop is a background job that fails silently — which is precisely how
the registry's integrity refresh went its entire life calling a 404 and
incrementing a counter instead of raising.
"""
def __init__(self, settings: Settings) -> None:
self._s = settings
@property
def configured(self) -> bool:
return bool(self._s.github_token and self._s.github_owner)
def _require(self) -> None:
if not self.configured:
raise RepairPointer(
status_code=503,
code="mirror_unconfigured",
speak="The off-site copy isn't switched on yet.",
machine_cause="GITHUB_TOKEN or GITHUB_OWNER is unset; refusing to claim a mirror",
remediation_tool=None,
)
async def ensure_github_repo(self, name: str, description: str, private: bool) -> str:
"""Idempotent. Returns the clone URL of the off-site copy."""
self._require()
headers = {
"Authorization": f"Bearer {self._s.github_token}",
"Accept": "application/vnd.github+json",
}
owner = self._s.github_owner
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
existing = await client.get(
f"https://api.github.com/repos/{owner}/{name}", headers=headers
)
if existing.status_code == 200:
return existing.json()["clone_url"]
created = await client.post(
"https://api.github.com/user/repos",
headers=headers,
json={
"name": name,
"description": f"{description} (Windy Git mirror)".strip(),
"private": private,
"auto_init": False,
},
)
if created.status_code not in (200, 201):
raise RepairPointer(
status_code=502,
code="mirror_target_failed",
speak="We couldn't set up the off-site copy. Your work here is safe.",
machine_cause=f"POST /user/repos -> {created.status_code}: {created.text[:200]}",
remediation_tool="windy_git.repair.resync_mirror",
)
return created.json()["clone_url"]
async def attach_push_mirror(self, owner: str, repo: str, remote_url: str) -> None:
"""Ask Gitea to keep the off-site copy in step on every save."""
self._require()
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
r = await client.post(
f"{self._s.gitea_base_url}/api/v1/repos/{owner}/{repo}/push_mirrors",
headers={
"Authorization": f"token {self._s.gitea_admin_token}",
"Content-Type": "application/json",
},
json={
"remote_address": remote_url,
"remote_username": self._s.github_owner,
"remote_password": self._s.github_token,
"interval": self._s.mirror_interval,
# The important one: mirror on every save, not just on a timer.
# An hourly timer means an hour of work can be the thing you
# lose, and the window is invisible until it costs you.
"sync_on_commit": True,
},
)
if r.status_code not in (200, 201):
raise RepairPointer(
status_code=502,
code="mirror_attach_failed",
speak="We couldn't keep the off-site copy in step. Your work here is safe.",
machine_cause=f"POST push_mirrors -> {r.status_code}: {r.text[:200]}",
remediation_tool="windy_git.repair.resync_mirror",
)
async def status(self, owner: str, repo: str) -> dict:
"""Report what is TRUE, including 'we do not know'.
`me-fleet.ts:22-25` in a sibling service refuses to say "online" when it
only knows "registered". Same posture here: an unconfigured mirror is
reported as unknown, never as healthy.
"""
if not self.configured:
return {"state": "unconfigured", "lag_seconds": None, "last_success_at": None}
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
r = await client.get(
f"{self._s.gitea_base_url}/api/v1/repos/{owner}/{repo}/push_mirrors",
headers={"Authorization": f"token {self._s.gitea_admin_token}"},
)
if r.status_code != 200:
return {"state": "unknown", "detail": f"gitea -> {r.status_code}"}
mirrors = r.json()
if not mirrors:
return {"state": "absent", "lag_seconds": None, "last_success_at": None}
m = mirrors[0]
last = m.get("last_update") or m.get("last_updated")
lag = None
if last:
try:
lag = int(
(datetime.now(UTC) - datetime.fromisoformat(last.replace("Z", "+00:00")))
.total_seconds()
)
except ValueError:
lag = None
# I-4: lag over the threshold is a P2, not a shrug.
if lag is None:
state = "unknown"
elif lag > self._s.mirror_lag_p2_seconds:
state = "degraded"
else:
state = "healthy"
return {
"state": state,
"lag_seconds": lag,
"last_success_at": last,
"remote": m.get("remote_address"),
}

34
api/tests/test_invariants.py
View File

@@ -349,3 +349,37 @@ def test_g55_shelter_strings_avoid_developer_vocabulary():
low = s.lower()
for jargon in ("commit", "repository", "branch", "sha", "push"):
assert jargon not in low, f"developer vocabulary in a user string: {s!r}"
# --------------------------------------------------------------------------
# I-4 — never a one-way door
# --------------------------------------------------------------------------
def test_i04_mirror_syncs_on_every_save_not_just_a_timer():
"""An hourly window means an hour of work can be the thing you lose, and the
window is invisible until it costs you."""
src = (ROOT / "api" / "app" / "services" / "mirror.py").read_text()
assert '"sync_on_commit": True' in src
def test_i04_unconfigured_mirror_is_never_reported_healthy():
"""A mirror nobody checks is a belief, not a backup. An unconfigured one
reports 'unconfigured' — never 'healthy'."""
src = (ROOT / "api" / "app" / "services" / "mirror.py").read_text()
assert '"state": "unconfigured"' in src
assert "if not self.configured:" in src
def test_i04_mirror_lag_threshold_is_set():
from api.app.config import Settings
assert Settings().mirror_lag_p2_seconds == 3600
def test_owner_namespace_is_derived_from_the_repo_not_the_caller():
"""Deriving the namespace from the caller is right only while the caller is
the owner, and addresses the wrong namespace the moment a collaborator asks
— surfacing as 'not found', which is the hardest kind of bug to see."""
src = (ROOT / "api" / "app" / "routes" / "repos.py").read_text()
body = src[src.index("async def list_versions") : src.index("async def create_grant")]
assert "_repo_owner_login(repo)" in body
assert "_owner_login(caller)" not in body