From 7af306701d72e1ea87986a759eb94122b711f3e3 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 16:57:28 -0500 Subject: [PATCH 01/54] fix(node): close the advisory-lock probe's session if it is dropped mid-acquire A cancelled .await does not cancel an already-sent SQL statement, so a pg_try_advisory_lock whose future is dropped still takes the lock server-side while the caller abandons the result, leaving nothing to release it. The connection then returns to the pool holding the lock and wedges that repo until sqlx recycles the session. Introduce LockProbe, which owns the connection across the in-flight try-lock and closes it in its own Drop if it is still held. close_on_drop is a one-way setter, so the arming lives in Drop rather than being set up front and cleared on success; disarming is Option::take, which is what into_conn does once an acquire is actually observed. This is now the only place that issues pg_try_advisory_lock. The committed gate drops a probe without taking its connection, which is the state a cancellation leaves behind, and polls a standalone observer until the lock frees. Deterministic on purpose: the timing sweep that found this window leaks roughly 1 in 600, which is not something a CI gate can rest on. Observed RED before this change with the lock still held for the full 10s window. Refs #279 --- crates/gitlawb-node/src/git/repo_store.rs | 210 ++++++++++++++++++++++ 1 file changed, 210 insertions(+) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 2aef6ff0..6975473b 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -544,6 +544,70 @@ fn validate_repo_name(repo_name: &str) -> Result<()> { Ok(()) } +/// Owns a lock-pool connection across an in-flight `pg_try_advisory_lock`. +/// +/// A cancelled `.await` does not cancel an already-sent SQL statement, so a +/// try-lock whose future is dropped still takes the lock server-side while the +/// caller abandons the result. Protection therefore has to exist *before* the +/// statement goes out, which is what this type is: its `Drop` closes any +/// connection still held, ending the session so Postgres frees the lock. +/// +/// `close_on_drop()` is a one-way setter, so the arming lives here in `Drop` +/// rather than being set up front and cleared on success; "disarming" is +/// `Option::take`, which is what `take_conn` does once an acquire is observed. +/// This is the only place that issues `pg_try_advisory_lock`. +// No production caller until U3 wires this into `acquire_write`; the attribute +// comes off in that unit. +#[allow(dead_code)] +struct LockProbe { + conn: Option>, +} + +#[allow(dead_code)] // ditto: U3 removes this with the wiring +impl LockProbe { + fn new(conn: sqlx::pool::PoolConnection) -> Self { + Self { conn: Some(conn) } + } + + /// Send the try-lock on the owned connection. + async fn try_lock(&mut self, key: i64) -> Result { + let conn = self + .conn + .as_mut() + .context("LockProbe::try_lock after the connection was taken")?; + let row: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut **conn) + .await + .context("trying advisory lock")?; + Ok(row.0) + } + + /// Hand the lock-owning connection out, leaving `Drop` with nothing to close. + /// Only call this after `try_lock` returned true. + /// + /// Named `take_` rather than `into_` deliberately: clippy expects an `into_*` + /// method to consume `self`, which a type implementing `Drop` cannot do + /// without tripping E0509. + fn take_conn(&mut self) -> Option> { + self.conn.take() + } +} + +impl Drop for LockProbe { + fn drop(&mut self) { + if let Some(mut conn) = self.conn.take() { + // Still holding the connection here means the try-lock's future was + // dropped before `take_conn` ran, so the statement may well have + // completed server-side and taken the lock with nobody left to + // release it. Close the connection instead of returning it to the + // pool: ending the session is what makes Postgres free the lock. + warn!("advisory-lock probe dropped before handing off its connection — closing the session to free the lock"); + conn.close_on_drop(); + } + } +} + /// Guard returned by `acquire_write()`. Holds the Postgres advisory lock and /// uploads to Tigris + releases the lock on `release()`. pub struct RepoWriteGuard { @@ -1839,5 +1903,151 @@ mod tests { .bind(key) .execute(&mut *checker) .await; + // ── U1: cancellation-safe lock probe ─────────────────────────────────── + + /// A pool with every reaping path disabled, so a leaked lock persists through + /// the observation window instead of being freed by ambient recycling. + async fn no_reap_pool(opts: &sqlx::postgres::PgConnectOptions, max: u32) -> PgPool { + sqlx::postgres::PgPoolOptions::new() + .max_connections(max) + .acquire_timeout(std::time::Duration::from_secs(5)) + .min_connections(0) + .idle_timeout(None) + .max_lifetime(None) + .test_before_acquire(false) + .connect_with(opts.clone()) + .await + .expect("no-reap pool") + } + + /// Poll a STANDALONE connection until the key is free, or the deadline passes. + /// + /// Standalone, never from the pool under test: pool reuse would hand the + /// observer the lock-holding session itself, where `pg_try_advisory_lock` + /// succeeds reentrantly and hides the very leak being measured. Polling rather + /// than asserting once because `PoolConnection::drop` spawns the close. + async fn poll_until_free( + opts: &sqlx::postgres::PgConnectOptions, + key: i64, + deadline: std::time::Duration, + ) -> bool { + use sqlx::Connection; + let start = std::time::Instant::now(); + let mut observer = sqlx::PgConnection::connect_with(opts) + .await + .expect("standalone observer connection"); + while start.elapsed() < deadline { + let got: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut observer) + .await + .expect("observer try-lock"); + if got.0 { + let _: (bool,) = sqlx::query_as("SELECT pg_advisory_unlock($1)") + .bind(key) + .fetch_one(&mut observer) + .await + .expect("observer unlock"); + return true; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + false + } + + /// THE COMMITTED GATE for the cancellation window (U1). + /// + /// Dropping the probe without taking its connection is exactly the state a + /// cancellation between the try-lock's send and the guard's construction + /// leaves behind. Deterministic on purpose: the timing sweep that first found + /// this window leaks about 1 in 600, which is not a signal a CI gate can rest + /// on. That sweep stays a local repro. + #[sqlx::test] + async fn lock_probe_dropped_without_taking_frees_the_lock(pool: PgPool) { + let opts = (*pool.connect_options()).clone(); + let lock_pool = no_reap_pool(&opts, 4).await; + let key: i64 = 990_001; + + { + let mut probe = LockProbe::new(lock_pool.acquire().await.unwrap()); + assert!( + probe.try_lock(key).await.unwrap(), + "probe should take a free key" + ); + // dropped here WITHOUT take_conn(): the cancellation shape + } + + assert!( + poll_until_free(&opts, key, std::time::Duration::from_secs(10)).await, + "lock must be freed after a probe is dropped without taking its connection" + ); + } + + /// Must-not: a successful acquire hands the connection out intact, so the + /// normal path does not pay a reconnect per write. + #[sqlx::test] + async fn lock_probe_take_conn_yields_a_usable_connection(pool: PgPool) { + let opts = (*pool.connect_options()).clone(); + let lock_pool = no_reap_pool(&opts, 4).await; + let key: i64 = 990_002; + + let mut probe = LockProbe::new(lock_pool.acquire().await.unwrap()); + assert!(probe.try_lock(key).await.unwrap()); + let mut conn = probe + .take_conn() + .expect("connection after a successful acquire"); + drop(probe); + + let one: (i32,) = sqlx::query_as("SELECT 1") + .fetch_one(&mut *conn) + .await + .expect("handed-out connection must still be usable"); + assert_eq!(one.0, 1); + + let released: (bool,) = sqlx::query_as("SELECT pg_advisory_unlock($1)") + .bind(key) + .fetch_one(&mut *conn) + .await + .unwrap(); + assert!(released.0, "the handed-out connection still owns the lock"); + } + + /// Must-not: a failed probe returns its connection without closing it. Nothing + /// was locked, so closing would be pure churn, and closing on every failed + /// probe would make a 60-attempt spinner tear down 60 backends. + #[sqlx::test] + async fn lock_probe_failed_acquire_does_not_hold_anything(pool: PgPool) { + let opts = (*pool.connect_options()).clone(); + let lock_pool = no_reap_pool(&opts, 4).await; + let key: i64 = 990_003; + + // a standalone holder takes the key first + use sqlx::Connection; + let mut holder = sqlx::PgConnection::connect_with(&opts).await.unwrap(); + let held: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut holder) + .await + .unwrap(); + assert!(held.0); + + { + let mut probe = LockProbe::new(lock_pool.acquire().await.unwrap()); + assert!( + !probe.try_lock(key).await.unwrap(), + "probe must observe false for a key held elsewhere" + ); + } + + // the holder still owns it: the failed probe neither took nor released it + let still: (i64,) = sqlx::query_as( + "SELECT count(*) FROM pg_locks WHERE locktype='advisory' \ + AND ((classid::bigint<<32)|objid::bigint) = $1", + ) + .bind(key) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(still.0, 1, "the original holder must still own the key"); } } From edcf0f2ccff7ecd2c270f4959bc4ccd71e136c69 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:12:48 -0500 Subject: [PATCH 02/54] fix(node): add a dedicated, lazily-connected advisory-lock pool Pinning a connection for the lock's lifetime is only safe if those connections come from somewhere other than the pool serving ordinary request handlers, otherwise a push burst starves every other query. Add GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS (default 32) and a Db::lock_pool builder, with the sizing tradeoff documented on the field and in .env.example: every in-flight write pins one connection here, so the value is a hard ceiling on simultaneous writes node-wide. The pool connects lazily on purpose. The main pool must connect eagerly because it runs migrations, which is why it needs connect_db_with_retry's backoff and degraded-server handoff; that function is not a generic retry helper and the lock pool is built well after the db-ready handoff has already resolved. A lazy pool has no startup work, so it adds no new way for the process to fail to boot and needs no second copy of that machinery. If Postgres is unreachable when the first write arrives, that write fails on the pool's own acquire timeout, like any other database-backed request. Pure configuration, so no proof-first cycle: the knob is covered by a parse/default/reject-zero test. Db::lock_pool has no caller until the guard wiring lands, hence the temporary dead_code attribute. Refs #279 --- .env.example | 15 ++++++++---- crates/gitlawb-node/src/config.rs | 38 +++++++++++++++++++++++++++++++ crates/gitlawb-node/src/db/mod.rs | 34 +++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 5 deletions(-) diff --git a/.env.example b/.env.example index b70d1117..eab6f3a9 100644 --- a/.env.example +++ b/.env.example @@ -24,11 +24,16 @@ DATABASE_URL=postgresql://gitlawb:changeme@localhost:5432/gitlawb # ── Database pool & startup resilience ──────────────────────────────────── # Maximum connections in the PostgreSQL pool. A cap, not a floor — # connections open lazily. Size against the DB server's max_connections, -# remembering admin tooling opens its own pool. Each concurrent write pins one -# connection for its whole duration (the connection-affine advisory lock), so the -# node REJECTS at boot any value below GITLAWB_MAX_CONCURRENT_GIT_PUSHES + 8 -# headroom — keep this comfortably above that (default 48 for pushes 32). -GITLAWB_DB_MAX_CONNECTIONS=48 +# remembering admin tooling opens its own pool. +GITLAWB_DB_MAX_CONNECTIONS=20 +# Maximum connections in the DEDICATED advisory-lock pool, separate from the +# pool above. Every in-flight repo write pins one connection here for its whole +# duration, so this is a hard ceiling on simultaneous writes node-wide: size it +# to expected peak concurrent writers, not small. Keeping it separate is what +# stops a push burst from starving ordinary request handlers. Budget +# (GITLAWB_DB_MAX_CONNECTIONS + this) per node against the server's +# max_connections, times node count, plus admin tooling. +GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS=32 # Seconds a request waits for a pool connection before failing with 503. GITLAWB_DB_ACQUIRE_TIMEOUT_SECS=5 # Upper bound on each startup connect+migrate attempt, in seconds. Keep it diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index fefa063e..a5e9dbc0 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -249,6 +249,25 @@ pub struct Config { )] pub db_max_connections: u32, + /// Maximum connections in the dedicated advisory-lock pool, which is separate + /// from the main pool above. + /// + /// Size this against the expected peak number of concurrent distinct-repo + /// writers, NOT small. Every in-flight repo write pins one connection here for + /// its whole duration (the write, its metadata tail, and the bounded archive + /// upload), so this value is a hard ceiling on simultaneous writes node-wide. + /// Keeping it separate from GITLAWB_DB_MAX_CONNECTIONS is what stops a push + /// burst from starving ordinary request handlers; the cost is that + /// (main pool + lock pool) must fit inside the database server's + /// max_connections, times the number of nodes, plus admin tooling. + #[arg( + long, + env = "GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS", + default_value_t = 32, + value_parser = clap::value_parser!(u32).range(1..) + )] + pub db_lock_pool_max_connections: u32, + /// Maximum time a request waits for a pool connection before failing with /// 503, in seconds. Bounds queueing when the database is slow or down. #[arg( @@ -591,6 +610,25 @@ impl Config { mod tests { use super::*; + #[test] + fn lock_pool_size_defaults_to_32_and_rejects_zero() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).db_lock_pool_max_connections, + 32 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--db-lock-pool-max-connections", "8"]) + .db_lock_pool_max_connections, + 8 + ); + // A zero-sized lock pool would deny every write, so clap must reject it + // rather than let a node boot into a state where no repo can be written. + assert!( + Config::try_parse_from(["gitlawb-node", "--db-lock-pool-max-connections", "0"]) + .is_err() + ); + } + #[test] fn git_service_timeout_defaults_to_600_and_rejects_zero() { assert_eq!( diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 50c3bdda..54e969c5 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -295,6 +295,40 @@ impl Db { Ok(db) } + /// Build the dedicated pool that advisory-lock connections come from. + /// + /// Deliberately **lazy**: connections open on first use rather than at boot. + /// The main pool has to connect eagerly because it runs migrations, which is + /// why it needs `connect_db_with_retry`'s backoff and degraded-server + /// handoff. This pool has no startup work at all, so an eager connect would + /// only add a new way for the process to fail to boot, and would need a + /// second copy of that retry machinery to be safe. Being lazy removes the + /// failure mode instead of handling it: if Postgres is unreachable when the + /// first write arrives, that write fails on the pool's own acquire timeout, + /// the same way any other database-backed request already does. + /// + /// Kept separate from the main pool so a burst of lock-holding connections + /// cannot starve ordinary request handlers. See + /// `GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS` for the sizing tradeoff. + // No caller until U3 wires this into main.rs; the attribute comes off there. + #[allow(dead_code)] + pub fn lock_pool( + database_url: &str, + max_connections: u32, + acquire_timeout: Duration, + ) -> Result { + info!( + max_connections, + acquire_timeout_secs = acquire_timeout.as_secs(), + "creating dedicated advisory-lock pool (lazy)" + ); + PgPoolOptions::new() + .max_connections(max_connections) + .acquire_timeout(acquire_timeout) + .connect_lazy(database_url) + .context("creating advisory-lock pool") + } + /// Cheap liveness probe against the pool, for readiness checks: one /// `SELECT 1` that fails fast when the database is unreachable. pub async fn ping(&self) -> Result<()> { From 5ce3c7f4132d5d63a426a3cf4de511b354633b26 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:24:18 -0500 Subject: [PATCH 03/54] fix(node): hold the lock-owning connection in RepoWriteGuard (#279) Postgres advisory locks are session-scoped: only the backend that took one can release it. acquire_write took the lock through fetch_one(&pool) and release unlocked through execute(&pool), two independent checkouts, so the unlock usually landed on a session that held nothing and returned false. Measured on main: two writers on one node and the same repo BOTH acquired, 50 of 50 sequential cycles leaked, and 100 writes left 100 orphaned advisory locks on the server. The guard now owns the PoolConnection that took the lock, drawn from the dedicated lock pool, and releases on that same session. The retry loop probes through LockProbe so a cancellation mid-acquire cannot strand the lock, and hands the connection back before each backoff so a spinner on a contended repo does not pin a slot while idle. Pool exhaustion is deliberately not retried. It is a different condition from lock contention, and retrying it would spend all 60 attempts on a capacity problem unrelated to this repo while reporting it as someone else holding the lock. Both #279 acceptance tests were observed RED first: the exclusion test admitted the second writer, and the leak test reported 1 lock held where 0 was required. Both GREEN after. Full crate suite 516 passed. Db::pool() is removed because this change was its only caller. Refs #279 --- crates/gitlawb-node/src/db/mod.rs | 7 - crates/gitlawb-node/src/git/repo_store.rs | 329 +++++++++------------- crates/gitlawb-node/src/main.rs | 11 +- 3 files changed, 146 insertions(+), 201 deletions(-) diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 54e969c5..57bafa3f 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -250,11 +250,6 @@ pub struct Db { } impl Db { - /// Access the underlying Postgres connection pool. - pub fn pool(&self) -> &PgPool { - &self.pool - } - #[cfg(test)] pub fn for_testing(pool: PgPool) -> Self { Self { pool } @@ -310,8 +305,6 @@ impl Db { /// Kept separate from the main pool so a burst of lock-holding connections /// cannot starve ordinary request handlers. See /// `GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS` for the sizing tradeoff. - // No caller until U3 wires this into main.rs; the attribute comes off there. - #[allow(dead_code)] pub fn lock_pool( database_url: &str, max_connections: u32, diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 6975473b..e6afe2a8 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -26,8 +26,12 @@ use super::tigris::TigrisClient; pub struct RepoStore { repos_dir: PathBuf, tigris: Option, - /// Shared Postgres pool for advisory locks. - pool: PgPool, + /// Dedicated Postgres pool that advisory-lock connections come from, kept + /// separate from the pool serving ordinary request handlers. Each write guard + /// pins one connection here for its whole lifetime, so a push burst consumes + /// this pool rather than starving application queries. Sized by + /// `GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS`. + lock_pool: PgPool, /// Tracks repos already confirmed to exist in Tigris — avoids redundant /// HEAD checks and background uploads for repos we've already migrated. migrated: Arc>>, @@ -41,30 +45,21 @@ pub struct RepoStore { impl RepoStore { #[cfg(test)] - pub fn for_testing(repos_dir: PathBuf, pool: PgPool) -> Self { + pub fn for_testing(repos_dir: PathBuf, lock_pool: PgPool) -> Self { Self { repos_dir, tigris: None, - pool, + lock_pool, migrated: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())), pre_unlock_gate: None, } } - /// Test-only: every guard from this store parks in `release` right before the - /// `pg_advisory_unlock` await, until `gate` is notified. Dropping the future - /// while it is parked reproduces a client disconnect inside `release`. - #[cfg(test)] - pub fn with_pre_unlock_gate(mut self, gate: Arc) -> Self { - self.pre_unlock_gate = Some(gate); - self - } - - pub fn new(repos_dir: PathBuf, tigris: Option, pool: PgPool) -> Self { + pub fn new(repos_dir: PathBuf, tigris: Option, lock_pool: PgPool) -> Self { Self { repos_dir, tigris, - pool, + lock_pool, migrated: Arc::new(Mutex::new(HashSet::new())), #[cfg(test)] pre_unlock_gate: None, @@ -203,58 +198,54 @@ impl RepoStore { let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; let lock_key = advisory_lock_key(&owner_slug, repo_name); - // Pin a dedicated pooled connection and build the guard holding it BEFORE - // issuing the lock query. Session-level pg advisory locks are - // connection-affine (they can only be released on the session that took - // them), so the guard must own the locking connection; and building the - // guard first means any cancellation after the lock is taken — a - // `tokio::time::timeout` firing during the Tigris download below — drops a - // guard that CAN release, closing the leak the outer timeout otherwise - // opened (#174 F1). - let conn = self - .pool - .acquire() - .await - .context("acquiring db connection for the write advisory lock")?; - let mut guard = RepoWriteGuard { - owner_slug: owner_slug.clone(), - repo_name: repo_name.to_string(), - local_path: local_path.clone(), - lock_key, - conn: Some(conn), - locked: false, - released: false, - tigris: self.tigris.clone(), - #[cfg(test)] - test_pre_unlock_gate: self.pre_unlock_gate.clone(), - }; - - // Acquire the advisory lock with retry, through the guard's OWN connection, - // so the matching unlock (in release, or the Drop backstop) runs on the same - // session — pg_advisory_unlock on a different pooled connection is a no-op. - let mut acquired = false; + // Take the lock on a connection this guard will own for its whole + // lifetime, so the release runs on the same session. `pg_try_advisory_lock` + // with retry rather than a blocking acquire, so a stale lock from a crashed + // connection cannot wedge us indefinitely. + // + // Each attempt checks a connection out and, on failure, returns it BEFORE + // sleeping: a writer spinning on a contended repo must not pin a lock-pool + // slot through its backoff, or a handful of spinners would starve the pool + // for everyone else. + // + // Pool exhaustion is a DIFFERENT condition from "someone else holds the + // lock" and is not retried here. Retrying it would burn all 60 attempts + // against a pool that is full for reasons unrelated to this repo, and would + // report a capacity problem as lock contention. It surfaces immediately with + // its own message instead. + let mut lock_conn = None; for attempt in 0..60 { - let c = guard - .conn - .as_deref_mut() - .expect("write guard holds its connection during acquisition"); - let row: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") - .bind(lock_key) - .fetch_one(&mut *c) + let conn = self + .lock_pool + .acquire() .await - .context("trying advisory lock")?; - if row.0 { - acquired = true; + .context("advisory-lock pool exhausted or unreachable")?; + let mut probe = LockProbe::new(conn); + if probe.try_lock(lock_key).await? { + lock_conn = probe.take_conn(); break; } + // Not acquired, and nothing is locked, so hand the connection back + // before the backoff rather than holding a slot while idle. + drop(probe); if attempt < 59 { tokio::time::sleep(std::time::Duration::from_secs(1)).await; } } - if !acquired { - anyhow::bail!("could not acquire advisory lock after 60s — possible stale lock for {owner_slug}/{repo_name}"); - } - guard.locked = true; + let Some(lock_conn) = lock_conn else { + anyhow::bail!("could not acquire advisory lock after 60 attempts — possible stale lock for {owner_slug}/{repo_name}"); + }; + // From here the lock is HELD. Any early return must not simply drop the + // connection back into the pool, so it is handed to the guard immediately + // below and every exit after this point goes through the guard. + let mut guard = RepoWriteGuard { + owner_slug: owner_slug.clone(), + repo_name: repo_name.to_string(), + local_path: local_path.clone(), + lock_key, + conn: Some(lock_conn), + tigris: self.tigris.clone(), + }; // Always download the latest from Tigris before writing. Local disk may be // stale if another machine pushed since our last access. The guard already @@ -276,6 +267,8 @@ impl RepoStore { } } + // Silence the unused-mut lint until U6 needs the binding mutable. + let _ = &mut guard; Ok(guard) } @@ -556,14 +549,10 @@ fn validate_repo_name(repo_name: &str) -> Result<()> { /// rather than being set up front and cleared on success; "disarming" is /// `Option::take`, which is what `take_conn` does once an acquire is observed. /// This is the only place that issues `pg_try_advisory_lock`. -// No production caller until U3 wires this into `acquire_write`; the attribute -// comes off in that unit. -#[allow(dead_code)] struct LockProbe { conn: Option>, } -#[allow(dead_code)] // ditto: U3 removes this with the wiring impl LockProbe { fn new(conn: sqlx::pool::PoolConnection) -> Self { Self { conn: Some(conn) } @@ -615,18 +604,11 @@ pub struct RepoWriteGuard { repo_name: String, pub local_path: PathBuf, lock_key: i64, - /// The pooled connection that took the advisory lock. Session-level pg - /// advisory locks are connection-affine, so the guard pins that connection - /// for its whole lifetime and unlocks on it (in `release`, or the `Drop` - /// backstop). `None` only after the connection has been taken, either to run - /// the detached unlock in `Drop` or to be closed when `release`'s unlock - /// errored (#174 F3b). - conn: Option>, - /// Set once the advisory lock has actually been taken. A guard dropped - /// before the lock is held (or after `release`) performs no unlock. - locked: bool, - /// Set once `release` has run its unlock, making the `Drop` backstop inert. - released: bool, + /// The connection that TOOK the lock. Postgres advisory locks are + /// session-scoped, so only this session can release it; holding it here is + /// what makes `release` land on the right backend instead of an arbitrary + /// pooled one. + conn: Option>, tigris: Option, /// Test-only seam: when set, `release` parks on this gate at the exact point /// it is about to await `pg_advisory_unlock` (connection still owned, not yet @@ -699,123 +681,14 @@ impl RepoWriteGuard { warn!(repo = %self.repo_name, "write failed — skipping tigris upload to avoid propagating an inconsistent repo"); } - // Release the advisory lock on the SAME connection that took it (session - // advisory locks are connection-affine). Unlock through the connection - // while it is STILL owned by `self` — do not `take()` it first. If this - // future is cancelled during the unlock await, `self` is dropped with - // `conn == Some(..)` and `released == false`, so the `Drop` backstop still - // runs the detached unlock. `released` is set only AFTER the await - // resolves, so a cancellation cannot make the backstop inert (#174 F4). - if self.locked { - #[cfg(test)] - let pre_unlock_gate = self.test_pre_unlock_gate.clone(); - let unlock = if let Some(conn) = self.conn.as_deref_mut() { - // Test-only: park right before the unlock await so a test can drop - // this future mid-unlock (connection owned, not yet released). - #[cfg(test)] - if let Some(gate) = pre_unlock_gate { - gate.notified().await; - } - Some( - sqlx::query("SELECT pg_advisory_unlock($1)") - .bind(self.lock_key) - .execute(&mut *conn) - .await, - ) - } else { - None - }; - // An unlock that ERRORS is a different failure from a cancellation: the - // await resolved, so `Drop` is about to be made inert by `released` - // below, but the session is still alive and still holds the lock - // (statement timeout, admin cancel, aborted transaction). Returning that - // `PoolConnection` to the pool would hand the next caller a connection - // holding a lock nobody tracks (#174 F3b). Connection disposal is the - // single mechanism here, and it is why we do not instead try to keep the - // `Drop` backstop armed: disposal needs `conn.take()`, and `Drop` - // early-returns on `conn == None`. Ending the session is what frees the - // lock, so `released = true` still holds. - if let Some(Err(e)) = unlock { - warn!(repo = %self.repo_name, err = %e, - "advisory unlock failed, closing the connection so the session ends and postgres drops the lock"); - if let Some(conn) = self.conn.take() { - // `close()` over `detach()`: both consume the `PoolConnection` by - // value in sqlx 0.8.6, but we are in an async fn, so `close()` - // sends Terminate and waits for the socket to go down before - // `release` returns. `detach()` would only end the session - // whenever the returned `PgConnection` is dropped and its - // background close completes. If this future is cancelled during - // `close()`, the connection is dropped mid-close, which still - // tears the session down. That last point is also why the await is - // safe to bound: see `close_conn_bounded`, which gives it the - // deadline sqlx does not. - close_conn_bounded(&self.repo_name, conn.close()).await; - } - } - } - self.released = true; - } -} - -impl Drop for RepoWriteGuard { - /// Cancellation-safe backstop: if the guard is dropped while still holding the - /// advisory lock (a `tokio::time::timeout` cancelled `acquire_write`, or a - /// handler future was dropped before `release`), unlock on the pinned - /// connection. This is NOT the backstop for an unlock that ran and returned an - /// error: that case is closed inside `release` by disposing of the connection, - /// because `Drop` early-returns on `conn == None` and the two mechanisms cannot - /// both apply (#174 F3b). `Drop` cannot await, so spawn a detached unlock — it runs on the - /// same session (connection-affine). An off-runtime drop has nothing to spawn onto, - /// so it disposes of the connection instead. On runtime - /// SHUTDOWN the spawned unlock task may be dropped before it polls, so the unlock - /// may not run — but shutdown tears down the pool, and closing the connection - /// releases the session-level advisory lock server-side, so this too is bounded. - fn drop(&mut self) { - if self.released || !self.locked { - return; - } - let Some(mut conn) = self.conn.take() else { - return; - }; - let lock_key = self.lock_key; - let repo_name = self.repo_name.clone(); - match tokio::runtime::Handle::try_current() { - Ok(handle) => { - handle.spawn(async move { - let unlock = sqlx::query("SELECT pg_advisory_unlock($1)") - .bind(lock_key) - .execute(&mut *conn) - .await; - // Same failure as `release`'s (#174 F3b), one level down: the await - // RESOLVED with an error, so the session is alive and still holds - // the lock. Letting this async block end here would drop `conn` and - // RETURN it to the pool, handing the next caller a connection - // holding a lock nobody tracks. Close it instead, which both keeps - // it out of the pool and ends the session that holds the lock. - if let Err(e) = unlock { - warn!(repo = %repo_name, err = %e, "detached advisory-unlock on write-guard drop failed, closing the connection so the session ends and postgres drops the lock"); - close_conn_bounded(&repo_name, conn.close()).await; - } - }); - } - Err(_) => { - // No runtime to spawn the unlock onto, and the connection is already - // out of the guard, so there is no path that unlocks on this session. - // Returning it to the pool would hand the next caller a connection - // still holding the lock. `PoolConnection`'s own drop also spawns its - // return-to-pool task, which panics with no runtime. `detach` gives up - // the pool slot and yields a plain `PgConnection`; dropping that closes - // the socket, which ends the session and is what frees the lock - // server-side. `Drop` cannot await, so this is the whole disposal: - // `close_conn_bounded` is not available here. - drop(conn.detach()); - warn!( - repo = %repo_name, - "RepoWriteGuard dropped off a Tokio runtime; no detached unlock is \ - possible, so the pinned connection is disposed of instead: ending \ - the session is what releases the advisory lock" - ); - } + // Release the advisory lock on the SAME session that took it, then let the + // connection return to the pool. Unlocking through the pool would land on an + // arbitrary backend, where the call is a silent no-op. + if let Some(mut conn) = self.conn.take() { + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(self.lock_key) + .execute(&mut *conn) + .await; } } } @@ -2050,4 +1923,76 @@ mod tests { .unwrap(); assert_eq!(still.0, 1, "the original holder must still own the key"); } + + // ── U3: the #279 acceptance tests ────────────────────────────────────── + + /// The store under test. Pre-U3 this ignores `opts` and shares the app pool, + /// which is exactly the broken shape; the wiring change swaps in a dedicated + /// no-reap lock pool without touching a single test body below. + async fn write_store(pool: &PgPool, opts: &sqlx::postgres::PgConnectOptions) -> RepoStore { + let _ = pool; + RepoStore::for_testing( + PathBuf::from("/tmp/gitlawb-u3"), + no_reap_pool(opts, 8).await, + ) + } + + fn advisory_locks_held(key: i64) -> String { + format!( + "SELECT count(*) FROM pg_locks WHERE locktype='advisory' \ + AND ((classid::bigint<<32)|objid::bigint) = {key}" + ) + } + + /// ACCEPTANCE 1 (#279): two writers on one node and the same repo must not + /// both hold the lock. On the pre-fix shape the second acquire succeeds + /// because the pool hands it the very session holding the lock, where + /// pg_try_advisory_lock is reentrant. + #[sqlx::test] + async fn two_writers_on_the_same_repo_are_not_both_admitted(pool: PgPool) { + let opts = (*pool.connect_options()).clone(); + let store = write_store(&pool, &opts).await; + + let _first = store + .acquire_write("did:key:z6MkU3Excl", "same-repo") + .await + .expect("first writer acquires"); + + let second = tokio::time::timeout( + std::time::Duration::from_secs(8), + store.acquire_write("did:key:z6MkU3Excl", "same-repo"), + ) + .await; + + assert!( + second.is_err(), + "second writer must NOT be admitted while the first holds the guard \ + (it should still be retrying when the deadline hits)" + ); + } + + /// ACCEPTANCE 2 (#279): a completed write leaves no advisory lock behind. + /// On the pre-fix shape the unlock runs on a different pooled session and + /// returns false, so the lock leaks on essentially every write. + #[sqlx::test] + async fn completed_write_releases_its_advisory_lock(pool: PgPool) { + let opts = (*pool.connect_options()).clone(); + let store = write_store(&pool, &opts).await; + + let guard = store + .acquire_write("did:key:z6MkU3Rel", "leak-check") + .await + .expect("acquire"); + guard.release(true).await; + + let key = advisory_lock_key("did_key_z6MkU3Rel", "leak-check"); + let held: (i64,) = sqlx::query_as(&advisory_locks_held(key)) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + held.0, 0, + "a completed write must leave zero advisory locks for its key" + ); + } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 10aaca5b..f8927324 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -287,8 +287,15 @@ async fn main() -> Result<()> { None }; - let repo_store = - git::repo_store::RepoStore::new(config.repos_dir.clone(), tigris, db.pool().clone()); + // Advisory-lock connections come from their own pool: a write guard pins one + // for its whole lifetime, and sharing the application pool would let a push + // burst starve ordinary request handlers. + let lock_pool = db::Db::lock_pool( + &config.database_url, + config.db_lock_pool_max_connections, + std::time::Duration::from_secs(config.db_acquire_timeout_secs), + )?; + let repo_store = git::repo_store::RepoStore::new(config.repos_dir.clone(), tigris, lock_pool); // Per-DID limiter for the creation endpoints. Keyed on the authenticated // DID (attacker-varied), so bound its key set to cap memory. From c49d3cdb57899d286efe75df2fefb2ec6867beec Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:31:41 -0500 Subject: [PATCH 04/54] fix(node): free the advisory lock when a write guard dies without releasing A guard can exit without reaching release(): an early ? on the pre-write download, a panic, or an axum handler future cancelled when the client disconnects. Its session still holds the lock, so returning that connection to the pool would block every future write to the repo until sqlx recycles it, which on the 0.8.6 defaults is ten minutes idle or thirty minutes lifetime. Close the session instead and let Postgres free the lock at session end. One hazard found by writing the teardown test rather than by reasoning: PoolConnection::drop spawns onto the runtime for both closing and returning, and panics outright when no runtime handle exists. That panic would fire inside a Drop and abort the process during unwind. It is not introduced by this commit, it comes with owning a PoolConnection at all, but this is where it becomes reachable. So Drop checks for a runtime first and, with none, leaks the handle deliberately rather than panicking: the process is already exiting and socket teardown ends the session, which is what frees the lock at exit anyway. Observed RED before the fix, with the lock still held for the full 10s poll window against a standalone observer on a no-reap pool, and the teardown case panicking in sqlx-core connection.rs:208. Proven load-bearing after: neutering the close_on_drop call turns the drop test RED again. The must-not case (a released guard reuses its backend pid across four writes on a pool sized 1) passes in both states, so the signal is specific to the abandoned-guard path. Full crate suite 519 passed. Refs #279 --- crates/gitlawb-node/src/git/repo_store.rs | 134 ++++++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index e6afe2a8..01ce3c4d 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -656,6 +656,22 @@ async fn close_conn_bounded( } impl RepoWriteGuard { + /// Backend pid of the session holding the lock. Test-only observable for the + /// must-not-over-close check: if `release` closed the session instead of + /// returning it, consecutive writes would report different pids. + #[cfg(test)] + async fn backend_pid_for_test(&mut self) -> i32 { + let conn = self + .conn + .as_mut() + .expect("guard still holds its connection"); + let pid: (i32,) = sqlx::query_as("SELECT pg_backend_pid()") + .fetch_one(&mut **conn) + .await + .expect("backend pid"); + pid.0 + } + /// Path to the bare repo on local disk. pub fn path(&self) -> &Path { &self.local_path @@ -693,6 +709,40 @@ impl RepoWriteGuard { } } +impl Drop for RepoWriteGuard { + fn drop(&mut self) { + let Some(mut conn) = self.conn.take() else { + // release() already unlocked and handed the connection back. + return; + }; + + // Reached on any exit that skipped release(): an early `?`, a panic, or an + // axum handler future cancelled when the client disconnected. The session + // still holds the advisory lock, so returning it to the pool would block + // every future write to this repo until sqlx recycles the connection. + // + // `PoolConnection::drop` spawns onto the runtime, both to close and to + // return, and panics outright when no runtime handle exists. A panic here + // would run inside a `Drop` and abort the process during unwind, so check + // for a runtime first. With none, the process is already going away: leak + // the handle deliberately rather than panic, and let socket teardown end + // the session, which is what frees the lock at exit anyway. + if tokio::runtime::Handle::try_current().is_ok() { + warn!( + repo = %self.repo_name, + "write guard dropped without release() — closing its session to free the advisory lock" + ); + conn.close_on_drop(); + } else { + warn!( + repo = %self.repo_name, + "write guard dropped with no runtime alive — leaking the connection handle so Drop cannot panic; the lock frees when the process exits" + ); + std::mem::forget(conn); + } + } +} + /// Compute a stable i64 hash for a Postgres advisory lock key. /// /// Uses SHA-256 (not `DefaultHasher`) so the same `(owner_slug, repo_name)` @@ -1995,4 +2045,88 @@ mod tests { "a completed write must leave zero advisory locks for its key" ); } + + // ── U4: a guard that dies without releasing must free the lock ────────── + + /// A guard dropped without `release()` (an early `?`, a panic, or a handler + /// future cancelled on client disconnect) must not return a lock-bearing + /// connection to the pool, where it would block every future write to that + /// repo until sqlx recycles the session. + #[sqlx::test] + async fn guard_dropped_without_release_frees_the_lock(pool: PgPool) { + let opts = (*pool.connect_options()).clone(); + let store = write_store(&pool, &opts).await; + let key = advisory_lock_key("did_key_z6MkU4Drop", "dropped"); + + { + let _guard = store + .acquire_write("did:key:z6MkU4Drop", "dropped") + .await + .expect("acquire"); + // dropped here without release() + } + + assert!( + poll_until_free(&opts, key, std::time::Duration::from_secs(10)).await, + "lock must be freed when a guard is dropped without release()" + ); + } + + /// Must-not over-close: the normal path returns its connection to the pool, so + /// a healthy write does not pay a reconnect. Sized to one connection so the + /// backend pid is a direct observable: if `release` were closing the session, + /// each cycle would land on a fresh backend. + #[sqlx::test] + async fn normal_release_reuses_the_same_backend(pool: PgPool) { + let opts = (*pool.connect_options()).clone(); + let store = RepoStore::for_testing( + PathBuf::from("/tmp/gitlawb-u4"), + no_reap_pool(&opts, 1).await, + ); + + let mut pids = Vec::new(); + for i in 0..4 { + let repo = format!("reuse-{i}"); + let mut guard = store + .acquire_write("did:key:z6MkU4Reuse", &repo) + .await + .expect("acquire"); + pids.push(guard.backend_pid_for_test().await); + guard.release(true).await; + } + assert!( + pids.windows(2).all(|w| w[0] == w[1]), + "a released guard must return its connection to the pool, so all four \ + writes share one backend; saw {pids:?}" + ); + } + + /// A guard abandoned while the runtime is tearing down must not panic. + /// `PoolConnection::drop` calls `crate::rt::spawn`, which panics without a + /// runtime handle, and a panic inside `Drop` during unwind aborts the process. + /// At real process exit the lock is freed by socket teardown, not by this Drop + /// body, so this asserts no-panic rather than lock release. + #[test] + fn guard_dropped_at_runtime_teardown_does_not_panic() { + let url = match std::env::var("DATABASE_URL") { + Ok(u) => u, + Err(_) => return, // no database configured; nothing to assert + }; + let rt = tokio::runtime::Runtime::new().unwrap(); + let guard = rt.block_on(async { + let lock_pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(2) + .connect(&url) + .await + .expect("lock pool"); + let store = RepoStore::for_testing(PathBuf::from("/tmp/gitlawb-u4b"), lock_pool); + store + .acquire_write("did:key:z6MkU4Teardown", "teardown") + .await + .expect("acquire") + }); + // Shut the runtime down first, then drop the guard with no runtime alive. + drop(rt); + drop(guard); + } } From 98f2daf80477b32b197551517e3b2e024dd65c90 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 18:03:22 -0500 Subject: [PATCH 05/54] fix(node): read the advisory unlock's result instead of discarding it pg_advisory_unlock reports "you did not hold this lock" as a false RETURN VALUE plus a server WARNING, never an error, so let _ = execute(...) could not distinguish a real release from a no-op. Three blindnesses stacked in those four lines: execute discards the row, the boolean lives in the row, and let _ discarded the Result too. Read it through fetch_one into (bool,). A false means this session's lock state is not what we believe it is, so the connection stays in the guard for Drop to close rather than being handed back to the pool as clean. Only a confirmed unlock returns it. A query error gets the same treatment, since the lock must not outlive a session we can no longer reason about. Note this is the only unlock site: the pre-write download's error path returns through the guard, so Drop covers it and there is no second place to keep in sync. RED before: the connection came back on the same backend pid after an unlock that returned false. GREEN after, and proven load-bearing by treating false as success, which turns it RED again. The must-not case (a normal release still reuses its backend) passes in both states, so this does not over-close the happy path. Full crate suite 520 passed. Refs #279 --- crates/gitlawb-node/src/git/repo_store.rs | 102 ++++++++++++++++++++-- 1 file changed, 94 insertions(+), 8 deletions(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 01ce3c4d..6ef907d3 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -697,14 +697,47 @@ impl RepoWriteGuard { warn!(repo = %self.repo_name, "write failed — skipping tigris upload to avoid propagating an inconsistent repo"); } - // Release the advisory lock on the SAME session that took it, then let the - // connection return to the pool. Unlocking through the pool would land on an - // arbitrary backend, where the call is a silent no-op. - if let Some(mut conn) = self.conn.take() { - let _ = sqlx::query("SELECT pg_advisory_unlock($1)") - .bind(self.lock_key) - .execute(&mut *conn) - .await; + // Release the advisory lock on the SAME session that took it. Unlocking + // through the pool would land on an arbitrary backend, where the call is a + // silent no-op. + // + // Read the boolean. `pg_advisory_unlock` reports "you did not hold this + // lock" as a false RETURN VALUE plus a server WARNING, never an error, so a + // discarded result cannot distinguish a real release from a no-op. A false + // here means this session's lock state is not what we believe it is, so the + // connection is left in `self.conn` for `Drop` to close rather than being + // handed back to the pool as clean. Only a confirmed unlock returns it. + let lock_key = self.lock_key; + let unlock = match self.conn.as_mut() { + Some(conn) => Some( + sqlx::query_as::<_, (bool,)>("SELECT pg_advisory_unlock($1)") + .bind(lock_key) + .fetch_one(&mut **conn) + .await, + ), + None => None, + }; + match unlock { + Some(Ok((true,))) => { + // Confirmed released: safe to return to the pool. + self.conn.take(); + } + Some(Ok((false,))) => { + warn!( + repo = %self.repo_name, + lock_key, + "advisory unlock reported the session did not hold this lock — closing the session instead of pooling it" + ); + } + Some(Err(e)) => { + warn!( + repo = %self.repo_name, + lock_key, + err = %e, + "advisory unlock failed — closing the session so the lock cannot outlive it" + ); + } + None => {} } } } @@ -2129,4 +2162,57 @@ mod tests { drop(rt); drop(guard); } + + // ── U5: the unlock's boolean result must be observed ──────────────────── + + /// `pg_advisory_unlock` reports "you did not hold this lock" as a `false` + /// RETURN VALUE plus a server WARNING, never an error, so a discarded result + /// cannot tell a real release from a no-op. A session that did not hold the + /// key must not be returned to the pool as if it were clean. + /// + /// The observable is the backend pid: on a one-connection pool, a session that + /// was closed forces the next acquire onto a fresh backend, while one returned + /// normally is handed straight back. + #[sqlx::test] + async fn release_that_did_not_hold_the_lock_closes_the_session(pool: PgPool) { + let opts = (*pool.connect_options()).clone(); + let lock_pool = no_reap_pool(&opts, 1).await; + + let pid_before = { + let mut c = lock_pool.acquire().await.unwrap(); + let pid: (i32,) = sqlx::query_as("SELECT pg_backend_pid()") + .fetch_one(&mut *c) + .await + .unwrap(); + pid.0 + }; + + // A guard whose key was never locked: release()'s unlock returns false. + let guard = RepoWriteGuard { + owner_slug: "did_key_z6MkU5".to_string(), + repo_name: "never-locked".to_string(), + local_path: PathBuf::from("/tmp/gitlawb-u5"), + lock_key: 995_001, + conn: Some(lock_pool.acquire().await.unwrap()), + tigris: None, + }; + guard.release(true).await; + + // Give the spawned close a moment, then see which backend we land on. + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + let pid_after = { + let mut c = lock_pool.acquire().await.unwrap(); + let pid: (i32,) = sqlx::query_as("SELECT pg_backend_pid()") + .fetch_one(&mut *c) + .await + .unwrap(); + pid.0 + }; + + assert_ne!( + pid_before, pid_after, + "an unlock that returned false means the session's lock state is not \ + what we think it is; that connection must be closed, not pooled" + ); + } } From 9321d3b9936bf5e19cb437d2bd24a44cc7161c8d Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 18:18:16 -0500 Subject: [PATCH 06/54] fix(node): bound the object-storage transfers that run under the write lock Two transfers happen while the per-repo advisory lock is held: the archive download inside acquire_write, which runs after the lock is taken and before the guard exists, and the upload inside release. Both were free before the guard pinned a lock-pool connection, because the lock's connection went back to the pool immediately. Now an unbounded stall holds a lock-pool slot for its whole duration, so enough concurrent stalls deny every write on the node with no reaping path. Add GITLAWB_LOCK_HELD_TRANSFER_TIMEOUT_SECS (default 300) and route both through it. A timed-out upload is UNKNOWABLE rather than failed, so the timeout arm takes no compensating action: the PUT may well have landed. The lock is released either way, which trades a narrow last-writer-wins window for not wedging the repo behind a stalled transfer. Deliberate, and stated here rather than discovered later. Note what is NOT bounded by this knob: acquire_fresh's download. It runs before any lock is taken, so it holds nothing. An earlier revision of this change put the bound there by mistake, because both functions contain an identical download call and the first match won. Two disclosures. Coverage: the committed tests cover the bound mechanism, not the wiring. Driving a genuinely stalled transfer through acquire_write needs either the object-store abstraction (out of scope) or a process-global AWS_ENDPOINT_URL_S3 mutation, which would make the suite order-dependent under the concurrent runner. That a stalled transfer is bounded is therefore verified by reading, not by execution. Behavior: on timeout with a local copy present, the download falls into the pre-existing self-healing fallback and the write proceeds against that local copy. This widens the conditions reaching that path from corrupt-or-unreachable to include merely-slow, so a stale tree could now be written and re-uploaded on a slow link. Kept consistent with the existing failed-download behavior rather than inventing new semantics here; changing it is its own decision. Full crate suite 522 passed. Refs #279 --- .env.example | 7 + crates/gitlawb-node/src/config.rs | 18 +++ crates/gitlawb-node/src/git/repo_store.rs | 171 +++++++++++++++------- crates/gitlawb-node/src/main.rs | 7 +- 4 files changed, 152 insertions(+), 51 deletions(-) diff --git a/.env.example b/.env.example index eab6f3a9..dd948317 100644 --- a/.env.example +++ b/.env.example @@ -34,6 +34,13 @@ GITLAWB_DB_MAX_CONNECTIONS=20 # (GITLAWB_DB_MAX_CONNECTIONS + this) per node against the server's # max_connections, times node count, plus admin tooling. GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS=32 +# Upper bound, in seconds, on any object-storage transfer that runs while a +# per-repo write lock is HELD (the archive download inside acquire_write and the +# upload inside release). These were free before the lock's connection was +# pinned to the guard; now an unbounded stall holds a lock-pool slot, and enough +# stalls deny every write on the node. Worst-case slot occupancy is roughly this +# value, so read it together with the pool size above. +GITLAWB_LOCK_HELD_TRANSFER_TIMEOUT_SECS=300 # Seconds a request waits for a pool connection before failing with 503. GITLAWB_DB_ACQUIRE_TIMEOUT_SECS=5 # Upper bound on each startup connect+migrate attempt, in seconds. Keep it diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index a5e9dbc0..12955b6f 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -268,6 +268,24 @@ pub struct Config { )] pub db_lock_pool_max_connections: u32, + /// Upper bound, in seconds, on any single object-storage transfer that runs + /// while the per-repo advisory lock is HELD. + /// + /// Two such transfers exist: the archive download in `acquire_write`, which + /// runs after the lock is taken and before the guard is constructed, and the + /// archive upload in `release`. Both used to be free, because the lock's + /// connection was returned to the pool immediately; now that a write guard + /// pins a lock-pool connection for its whole lifetime, an unbounded transfer + /// holds that slot, and enough stalled transfers deny every write on the node. + /// This is the bound that keeps a stall from becoming an outage. + #[arg( + long, + env = "GITLAWB_LOCK_HELD_TRANSFER_TIMEOUT_SECS", + default_value_t = 300, + value_parser = clap::value_parser!(u64).range(1..) + )] + pub lock_held_transfer_timeout_secs: u64, + /// Maximum time a request waits for a pool connection before failing with /// 503, in seconds. Bounds queueing when the database is slow or down. #[arg( diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 6ef907d3..db6bba71 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -11,6 +11,7 @@ use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::sync::Arc; +use std::time::Duration; use anyhow::{Context, Result}; use sqlx::pool::PoolConnection; @@ -32,6 +33,8 @@ pub struct RepoStore { /// this pool rather than starving application queries. Sized by /// `GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS`. lock_pool: PgPool, + /// Bound on any object-storage transfer that runs while the lock is HELD. + lock_held_transfer_timeout: Duration, /// Tracks repos already confirmed to exist in Tigris — avoids redundant /// HEAD checks and background uploads for repos we've already migrated. migrated: Arc>>, @@ -50,16 +53,23 @@ impl RepoStore { repos_dir, tigris: None, lock_pool, + lock_held_transfer_timeout: Duration::from_secs(300), migrated: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())), pre_unlock_gate: None, } } - pub fn new(repos_dir: PathBuf, tigris: Option, lock_pool: PgPool) -> Self { + pub fn new( + repos_dir: PathBuf, + tigris: Option, + lock_pool: PgPool, + lock_held_transfer_timeout: Duration, + ) -> Self { Self { repos_dir, tigris, lock_pool, + lock_held_transfer_timeout, migrated: Arc::new(Mutex::new(HashSet::new())), #[cfg(test)] pre_unlock_gate: None, @@ -245,6 +255,7 @@ impl RepoStore { lock_key, conn: Some(lock_conn), tigris: self.tigris.clone(), + lock_held_transfer_timeout: self.lock_held_transfer_timeout, }; // Always download the latest from Tigris before writing. Local disk may be @@ -253,7 +264,24 @@ impl RepoStore { if let Some(ref tigris) = self.tigris { if tigris.exists(&owner_slug, repo_name).await.unwrap_or(false) { debug!(repo = %repo_name, "write acquire: downloading latest from tigris"); - if let Err(e) = tigris.download(&owner_slug, repo_name, &local_path).await { + // The lock is already HELD at this point and the guard owns a + // lock-pool slot, so this transfer is bounded: an unbounded stall + // here would hold both, and enough of them deny every write on the + // node. acquire_fresh's download is deliberately NOT bounded by + // this knob, because it runs before any lock is taken. + let downloaded = bounded_transfer( + "acquire-download", + repo_name, + self.lock_held_transfer_timeout, + tigris.download(&owner_slug, repo_name, &local_path), + ) + .await + .unwrap_or_else(|| { + Err(anyhow::anyhow!( + "archive download exceeded the under-lock transfer bound" + )) + }); + if let Err(e) = downloaded { // Same self-healing fallback as acquire_fresh: a corrupt/unreadable // Tigris archive must not block a write when a valid local copy // exists — release(success) will re-upload a good archive. @@ -610,49 +638,8 @@ pub struct RepoWriteGuard { /// pooled one. conn: Option>, tigris: Option, - /// Test-only seam: when set, `release` parks on this gate at the exact point - /// it is about to await `pg_advisory_unlock` (connection still owned, not yet - /// released). Dropping the `release` future while it is parked reproduces a - /// mid-unlock cancellation, so a test can assert the `Drop` backstop still - /// frees the session lock. Never set outside tests. - #[cfg(test)] - test_pre_unlock_gate: Option>, -} - -/// Deadline for tearing down the connection that saw a failing `pg_advisory_unlock`. -/// Long enough that a healthy socket always finishes well inside it, short enough that -/// a blackholed one does not pin admission resources for a TCP timeout. -const UNLOCK_ERROR_CLOSE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); - -/// Await `close` under a deadline (#174 F3c). -/// -/// `release` awaits this INLINE while the global write permit, the per-source permit -/// and the write lease are all still held, and sqlx puts no deadline on `close()`: -/// it writes Terminate and then tears the socket down. The branch that reaches here is -/// by definition a connection whose last statement errored, and a blackholed TCP path -/// to Postgres (a cloud failover that drops packets without an RST) is a plausible -/// cause, so an unbounded await here parks every later push to the repo behind three -/// pinned admission resources until the steal bound. -/// -/// On elapsed the future is simply dropped, which drops the `PoolConnection` it owns. -/// Dropping it closes the socket, and closing the socket is what actually ends the -/// session and makes Postgres release the lock, so the deadline costs nothing the -/// graceful path was buying. -async fn close_conn_bounded( - repo_name: &str, - close: impl std::future::Future>, -) { - match tokio::time::timeout(UNLOCK_ERROR_CLOSE_TIMEOUT, close).await { - Ok(Ok(())) => {} - Ok(Err(e)) => { - warn!(repo = %repo_name, err = %e, - "closing the write-lock connection failed, the session teardown still frees the lock server-side"); - } - Err(_) => { - warn!(repo = %repo_name, timeout_secs = UNLOCK_ERROR_CLOSE_TIMEOUT.as_secs(), - "closing the write-lock connection timed out, dropping it instead; the socket goes down either way, which is what frees the lock server-side"); - } - } + /// Bound on the release-side upload, which runs with the lock still held. + lock_held_transfer_timeout: Duration, } impl RepoWriteGuard { @@ -686,11 +673,28 @@ impl RepoWriteGuard { // Upload to Tigris only on success. if success { if let Some(ref tigris) = self.tigris { - if let Err(e) = tigris - .upload(&self.owner_slug, &self.repo_name, &self.local_path) - .await + // Bounded for the same reason as the acquire-side download: this + // runs with the lock held and a lock-pool slot pinned. + match bounded_transfer( + "release-upload", + &self.repo_name, + self.lock_held_transfer_timeout, + tigris.upload(&self.owner_slug, &self.repo_name, &self.local_path), + ) + .await { - warn!(repo = %self.repo_name, err = %e, "failed to upload repo to tigris after write"); + Some(Ok(())) => {} + Some(Err(e)) => { + warn!(repo = %self.repo_name, err = %e, "failed to upload repo to tigris after write"); + } + None => { + // Timed out is UNKNOWABLE, not failed: the PUT may well + // have landed, so there is deliberately no compensating + // action. The lock releases either way, so the repo is not + // wedged behind a stalled transfer. The tradeoff is a narrow + // last-writer-wins window if the slow PUT lands after + // another writer takes the lock. + } } } } else { @@ -776,6 +780,33 @@ impl Drop for RepoWriteGuard { } } +/// Run a future under a wall-clock bound, returning `None` if it did not finish. +/// +/// For the object-storage transfers that run while the per-repo advisory lock is +/// held. Those were free before the lock's connection was pinned to the guard; +/// now an unbounded transfer holds a lock-pool slot for as long as it stalls, and +/// enough of them deny every write on the node. +/// +/// A timed-out transfer is **unknowable**, not failed: it may well have landed. +/// Callers must not compensate as though it definitely failed. +async fn bounded_transfer(label: &str, repo: &str, limit: Duration, fut: F) -> Option +where + F: std::future::Future, +{ + match tokio::time::timeout(limit, fut).await { + Ok(v) => Some(v), + Err(_) => { + warn!( + repo = %repo, + transfer = label, + limit_secs = limit.as_secs(), + "object-storage transfer exceeded its under-lock bound — giving up so the advisory lock and its pool slot are not held longer" + ); + None + } + } +} + /// Compute a stable i64 hash for a Postgres advisory lock key. /// /// Uses SHA-256 (not `DefaultHasher`) so the same `(owner_slug, repo_name)` @@ -1165,7 +1196,12 @@ mod tests { // the pool or the network. Fabricate a pool reference via PgPool::connect_lazy // so we don't need a live DB. let pool = sqlx::PgPool::connect_lazy("postgres://invalid").unwrap(); - RepoStore::new(PathBuf::from("/var/lib/gitlawb/repos"), None, pool) + RepoStore::new( + PathBuf::from("/var/lib/gitlawb/repos"), + None, + pool, + Duration::from_secs(300), + ) } #[tokio::test] @@ -2195,6 +2231,7 @@ mod tests { lock_key: 995_001, conn: Some(lock_pool.acquire().await.unwrap()), tigris: None, + lock_held_transfer_timeout: Duration::from_secs(300), }; guard.release(true).await; @@ -2215,4 +2252,38 @@ mod tests { what we think it is; that connection must be closed, not pooled" ); } + + // ── U6: under-lock transfers are bounded ──────────────────────────────── + + /// The bound itself. Driving a real stalled transfer through `acquire_write` + /// would need either the object-store abstraction (out of scope here) or a + /// process-global `AWS_ENDPOINT_URL_S3` mutation, which would make the suite + /// order-dependent under the concurrent test runner. So this covers the + /// mechanism deterministically and the wiring is verified by reading, which is + /// recorded as a coverage gap rather than papered over. + #[tokio::test] + async fn bounded_transfer_gives_up_past_the_limit() { + let slow = async { + tokio::time::sleep(std::time::Duration::from_secs(30)).await; + Ok::<(), anyhow::Error>(()) + }; + let out = + bounded_transfer("test", "repo", std::time::Duration::from_millis(50), slow).await; + assert!( + out.is_none(), + "a transfer past its limit must report None so the caller stops holding the lock" + ); + } + + /// Must-not: a transfer that finishes inside the limit is returned intact and + /// is not truncated by the bound. + #[tokio::test] + async fn bounded_transfer_passes_through_a_prompt_result() { + let quick = async { Ok::(7) }; + let out = bounded_transfer("test", "repo", std::time::Duration::from_secs(30), quick).await; + assert!( + matches!(out, Some(Ok(7))), + "a prompt transfer must pass through untouched" + ); + } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index f8927324..74ccfdc4 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -295,7 +295,12 @@ async fn main() -> Result<()> { config.db_lock_pool_max_connections, std::time::Duration::from_secs(config.db_acquire_timeout_secs), )?; - let repo_store = git::repo_store::RepoStore::new(config.repos_dir.clone(), tigris, lock_pool); + let repo_store = git::repo_store::RepoStore::new( + config.repos_dir.clone(), + tigris, + lock_pool, + std::time::Duration::from_secs(config.lock_held_transfer_timeout_secs), + ); // Per-DID limiter for the creation endpoints. Keyed on the authenticated // DID (attacker-varied), so bound its key set to cap memory. From 9da0939312fd9c2e0ae2cfcd37a4c20ea444f7b5 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 18:26:11 -0500 Subject: [PATCH 07/54] fix(node): authorize before taking the write lock in close_issue close_issue acquired the per-repo advisory lock, then ran the owner-or-author check, then returned 403 with the lock still held. That was harmless only because the lock excluded nothing. Making it work, as the preceding commits do, turns this ordering into a denial primitive: any caller with repo read access could take the write lock on demand and be refused the write, while a legitimate writer burned its 60-attempt retry budget against a lock held by someone with no write authorization. On a public repo that is every permissionless identity. This series creates the exposure, so it closes it in the same series. The owner check is cheap and moves above the lock outright. The author fallback needs the issue's git-JSON blob, since there is no author column, so it now reads through acquire() rather than acquire_write(): an issue's author is set at creation and never changes, so reading it outside the lock races nothing, and only the mutation needs exclusion. Two deliberate behavior choices. A non-owner whose authorship cannot be established gets 403 rather than 404, so this route does not tell an unauthorized caller whether an issue exists. And the owner's existing 404-for-a-missing-issue path is preserved by re-reading under the guard, which the mutation wants anyway. Swept the other three acquire_write sites: merge_pr owner-gates at :200 before acquiring at :214, the repo write path checks did_matches at :916 before :933, and create_issue's read gate is legitimate because that caller IS authorized for the action it performs. close_issue was the only one with the wrong order. RED before: with an independent session holding the repo's lock, a stranger's request sat in the retry loop until the 3s deadline fired (Elapsed), which is the wedge itself. GREEN after, refused immediately. Proven load-bearing: disabling the pre-lock refusal restores the Elapsed. Full crate suite 523 passed. Refs #279 --- crates/gitlawb-node/src/api/issues.rs | 138 +++++++++++++++++++--- crates/gitlawb-node/src/git/repo_store.rs | 7 ++ 2 files changed, 128 insertions(+), 17 deletions(-) diff --git a/crates/gitlawb-node/src/api/issues.rs b/crates/gitlawb-node/src/api/issues.rs index 17acf9af..03811310 100644 --- a/crates/gitlawb-node/src/api/issues.rs +++ b/crates/gitlawb-node/src/api/issues.rs @@ -229,6 +229,44 @@ pub async fn close_issue( .await? .ok_or_else(|| AppError::RepoNotFound(format!("{owner}/{repo}")))?; + // AUTHORIZE BEFORE ACQUIRING. The per-repo advisory lock genuinely excludes + // now, so taking it first would hand any caller with read access a way to hold + // that lock on demand and be refused afterwards, while a legitimate writer + // burned its retry budget against it. On a public repo that is every + // permissionless identity. The lock must not be reachable by a caller who is + // about to be refused the write. + let is_owner = crate::api::require_repo_owner(&record, &auth.0).is_ok(); + if !is_owner { + // Not the owner, so the author fallback decides it, and the author lives in + // the issue's git-JSON blob rather than a DB column. Read it WITHOUT the + // write lock: an issue's author is set at creation and never changes, so + // reading it outside the lock races nothing. `acquire` ensures the repo is + // on disk without taking the lock. + let disk_path = state + .repo_store + .acquire(&record.owner_did, &record.name) + .await + .map_err(|e| AppError::Git(e.to_string()))?; + let author_did: Option = match git_issues::get_issue(&disk_path, &issue_id) { + Ok(Some(raw)) => serde_json::from_str::(&raw) + .ok() + .and_then(|i| i.author), + // Cannot establish authorship, so fail closed. Deliberately 403 rather + // than 404 for a non-owner: a caller who is not authorized to write + // should not learn from this route whether the issue exists. + Ok(None) | Err(_) => None, + }; + let is_author = author_did + .as_deref() + .is_some_and(|a| crate::api::did_matches(&auth.0, a)); + if !is_author { + return Err(AppError::Forbidden( + "only the repo owner or the issue author can close this issue".into(), + )); + } + } + + // Authorized. Only now is the lock taken. let guard = state .repo_store .acquire_write(&record.owner_did, &record.name) @@ -236,13 +274,10 @@ pub async fn close_issue( .map_err(|e| AppError::Git(e.to_string()))?; let disk_path = guard.path().to_path_buf(); - // Owner OR issue author may close. The author lives in the issue's git-JSON - // blob (not a DB column); a None author (legacy issues) falls back to - // owner-only. Read it under the write guard, before mutating. - let author_did: Option = match git_issues::get_issue(&disk_path, &issue_id) { - Ok(Some(raw)) => serde_json::from_str::(&raw) - .ok() - .and_then(|i| i.author), + // Re-read under the guard so the mutation acts on current state, and keep the + // owner's existing 404-for-a-missing-issue behavior. + match git_issues::get_issue(&disk_path, &issue_id) { + Ok(Some(_)) => {} Ok(None) => { guard.release(false).await; return Err(AppError::NotFound(format!("issue {issue_id} not found"))); @@ -251,16 +286,6 @@ pub async fn close_issue( guard.release(false).await; return Err(AppError::Git(e.to_string())); } - }; - let is_owner = crate::api::require_repo_owner(&record, &auth.0).is_ok(); - let is_author = author_did - .as_deref() - .is_some_and(|a| crate::api::did_matches(&auth.0, a)); - if !is_owner && !is_author { - guard.release(false).await; - return Err(AppError::Forbidden( - "only the repo owner or the issue author can close this issue".into(), - )); } let close_result = git_issues::close_issue(&disk_path, &issue_id); @@ -279,3 +304,82 @@ pub async fn close_issue( Ok(Json(issue)) } + +#[cfg(test)] +mod tests { + use super::*; + use sqlx::PgPool; + + /// U7: once the advisory lock actually excludes, taking it BEFORE authorizing + /// turns close_issue into a wedge primitive. Any caller with repo read access + /// (on a public repo, any permissionless identity) could take the per-repo + /// write lock on demand and be refused the write afterwards, while the owner's + /// push burned its retry budget against a lock held by someone with no write + /// authorization. + /// + /// The observable: hold the lock from an independent session, then call the + /// handler as a stranger. If it authorizes first it refuses immediately; if it + /// acquires first it sits in the 60-attempt retry loop and the deadline fires. + #[sqlx::test] + async fn stranger_is_refused_without_waiting_on_the_write_lock(pool: PgPool) { + use sqlx::Connection; + let opts = (*pool.connect_options()).clone(); + let state = crate::test_support::test_state(pool.clone()).await; + + let owner = "did:key:z6MkU7Owner"; + state + .db + .upsert_mirror_repo("z6MkU7Owner", "u7repo", "/tmp/u7repo", None, true) + .await + .expect("seed repo"); + let record = state + .db + .get_repo("z6MkU7Owner", "u7repo") + .await + .expect("get_repo") + .expect("repo exists"); + + // An independent session holds the repo's write lock for the whole call. + let key = crate::git::repo_store::advisory_lock_key_for_test( + &record.owner_did.replace([':', '/'], "_"), + &record.name, + ); + let mut holder = sqlx::PgConnection::connect_with(&opts).await.unwrap(); + let held: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut holder) + .await + .unwrap(); + assert!( + held.0, + "the test must hold the lock for this to mean anything" + ); + let _ = owner; + + let stranger = crate::auth::AuthenticatedDid("did:key:z6MkU7Stranger".to_string()); + let outcome = tokio::time::timeout( + std::time::Duration::from_secs(3), + close_issue( + axum::extract::State(state.clone()), + axum::Extension(stranger), + axum::extract::Path(( + "z6MkU7Owner".to_string(), + "u7repo".to_string(), + "1".to_string(), + )), + ), + ) + .await; + + let refused = outcome.expect( + "a caller with no write authorization must be refused WITHOUT waiting on the \ + write lock; hitting this deadline means the handler tried to acquire first, \ + which is the wedge primitive", + ); + assert!( + matches!(refused, Err(AppError::Forbidden(_))), + "expected 403 Forbidden for a stranger, got {:?}", + refused.err().map(|e| format!("{e:?}")) + ); + } +} diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index db6bba71..9f96330c 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -807,6 +807,13 @@ where } } +/// Test-only re-export of the advisory-lock key derivation, so handler tests can +/// hold a repo's lock from an independent session. +#[cfg(test)] +pub fn advisory_lock_key_for_test(owner_slug: &str, repo_name: &str) -> i64 { + advisory_lock_key(owner_slug, repo_name) +} + /// Compute a stable i64 hash for a Postgres advisory lock key. /// /// Uses SHA-256 (not `DefaultHasher`) so the same `(owner_slug, repo_name)` From 5d254fb336daf17043ad89992e7412f2462ac95e Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 20:27:59 -0500 Subject: [PATCH 08/54] fix(node): address the review findings on the advisory-lock series Seven fixes from a seven-reviewer pass. Eight of the nine substantive findings were defects introduced by this branch, and two contradicted claims made in its own earlier commit messages. P0, the worst of them. The under-lock download's timeout was folded into the same Err as a corrupt archive, so it fell through to the use-the-local-copy fallback and the write proceeded. Two ways that destroys data: we do not know whether we hold the latest tree, so re-uploading can overwrite another node's newer archive; and the abandoned download's extraction runs in an uncancellable spawn_blocking that ends in remove_dir_all + rename over the same path, so git would have been running against a directory a background task was about to delete. Now the Option is branched rather than collapsed: a genuine fetch error keeps the local-copy fallback, a timeout refuses the acquire and lets the guard's Drop free the lock. LockProbe closed its connection on ordinary contention, so a 60-attempt spinner tore down 60 backends -- the exact behavior an earlier commit claimed it avoided. The test written to guarantee that asserted the HOLDER's lock count, which cannot observe the probe's own connection, so it passed throughout. The correct predicate turned out to be narrower than the first attempt at this fix: not "we saw an answer" but "we positively know nothing was acquired," because a true answer means the lock IS held and dropping without handoff leaks it exactly as a cancellation would. The first attempt reopened U1's leak and U1's gate test caught it. tigris.exists() ran unbounded inside the lock-held span, so the knob's promise was not kept and the docs were wrong about the shape. The HEAD and the download now share one budget, so worst-case occupancy is one budget per span rather than two. close_issue's pre-lock author read used acquire(), whose fast path returns on a cached directory and never contacts object storage, so on a multi-node deploy an issue author would be refused their own issue. Now acquire_fresh, which refreshes without taking the lock -- the property the comment already claimed. That comment also asserted authorship is immutable, which a reviewer disproved by force-pushing a forged blob at refs/gitlawb/**; reworded to the real justification (a pre-check whose mutation re-reads under the guard). The underlying pushability is pre-existing and larger than this branch. The retry loop had no wall-clock cap, reaching ~360s against a 120s proxy idle timeout that was itself lowered from 600s after held connection slots caused a production outage. Capped at 90s, with the bail message naming the real bound. All four acquire_write call sites stringified their error, destroying the anyhow downcast that error.rs documents explicitly ("without this, every database outage surfaces as a 500 instead of a 503"). PoolTimedOut is in that 503 set, so a saturated lock pool told clients not to retry something transient. Now propagated. Tests: six that the plan required and the first pass never wrote. The R4 pool-isolation test (named in the plan as proving the pool split, entirely absent), close_issue's owner and non-owner-author twins (INV-21c, a two-principal gate with only its deny arm covered), a waiter-does-not-block- an-unrelated-repo case, and a config test for the transfer knob. The runtime-teardown test no longer returns green when DATABASE_URL is unset. Drop's no-runtime branch now detaches via leak() instead of mem::forget: PgConnection has no Drop impl, so the socket closes synchronously and the lock frees immediately rather than at process exit. Evidence. The probe predicate is proven load-bearing in BOTH directions: forcing always-close reddens the churn regression, forcing never-close reddens the cancellation gate, and only the correct predicate satisfies both. The author twin reddens when the fallback is removed. One honest limit: the author twin does NOT redden when acquire_fresh is reverted to acquire. With tigris disabled in tests the two calls are identical, so that fix is correct by reading, not by execution -- the same seam that leaves the transfer bound's wiring untested. Full crate suite 529 passed, clippy -D warnings clean. Refs #279 --- .env.example | 7 +- crates/gitlawb-node/src/api/issues.rs | 135 +++++++++- crates/gitlawb-node/src/api/pulls.rs | 3 +- crates/gitlawb-node/src/config.rs | 9 +- crates/gitlawb-node/src/git/repo_store.rs | 299 ++++++++++++++++++---- 5 files changed, 392 insertions(+), 61 deletions(-) diff --git a/.env.example b/.env.example index dd948317..6cdb407e 100644 --- a/.env.example +++ b/.env.example @@ -38,8 +38,11 @@ GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS=32 # per-repo write lock is HELD (the archive download inside acquire_write and the # upload inside release). These were free before the lock's connection was # pinned to the guard; now an unbounded stall holds a lock-pool slot, and enough -# stalls deny every write on the node. Worst-case slot occupancy is roughly this -# value, so read it together with the pool size above. +# stalls deny every write on the node. The bound applies PER SPAN and there are +# two (the acquire-side refresh, which covers the existence check and download +# together, and the release-side upload), so worst-case slot occupancy is about +# twice this value plus the git work between them. Read it together with the pool +# size above and with GITLAWB_GIT_SERVICE_TIMEOUT_SECS. GITLAWB_LOCK_HELD_TRANSFER_TIMEOUT_SECS=300 # Seconds a request waits for a pool connection before failing with 503. GITLAWB_DB_ACQUIRE_TIMEOUT_SECS=5 diff --git a/crates/gitlawb-node/src/api/issues.rs b/crates/gitlawb-node/src/api/issues.rs index 03811310..1cb7c009 100644 --- a/crates/gitlawb-node/src/api/issues.rs +++ b/crates/gitlawb-node/src/api/issues.rs @@ -64,8 +64,7 @@ pub async fn create_issue( let guard = state .repo_store .acquire_write(&record.owner_did, &record.name) - .await - .map_err(|e| AppError::Git(e.to_string()))?; + .await?; let disk_path = guard.path().to_path_buf(); let create_result = git_issues::create_issue(&disk_path, &issue_id, &json_str); @@ -238,15 +237,25 @@ pub async fn close_issue( let is_owner = crate::api::require_repo_owner(&record, &auth.0).is_ok(); if !is_owner { // Not the owner, so the author fallback decides it, and the author lives in - // the issue's git-JSON blob rather than a DB column. Read it WITHOUT the - // write lock: an issue's author is set at creation and never changes, so - // reading it outside the lock races nothing. `acquire` ensures the repo is - // on disk without taking the lock. + // the issue's git-JSON blob rather than a DB column. + // + // Read it WITHOUT the write lock. The justification is NOT that authorship + // is immutable — it is not: `refs/gitlawb/**` is pushable, so a forged + // author blob can be pushed (tracked separately; it is what makes this + // fallback only as trustworthy as push authorization). The justification is + // that this read is a PRE-CHECK: it decides whether to take the lock at all, + // and the mutation below re-reads under the guard, so a change landing + // between the two cannot cause a write against state we never looked at. + // + // `acquire_fresh`, not `acquire`: acquire's fast path returns as soon as the + // directory exists and never contacts object storage, so on a node with a + // stale copy the author's own issue would be invisible and the + // cannot-establish-authorship arm below would 403 a legitimate author. + // acquire_fresh refreshes first and still takes no lock. let disk_path = state .repo_store - .acquire(&record.owner_did, &record.name) - .await - .map_err(|e| AppError::Git(e.to_string()))?; + .acquire_fresh(&record.owner_did, &record.name) + .await?; let author_did: Option = match git_issues::get_issue(&disk_path, &issue_id) { Ok(Some(raw)) => serde_json::from_str::(&raw) .ok() @@ -267,11 +276,13 @@ pub async fn close_issue( } // Authorized. Only now is the lock taken. + // Propagate rather than stringify: AppError's From downcasts to + // sqlx::Error so a pool timeout or a database outage surfaces as a retryable + // 503. Calling .to_string() first destroys that and reports both as a 500. let guard = state .repo_store .acquire_write(&record.owner_did, &record.name) - .await - .map_err(|e| AppError::Git(e.to_string()))?; + .await?; let disk_path = guard.path().to_path_buf(); // Re-read under the guard so the mutation acts on current state, and keep the @@ -382,4 +393,106 @@ mod tests { refused.err().map(|e| format!("{e:?}")) ); } + + /// Seed a real bare repo with one issue blob whose author is `author_did`, at + /// the on-disk path the store will resolve for (owner_did, repo). + async fn seed_repo_with_issue( + state: &crate::state::AppState, + owner_slug: &str, + owner_did: &str, + repo: &str, + issue_id: &str, + author_did: &str, + ) -> std::path::PathBuf { + state + .db + .upsert_mirror_repo(owner_slug, repo, "/unused", None, true) + .await + .expect("seed repo row"); + // Seed at the path the HANDLER will resolve. upsert_mirror_repo stores the + // bare slug in owner_did, and close_issue resolves from record.owner_did, so + // seeding from the full did:key would create the repo in a different + // directory and the handler would find nothing. + let record = state + .db + .get_repo(owner_slug, repo) + .await + .expect("get_repo") + .expect("seeded repo exists"); + let _ = owner_did; + let path = state + .repo_store + .acquire(&record.owner_did, &record.name) + .await + .expect("resolve disk path"); + let _ = std::fs::remove_dir_all(&path); + crate::git::store::init_bare(&path).expect("init bare repo"); + // Must deserialize as a real IssueRecord: `created_at` and `status` are + // required, and a parse failure would silently drop the author (the + // `.ok()` on from_str), which reads as a 403 rather than as a broken fixture. + let json = serde_json::to_string(&IssueRecord { + id: issue_id.to_string(), + title: "seeded".to_string(), + body: Some(String::new()), + author: Some(author_did.to_string()), + created_at: chrono::Utc::now().to_rfc3339(), + status: "open".to_string(), + signed_payload: None, + }) + .expect("serialize seeded issue"); + crate::git::issues::create_issue(&path, issue_id, &json).expect("seed issue blob"); + path + } + + /// INV-21(c) positive twin 1: the OWNER can still close. The reorder moved the + /// owner check above the lock, so this is the arm most likely to have broken, + /// and the deny test alone could not see it. + #[sqlx::test] + async fn owner_can_still_close_after_the_reorder(pool: PgPool) { + let state = crate::test_support::test_state(pool.clone()).await; + let owner_did = "did:key:z6MkT1Owner"; + seed_repo_with_issue(&state, "z6MkT1Owner", owner_did, "t1repo", "1", owner_did).await; + + let res = close_issue( + axum::extract::State(state.clone()), + axum::Extension(crate::auth::AuthenticatedDid(owner_did.to_string())), + axum::extract::Path(( + "z6MkT1Owner".to_string(), + "t1repo".to_string(), + "1".to_string(), + )), + ) + .await; + assert!( + res.is_ok(), + "the owner must still be able to close: {:?}", + res.err().map(|e| format!("{e:?}")) + ); + } + + /// INV-21(c) positive twin 2: the non-owner AUTHOR can still close. This is the + /// arm the acquire-vs-acquire_fresh regression broke, and nothing caught it. + #[sqlx::test] + async fn issue_author_who_is_not_the_owner_can_still_close(pool: PgPool) { + let state = crate::test_support::test_state(pool.clone()).await; + let owner_did = "did:key:z6MkT2Owner"; + let author_did = "did:key:z6MkT2Author"; + seed_repo_with_issue(&state, "z6MkT2Owner", owner_did, "t2repo", "1", author_did).await; + + let res = close_issue( + axum::extract::State(state.clone()), + axum::Extension(crate::auth::AuthenticatedDid(author_did.to_string())), + axum::extract::Path(( + "z6MkT2Owner".to_string(), + "t2repo".to_string(), + "1".to_string(), + )), + ) + .await; + assert!( + res.is_ok(), + "the issue author, who is NOT the repo owner, must still be able to close: {:?}", + res.err().map(|e| format!("{e:?}")) + ); + } } diff --git a/crates/gitlawb-node/src/api/pulls.rs b/crates/gitlawb-node/src/api/pulls.rs index 26be6109..adabd146 100644 --- a/crates/gitlawb-node/src/api/pulls.rs +++ b/crates/gitlawb-node/src/api/pulls.rs @@ -212,8 +212,7 @@ pub async fn merge_pr( let guard = state .repo_store .acquire_write(&record.owner_did, &record.name) - .await - .map_err(|e| AppError::Git(e.to_string()))?; + .await?; let disk_path = guard.path().to_path_buf(); let merger_did = auth.0; let merge_result = store::merge_branch( diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 12955b6f..da6bec38 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -271,9 +271,12 @@ pub struct Config { /// Upper bound, in seconds, on any single object-storage transfer that runs /// while the per-repo advisory lock is HELD. /// - /// Two such transfers exist: the archive download in `acquire_write`, which - /// runs after the lock is taken and before the guard is constructed, and the - /// archive upload in `release`. Both used to be free, because the lock's + /// Two bounded spans exist, and the bound applies per span. The acquire-side + /// refresh in `acquire_write` covers the existence HEAD and the download + /// together under ONE budget (it runs after the lock is taken and before the + /// guard is constructed), and the archive upload in `release` gets its own. + /// Worst-case slot occupancy is therefore about twice this value plus the git + /// work between them, not one times this value. Both used to be free, because the lock's /// connection was returned to the pool immediately; now that a write guard /// pins a lock-pool connection for its whole lifetime, an unbounded transfer /// holds that slot, and enough stalled transfers deny every write on the node. diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 9f96330c..c63c20e4 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -223,8 +223,18 @@ impl RepoStore { // against a pool that is full for reasons unrelated to this repo, and would // report a capacity problem as lock contention. It surfaces immediately with // its own message instead. + // Cap the WALL CLOCK, not just the attempt count. 60 attempts each pay a + // pool acquire (up to db_acquire_timeout_secs) plus a 1s sleep, so an + // attempt-only bound reaches ~360s — far past the 120s proxy idle timeout + // that was deliberately lowered from 600s after held connection slots + // caused a production outage. A caller must not be able to sit here longer + // than the proxy will hold its connection. + let deadline = std::time::Instant::now() + LOCK_ACQUIRE_DEADLINE; let mut lock_conn = None; for attempt in 0..60 { + if std::time::Instant::now() >= deadline { + break; + } let conn = self .lock_pool .acquire() @@ -243,12 +253,15 @@ impl RepoStore { } } let Some(lock_conn) = lock_conn else { - anyhow::bail!("could not acquire advisory lock after 60 attempts — possible stale lock for {owner_slug}/{repo_name}"); + anyhow::bail!( + "could not acquire advisory lock within {}s — possible stale lock for {owner_slug}/{repo_name}", + LOCK_ACQUIRE_DEADLINE.as_secs() + ); }; // From here the lock is HELD. Any early return must not simply drop the // connection back into the pool, so it is handed to the guard immediately // below and every exit after this point goes through the guard. - let mut guard = RepoWriteGuard { + let guard = RepoWriteGuard { owner_slug: owner_slug.clone(), repo_name: repo_name.to_string(), local_path: local_path.clone(), @@ -262,41 +275,62 @@ impl RepoStore { // stale if another machine pushed since our last access. The guard already // owns the lock + its connection, so a cancellation here drops through Drop. if let Some(ref tigris) = self.tigris { - if tigris.exists(&owner_slug, repo_name).await.unwrap_or(false) { - debug!(repo = %repo_name, "write acquire: downloading latest from tigris"); - // The lock is already HELD at this point and the guard owns a - // lock-pool slot, so this transfer is bounded: an unbounded stall - // here would hold both, and enough of them deny every write on the - // node. acquire_fresh's download is deliberately NOT bounded by - // this knob, because it runs before any lock is taken. - let downloaded = bounded_transfer( - "acquire-download", - repo_name, - self.lock_held_transfer_timeout, - tigris.download(&owner_slug, repo_name, &local_path), - ) - .await - .unwrap_or_else(|| { - Err(anyhow::anyhow!( - "archive download exceeded the under-lock transfer bound" - )) - }); - if let Err(e) = downloaded { - // Same self-healing fallback as acquire_fresh: a corrupt/unreadable - // Tigris archive must not block a write when a valid local copy - // exists — release(success) will re-upload a good archive. + // ONE budget for the whole refresh, covering the HEAD and the download + // together. Both run with the lock held and a lock-pool slot pinned, so + // bounding only the download would leave a mute endpoint able to hold + // both indefinitely on the HEAD, and bounding them separately would make + // worst-case occupancy two budgets instead of one. + let refreshed = bounded_transfer( + "acquire-refresh", + repo_name, + self.lock_held_transfer_timeout, + async { + if tigris.exists(&owner_slug, repo_name).await.unwrap_or(false) { + debug!(repo = %repo_name, "write acquire: downloading latest from tigris"); + tigris.download(&owner_slug, repo_name, &local_path).await + } else { + Ok(()) + } + }, + ) + .await; + + match refreshed { + Some(Ok(())) => {} + Some(Err(e)) => { + // The archive is present but unreadable: a corrupt or partial + // upload, or a transient GET failure. We KNOW the fetch failed, + // so falling back to a valid local copy is sound and + // release(success) re-uploads a good archive. Only hard-fail + // when there is no local copy to fall back to. if local_path.exists() { warn!(repo = %repo_name, err = %e, - "write acquire: tigris download failed — falling back to local copy"); + "write acquire: tigris refresh failed — falling back to local copy"); } else { return Err(e).context("downloading repo from tigris for write"); } } + None => { + // TIMED OUT, which is NOT the same as failed, and must not reach + // the fallback above. Two reasons. We do not know whether we have + // the latest tree, so writing against the local copy and then + // re-uploading can silently overwrite another node's newer + // archive. Worse, the abandoned download's extraction runs in an + // uncancellable spawn_blocking that ends in remove_dir_all + + // rename over local_path, so proceeding would run git against a + // directory that a background task is about to delete. + // + // Refuse the acquire. Returning here drops the guard, whose Drop + // frees the lock and its pool slot. + return Err(anyhow::anyhow!( + "tigris refresh exceeded the {}s under-lock bound for {owner_slug}/{repo_name}; \ + refusing the write rather than proceeding against a possibly-stale tree", + self.lock_held_transfer_timeout.as_secs() + )); + } } } - // Silence the unused-mut lint until U6 needs the binding mutable. - let _ = &mut guard; Ok(guard) } @@ -579,11 +613,24 @@ fn validate_repo_name(repo_name: &str) -> Result<()> { /// This is the only place that issues `pg_try_advisory_lock`. struct LockProbe { conn: Option>, + /// True only when we have POSITIVELY established that this session does not + /// hold the lock, i.e. `try_lock` came back `false`. + /// + /// The predicate has to be "we know nothing was acquired," not "we saw an + /// answer." A `true` answer means the lock IS held, so dropping without handing + /// the connection to a guard leaks it exactly as a cancellation would; an + /// earlier version of this flag meant "settled" and reopened that leak. Default + /// false so both the cancelled-mid-flight and lock-acquired cases close, and + /// only ordinary contention returns the connection. + lock_not_taken: bool, } impl LockProbe { fn new(conn: sqlx::pool::PoolConnection) -> Self { - Self { conn: Some(conn) } + Self { + conn: Some(conn), + lock_not_taken: false, + } } /// Send the try-lock on the owned connection. @@ -597,6 +644,11 @@ impl LockProbe { .fetch_one(&mut **conn) .await .context("trying advisory lock")?; + // Only a false answer licenses returning the connection: it means the + // statement completed and took nothing. A true answer means this session + // now holds the lock, so Drop must still close unless `take_conn` hands it + // to a guard. + self.lock_not_taken = !row.0; Ok(row.0) } @@ -613,15 +665,23 @@ impl LockProbe { impl Drop for LockProbe { fn drop(&mut self) { - if let Some(mut conn) = self.conn.take() { - // Still holding the connection here means the try-lock's future was - // dropped before `take_conn` ran, so the statement may well have - // completed server-side and taken the lock with nobody left to - // release it. Close the connection instead of returning it to the - // pool: ending the session is what makes Postgres free the lock. - warn!("advisory-lock probe dropped before handing off its connection — closing the session to free the lock"); - conn.close_on_drop(); + let Some(mut conn) = self.conn.take() else { + // take_conn already handed the connection to the guard. + return; + }; + if self.lock_not_taken { + // The probe ran and reported that someone else holds the key, so nothing + // was acquired here. Return the connection to the pool: closing would + // make a 60-attempt spinner tear down 60 backends for ordinary + // contention. Dropping `conn` unarmed does exactly that. + return; } + // Either the future was dropped before we saw an answer, or the answer was + // that we DID take the lock and nobody took the connection off us. Both mean + // a session may be holding the lock with no one to release it, so end the + // session — which is what makes Postgres free it. + warn!("advisory-lock probe dropped while its session may hold the lock — closing the session to free it"); + conn.close_on_drop(); } } @@ -773,13 +833,28 @@ impl Drop for RepoWriteGuard { } else { warn!( repo = %self.repo_name, - "write guard dropped with no runtime alive — leaking the connection handle so Drop cannot panic; the lock frees when the process exits" + "write guard dropped with no runtime alive — detaching the connection so Drop cannot panic" ); - std::mem::forget(conn); + // `PoolConnection::drop` spawns onto the runtime for BOTH closing and + // returning, and panics without a handle; a panic inside Drop aborts the + // process during unwind. `leak()` detaches the raw `PgConnection`, which + // has no Drop impl of its own, so dropping it closes the socket + // synchronously with no runtime involved. That frees the lock + // immediately rather than at process exit, and leaks no fd — strictly + // better than the mem::forget this replaced. + drop(conn.leak()); } } } +/// Overall wall-clock cap on acquiring the per-repo advisory lock. +/// +/// Deliberately under the 120s proxy idle timeout (`infra/fly/fly.toml`), which was +/// itself lowered from 600s after long-held connection slots caused a production +/// outage. An attempt-count bound alone is not enough: 60 attempts each paying a +/// pool acquire plus a 1s sleep reach roughly 360s. +const LOCK_ACQUIRE_DEADLINE: Duration = Duration::from_secs(90); + /// Run a future under a wall-clock bound, returning `None` if it did not finish. /// /// For the object-storage transfers that run while the per-repo advisory lock is @@ -2184,10 +2259,12 @@ mod tests { /// body, so this asserts no-panic rather than lock release. #[test] fn guard_dropped_at_runtime_teardown_does_not_panic() { - let url = match std::env::var("DATABASE_URL") { - Ok(u) => u, - Err(_) => return, // no database configured; nothing to assert - }; + // No silent skip: a test that returns green when its precondition is + // absent is worse than one that fails, because it reports coverage it does + // not have. CI provisions Postgres, so an absent DATABASE_URL is a broken + // environment rather than an expected one. + let url = std::env::var("DATABASE_URL") + .expect("DATABASE_URL must be set; this test cannot pass vacuously"); let rt = tokio::runtime::Runtime::new().unwrap(); let guard = rt.block_on(async { let lock_pool = sqlx::postgres::PgPoolOptions::new() @@ -2293,4 +2370,140 @@ mod tests { "a prompt transfer must pass through untouched" ); } + + /// F2 regression: an ordinary failed probe must RETURN its connection, not + /// close it. The old test asserted the holder's pg_locks count, which cannot + /// see what happened to the probe's own connection — so it passed while a + /// 60-attempt spinner tore down 60 backends. The observable that discriminates + /// is the backend pid on a one-connection pool. + #[sqlx::test] + async fn failed_probe_returns_its_connection_to_the_pool(pool: PgPool) { + use sqlx::Connection; + let opts = (*pool.connect_options()).clone(); + let lock_pool = no_reap_pool(&opts, 1).await; + let key: i64 = 991_100; + + // someone else holds the key, from an independent session + let mut holder = sqlx::PgConnection::connect_with(&opts).await.unwrap(); + let held: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut holder) + .await + .unwrap(); + assert!(held.0); + + let mut pids = Vec::new(); + for _ in 0..3 { + let mut probe = LockProbe::new(lock_pool.acquire().await.unwrap()); + let pid: (i32,) = sqlx::query_as("SELECT pg_backend_pid()") + .fetch_one(&mut **probe.conn.as_mut().unwrap()) + .await + .unwrap(); + pids.push(pid.0); + assert!(!probe.try_lock(key).await.unwrap(), "key is held elsewhere"); + drop(probe); + tokio::time::sleep(std::time::Duration::from_millis(150)).await; + } + assert!( + pids.windows(2).all(|w| w[0] == w[1]), + "a failed probe must return its connection so a spinner does not churn \ + backends; saw {pids:?}" + ); + } + + // ── F5: the tests the plan required and the first pass never wrote ──────── + + /// R4, the test the plan named as proving the pool split and the one that would + /// have caught PR #215's node-wide two-write ceiling. Holding N guards on + /// DISTINCT repos must pin N lock-pool connections while leaving the app pool + /// free to serve ordinary queries. + #[sqlx::test] + async fn lock_pool_exhaustion_does_not_starve_the_app_pool(pool: PgPool) { + const N: u32 = 3; + let opts = (*pool.connect_options()).clone(); + let lock_pool = no_reap_pool(&opts, N).await; + let store = RepoStore::for_testing(PathBuf::from("/tmp/gitlawb-f5"), lock_pool.clone()); + + let mut guards = Vec::new(); + for i in 0..N { + guards.push( + store + .acquire_write(&format!("did:key:z6MkF5Iso{i}"), "iso") + .await + .expect("distinct repos each acquire"), + ); + } + + // The lock pool is now exhausted: a further checkout must time out. + let starved = + tokio::time::timeout(std::time::Duration::from_secs(8), lock_pool.acquire()).await; + assert!( + matches!(starved, Ok(Err(_)) | Err(_)), + "with N guards held, an N+1th lock-pool checkout must not succeed" + ); + + // ...while the APP pool still serves queries. This is the whole point of + // the split: write pressure must not deny ordinary reads. + let alive: (i32,) = sqlx::query_as("SELECT 1") + .fetch_one(&pool) + .await + .expect("app pool must remain usable while the lock pool is exhausted"); + assert_eq!(alive.0, 1); + + for g in guards.drain(..) { + g.release(true).await; + } + } + + /// A waiter spinning on a contended repo must not block a write to an + /// unrelated repo (R5's user-visible half). + #[sqlx::test] + async fn waiter_on_one_repo_does_not_block_another(pool: PgPool) { + let opts = (*pool.connect_options()).clone(); + let lock_pool = no_reap_pool(&opts, 3).await; + let store = std::sync::Arc::new(RepoStore::for_testing( + PathBuf::from("/tmp/gitlawb-f5b"), + lock_pool, + )); + + let held = store + .acquire_write("did:key:z6MkF5Cont", "contended") + .await + .unwrap(); + + let spinner = { + let s = store.clone(); + tokio::spawn(async move { s.acquire_write("did:key:z6MkF5Cont", "contended").await }) + }; + tokio::time::sleep(std::time::Duration::from_millis(1200)).await; + + let unrelated = tokio::time::timeout( + std::time::Duration::from_secs(8), + store.acquire_write("did:key:z6MkF5Other", "innocent"), + ) + .await + .expect("an unrelated repo must not wait on someone else's contention") + .expect("and must acquire"); + unrelated.release(true).await; + + spinner.abort(); + held.release(true).await; + } + + /// The transfer bound is a knob, so it gets the same parse/default/reject-zero + /// coverage its sibling lock-pool-size knob has. + #[test] + fn lock_held_transfer_timeout_defaults_and_rejects_zero() { + use clap::Parser; + assert_eq!( + crate::config::Config::parse_from(["gitlawb-node"]).lock_held_transfer_timeout_secs, + 300 + ); + assert!(crate::config::Config::try_parse_from([ + "gitlawb-node", + "--lock-held-transfer-timeout-secs", + "0" + ]) + .is_err()); + } } From d346bde53289f63ae2dfe51bb454b2c1990a1cf0 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 21:10:45 -0500 Subject: [PATCH 09/54] fix(node): log lock-pool saturation in the request path, not via readiness Completes the shed-in-path half of the availability story. F7 already made a pool timeout a retryable 503 by letting the sqlx downcast through; this adds the operator-facing half, with the pool's own size/idle counters so an incident can distinguish "the pool is full" from "the database is gone" without reproducing it. Deliberately NOT a readiness probe. /ready gates Fly routing with no fail-open, so failing it on a saturated pool would pull this node's READS out of service too and push its write load onto peers carrying the same load. That is the downward spiral AWS's health-check guidance names and the SRE Book documents independently. A lock-pool readiness probe would also add nothing on the reachability axis, because both pools are built from the same database_url, so the existing app-pool ping already answers it. The error is still returned via .context() rather than replaced, because anyhow preserves downcastability through context layers and the 503 mapping depends on it. Cost is bounded by construction: one line per failed acquire, and a failed acquire has already paid a multi-second pool timeout. Refs #279 --- crates/gitlawb-node/src/git/repo_store.rs | 33 +++++++++++++++++++---- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index c63c20e4..e4ebe06b 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -235,11 +235,34 @@ impl RepoStore { if std::time::Instant::now() >= deadline { break; } - let conn = self - .lock_pool - .acquire() - .await - .context("advisory-lock pool exhausted or unreachable")?; + let conn = match self.lock_pool.acquire().await { + Ok(c) => c, + Err(e) => { + // Saturation is surfaced HERE, in the request path, and + // deliberately not through /ready. Failing readiness on a full + // pool would pull this node out of routing, taking its reads + // with it and pushing its write load onto peers carrying the + // same load — the documented downward spiral. So the signals + // are: a retryable 503 to the caller (via the sqlx downcast on + // this error) and this log line for the operator. + // + // Logged at warn with the pool's own counters so an incident can + // tell "the pool is full" from "the database is gone" without + // reproducing it. Once per failed acquire, and a failed acquire + // already costs a multi-second timeout, so this cannot itself + // become a log flood. + warn!( + repo = %repo_name, + owner = %owner_slug, + pool_size = self.lock_pool.size(), + pool_idle = self.lock_pool.num_idle(), + err = %e, + "advisory-lock pool acquire failed — writes are being shed; \ + raise GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS or investigate long-held write locks" + ); + return Err(e).context("advisory-lock pool exhausted or unreachable"); + } + }; let mut probe = LockProbe::new(conn); if probe.try_lock(lock_key).await? { lock_conn = probe.take_conn(); From 82143ad692dcb9e9aca53fcf303146d904d2bd71 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 22:11:14 -0500 Subject: [PATCH 10/54] fix(node): refuse a write when object storage is unknowable, and shed contention as 503 Four defects a seven-reviewer pass found in the previous two commits, three of them cases where one arm of a match contradicted its neighbour. The under-lock refresh read a failed Tigris HEAD as "no archive" (`exists().await.unwrap_or(false)`), skipped the refresh, and wrote against a possibly-stale tree, then re-uploaded over it. That is the outcome the timeout arm twenty lines below explicitly refuses. A failed HEAD and a failed download leave us knowing different things, so they no longer share a branch: only a download failure after a successful HEAD establishes that the local copy is a sound thing to fall back to and re-upload. A HEAD failure now refuses the write. Lock contention that ran out the acquire deadline surfaced as a 500 carrying the owner slug and repo name in the client-visible body, while pool exhaustion fifteen lines up returned a deliberate 503. Contention is transient and ordinary, so it now maps through a typed `RepoBusy` to a retryable 503 with a fixed body; the detail stays in the log at the raise site. `lock_not_taken` was assigned after the try-lock answered, so an error left a previous `true`-derived value standing and could return a lock-holding session to the pool. It is now cleared before the statement is sent. `leak()`'s no-panic property in the guard's no-runtime Drop branch depended on `min_connections == 0` without saying so; a future tuning change would have silently re-armed a panic inside Drop. Made explicit with the reason. Two tests that did not bind: - `waiter_on_one_repo_does_not_block_another` proved only that two lock keys do not collide. It now samples the pool counters across more than two backoff cycles. RED at 0/50 samples with `drop(probe)` moved after the sleep, the exact defect it names, which the previous version survived. - the exhaustion test's `Ok(Err(_)) | Err(_)` was satisfied by its own outer timeout. It now requires `PoolTimedOut` and asserts the slots are accounted for. RED with the pool sized N+1. `contended_acquire_sheds_as_repo_busy_not_internal_error` is new and drives the deadline path for real; the deadline became a field so it does not wait 90s. RED at 500-vs-503 with the downcast removed. The acquire deadline's docstring claimed a total under the 120s proxy idle timeout, which the 300s under-lock transfer bound in the same function contradicts. It bounds the wait only, and now says so. The backoff is clamped to the remaining budget. --- crates/gitlawb-node/src/db/mod.rs | 8 + crates/gitlawb-node/src/error.rs | 19 +- crates/gitlawb-node/src/git/repo_store.rs | 258 +++++++++++++++++++--- 3 files changed, 251 insertions(+), 34 deletions(-) diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 57bafa3f..4fbf7e98 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -317,6 +317,14 @@ impl Db { ); PgPoolOptions::new() .max_connections(max_connections) + // Explicit, and load-bearing rather than cosmetic: `RepoWriteGuard::Drop` + // has a no-runtime branch that calls `PoolConnection::leak()` and relies + // on the husk's own drop doing nothing. With `min_connections > 0` that + // drop still spawns a pool-replenish task, and spawning without a runtime + // panics inside Drop, which aborts the process during unwind. It is 0 by + // default, so this line exists to keep a future tuning change from + // silently re-arming that panic. + .min_connections(0) .acquire_timeout(acquire_timeout) .connect_lazy(database_url) .context("creating advisory-lock pool") diff --git a/crates/gitlawb-node/src/error.rs b/crates/gitlawb-node/src/error.rs index f5e14df1..ecc46309 100644 --- a/crates/gitlawb-node/src/error.rs +++ b/crates/gitlawb-node/src/error.rs @@ -59,6 +59,9 @@ pub enum AppError { #[error("server overloaded: {0}")] Overloaded(String), + #[error("repository is busy")] + RepoBusy, + #[error("database error: {0}")] Db(#[from] sqlx::Error), @@ -100,7 +103,14 @@ impl From for AppError { fn from(err: anyhow::Error) -> Self { match err.downcast::() { Ok(sql) => AppError::Db(sql), - Err(err) => AppError::Internal(err), + // Lock contention is transient and ordinary, so it must not land as a + // 500. The internal message names the owner slug and repo, so the + // variant carries nothing: the detail stays in the log at the raise + // site and the client gets a fixed retryable body. + Err(err) => match err.downcast::() { + Ok(_) => AppError::RepoBusy, + Err(err) => AppError::Internal(err), + }, } } } @@ -165,6 +175,13 @@ impl IntoResponse for AppError { // 504, distinct from the 500 git_error and from the read-gate's 404 / // the auth 401, so the client can tell a deadline from a failure. AppError::Timeout(msg) => (StatusCode::GATEWAY_TIMEOUT, "git_timeout", msg.clone()), + // 503 with a FIXED body: the caller should retry, and must not be told + // which repo is contended or for how long. + AppError::RepoBusy => ( + StatusCode::SERVICE_UNAVAILABLE, + "repo_busy", + "repository is busy — retry".into(), + ), AppError::Db(e) if db_unavailable(e) => ( StatusCode::SERVICE_UNAVAILABLE, DB_UNAVAILABLE_CODE, diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index e4ebe06b..da5daec9 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -35,6 +35,9 @@ pub struct RepoStore { lock_pool: PgPool, /// Bound on any object-storage transfer that runs while the lock is HELD. lock_held_transfer_timeout: Duration, + /// Wall-clock cap on WAITING for the lock. A field rather than a bare const so + /// the busy path can be driven in a test without a 90s wait. + lock_acquire_deadline: Duration, /// Tracks repos already confirmed to exist in Tigris — avoids redundant /// HEAD checks and background uploads for repos we've already migrated. migrated: Arc>>, @@ -54,11 +57,20 @@ impl RepoStore { tigris: None, lock_pool, lock_held_transfer_timeout: Duration::from_secs(300), + lock_acquire_deadline: LOCK_ACQUIRE_DEADLINE, migrated: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())), pre_unlock_gate: None, } } + /// Shorten the lock-acquire deadline so the busy path is reachable in a test + /// without waiting out the production default. + #[cfg(test)] + pub fn with_lock_acquire_deadline(mut self, deadline: Duration) -> Self { + self.lock_acquire_deadline = deadline; + self + } + pub fn new( repos_dir: PathBuf, tigris: Option, @@ -70,6 +82,7 @@ impl RepoStore { tigris, lock_pool, lock_held_transfer_timeout, + lock_acquire_deadline: LOCK_ACQUIRE_DEADLINE, migrated: Arc::new(Mutex::new(HashSet::new())), #[cfg(test)] pre_unlock_gate: None, @@ -223,16 +236,19 @@ impl RepoStore { // against a pool that is full for reasons unrelated to this repo, and would // report a capacity problem as lock contention. It surfaces immediately with // its own message instead. - // Cap the WALL CLOCK, not just the attempt count. 60 attempts each pay a - // pool acquire (up to db_acquire_timeout_secs) plus a 1s sleep, so an - // attempt-only bound reaches ~360s — far past the 120s proxy idle timeout - // that was deliberately lowered from 600s after held connection slots - // caused a production outage. A caller must not be able to sit here longer - // than the proxy will hold its connection. - let deadline = std::time::Instant::now() + LOCK_ACQUIRE_DEADLINE; + // Cap the WALL CLOCK of the WAIT, not just the attempt count. 60 attempts + // each pay a pool acquire (up to db_acquire_timeout_secs) plus a 1s sleep, + // so an attempt-only bound reaches ~360s. This bounds the wait only; the + // under-lock refresh below carries its own separate bound, so do not read + // this as a total for `acquire_write` (see LOCK_ACQUIRE_DEADLINE). + let deadline_budget = self.lock_acquire_deadline; + let deadline = std::time::Instant::now() + deadline_budget; let mut lock_conn = None; for attempt in 0..60 { - if std::time::Instant::now() >= deadline { + let Some(left) = deadline.checked_duration_since(std::time::Instant::now()) else { + break; + }; + if left.is_zero() { break; } let conn = match self.lock_pool.acquire().await { @@ -271,15 +287,28 @@ impl RepoStore { // Not acquired, and nothing is locked, so hand the connection back // before the backoff rather than holding a slot while idle. drop(probe); + // Clamp the backoff to what is left of the budget: sleeping a full + // second past the deadline would turn a short deadline into a longer + // wait than the caller was promised. if attempt < 59 { - tokio::time::sleep(std::time::Duration::from_secs(1)).await; + tokio::time::sleep(left.min(std::time::Duration::from_secs(1))).await; } } let Some(lock_conn) = lock_conn else { - anyhow::bail!( - "could not acquire advisory lock within {}s — possible stale lock for {owner_slug}/{repo_name}", - LOCK_ACQUIRE_DEADLINE.as_secs() + // Contention is transient, so this must NOT land as a 500. The detail + // (which repo, which key, how long) goes to the log; the client gets a + // retryable 503 with a fixed body via the `RepoBusy` downcast. + warn!( + repo = %repo_name, + owner = %owner_slug, + lock_key, + waited_secs = deadline_budget.as_secs(), + "advisory lock not acquired within the deadline — shedding the write as busy" ); + return Err(anyhow::Error::new(RepoBusy).context(format!( + "could not acquire advisory lock within {}s for {owner_slug}/{repo_name}", + deadline_budget.as_secs() + ))); }; // From here the lock is HELD. Any early return must not simply drop the // connection back into the pool, so it is handed to the guard immediately @@ -308,11 +337,25 @@ impl RepoStore { repo_name, self.lock_held_transfer_timeout, async { - if tigris.exists(&owner_slug, repo_name).await.unwrap_or(false) { - debug!(repo = %repo_name, "write acquire: downloading latest from tigris"); - tigris.download(&owner_slug, repo_name, &local_path).await - } else { - Ok(()) + // The HEAD and the download fail for epistemically DIFFERENT + // reasons, so they are kept apart rather than collapsed into one + // `Result`. A failed HEAD leaves us not knowing whether an archive + // exists at all, which is the same state a timeout leaves us in; + // a failed download after a successful HEAD tells us an archive is + // there and unreadable. Only the second licenses the local + // fallback. Collapsing them (the `unwrap_or(false)` this replaced + // read a HEAD error as "no archive") skipped the refresh silently + // and then re-uploaded over a possibly-newer archive. + match tigris.exists(&owner_slug, repo_name).await { + Ok(true) => { + debug!(repo = %repo_name, "write acquire: downloading latest from tigris"); + tigris + .download(&owner_slug, repo_name, &local_path) + .await + .map_err(RefreshFailure::Download) + } + Ok(false) => Ok(()), + Err(e) => Err(RefreshFailure::Unknown(e)), } }, ) @@ -320,7 +363,7 @@ impl RepoStore { match refreshed { Some(Ok(())) => {} - Some(Err(e)) => { + Some(Err(RefreshFailure::Download(e))) => { // The archive is present but unreadable: a corrupt or partial // upload, or a transient GET failure. We KNOW the fetch failed, // so falling back to a valid local copy is sound and @@ -333,6 +376,18 @@ impl RepoStore { return Err(e).context("downloading repo from tigris for write"); } } + Some(Err(RefreshFailure::Unknown(e))) => { + // The HEAD itself failed, so we do not know whether a newer + // archive exists. Refuse for the same reason the timeout arm + // below refuses: proceeding would write against a possibly-stale + // tree and then re-upload over another node's newer archive. A + // transient object-storage blip costs a retryable refusal here, + // which is the cheaper failure than silent overwrite. + warn!(repo = %repo_name, err = %e, + "write acquire: tigris HEAD failed — refusing the write rather than \ + guessing the archive is absent"); + return Err(e).context("checking tigris for the repo archive before a write"); + } None => { // TIMED OUT, which is NOT the same as failed, and must not reach // the fallback above. Two reasons. We do not know whether we have @@ -662,6 +717,12 @@ impl LockProbe { .conn .as_mut() .context("LockProbe::try_lock after the connection was taken")?; + // Cleared BEFORE the statement is sent, not after it answers. Once the + // statement is in flight this session may hold the lock, and an error or a + // cancellation gives us no way to find out, so the connection must not be + // returned to the pool on any path but a positive `false`. Assigning only on + // success would leave a previous `true`-derived value standing. + self.lock_not_taken = false; let row: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") .bind(key) .fetch_one(&mut **conn) @@ -870,14 +931,46 @@ impl Drop for RepoWriteGuard { } } -/// Overall wall-clock cap on acquiring the per-repo advisory lock. +/// Default wall-clock cap on WAITING for the per-repo advisory lock. /// -/// Deliberately under the 120s proxy idle timeout (`infra/fly/fly.toml`), which was -/// itself lowered from 600s after long-held connection slots caused a production -/// outage. An attempt-count bound alone is not enough: 60 attempts each paying a -/// pool acquire plus a 1s sleep reach roughly 360s. +/// An attempt-count bound alone is not enough: 60 attempts each paying a pool +/// acquire plus a 1s sleep reach roughly 360s. +/// +/// This bounds the wait only, NOT the whole of `acquire_write`. The under-lock +/// refresh carries its own separate bound (`lock_held_transfer_timeout`, default +/// 300s), so the two compose rather than nest and a caller can legitimately spend +/// this deadline waiting and then that bound refreshing. Do not read 90s as a +/// promise that `acquire_write` returns inside the 120s proxy idle timeout in +/// `infra/fly/fly.toml`; it is not, and reconciling the two is tracked separately. const LOCK_ACQUIRE_DEADLINE: Duration = Duration::from_secs(90); +/// Why an under-lock refresh did not complete, split by what it leaves us knowing. +/// +/// `Unknown` (the existence check failed) and `Download` (the archive is there and +/// unreadable) must not share a branch: only the second establishes that the local +/// copy is a sound thing to fall back to and re-upload. +enum RefreshFailure { + Unknown(anyhow::Error), + Download(anyhow::Error), +} + +/// The per-repo advisory lock was not obtained within the acquire deadline. +/// +/// A distinct type rather than a bare `anyhow` string so the handler layer can map +/// it to a retryable 503 with a FIXED body. Contention is transient and ordinary, +/// and the internal message names the owner slug and repo, which must stay in the +/// log rather than reaching the client. +#[derive(Debug)] +pub struct RepoBusy; + +impl std::fmt::Display for RepoBusy { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("repository is busy") + } +} + +impl std::error::Error for RepoBusy {} + /// Run a future under a wall-clock bound, returning `None` if it did not finish. /// /// For the object-storage transfers that run while the per-repo advisory lock is @@ -2457,16 +2550,36 @@ mod tests { ); } - // The lock pool is now exhausted: a further checkout must time out. + // Every slot is accounted for by a guard, so the pool really is exhausted + // rather than merely slow. Asserted directly, because the starvation check + // below cannot tell the two apart on its own. + assert_eq!( + lock_pool.size() as usize - lock_pool.num_idle(), + N as usize, + "all N slots must be checked out by the guards" + ); + + // An N+1th checkout must be refused BY THE POOL. The specific error matters: + // `Ok(Err(_)) | Err(_)` would also be satisfied by the outer tokio timeout + // firing for an unrelated reason, which would let this pass without the pool + // ever having refused anything. let starved = tokio::time::timeout(std::time::Duration::from_secs(8), lock_pool.acquire()).await; - assert!( - matches!(starved, Ok(Err(_)) | Err(_)), - "with N guards held, an N+1th lock-pool checkout must not succeed" - ); + match starved { + Ok(Err(sqlx::Error::PoolTimedOut)) => {} + Ok(Err(e)) => panic!("expected the pool's own timeout, got {e:?}"), + Ok(Ok(_)) => panic!("with N guards held, an N+1th lock-pool checkout must not succeed"), + Err(_) => panic!( + "the pool must refuse the checkout itself within its acquire_timeout; \ + the outer timeout firing means it never did" + ), + } // ...while the APP pool still serves queries. This is the whole point of - // the split: write pressure must not deny ordinary reads. + // the split: write pressure must not deny ordinary reads. Weak on its own (it + // is a different pool object, so it would serve regardless), so it is the + // exhaustion assertions above that carry the isolation claim; this only + // confirms the reads are actually reachable in that state. let alive: (i32,) = sqlx::query_as("SELECT 1") .fetch_one(&pool) .await @@ -2478,15 +2591,23 @@ mod tests { } } - /// A waiter spinning on a contended repo must not block a write to an - /// unrelated repo (R5's user-visible half). + /// A waiter spinning on a contended repo must hand its pool slot back for the + /// duration of each backoff, and must not block a write to an unrelated repo + /// (R5, both halves). + /// + /// The pool-counter sampling is the load-bearing half. A second `acquire_write` + /// succeeding proves only that two different lock keys do not collide, which is + /// true whether or not the spinner released anything: with the slot held through + /// the sleep, a pool of 3 still has room for it. So this samples what the + /// spinner actually occupies across more than two backoff cycles. Moving + /// `drop(probe)` after the backoff sleep turns it red. #[sqlx::test] async fn waiter_on_one_repo_does_not_block_another(pool: PgPool) { let opts = (*pool.connect_options()).clone(); let lock_pool = no_reap_pool(&opts, 3).await; let store = std::sync::Arc::new(RepoStore::for_testing( PathBuf::from("/tmp/gitlawb-f5b"), - lock_pool, + lock_pool.clone(), )); let held = store @@ -2498,7 +2619,25 @@ mod tests { let s = store.clone(); tokio::spawn(async move { s.acquire_write("did:key:z6MkF5Cont", "contended").await }) }; - tokio::time::sleep(std::time::Duration::from_millis(1200)).await; + + // Sample across >2 backoff cycles. `held` accounts for exactly one + // checked-out connection throughout, so every sample above that is the + // spinner sitting on a slot it is not using. + let mut spinner_idle = 0; + let mut samples = 0; + for _ in 0..50 { + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + let checked_out = lock_pool.size() as usize - lock_pool.num_idle(); + if checked_out == 1 { + spinner_idle += 1; + } + samples += 1; + } + assert!( + spinner_idle * 10 >= samples * 7, + "a spinner must hold no lock-pool slot through its backoff: only {spinner_idle}/{samples} \ + samples showed just the held guard checked out" + ); let unrelated = tokio::time::timeout( std::time::Duration::from_secs(8), @@ -2513,6 +2652,59 @@ mod tests { held.release(true).await; } + /// Lock contention that runs out the acquire deadline must surface as a + /// retryable 503 with a fixed body, not a 500 carrying the owner slug and repo + /// name. The deadline is a field so this does not wait out the 90s default. + #[sqlx::test] + async fn contended_acquire_sheds_as_repo_busy_not_internal_error(pool: PgPool) { + use axum::response::IntoResponse; + + let opts = (*pool.connect_options()).clone(); + let lock_pool = no_reap_pool(&opts, 4).await; + let store = RepoStore::for_testing(PathBuf::from("/tmp/gitlawb-busy"), lock_pool) + .with_lock_acquire_deadline(std::time::Duration::from_millis(300)); + + let held = store + .acquire_write("did:key:z6MkBusyOwner", "busyrepo") + .await + .expect("first writer acquires"); + + // Not `expect_err`: the guard is not Debug, and a guard obtained here must be + // released rather than dropped on a panic path. + let err = match store.acquire_write("did:key:z6MkBusyOwner", "busyrepo").await { + Err(e) => e, + Ok(second) => { + second.release(false).await; + panic!("a second writer must be shed once the deadline expires"); + } + }; + + // The internal chain keeps the operator detail... + let chain = format!("{err:#}"); + assert!( + chain.contains("busyrepo"), + "the log-side error must name the repo, got {chain}" + ); + + // ...and the client-visible mapping must carry neither it nor a 500. + let resp = crate::error::AppError::from(err).into_response(); + assert_eq!( + resp.status(), + axum::http::StatusCode::SERVICE_UNAVAILABLE, + "contention is transient and must be retryable" + ); + let body = axum::body::to_bytes(resp.into_body(), 64 * 1024) + .await + .expect("body"); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("repo_busy") && !body.contains("busyrepo"), + "the 503 body must be fixed and must not name the repo, got {body}" + ); + + held.release(true).await; + } + /// The transfer bound is a knob, so it gets the same parse/default/reject-zero /// coverage its sibling lock-pool-size knob has. #[test] From 8cf6b7ae465d4411da82b3b48d5d1da55e2ecc9a Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 22:11:27 -0500 Subject: [PATCH 11/54] fix(node): re-authorize close_issue under the guard instead of only re-checking existence The guarded re-read matched `Ok(Some(_))` and discarded the blob, so the authorization decision rested entirely on the pre-lock read. A comment above claimed the opposite. That is not a narrow window: `acquire_write` re-downloads the archive after locking, so the tree that gets mutated is routinely not the one the author was read from. With owner-push enforcement defaulting to false and branch protection covering only `refs/heads/*`, `refs/gitlawb/issues/*` is pushable, so a forged author blob landing between the two reads was honored. The blob is already in hand under the guard, so re-asserting owner-or-author costs a deserialize. A non-owner whose issue has vanished gets 403 rather than 404, matching the pre-check's existing refusal to reveal existence. `owner_can_still_close_after_the_reorder` seeded the owner as their own issue's author, which made it unable to fail: with the owner check disabled the author fallback granted the close and the test stayed green. It now seeds a third party, so only the owner arm can grant. RED with `is_owner = false`. Also drops the claim that the author twin covers the acquire-vs-acquire_fresh distinction. `RepoStore::for_testing` hardcodes `tigris: None`, so the two calls are identical in every test here and reverting that line leaves the twin green. Separating them needs an object-storage seam. Stating the gap beats asserting coverage that does not exist. --- crates/gitlawb-node/src/api/issues.rs | 69 +++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 10 deletions(-) diff --git a/crates/gitlawb-node/src/api/issues.rs b/crates/gitlawb-node/src/api/issues.rs index 1cb7c009..f85d45b2 100644 --- a/crates/gitlawb-node/src/api/issues.rs +++ b/crates/gitlawb-node/src/api/issues.rs @@ -243,9 +243,12 @@ pub async fn close_issue( // is immutable — it is not: `refs/gitlawb/**` is pushable, so a forged // author blob can be pushed (tracked separately; it is what makes this // fallback only as trustworthy as push authorization). The justification is - // that this read is a PRE-CHECK: it decides whether to take the lock at all, - // and the mutation below re-reads under the guard, so a change landing - // between the two cannot cause a write against state we never looked at. + // that this read is only a PRE-CHECK, deciding whether to take the lock at + // all. It is NOT the authorization decision: `acquire_write` re-downloads the + // archive after locking, so the tree that gets mutated is routinely not this + // one, and the authoritative owner-or-author check runs again under the guard + // below. Refusing here early just keeps a caller who is already visibly + // unauthorized from reaching the lock. // // `acquire_fresh`, not `acquire`: acquire's fast path returns as soon as the // directory exists and never contacts object storage, so on a node with a @@ -285,13 +288,39 @@ pub async fn close_issue( .await?; let disk_path = guard.path().to_path_buf(); - // Re-read under the guard so the mutation acts on current state, and keep the - // owner's existing 404-for-a-missing-issue behavior. + // Re-read under the guard and RE-AUTHORIZE against what we read, rather than + // only confirming the issue still exists. The pre-lock read decided whether to + // take the lock; it cannot be the authorization decision, because acquire_write + // re-downloads the archive after locking, so this is frequently a different tree + // than the one the author was read from. Checking existence alone would leave the + // whole decision resting on the earlier read of a tree we are no longer looking + // at. The blob is already in hand here, so this costs a deserialize. match git_issues::get_issue(&disk_path, &issue_id) { - Ok(Some(_)) => {} + Ok(Some(raw)) => { + let author_now: Option = serde_json::from_str::(&raw) + .ok() + .and_then(|i| i.author); + let is_author_now = author_now + .as_deref() + .is_some_and(|a| crate::api::did_matches(&auth.0, a)); + if !is_owner && !is_author_now { + guard.release(false).await; + return Err(AppError::Forbidden( + "only the repo owner or the issue author can close this issue".into(), + )); + } + } Ok(None) => { guard.release(false).await; - return Err(AppError::NotFound(format!("issue {issue_id} not found"))); + // The owner keeps the informative 404; a non-owner must not learn from + // this route whether the issue exists, matching the pre-check above. + return Err(if is_owner { + AppError::NotFound(format!("issue {issue_id} not found")) + } else { + AppError::Forbidden( + "only the repo owner or the issue author can close this issue".into(), + ) + }); } Err(e) => { guard.release(false).await; @@ -447,11 +476,24 @@ mod tests { /// INV-21(c) positive twin 1: the OWNER can still close. The reorder moved the /// owner check above the lock, so this is the arm most likely to have broken, /// and the deny test alone could not see it. + /// + /// The issue is seeded with a THIRD party as its author, deliberately. Seeding + /// the owner as their own author made this test unable to fail: with the owner + /// check disabled, the author fallback granted the close anyway and the test + /// stayed green. Only the owner arm can grant here now. #[sqlx::test] async fn owner_can_still_close_after_the_reorder(pool: PgPool) { let state = crate::test_support::test_state(pool.clone()).await; let owner_did = "did:key:z6MkT1Owner"; - seed_repo_with_issue(&state, "z6MkT1Owner", owner_did, "t1repo", "1", owner_did).await; + seed_repo_with_issue( + &state, + "z6MkT1Owner", + owner_did, + "t1repo", + "1", + "did:key:z6MkT1Stranger", + ) + .await; let res = close_issue( axum::extract::State(state.clone()), @@ -470,8 +512,15 @@ mod tests { ); } - /// INV-21(c) positive twin 2: the non-owner AUTHOR can still close. This is the - /// arm the acquire-vs-acquire_fresh regression broke, and nothing caught it. + /// INV-21(c) positive twin 2: the non-owner AUTHOR can still close, through both + /// the pre-lock check and the re-assertion under the guard. + /// + /// It does NOT cover the acquire-vs-acquire_fresh distinction, despite that being + /// the reason the call changed. `RepoStore::for_testing` hardcodes `tigris: None`, + /// which makes `acquire` and `acquire_fresh` identical in every test here, so + /// reverting that line leaves this green. Separating them needs an object-storage + /// seam, which is out of scope for this change and tracked separately. Claiming + /// the coverage here would be worse than admitting the gap. #[sqlx::test] async fn issue_author_who_is_not_the_owner_can_still_close(pool: PgPool) { let state = crate::test_support::test_state(pool.clone()).await; From af5948b2040172795f278255969f0177adda15f2 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 22:26:08 -0500 Subject: [PATCH 12/54] style(node): rustfmt the new contention test Whitespace only; cargo fmt --check gates the push. --- crates/gitlawb-node/src/git/repo_store.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index da5daec9..6085e073 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -2671,7 +2671,10 @@ mod tests { // Not `expect_err`: the guard is not Debug, and a guard obtained here must be // released rather than dropped on a panic path. - let err = match store.acquire_write("did:key:z6MkBusyOwner", "busyrepo").await { + let err = match store + .acquire_write("did:key:z6MkBusyOwner", "busyrepo") + .await + { Err(e) => e, Ok(second) => { second.release(false).await; From e5558fbb2784ba4a5bd890eb46d88c619f14420d Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 3 Aug 2026 11:38:03 -0500 Subject: [PATCH 13/54] fix(node): shed an unrefreshable write as a retryable 503 with a fixed body The two refusal arms added for the under-lock transfer bound returned bare anyhow errors, so AppError's From impl (which downcasts only sqlx::Error and RepoBusy) landed them in Internal: a 500 internal_error whose body was the error string. Two defects in one. A transient object-storage failure told the client not to retry, and the timeout arm's message interpolated the owner slug and repo name straight into the response body, contradicting the fixed-body policy the RepoBusy arm sets six lines above it. RepoUnavailable follows RepoBusy exactly: a fieldless type raised with the operator detail in a context string, downcast to its own rung, mapped to a 503 whose body interpolates nothing. The detail stays in the log at the raise site. The timeout arm logs at error! where its sibling logs at warn!, deliberately: a 300s stall pinned a lock-pool slot for five minutes and is the condition the transfer bound exists to surface, so it must keep paging when the handler classifier demotes the ordinary blip case. --- crates/gitlawb-node/src/error.rs | 18 +++- crates/gitlawb-node/src/git/repo_store.rs | 101 ++++++++++++++++++++-- 2 files changed, 113 insertions(+), 6 deletions(-) diff --git a/crates/gitlawb-node/src/error.rs b/crates/gitlawb-node/src/error.rs index ecc46309..d8362af8 100644 --- a/crates/gitlawb-node/src/error.rs +++ b/crates/gitlawb-node/src/error.rs @@ -62,6 +62,9 @@ pub enum AppError { #[error("repository is busy")] RepoBusy, + #[error("repository is temporarily unavailable")] + RepoUnavailable, + #[error("database error: {0}")] Db(#[from] sqlx::Error), @@ -109,7 +112,13 @@ impl From for AppError { // site and the client gets a fixed retryable body. Err(err) => match err.downcast::() { Ok(_) => AppError::RepoBusy, - Err(err) => AppError::Internal(err), + // Same reasoning one rung down: a refused under-lock refresh is a + // transient storage condition, and its internal message names the + // owner slug and repo, so the variant carries nothing. + Err(err) => match err.downcast::() { + Ok(_) => AppError::RepoUnavailable, + Err(err) => AppError::Internal(err), + }, }, } } @@ -182,6 +191,13 @@ impl IntoResponse for AppError { "repo_busy", "repository is busy — retry".into(), ), + // 503 with a FIXED body for the same reason: the caller should retry, and + // must not be told which repo could not be refreshed or why. + AppError::RepoUnavailable => ( + StatusCode::SERVICE_UNAVAILABLE, + "repo_unavailable", + "repository is temporarily unavailable, retry".into(), + ), AppError::Db(e) if db_unavailable(e) => ( StatusCode::SERVICE_UNAVAILABLE, DB_UNAVAILABLE_CODE, diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 6085e073..6f1e362e 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -386,7 +386,9 @@ impl RepoStore { warn!(repo = %repo_name, err = %e, "write acquire: tigris HEAD failed — refusing the write rather than \ guessing the archive is absent"); - return Err(e).context("checking tigris for the repo archive before a write"); + return Err(anyhow::Error::new(RepoUnavailable).context(format!( + "tigris HEAD failed before a write for {owner_slug}/{repo_name}" + ))); } None => { // TIMED OUT, which is NOT the same as failed, and must not reach @@ -400,11 +402,23 @@ impl RepoStore { // // Refuse the acquire. Returning here drops the guard, whose Drop // frees the lock and its pool slot. - return Err(anyhow::anyhow!( - "tigris refresh exceeded the {}s under-lock bound for {owner_slug}/{repo_name}; \ - refusing the write rather than proceeding against a possibly-stale tree", + // + // `error!`, not the sibling `warn!` above, and that is deliberate. + // The handler layer demotes every `RepoUnavailable` to warn because + // the common cause is an ordinary storage blip. A stall that ran out + // the whole bound is not that: it pinned a lock-pool slot for the + // full duration, and this raise-site `error!` is what keeps it + // paging. Do NOT "fix" it to match the arm above. + tracing::error!( + repo = %repo_name, + owner = %owner_slug, + bound_secs = self.lock_held_transfer_timeout.as_secs(), + "under-lock tigris refresh exceeded the transfer bound, refusing the write" + ); + return Err(anyhow::Error::new(RepoUnavailable).context(format!( + "tigris refresh exceeded the {}s under-lock bound for {owner_slug}/{repo_name}", self.lock_held_transfer_timeout.as_secs() - )); + ))); } } } @@ -971,6 +985,24 @@ impl std::fmt::Display for RepoBusy { impl std::error::Error for RepoBusy {} +/// The under-lock refresh could not establish what is in object storage, so the +/// write was refused rather than run against a possibly-stale tree. +/// +/// A distinct type rather than a bare `anyhow` string so the handler layer can map +/// it to a retryable 503 with a FIXED body. The internal message names the owner +/// slug and repo, which must stay in the log at the raise site rather than reaching +/// the client. +#[derive(Debug)] +pub struct RepoUnavailable; + +impl std::fmt::Display for RepoUnavailable { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("repository is temporarily unavailable") + } +} + +impl std::error::Error for RepoUnavailable {} + /// Run a future under a wall-clock bound, returning `None` if it did not finish. /// /// For the object-storage transfers that run while the per-repo advisory lock is @@ -2708,6 +2740,65 @@ mod tests { held.release(true).await; } + /// An under-lock refresh refusal must surface as a retryable 503 with a fixed + /// body, not a 500 carrying the owner slug and repo name. Built directly from + /// the typed error so it needs no database; the `.context()` layer is kept + /// deliberately, because the real raise path wraps one and this proves anyhow + /// preserves downcastability through it. + #[tokio::test] + async fn repo_unavailable_maps_to_retryable_503_with_fixed_body() { + use axum::response::IntoResponse; + + let err = anyhow::Error::new(RepoUnavailable) + .context("tigris HEAD failed before a write for did_key_z6MkTest/secret-repo"); + + let resp = crate::error::AppError::from(err).into_response(); + assert_eq!( + resp.status(), + axum::http::StatusCode::SERVICE_UNAVAILABLE, + "a storage blip is transient and must be retryable, not a 500" + ); + let body = axum::body::to_bytes(resp.into_body(), 64 * 1024) + .await + .expect("body"); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("repo_unavailable"), + "the 503 must carry the repo_unavailable code, got {body}" + ); + assert!( + !body.contains("secret-repo"), + "the 503 body must be fixed and must not name the repo, got {body}" + ); + assert!( + !body.contains("did_key_z6MkTest"), + "the 503 body must be fixed and must not name the owner, got {body}" + ); + } + + /// The new downcast rung must be additive: an unrelated anyhow error still + /// falls through to the internal 500. + #[tokio::test] + async fn repo_unavailable_rung_does_not_swallow_unrelated_errors() { + use axum::response::IntoResponse; + + let resp = + crate::error::AppError::from(anyhow::anyhow!("some other failure")).into_response(); + assert_eq!( + resp.status(), + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + "an unrelated failure must not be reclassified as retryable" + ); + let body = axum::body::to_bytes(resp.into_body(), 64 * 1024) + .await + .expect("body"); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("internal_error"), + "an unrelated failure must keep the internal_error code, got {body}" + ); + } + /// The transfer bound is a knob, so it gets the same parse/default/reject-zero /// coverage its sibling lock-pool-size knob has. #[test] From 63a9ef8b1b8093552db2766436e3f8dd02834f3f Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 3 Aug 2026 11:45:51 -0500 Subject: [PATCH 14/54] fix(node): refuse a fresh-copy write when the archive HEAD cannot be read acquire_fresh collapsed a failed HEAD into "no archive exists" via unwrap_or(false) and served the local copy, while the sibling path under the lock already refused on the same condition. A push that hit a storage blip got an advertisement built from a possibly-stale tree, uploaded a whole pack, and was then refused by acquire_write for the reason the advertisement had already swallowed. The authorship pre-check on close_issue read the same way, which is an infrastructure failure resolving toward a denial. A failed HEAD now raises RepoUnavailable, so both callers surface it as the retryable 503 rather than a stale success. The download-failure fallback is unchanged: a present-but-unreadable archive is still self-healed from local. info_refs keeps its map_err for every other failure so the read path's error vocabulary does not move; only the typed error takes the From chain. A test-only TigrisClient constructor pointed at a closed port makes both refusals executable, so this is no longer verified by reading. It also proves the under-lock arm end to end, which the earlier draft had recorded as an untestable gap. --- crates/gitlawb-node/src/git/repo_store.rs | 129 +++++++++++++++++++--- crates/gitlawb-node/src/git/tigris.rs | 34 +++--- 2 files changed, 136 insertions(+), 27 deletions(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 6f1e362e..272fe912 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -63,6 +63,17 @@ impl RepoStore { } } + /// Same as [`RepoStore::for_testing`] but with Tigris enabled, so the paths + /// that only run when a backend is configured are reachable in a test. + #[cfg(test)] + pub fn for_testing_with_tigris( + repos_dir: PathBuf, + lock_pool: PgPool, + tigris: TigrisClient, + ) -> Self { + Self::new(repos_dir, Some(tigris), lock_pool, Duration::from_secs(300)) + } + /// Shorten the lock-acquire deadline so the busy path is reachable in a test /// without waiting out the production default. #[cfg(test)] @@ -161,26 +172,51 @@ impl RepoStore { /// Use this for operations that precede a write (e.g. `info/refs` for /// `git-receive-pack`) so the client sees the same refs that `acquire_write()` /// will operate on. + /// + /// A failed existence check refuses the acquire rather than guessing the + /// archive is absent, matching the under-lock path in `acquire_write()`. pub async fn acquire_fresh(&self, owner_did: &str, repo_name: &str) -> Result { let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; if let Some(ref tigris) = self.tigris { - if tigris.exists(&owner_slug, repo_name).await.unwrap_or(false) { - debug!(repo = %repo_name, "acquire_fresh: downloading latest from tigris"); - if let Err(e) = tigris.download(&owner_slug, repo_name, &local_path).await { - // The Tigris archive is present (HEAD ok) but unreadable — a - // corrupt/partial upload, or a transient GET failure. If we have a - // valid local copy, proceed with it rather than blocking the write; - // the post-write upload re-syncs (self-heals) Tigris. Only hard-fail - // when there is no local copy to fall back to. - if local_path.exists() { - warn!(repo = %repo_name, err = %e, - "acquire_fresh: tigris download failed — falling back to local copy"); - return Ok(local_path); + // The HEAD and the download fail for epistemically DIFFERENT reasons, + // so they are kept apart rather than collapsed into one `Result`. The + // `unwrap_or(false)` this replaced read a HEAD error as "no archive" + // and silently advertised a possibly-stale local copy to a client that + // is about to push against it. + match tigris.exists(&owner_slug, repo_name).await { + Ok(true) => { + debug!(repo = %repo_name, "acquire_fresh: downloading latest from tigris"); + if let Err(e) = tigris.download(&owner_slug, repo_name, &local_path).await { + // The Tigris archive is present (HEAD ok) but unreadable — a + // corrupt/partial upload, or a transient GET failure. If we have a + // valid local copy, proceed with it rather than blocking the write; + // the post-write upload re-syncs (self-heals) Tigris. Only hard-fail + // when there is no local copy to fall back to. + if local_path.exists() { + warn!(repo = %repo_name, err = %e, + "acquire_fresh: tigris download failed — falling back to local copy"); + return Ok(local_path); + } + return Err(e).context("downloading repo from tigris (fresh)"); } - return Err(e).context("downloading repo from tigris (fresh)"); + return Ok(local_path); + } + Ok(false) => {} + Err(e) => { + // We do not know whether a newer archive exists, so we cannot + // tell whether the local copy is current. Advertising stale refs + // here sends the client into a push computed against the wrong + // base, so refuse for the same reason `acquire_write` refuses on + // this condition. A transient storage blip costs a retryable + // refusal, which is the cheaper failure. + warn!(repo = %repo_name, err = %e, + "acquire_fresh: tigris HEAD failed — refusing rather than \ + guessing the archive is absent"); + return Err(anyhow::Error::new(RepoUnavailable).context(format!( + "tigris HEAD failed during acquire_fresh for {owner_slug}/{repo_name}" + ))); } - return Ok(local_path); } } @@ -2799,6 +2835,71 @@ mod tests { ); } + /// A Tigris client aimed at a closed port, so every call fails at the + /// transport layer promptly and `exists()` returns `Err` rather than + /// `Ok(false)`. + #[cfg(test)] + fn unreachable_tigris() -> TigrisClient { + TigrisClient::for_testing_with_endpoint("test-bucket", "http://127.0.0.1:1") + } + + /// A failed HEAD tells us nothing about whether a newer archive exists, so + /// the pre-write refresh must refuse rather than read the failure as "no + /// archive" and serve a possibly-stale local copy to the pushing client. + /// + /// Asserts on the downcast, not the message, so a context rewrite cannot + /// quietly make this vacuous. + #[sqlx::test] + async fn acquire_fresh_refuses_when_the_head_check_fails(pool: PgPool) { + let opts = (*pool.connect_options()).clone(); + let lock_pool = no_reap_pool(&opts, 2).await; + let store = RepoStore::for_testing_with_tigris( + PathBuf::from("/tmp/gitlawb-headfail-fresh"), + lock_pool, + unreachable_tigris(), + ); + + let err = store + .acquire_fresh("did:key:z6MkHeadFail", "freshrepo") + .await + .expect_err("a failed HEAD must refuse rather than serve the local copy"); + assert!( + err.downcast_ref::().is_some(), + "the refusal must be typed so the handler layer maps it to a retryable 503, got {err:#}" + ); + } + + /// The under-lock sibling of the above. `acquire_write` already refuses on + /// this condition; this proves the `RefreshFailure::Unknown` arm end to end + /// against a real failing HEAD rather than by reading the code. + #[sqlx::test] + async fn acquire_write_refuses_when_the_head_check_fails(pool: PgPool) { + let opts = (*pool.connect_options()).clone(); + let lock_pool = no_reap_pool(&opts, 2).await; + let store = RepoStore::for_testing_with_tigris( + PathBuf::from("/tmp/gitlawb-headfail-write"), + lock_pool, + unreachable_tigris(), + ); + + // Not `expect_err`: the guard is not Debug, and a guard obtained here + // must be released rather than dropped on a panic path. + let err = match store + .acquire_write("did:key:z6MkHeadFail", "writerepo") + .await + { + Err(e) => e, + Ok(guard) => { + guard.release(false).await; + panic!("a failed HEAD must refuse the write rather than proceed on a stale tree"); + } + }; + assert!( + err.downcast_ref::().is_some(), + "the refusal must be typed so the handler layer maps it to a retryable 503, got {err:#}" + ); + } + /// The transfer bound is a knob, so it gets the same parse/default/reject-zero /// coverage its sibling lock-pool-size knob has. #[test] diff --git a/crates/gitlawb-node/src/git/tigris.rs b/crates/gitlawb-node/src/git/tigris.rs index cf7abfd5..cba8867c 100644 --- a/crates/gitlawb-node/src/git/tigris.rs +++ b/crates/gitlawb-node/src/git/tigris.rs @@ -31,21 +31,29 @@ impl TigrisClient { }) } - /// Test-only constructor with an explicit S3 endpoint, region, and static - /// credentials — no env-var reads, so parallel tests cannot race each other's - /// `AWS_*` environment the way the env-based `new` would. Lets a test point - /// the client at a non-routable endpoint to exercise acquire-stall paths. + /// Build a client pointed at an arbitrary endpoint, for tests. + /// + /// The production constructor reads the endpoint and credentials from the + /// environment, which a test cannot steer without mutating process-global + /// state. This takes both explicitly so a test can aim the client at a + /// closed port and get a prompt transport error out of `exists()`. + /// + /// `RetryConfig::disabled()` is load-bearing, not tidiness: the SDK's default + /// policy retries a connection refusal with backoff, which turns each failing + /// call into seconds of waiting. #[cfg(test)] - pub(crate) async fn for_testing_with_endpoint(bucket: &str, endpoint_url: &str) -> Self { - let creds = aws_sdk_s3::config::Credentials::new("test", "test", None, None, "test"); - let config = aws_config::defaults(aws_config::BehaviorVersion::latest()) - .endpoint_url(endpoint_url) - .region(aws_config::Region::new("auto")) - .credentials_provider(creds) - .load() - .await; + pub fn for_testing_with_endpoint(bucket: &str, endpoint: &str) -> Self { + use aws_sdk_s3::config::{retry::RetryConfig, Credentials, Region}; + + let config = aws_sdk_s3::config::Config::builder() + .endpoint_url(endpoint) + .credentials_provider(Credentials::new("test", "test", None, None, "test")) + .region(Region::new("auto")) + .retry_config(RetryConfig::disabled()) + .behavior_version_latest() + .build(); Self { - s3: S3Client::new(&config), + s3: S3Client::from_conf(config), bucket: bucket.to_string(), } } From 10230ac50ef04a8388367a5129e83018b4d4ed84 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 3 Aug 2026 11:49:45 -0500 Subject: [PATCH 15/54] fix(node): log expected transient acquire failures at warn, not error Both acquire call sites logged every failure at error, including RepoBusy, which the raise site already logs at warn. Ordinary write contention paged. The previous commit widened the problem: info_refs now raises RepoUnavailable on a storage blip, so that site would have started paging on the condition this series just classified as transient and retryable. A classifier over the two typed refusals picks the level at both sites, mirroring the startup path's permanent-vs-transient split. Anything it cannot classify still logs at error, so an unknown failure keeps paging. --- crates/gitlawb-node/src/api/repos.rs | 70 ++++++++++++++++++++++++---- 1 file changed, 62 insertions(+), 8 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index b09cb6da..7634cc3c 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -703,7 +703,7 @@ pub async fn git_info_refs( // so the shed frees the slot; return a bounded 503. let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); let acquire_fut = async { - if service == "git-receive-pack" { + let res = if service == "git-receive-pack" { state .repo_store .acquire_fresh(&record.owner_did, &record.name) @@ -713,18 +713,31 @@ pub async fn git_info_refs( .repo_store .acquire(&record.owner_did, &record.name) .await - } + }; + res.map_err(|e| { + if is_expected_transient_acquire_failure(&e) { + tracing::warn!(repo = %name, service = %service, err = %e, "repo acquire failed"); + } else { + tracing::error!(repo = %name, service = %service, err = %e, "repo acquire failed"); + } + // This closure bypasses the `From` chain, so a typed + // refusal would otherwise be stringified into a 500 `git_error`. Route + // just that one case through `From` and leave every other failure on + // exactly today's behavior: this call site also serves the read path via + // `acquire`, whose error vocabulary is out of scope here. + if e.is::() { + AppError::from(e) + } else { + AppError::Git(e.to_string()) + } + }) }; let disk_path = tokio::time::timeout(acquire_deadline, acquire_fut) .await .map_err(|_elapsed| { tracing::warn!(repo = %name, service = %service, "repo acquire timed out; shedding with 503"); AppError::Overloaded("git service acquisition timed out, retry shortly".into()) - })? - .map_err(|e| { - tracing::error!(repo = %name, service = %service, err = %e, "repo acquire failed"); - AppError::Git(e.to_string()) - })?; + })??; // Move the admission permits into the guard so they release only after the spawned // git process group is confirmed reaped, on complete/timeout/disconnect — not the @@ -1273,6 +1286,23 @@ async fn pin_and_encrypt_objects( /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; /// callers add their own tracing. +/// Acquire failures that are ordinary and transient: lock contention +/// ([`RepoBusy`]) and an under-lock refresh that could not reach object storage +/// ([`RepoUnavailable`]). Both already log at their raise site and both map to a +/// retryable 503, so the handler layer logs them at warn rather than paging. +/// Best-effort, like the database startup classifier: anything this cannot +/// recognize counts as NOT transient and keeps its error-level log. +/// +/// [`RepoBusy`]: crate::git::repo_store::RepoBusy +/// [`RepoUnavailable`]: crate::git::repo_store::RepoUnavailable +fn is_expected_transient_acquire_failure(err: &anyhow::Error) -> bool { + err.downcast_ref::() + .is_some() + || err + .downcast_ref::() + .is_some() +} + fn git_service_app_error(err: &anyhow::Error) -> AppError { if err .downcast_ref::() @@ -1941,7 +1971,11 @@ pub async fn git_receive_pack( AppError::Overloaded("git service acquisition timed out, retry shortly".into()) })? .map_err(|e| { - tracing::error!(repo = %name, err = %e, "acquire_write failed"); + if is_expected_transient_acquire_failure(&e) { + tracing::warn!(repo = %name, err = %e, "acquire_write failed"); + } else { + tracing::error!(repo = %name, err = %e, "acquire_write failed"); + } AppError::Git(e.to_string()) })?; let disk_path = guard.path().to_path_buf(); @@ -3228,6 +3262,26 @@ mod tests { assert!(git_permit(&sem).is_ok()); } + #[test] + fn is_expected_transient_matches_both_typed_refusals() { + // The real raise shape wraps the marker in a `.context()` layer naming the + // owner slug and repo, so the downcast has to survive that wrapping. + let busy = anyhow::Error::new(crate::git::repo_store::RepoBusy) + .context("another write is in progress for alice/demo"); + assert!(is_expected_transient_acquire_failure(&busy)); + + let unavailable = anyhow::Error::new(crate::git::repo_store::RepoUnavailable) + .context("could not read the archive HEAD for alice/demo"); + assert!(is_expected_transient_acquire_failure(&unavailable)); + } + + #[test] + fn is_expected_transient_rejects_unrelated_failures() { + // Anything the classifier cannot recognize keeps paging at error level. + let other = anyhow::anyhow!("disk on fire"); + assert!(!is_expected_transient_acquire_failure(&other)); + } + fn repo_owned_by(owner_did: &str) -> crate::db::RepoRecord { let now = chrono::Utc::now(); crate::db::RepoRecord { From 259d5873ec625464d262252709ccb556139d4e13 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 3 Aug 2026 11:55:17 -0500 Subject: [PATCH 16/54] fix(node): log the read failure the close_issue pre-check folds into a denial The authorship pre-check treated a get_issue I/O error and a genuinely absent issue as the same None, with no log line. The client answer is deliberately identical, since a caller who cannot write must not learn whether the issue exists, but the two are not the same event and an operator had no way to tell a real authorization denial from a filesystem or parse failure behind it. Splitting the arm leaves the 403 exactly where it was and makes the read failure visible in the log. --- crates/gitlawb-node/src/api/issues.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/crates/gitlawb-node/src/api/issues.rs b/crates/gitlawb-node/src/api/issues.rs index f85d45b2..e0e58fa5 100644 --- a/crates/gitlawb-node/src/api/issues.rs +++ b/crates/gitlawb-node/src/api/issues.rs @@ -265,8 +265,20 @@ pub async fn close_issue( .and_then(|i| i.author), // Cannot establish authorship, so fail closed. Deliberately 403 rather // than 404 for a non-owner: a caller who is not authorized to write - // should not learn from this route whether the issue exists. - Ok(None) | Err(_) => None, + // should not learn from this route whether the issue exists. Both arms + // below return None; they are split only so a read failure is visible + // to operators, since a genuinely absent issue and an unreadable one + // are the same answer to the client but not the same event. + Ok(None) => None, + Err(e) => { + tracing::warn!( + repo = %repo, + issue = %issue_id, + err = %e, + "get_issue failed during close_issue authorship pre-check" + ); + None + } }; let is_author = author_did .as_deref() From 999ee72f3c3ad6341f0cc3149784c2e0932e378e Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 3 Aug 2026 11:58:49 -0500 Subject: [PATCH 17/54] test(node): poll for the closed session instead of sleeping a fixed 300ms The release-invariant test slept 300ms for a Drop-spawned close before comparing backend pids, which is a coin flip on a loaded CI runner. It now polls pg_stat_activity for the captured pid on a standalone connection, the same discipline poll_until_free documents: a pooled observer would be handed the session under measurement and hide the effect. The conversion was checked against the failure it exists to catch rather than assumed. Pooling the session on an unlock that returned false makes the test fail after the poll deadline, not hang, and no_reap_pool disables idle timeout and max lifetime so nothing but the close under test can retire that backend. A generous deadline would have been the same defect as the sleep. --- crates/gitlawb-node/src/git/repo_store.rs | 30 +++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 272fe912..1bdabc55 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -2503,8 +2503,34 @@ mod tests { }; guard.release(true).await; - // Give the spawned close a moment, then see which backend we land on. - tokio::time::sleep(std::time::Duration::from_millis(300)).await; + // Wait for the backend to actually go away rather than sleeping a fixed + // span, which is flaky on slow CI. The observer is a STANDALONE + // connection for the same reason `poll_until_free` uses one: taking it + // from the pool under test would hand us the very session being measured. + // Nothing but the close under test can retire that backend, because + // `no_reap_pool` disables idle timeout and max lifetime, so a zero count + // here is attributable to `release()` and to nothing else. + { + use sqlx::Connection; + let deadline = std::time::Duration::from_secs(5); + let start = std::time::Instant::now(); + let mut observer = sqlx::PgConnection::connect_with(&opts) + .await + .expect("standalone observer connection"); + while start.elapsed() < deadline { + let alive: (i64,) = + sqlx::query_as("SELECT count(*) FROM pg_stat_activity WHERE pid = $1") + .bind(pid_before) + .fetch_one(&mut observer) + .await + .expect("observer pg_stat_activity probe"); + if alive.0 == 0 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + } + let pid_after = { let mut c = lock_pool.acquire().await.unwrap(); let pid: (i32,) = sqlx::query_as("SELECT pg_backend_pid()") From 5fdb902096898877256c2c96ab766b9f063fb821 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sun, 9 Aug 2026 00:13:57 -0500 Subject: [PATCH 18/54] fix(node): raise a failed fresh download with no local copy as RepoUnavailable acquire_fresh already refused a failed Tigris HEAD as RepoUnavailable, so the handler layer mapped it to a retryable 503. A failed GET on the same path still returned a bare anyhow error, which the info/refs map_err closure stringified to AppError::Git and answered 500. One endpoint therefore told the client a transient object-storage blip was permanent or retryable depending on which call failed, and close_issue's pre-check inherited the same split through its bare ?. Raise it at the source instead of at each consumer: From for AppError already downcasts RepoUnavailable out of the context chain, so both callers pick up the retryable mapping without touching either. The new test drives HEAD 200 with GET 500, which is the exact state the refusal is for: archive present per HEAD, GET failed, no local copy to fall back on. --- crates/gitlawb-node/src/git/repo_store.rs | 61 ++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 1bdabc55..f2b7b530 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -198,7 +198,18 @@ impl RepoStore { "acquire_fresh: tigris download failed — falling back to local copy"); return Ok(local_path); } - return Err(e).context("downloading repo from tigris (fresh)"); + // No local copy, so the write cannot proceed and the archive's + // readability is unknowable. Same epistemic class as the HEAD arm + // and the under-lock refresh: a transient storage blip must be a + // retryable refusal, not a 500 that tells the client the failure + // is permanent. Wrap so the handler layer's `RepoUnavailable` + // downcast maps this to a retryable 503 with a fixed body; the + // detail (which repo, why) stays in this warn and the context. + warn!(repo = %repo_name, err = %e, + "acquire_fresh: tigris download failed and no local copy exists — refusing"); + return Err(anyhow::Error::new(RepoUnavailable).context(format!( + "tigris download failed during acquire_fresh for {owner_slug}/{repo_name}: {e:#}" + ))); } return Ok(local_path); } @@ -2895,6 +2906,54 @@ mod tests { ); } + /// A download that fails when the HEAD succeeded tells us the archive is + /// present but unreadable, and with no local copy to fall back on the + /// pre-write refresh must refuse as `RepoUnavailable` — not leak a bare + /// Tigris error that the handler layer would map to a non-retryable 500. + /// + /// The server answers HEAD 200 and GET 500, so `exists()` returns + /// `Ok(true)` while `download()` fails at the transport layer, exactly the + /// "archive present per HEAD, GET failed, no local fallback" state. + #[sqlx::test] + async fn acquire_fresh_refuses_when_the_download_fails_and_no_local_copy_exists(pool: PgPool) { + use axum::response::IntoResponse; + + let app = axum::Router::new().route( + "/{*key}", + axum::routing::any(|method: axum::http::Method| async move { + if method == axum::http::Method::HEAD { + axum::http::StatusCode::OK.into_response() + } else { + axum::http::StatusCode::INTERNAL_SERVER_ERROR.into_response() + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port()); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + + let opts = (*pool.connect_options()).clone(); + let lock_pool = no_reap_pool(&opts, 2).await; + let store = RepoStore::for_testing_with_tigris( + PathBuf::from("/tmp/gitlawb-getfail-fresh"), + lock_pool, + TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint), + ); + + let err = store + .acquire_fresh("did:key:z6MkGetFail", "freshrepo") + .await + .expect_err("a failed download with no local copy must refuse"); + assert!( + err.downcast_ref::().is_some(), + "the refusal must be typed so the handler layer maps it to a retryable 503, got {err:#}" + ); + + server.abort(); + } + /// The under-lock sibling of the above. `acquire_write` already refuses on /// this condition; this proves the `RefreshFailure::Unknown` arm end to end /// against a real failing HEAD rather than by reading the code. From f81ebbe61449579f6d1b4785fa555b488c1c525b Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sun, 9 Aug 2026 00:13:57 -0500 Subject: [PATCH 19/54] test(node): assert the second writer sheds as RepoBusy instead of timing out two_writers_on_the_same_repo_are_not_both_admitted wrapped the second acquire_write in an 8-second outer timeout and only checked the future had not finished. That passes for any stall, including lock-pool saturation or a slow CI box, so it could not tell a working shed from an unrelated hang, and it cost 8 seconds on every suite run. Use with_lock_acquire_deadline and assert the typed RepoBusy downcast while the first guard is still held, matching what contended_acquire_sheds_as_repo_busy_ not_internal_error already does. It now also fails loudly if a second writer is admitted, which the timeout version could not distinguish. --- crates/gitlawb-node/src/git/repo_store.rs | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index f2b7b530..8fe0ad32 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -2347,23 +2347,25 @@ mod tests { #[sqlx::test] async fn two_writers_on_the_same_repo_are_not_both_admitted(pool: PgPool) { let opts = (*pool.connect_options()).clone(); - let store = write_store(&pool, &opts).await; + let store = write_store(&pool, &opts) + .await + .with_lock_acquire_deadline(std::time::Duration::from_millis(300)); let _first = store .acquire_write("did:key:z6MkU3Excl", "same-repo") .await .expect("first writer acquires"); - let second = tokio::time::timeout( - std::time::Duration::from_secs(8), - store.acquire_write("did:key:z6MkU3Excl", "same-repo"), - ) - .await; - + let err = match store.acquire_write("did:key:z6MkU3Excl", "same-repo").await { + Err(e) => e, + Ok(second) => { + second.release(false).await; + panic!("a second writer must NOT be admitted while the first holds the guard"); + } + }; assert!( - second.is_err(), - "second writer must NOT be admitted while the first holds the guard \ - (it should still be retrying when the deadline hits)" + err.downcast_ref::().is_some(), + "the second writer must be shed as RepoBusy, got {err:#}" ); } From c4dbd8a7976cfe3f645c019c8cb8ad1e6cfabc44 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sun, 9 Aug 2026 21:56:55 -0500 Subject: [PATCH 20/54] fix(node): read the close_issue author pre-check from a non-mutating snapshot The non-owner author fallback refreshed through acquire_fresh, which publishes into the live repo directory: its extract step removes the existing directory and renames the new one into place. That runs with no write lock held, so any signed non-owner could trigger a directory swap underneath an in-flight guarded write on the same path. read_snapshot downloads to a throwaway temp dir and hands back a RepoSnapshot that removes it on drop, so the pre-check still sees fresh data and the live path is never touched. download_to grows a publish flag to serve both shapes from one path. The wedge invariant still holds: a stranger is refused without waiting on the write lock. --- crates/gitlawb-node/src/api/issues.rs | 37 +++-- crates/gitlawb-node/src/git/repo_store.rs | 188 ++++++++++++++++++++++ crates/gitlawb-node/src/git/tigris.rs | 63 +++++++- 3 files changed, 265 insertions(+), 23 deletions(-) diff --git a/crates/gitlawb-node/src/api/issues.rs b/crates/gitlawb-node/src/api/issues.rs index e0e58fa5..c34e41ad 100644 --- a/crates/gitlawb-node/src/api/issues.rs +++ b/crates/gitlawb-node/src/api/issues.rs @@ -239,27 +239,32 @@ pub async fn close_issue( // Not the owner, so the author fallback decides it, and the author lives in // the issue's git-JSON blob rather than a DB column. // - // Read it WITHOUT the write lock. The justification is NOT that authorship - // is immutable — it is not: `refs/gitlawb/**` is pushable, so a forged - // author blob can be pushed (tracked separately; it is what makes this - // fallback only as trustworthy as push authorization). The justification is - // that this read is only a PRE-CHECK, deciding whether to take the lock at - // all. It is NOT the authorization decision: `acquire_write` re-downloads the - // archive after locking, so the tree that gets mutated is routinely not this - // one, and the authoritative owner-or-author check runs again under the guard - // below. Refusing here early just keeps a caller who is already visibly + // Read it WITHOUT the write lock, from a NON-MUTATING SNAPSHOT. The + // justification is NOT that authorship is immutable — it is not: + // `refs/gitlawb/**` is pushable, so a forged author blob can be pushed + // (tracked separately; it is what makes this fallback only as trustworthy + // as push authorization). The justification is that this read is only a + // PRE-CHECK, deciding whether to take the lock at all. It is NOT the + // authorization decision: `acquire_write` re-downloads the archive after + // locking, so the tree that gets mutated is routinely not this one, and the + // authoritative owner-or-author check runs again under the guard below. + // Refusing here early just keeps a caller who is already visibly // unauthorized from reaching the lock. // - // `acquire_fresh`, not `acquire`: acquire's fast path returns as soon as the - // directory exists and never contacts object storage, so on a node with a - // stale copy the author's own issue would be invisible and the + // `read_snapshot`, not `acquire_fresh`: acquire's fast path returns as soon + // as the directory exists and never contacts object storage, so on a node + // with a stale copy the author's own issue would be invisible and the // cannot-establish-authorship arm below would 403 a legitimate author. - // acquire_fresh refreshes first and still takes no lock. - let disk_path = state + // read_snapshot refreshes the same way, but unpacks into a throwaway temp + // dir instead of publishing into the live repo path — an unlocked + // pre-check must not delete or swap the directory under a concurrent + // guarded write on the same path. + let snapshot = state .repo_store - .acquire_fresh(&record.owner_did, &record.name) + .read_snapshot(&record.owner_did, &record.name) .await?; - let author_did: Option = match git_issues::get_issue(&disk_path, &issue_id) { + let snapshot_path = snapshot.path().to_path_buf(); + let author_did: Option = match git_issues::get_issue(&snapshot_path, &issue_id) { Ok(Some(raw)) => serde_json::from_str::(&raw) .ok() .and_then(|i| i.author), diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 8fe0ad32..320d3479 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -235,6 +235,59 @@ impl RepoStore { Ok(local_path) } + /// Non-mutating snapshot of a repo's **latest** Tigris state, for reads that + /// must see fresh data but must NOT write into the live repo path. + /// + /// Unlike `acquire_fresh`, which downloads and PUBLISHES into the live + /// directory (removing the existing dir and renaming the extract into + /// place), this unpacks into a throwaway temp dir and returns it. The live + /// path is never touched, so an unlocked caller cannot delete or swap the + /// directory under a concurrent guarded write. + /// + /// The returned snapshot owns its temp dir and removes it on drop; when + /// there is no Tigris backend (or no archive), the snapshot borrows the live + /// local path and owns nothing. A HEAD failure refuses rather than guessing, + /// matching `acquire_fresh` and the under-lock refresh path: a transient + /// storage blip must be a retryable refusal (`RepoUnavailable`), not a 500 + /// or a silently stale read. + pub async fn read_snapshot(&self, owner_did: &str, repo_name: &str) -> Result { + let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; + + if let Some(ref tigris) = self.tigris { + match tigris.exists(&owner_slug, repo_name).await { + Ok(true) => { + // Snapshot form: unpack into a temp dir, never the live path. + let snapshot = tigris + .download_to(&owner_slug, repo_name, &local_path, false) + .await + .map_err(|e| { + anyhow::Error::new(RepoUnavailable).context(format!( + "tigris snapshot download failed during read_snapshot for {owner_slug}/{repo_name}: {e:#}" + )) + })?; + return Ok(RepoSnapshot { + path: snapshot.clone(), + owned: true, + }); + } + Ok(false) => {} + Err(e) => { + warn!(repo = %repo_name, err = %e, + "read_snapshot: tigris HEAD failed — refusing rather than guessing the archive is absent"); + return Err(anyhow::Error::new(RepoUnavailable).context(format!( + "tigris HEAD failed during read_snapshot for {owner_slug}/{repo_name}" + ))); + } + } + } + + // Tigris disabled or repo not in Tigris — fall back to local. + Ok(RepoSnapshot { + path: local_path, + owned: false, + }) + } + /// Take a write lock (Postgres advisory lock), ensure repo is local, return guard. /// /// # Cross-machine guarantee @@ -830,6 +883,29 @@ impl Drop for LockProbe { } } +/// Non-mutating snapshot of a repo's latest Tigris state. Owns the throwaway +/// temp dir it was unpacked into and removes it on drop; a snapshot that +/// borrowed the live local path owns nothing and drops as a no-op. +pub struct RepoSnapshot { + path: PathBuf, + owned: bool, +} + +impl RepoSnapshot { + /// Path to the snapshot's bare repo directory. + pub fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for RepoSnapshot { + fn drop(&mut self) { + if self.owned { + let _ = std::fs::remove_dir_all(&self.path); + } + } +} + /// Guard returned by `acquire_write()`. Holds the Postgres advisory lock and /// uploads to Tigris + releases the lock on `release()`. pub struct RepoWriteGuard { @@ -3003,4 +3079,116 @@ mod tests { ]) .is_err()); } + + /// P1a: the non-owner pre-check must refresh from a NON-MUTATING snapshot. + /// A snapshot download must unpack into a throwaway temp dir and leave the + /// live repo path untouched, so an unlocked pre-check cannot delete or swap + /// the directory under a concurrent guarded write. + /// + /// Real S3 server (not a mock): upload an archive, then `read_snapshot` it, + /// and assert the snapshot path is a fresh temp dir distinct from the live + /// path, that the live path was never created, and that the snapshot reads + /// the same content. + #[sqlx::test] + async fn read_snapshot_is_non_mutating(pool: PgPool) { + use axum::response::IntoResponse; + + // A real in-process S3-compatible server via the SDK against an axum + // router is more plumbing than this test needs; instead, upload through + // the real Tigris client against an axum server that stores the object + // in memory, then snapshot through the same store. + // + // Simpler and equally load-bearing: build the archive bytes, serve them + // with a real HTTP server that answers HEAD 200 and GET with the bytes, + // then call read_snapshot and assert the live path is untouched and the + // snapshot content matches. + let mut archive_bytes = Vec::new(); + { + let dir = + std::env::temp_dir().join(format!("gitlawb-snap-src-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(dir.join("objects/info")).unwrap(); + std::fs::create_dir_all(dir.join("refs/heads")).unwrap(); + std::fs::write(dir.join("HEAD"), "ref: refs/heads/main\n").unwrap(); + std::fs::write(dir.join("objects/info/packs"), "").unwrap(); + let encoder = zstd::stream::Encoder::new(&mut archive_bytes, 3).unwrap(); + let mut tar = tar::Builder::new(encoder); + tar.append_dir_all(".", &dir).unwrap(); + tar.into_inner().unwrap().finish().unwrap(); + std::fs::remove_dir_all(&dir).unwrap(); + } + let archive = std::sync::Arc::new(archive_bytes); + + let app = axum::Router::new().route( + "/{*key}", + axum::routing::any(move |method: axum::http::Method| { + let archive = archive.clone(); + async move { + match method { + axum::http::Method::HEAD => axum::http::StatusCode::OK.into_response(), + axum::http::Method::GET => { + use axum::body::Body; + ( + [(axum::http::header::CONTENT_TYPE, "application/zstd")], + Body::from(archive.as_ref().clone()), + ) + .into_response() + } + _ => axum::http::StatusCode::METHOD_NOT_ALLOWED.into_response(), + } + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port()); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + + let opts = (*pool.connect_options()).clone(); + let lock_pool = no_reap_pool(&opts, 2).await; + let store = RepoStore::for_testing_with_tigris( + PathBuf::from("/tmp/gitlawb-snapshot-nonmut"), + lock_pool, + TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint), + ); + + let owner_did = "did:key:z6MkSnap"; + let (owner_slug, live_path) = store.local_path(owner_did, "snaprepo").unwrap(); + assert!( + !live_path.exists(), + "the live path must not exist before the snapshot" + ); + + let snap = store + .read_snapshot(owner_did, "snaprepo") + .await + .expect("snapshot reads the archive"); + let snap_path = snap.path().to_path_buf(); + assert_ne!( + snap_path, live_path, + "the snapshot must unpack into a temp dir, not the live path" + ); + assert!( + snap_path.starts_with(live_path.parent().unwrap()), + "the snapshot temp dir must live under the repo parent" + ); + assert!( + !live_path.exists(), + "the live path must remain untouched by a snapshot read" + ); + assert_eq!( + std::fs::read_to_string(snap_path.join("HEAD")).unwrap(), + "ref: refs/heads/main\n", + "the snapshot must contain the archive's content" + ); + drop(snap); + assert!( + !snap_path.exists(), + "dropping the snapshot must clean up its temp dir" + ); + let _ = owner_slug; + + server.abort(); + } + } diff --git a/crates/gitlawb-node/src/git/tigris.rs b/crates/gitlawb-node/src/git/tigris.rs index cba8867c..a154d614 100644 --- a/crates/gitlawb-node/src/git/tigris.rs +++ b/crates/gitlawb-node/src/git/tigris.rs @@ -122,8 +122,28 @@ impl TigrisClient { repo_name: &str, local_path: &Path, ) -> Result<()> { + self.download_to(owner_slug, repo_name, local_path, true) + .await + .map(|_| ()) + } + + /// Download a repo archive from Tigris and extract it, returning the + /// directory that was populated. + /// + /// `publish` controls whether the extract is swapped into `target` in place + /// (the live-path mutation used by writes; returns `target`) or unpacked + /// into a fresh temp directory under `target`'s parent (a non-mutating + /// snapshot read; returns the temp dir, which the caller owns and cleans + /// up). The snapshot form never touches the live repo path. + pub async fn download_to( + &self, + owner_slug: &str, + repo_name: &str, + target: &Path, + publish: bool, + ) -> Result { let key = Self::repo_key(owner_slug, repo_name); - debug!(key = %key, path = %local_path.display(), "downloading repo from tigris"); + debug!(key = %key, path = %target.display(), "downloading repo from tigris"); let resp = self .s3 @@ -141,17 +161,46 @@ impl TigrisClient { .context("reading tigris response body")? .into_bytes(); - // Extract tar.zst to local path - tokio::task::spawn_blocking({ - let local_path = local_path.to_path_buf(); - move || decompress_repo(&data, &local_path) + // Extract tar.zst to a directory. + let extracted = tokio::task::spawn_blocking({ + let target = target.to_path_buf(); + move || -> Result { + if publish { + decompress_repo(&data, &target)?; + return Ok(target); + } + // Non-mutating snapshot: unpack into a fresh temp dir under the + // target's parent. The live repo path is never touched. + let parent = target.parent().context("snapshot path has no parent")?; + std::fs::create_dir_all(parent).context("creating parent dir")?; + let file_name = target + .file_name() + .context("snapshot path has no file name")? + .to_string_lossy(); + let tmp_dir = parent.join(format!( + ".{file_name}.tmp-snapshot.{}", + uuid::Uuid::new_v4() + )); + std::fs::create_dir_all(&tmp_dir).context("creating temp extract dir")?; + let unpack = (|| -> Result<()> { + let decoder = zstd::stream::Decoder::new(&data[..])?; + let mut archive = tar::Archive::new(decoder); + archive.unpack(&tmp_dir).context("unpacking tar.zst")?; + Ok(()) + })(); + if let Err(e) = unpack { + let _ = std::fs::remove_dir_all(&tmp_dir); + return Err(e); + } + Ok(tmp_dir) + } }) .await .context("extract task panicked")? .context("extracting repo")?; - info!(key = %key, path = %local_path.display(), "downloaded repo from tigris"); - Ok(()) + info!(key = %key, path = %target.display(), "downloaded repo from tigris"); + Ok(extracted) } /// Delete a repo archive from Tigris. From 9f313b9bf78c3e84f5fee0803dc08b8d84215f0a Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sun, 9 Aug 2026 21:59:05 -0500 Subject: [PATCH 21/54] fix(node): bound every await in the lock-acquire loop by the deadline The remaining budget was checked before the pool checkout but bounded neither the checkout nor the pg_try_advisory_lock query that follows it. A checkout starting just under the deadline could wait out the pool's own acquire timeout, and a slow query could be accepted after the budget was spent, so the advertised wall-clock cap held only on the fast path. Both awaits now run under the remaining budget and shed as RepoBusy when it runs out. The probe's Drop closes its session, which cannot hold a lock it never confirmed taking, so the query-timeout arm is a plain shed. --- crates/gitlawb-node/src/git/repo_store.rs | 123 ++++++++++++++++++++-- 1 file changed, 114 insertions(+), 9 deletions(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 320d3479..33ec71dd 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -345,15 +345,22 @@ impl RepoStore { let deadline = std::time::Instant::now() + deadline_budget; let mut lock_conn = None; for attempt in 0..60 { - let Some(left) = deadline.checked_duration_since(std::time::Instant::now()) else { - break; + // The advertised cap is WALL CLOCK, so the remaining budget must bound + // every await in the loop, not just the sleep between attempts. A pool + // checkout or a slow advisory query that starts just before the deadline + // and lands after it would otherwise hold the write task past the budget + // it was promised, which is exactly what the deadline exists to prevent. + let left = match deadline.checked_duration_since(std::time::Instant::now()) { + Some(left) if !left.is_zero() => left, + _ => break, }; - if left.is_zero() { - break; - } - let conn = match self.lock_pool.acquire().await { - Ok(c) => c, - Err(e) => { + // Bound the pool checkout by the remaining budget. A checkout that + // would outlive the deadline is not worth starting: it either waits out + // the full DB acquire timeout and fails anyway, or lands a connection + // with no budget left to use it. + let conn = match tokio::time::timeout(left, self.lock_pool.acquire()).await { + Ok(Ok(c)) => c, + Ok(Err(e)) => { // Saturation is surfaced HERE, in the request path, and // deliberately not through /ready. Failing readiness on a full // pool would pull this node out of routing, taking its reads @@ -378,9 +385,51 @@ impl RepoStore { ); return Err(e).context("advisory-lock pool exhausted or unreachable"); } + Err(_) => { + // The pool checkout itself outlived the remaining budget. Same + // refusal as running out of attempts: the wall-clock cap is what + // is advertised, so a checkout that blows past it is contention + // the caller was promised would not happen. + warn!( + repo = %repo_name, + owner = %owner_slug, + waited_secs = deadline_budget.as_secs(), + "advisory-lock pool checkout exceeded the acquire deadline — shedding the write as busy" + ); + return Err(anyhow::Error::new(RepoBusy).context(format!( + "advisory-lock pool checkout exceeded the {}s deadline for {owner_slug}/{repo_name}", + deadline_budget.as_secs() + ))); + } + }; + // Bound the advisory query by the remaining budget too: a query that + // starts with budget left but answers after the deadline must not be + // accepted, or the cap is only as good as the fast path. + let left = match deadline.checked_duration_since(std::time::Instant::now()) { + Some(left) if !left.is_zero() => left, + _ => break, }; let mut probe = LockProbe::new(conn); - if probe.try_lock(lock_key).await? { + let acquired = match tokio::time::timeout(left, probe.try_lock(lock_key)).await { + Ok(Ok(acquired)) => acquired, + Ok(Err(e)) => return Err(e).context("trying advisory lock"), + Err(_) => { + // The query outlived the remaining budget. The probe's Drop + // closes its session, which cannot hold the lock it never + // confirmed taking, so this is a plain shed. + warn!( + repo = %repo_name, + owner = %owner_slug, + waited_secs = deadline_budget.as_secs(), + "advisory-lock query exceeded the acquire deadline — shedding the write as busy" + ); + return Err(anyhow::Error::new(RepoBusy).context(format!( + "advisory-lock query exceeded the {}s deadline for {owner_slug}/{repo_name}", + deadline_budget.as_secs() + ))); + } + }; + if acquired { lock_conn = probe.take_conn(); break; } @@ -3191,4 +3240,60 @@ mod tests { server.abort(); } + /// P2: the lock-acquire deadline must bound EVERY await in the retry loop, + /// not just the sleep between attempts. A pool checkout that would exceed + /// the deadline must shed as `RepoBusy` rather than wait out the pool's own + /// acquire timeout past the promised wall-clock cap. + /// + /// Observable: hold every lock-pool slot from an independent store, then + /// acquire with a short deadline. The pool checkout will not complete within + /// the deadline, so `acquire_write` must refuse as `RepoBusy` once the + /// deadline fires — not hang for the pool's 5s acquire timeout. + #[sqlx::test] + async fn pool_checkout_past_the_deadline_sheds_as_repo_busy(pool: PgPool) { + let opts = (*pool.connect_options()).clone(); + + // Exhaust every slot of the lock pool. The checkouts must come from the + // pool the store will use, not from independent connections: a separate + // `PgConnection::connect_with` consumes no slot, so the store's checkout + // would succeed immediately and the deadline would never be reached. + const N: u32 = 2; + let lock_pool = no_reap_pool(&opts, N).await; + let mut holders = Vec::new(); + for _ in 0..N { + holders.push(lock_pool.acquire().await.expect("hold a lock-pool slot")); + } + + // The store shares that exhausted pool (`PgPool` is a handle to one + // inner pool, so the clone is the same set of slots). + let store = + RepoStore::for_testing(PathBuf::from("/tmp/gitlawb-deadline"), lock_pool.clone()) + .with_lock_acquire_deadline(std::time::Duration::from_millis(400)); + + // The pool is exhausted, so the checkout cannot complete within the + // deadline; the deadline must fire and shed as RepoBusy rather than let + // the pool's own 5s acquire timeout run. + let started = std::time::Instant::now(); + let err = match store + .acquire_write("did:key:z6MkDeadline", "deadline-repo") + .await + { + Err(e) => e, + Ok(guard) => { + guard.release(false).await; + panic!("with the pool exhausted, the deadline must shed, not succeed"); + } + }; + assert!( + err.downcast_ref::().is_some(), + "a checkout past the deadline must shed as RepoBusy, got {err:#}" + ); + assert!( + started.elapsed() < std::time::Duration::from_secs(5), + "the refusal must come from the deadline, not the pool's own 5s acquire timeout" + ); + + // Release the holders so the test's pool can be torn down cleanly. + drop(holders); + } } From 60d70c7fc78a4c03510af1bc54f9e15f39ebf57e Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sun, 9 Aug 2026 22:00:18 -0500 Subject: [PATCH 22/54] test(node): add an S3 mock that really enforces conditional writes The fence work landing next is only as good as what the tests can observe, and a mock that answers 200 to every PUT would make the whole suite vacuous. This one holds the object and its ETag, refuses a mismatched If-Match and an If-None-Match "*" over an existing object with 412, and mints a fresh ETag per successful PUT so two byte-identical archives never share a token. Capture-then-replay is deliberate rather than parking a handler and hoping it resumes: when tokio drops an SDK future the client can tear the connection down and cancel the server task with it. Replaying what arrived models the arm that matters (body fully transmitted, commit decided later) with no timing in it. Six tests pin the semantics in both directions so a hollowed mock cannot hide. --- crates/gitlawb-node/src/git/repo_store.rs | 537 ++++++++++++++++++++++ 1 file changed, 537 insertions(+) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 33ec71dd..bee1b139 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -3296,4 +3296,541 @@ mod tests { // Release the holders so the test's pool can be torn down cleanly. drop(holders); } + + // ── conditional-semantics S3 mock (#279) ─────────────────────────────── + + /// A captured PUT: the body and the conditional headers as they arrived. + /// + /// Capture is deliberately separate from evaluation. When tokio drops an + /// SDK future the client can tear the TCP connection down and the server + /// side handler task is cancelled with it, so a parked handler that resumes + /// on its own is not something a test can depend on. Replaying what arrived + /// models the real S3 arm we care about (body fully transmitted, commit + /// decided later) with no timing in it. + #[derive(Clone, Debug)] + struct CapturedPut { + body: Vec, + if_match: Option, + if_none_match: Option, + } + + /// One PUT as the mock judged it, for tests that assert on attempt counts. + /// `status` is `None` while a PUT is parked: it arrived and was logged, but + /// no precondition has been evaluated for it yet. + #[derive(Clone, Debug, PartialEq)] + struct PutAttempt { + if_match: Option, + if_none_match: Option, + status: Option, + } + + #[derive(Default)] + struct MockState { + object: Option>, + etag: Option, + next_etag: u64, + puts: Vec, + /// Set by `park_next_put`, consumed by the next arriving PUT. + park_next_put: bool, + captured: Option, + } + + /// An in-process S3-compatible server with REAL conditional semantics. + /// + /// The fence tests downstream are only worth anything if a precondition can + /// actually fail here, so this helper carries its own semantics tests below. + struct S3Mock { + endpoint: String, + state: Arc>, + gate: Arc, + server: tokio::task::JoinHandle<()>, + } + + /// S3 quotes ETags. Compare unquoted so a value that round-tripped through + /// the SDK (which surfaces `e_tag()` with the quotes intact) matches what + /// the mock minted. + fn unquote_etag(raw: &str) -> &str { + raw.trim().trim_matches('"') + } + + /// The conditional evaluation, in one place so a live PUT and a replayed + /// one cannot drift apart. Returns the status, and on success the fresh + /// ETag. Preconditions are read against the state passed in, which is + /// always the state as of the CALL, never as of capture. + fn evaluate_put( + st: &mut MockState, + body: Vec, + if_match: Option<&str>, + if_none_match: Option<&str>, + ) -> (u16, Option) { + let refuse = |st: &mut MockState| { + st.puts.push(PutAttempt { + if_match: if_match.map(str::to_string), + if_none_match: if_none_match.map(str::to_string), + status: Some(412), + }); + (412u16, None) + }; + + if let Some(want) = if_match { + // An absent object matches nothing, so If-Match cannot pass. + match st.etag.as_deref() { + Some(have) if unquote_etag(have) == unquote_etag(want) => {} + _ => return refuse(st), + } + } + if if_none_match.map(str::trim) == Some("*") && st.object.is_some() { + return refuse(st); + } + + // A fresh ETag per successful PUT, from a counter rather than a content + // hash: two writers can publish byte-identical archives, and an ETag + // that repeated across them would let a fence pass on a generation it + // never observed. + st.next_etag += 1; + let etag = format!("\"mock-etag-{}\"", st.next_etag); + st.object = Some(body); + st.etag = Some(etag.clone()); + st.puts.push(PutAttempt { + if_match: if_match.map(str::to_string), + if_none_match: if_none_match.map(str::to_string), + status: Some(200), + }); + (200, Some(etag)) + } + + impl S3Mock { + async fn start() -> Self { + use axum::response::IntoResponse; + + let state = Arc::new(std::sync::Mutex::new(MockState::default())); + let gate = Arc::new(tokio::sync::Notify::new()); + + let app = axum::Router::new().route( + "/{*key}", + axum::routing::any({ + let state = state.clone(); + let gate = gate.clone(); + move |method: axum::http::Method, + headers: axum::http::HeaderMap, + body: axum::body::Bytes| { + let state = state.clone(); + let gate = gate.clone(); + async move { + let header = |name: &str| { + headers + .get(name) + .and_then(|v| v.to_str().ok()) + .map(str::to_string) + }; + match method { + axum::http::Method::PUT => { + let if_match = header("if-match"); + let if_none_match = header("if-none-match"); + + // A parked PUT records what arrived and then + // waits. The client will usually be gone by + // the time the gate opens, which is exactly + // why the deterministic arm is the replay. + let parked = { + let mut st = state.lock().unwrap(); + if st.park_next_put { + st.park_next_put = false; + st.puts.push(PutAttempt { + if_match: if_match.clone(), + if_none_match: if_none_match.clone(), + status: None, + }); + st.captured = Some(CapturedPut { + body: body.to_vec(), + if_match: if_match.clone(), + if_none_match: if_none_match.clone(), + }); + true + } else { + false + } + }; + if parked { + gate.notified().await; + return axum::http::StatusCode::OK.into_response(); + } + + let (status, etag) = { + let mut st = state.lock().unwrap(); + evaluate_put( + &mut st, + body.to_vec(), + if_match.as_deref(), + if_none_match.as_deref(), + ) + }; + match etag { + Some(etag) => ( + axum::http::StatusCode::OK, + [(axum::http::header::ETAG, etag)], + ) + .into_response(), + None => axum::http::StatusCode::from_u16(status) + .unwrap() + .into_response(), + } + } + axum::http::Method::HEAD | axum::http::Method::GET => { + let st = state.lock().unwrap(); + match (st.object.clone(), st.etag.clone()) { + (Some(bytes), Some(etag)) => ( + axum::http::StatusCode::OK, + [(axum::http::header::ETAG, etag)], + axum::body::Body::from(bytes), + ) + .into_response(), + _ => axum::http::StatusCode::NOT_FOUND.into_response(), + } + } + _ => axum::http::StatusCode::METHOD_NOT_ALLOWED.into_response(), + } + } + } + }), + ); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port()); + let server = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + + Self { + endpoint, + state, + gate, + server, + } + } + + fn endpoint(&self) -> &str { + &self.endpoint + } + + fn current_etag(&self) -> Option { + self.state.lock().unwrap().etag.clone() + } + + fn object(&self) -> Option> { + self.state.lock().unwrap().object.clone() + } + + fn put_attempts(&self) -> Vec { + self.state.lock().unwrap().puts.clone() + } + + /// Park the next arriving PUT so the caller's transfer bound elapses + /// with the request in flight (the abandoned-writer arm). + fn park_next_put(&self) { + self.state.lock().unwrap().park_next_put = true; + } + + /// Let a parked handler go. Only the socket-level arm needs this; the + /// deterministic assertion is `replay_captured`. + fn open_gate(&self) { + self.gate.notify_waiters(); + } + + fn captured_put(&self) -> Option { + self.state.lock().unwrap().captured.clone() + } + + /// Re-run the captured PUT through the SAME evaluation the handler uses, + /// against the state as it is NOW. + fn replay_captured(&self) -> u16 { + let mut st = self.state.lock().unwrap(); + let captured = st.captured.clone().expect("a PUT was captured"); + evaluate_put( + &mut st, + captured.body, + captured.if_match.as_deref(), + captured.if_none_match.as_deref(), + ) + .0 + } + + fn shutdown(&self) { + self.server.abort(); + } + } + + /// An SDK client aimed at the mock. Built here rather than through + /// `TigrisClient` because these tests exercise raw conditional PUTs, which + /// the storage client does not expose. + fn mock_s3_client(endpoint: &str) -> aws_sdk_s3::Client { + use aws_sdk_s3::config::{retry::RetryConfig, Credentials, Region}; + + let config = aws_sdk_s3::config::Config::builder() + .endpoint_url(endpoint) + .credentials_provider(Credentials::new("test", "test", None, None, "test")) + .region(Region::new("auto")) + .retry_config(RetryConfig::disabled()) + .behavior_version_latest() + .build(); + aws_sdk_s3::Client::from_conf(config) + } + + /// PUT through the SDK, returning either the fresh ETag or the HTTP status + /// the mock refused with. + async fn mock_put( + client: &aws_sdk_s3::Client, + body: &[u8], + if_match: Option<&str>, + if_none_match: Option<&str>, + ) -> Result { + let mut req = client + .put_object() + .bucket("test-bucket") + .key("repos/v1/owner/repo.tar.zst") + .body(aws_sdk_s3::primitives::ByteStream::from(body.to_vec())); + if let Some(v) = if_match { + req = req.if_match(v); + } + if let Some(v) = if_none_match { + req = req.if_none_match(v); + } + match req.send().await { + Ok(out) => Ok(out + .e_tag() + .expect("a successful PUT returns an ETag") + .to_string()), + Err(e) => Err(e + .raw_response() + .map(|r| r.status().as_u16()) + .unwrap_or_else(|| panic!("expected an HTTP response from the mock, got {e:?}"))), + } + } + + /// HEAD through the SDK, reported the way `TigrisClient::exists` reports it: + /// `Ok(false)` for a not-found, `Ok(true)` for a hit whose ETag is present. + async fn mock_head(client: &aws_sdk_s3::Client) -> Result { + match client + .head_object() + .bucket("test-bucket") + .key("repos/v1/owner/repo.tar.zst") + .send() + .await + { + Ok(out) => { + out.e_tag().ok_or("a HEAD hit must carry an ETag")?; + Ok(true) + } + Err(e) if e.as_service_error().is_some_and(|e| e.is_not_found()) => Ok(false), + Err(e) => Err(format!("unexpected HEAD failure: {e}")), + } + } + + /// 1. A stale If-Match must be refused, and the refusal must not write. + #[tokio::test] + async fn mock_refuses_a_wrong_if_match_and_leaves_the_object_unchanged() { + let mock = S3Mock::start().await; + let client = mock_s3_client(mock.endpoint()); + + let etag = mock_put(&client, b"first", None, None) + .await + .expect("the seeding PUT succeeds"); + + let status = mock_put(&client, b"second", Some("\"not-the-current-etag\""), None) + .await + .expect_err("a stale If-Match must be refused"); + assert_eq!(status, 412, "a stale If-Match must answer 412"); + assert_eq!( + mock.object().as_deref(), + Some(b"first".as_slice()), + "a refused PUT must leave the stored object unchanged" + ); + assert_eq!( + mock.current_etag().as_deref().map(unquote_etag), + Some(unquote_etag(&etag)), + "a refused PUT must leave the ETag unchanged" + ); + + mock.shutdown(); + } + + /// 2. The matching If-Match is the write that must go through. + #[tokio::test] + async fn mock_accepts_a_matching_if_match_and_rotates_the_etag() { + let mock = S3Mock::start().await; + let client = mock_s3_client(mock.endpoint()); + + let first = mock_put(&client, b"first", None, None).await.expect("seed"); + let second = mock_put(&client, b"second", Some(&first), None) + .await + .expect("a matching If-Match must succeed"); + + assert_ne!( + unquote_etag(&first), + unquote_etag(&second), + "a successful conditional PUT must mint a fresh ETag" + ); + assert_eq!( + mock.object().as_deref(), + Some(b"second".as_slice()), + "the accepted body must be what is stored" + ); + assert_eq!( + mock.current_etag().as_deref().map(unquote_etag), + Some(unquote_etag(&second)), + "HEAD/GET must report the ETag the PUT returned" + ); + + mock.shutdown(); + } + + /// 3. If-None-Match `*` is the create-only fence, so an existing object + /// must refuse it. + #[tokio::test] + async fn mock_refuses_if_none_match_star_against_an_existing_object() { + let mock = S3Mock::start().await; + let client = mock_s3_client(mock.endpoint()); + + mock_put(&client, b"first", None, None).await.expect("seed"); + let status = mock_put(&client, b"second", None, Some("*")) + .await + .expect_err("create-only against an existing object must be refused"); + + assert_eq!(status, 412, "If-None-Match * on an existing object is 412"); + assert_eq!( + mock.object().as_deref(), + Some(b"first".as_slice()), + "the refused create-only PUT must not overwrite" + ); + + mock.shutdown(); + } + + /// 4. The same fence must ADMIT the first writer, or the fresh-repo path + /// could never publish. + #[tokio::test] + async fn mock_accepts_if_none_match_star_against_an_empty_store() { + let mock = S3Mock::start().await; + let client = mock_s3_client(mock.endpoint()); + + // HEAD both ways, because `exists()` reads a not-found as "fresh repo" + // and any other status as a hard refusal. A mock that answered 200 on + // an empty store would send every fresh-repo test down the wrong arm. + assert!( + !mock_head(&client).await.expect("HEAD on an empty store"), + "an absent object must HEAD 404" + ); + + let etag = mock_put(&client, b"first", None, Some("*")) + .await + .expect("create-only against an empty store must succeed"); + assert_eq!(mock.object().as_deref(), Some(b"first".as_slice())); + assert_eq!( + mock.current_etag().as_deref().map(unquote_etag), + Some(unquote_etag(&etag)) + ); + assert!( + mock_head(&client).await.expect("HEAD after the create"), + "a stored object must HEAD 200 with the ETag the PUT returned" + ); + + mock.shutdown(); + } + + /// 5. Identical bytes must still produce a new ETag. Without this, an + /// If-Match fence would pass on a generation it never observed. + #[tokio::test] + async fn mock_mints_a_distinct_etag_per_successful_put() { + let mock = S3Mock::start().await; + let client = mock_s3_client(mock.endpoint()); + + let first = mock_put(&client, b"same", None, None).await.expect("first"); + let second = mock_put(&client, b"same", Some(&first), None) + .await + .expect("second"); + + assert_ne!( + unquote_etag(&first), + unquote_etag(&second), + "successive successful PUTs of identical bytes must still differ in ETag" + ); + + mock.shutdown(); + } + + /// 6. The whole point of capture-and-replay: the commit is judged when it + /// is replayed, not when the bytes arrived. A capture that was valid on + /// arrival must lose to a write that landed in between. + #[tokio::test] + async fn mock_judges_a_replayed_put_against_the_state_at_replay_time() { + let mock = S3Mock::start().await; + let client = mock_s3_client(mock.endpoint()); + + let first = mock_put(&client, b"first", None, None).await.expect("seed"); + + // Park the abandoned writer's PUT. Its If-Match is valid at ARRIVAL. + mock.park_next_put(); + let parked = tokio::time::timeout( + std::time::Duration::from_millis(300), + mock_put(&client, b"abandoned", Some(&first), None), + ) + .await; + assert!( + parked.is_err(), + "the parked PUT must still be in flight when the caller's bound elapses" + ); + let captured = mock + .captured_put() + .expect("the parked PUT must be captured at arrival"); + assert_eq!(captured.body, b"abandoned".to_vec()); + assert_eq!( + captured.if_match.as_deref().map(unquote_etag), + Some(unquote_etag(&first)), + "the capture must record the conditional headers as they arrived" + ); + assert_eq!(captured.if_none_match, None); + + // A successor commits while the capture sits parked. + let second = mock_put(&client, b"successor", Some(&first), None) + .await + .expect("the successor's PUT is the one that lands"); + + // Replaying now must be judged against the successor's state. + assert_eq!( + mock.replay_captured(), + 412, + "a replayed PUT must be evaluated against the state at replay time, \ + not the state it was captured against" + ); + assert_eq!( + mock.object().as_deref(), + Some(b"successor".as_slice()), + "the refused replay must not clobber the successor's object" + ); + assert_eq!( + mock.current_etag().as_deref().map(unquote_etag), + Some(unquote_etag(&second)) + ); + // Seed, parked, successor, replay. The log is what later tests assert + // attempt counts against, so it is checked here rather than trusted. + let attempts = mock.put_attempts(); + assert_eq!( + attempts.len(), + 4, + "every PUT attempt must be logged, got {attempts:?}" + ); + assert_eq!( + attempts.iter().map(|a| a.status).collect::>(), + vec![Some(200), None, Some(200), Some(412)], + "the parked attempt is logged undecided; the replay is the 412" + ); + assert_eq!( + attempts[1].if_match.as_deref().map(unquote_etag), + Some(unquote_etag(&first)), + "the parked attempt must be logged with the headers it arrived with" + ); + + mock.open_gate(); + mock.shutdown(); + } } From 31f90c4e05d4c265f71dd62c4afa86616975898b Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sun, 9 Aug 2026 22:01:23 -0500 Subject: [PATCH 23/54] fix(node): stop pretending a held lock fences a late upload The timeout arm claimed that keeping the advisory lock protected a successor from an abandoned PUT. It does not. release takes mut self, so the guard drops the moment it returns and Drop closes the session; measured on this branch, a successor took the same repo's lock 5ms after release returned while the PUT was still in flight. The comment and warn now say what is actually true: the outcome is unknowable, the PUT may still land, the lock releases normally, and a conditional upload is what keeps a late publish from overwriting a successor's archive. Two tests replace the claim. Session disposition is the observable that separates the two shapes, so the unlock is pinned to run and be confirmed on the guard's own session with the connection returned to the pool, checked by backend pid. Successor admission is pinned too, but noted as not what proves the point, since the lock frees within milliseconds either way. --- crates/gitlawb-node/src/git/repo_store.rs | 136 +++++++++++++++++++++- 1 file changed, 131 insertions(+), 5 deletions(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index bee1b139..0d92cfda 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -1019,11 +1019,21 @@ impl RepoWriteGuard { } None => { // Timed out is UNKNOWABLE, not failed: the PUT may well - // have landed, so there is deliberately no compensating - // action. The lock releases either way, so the repo is not - // wedged behind a stalled transfer. The tradeoff is a narrow - // last-writer-wins window if the slow PUT lands after - // another writer takes the lock. + // still land after this returns, so there is deliberately + // no compensating action here. The lock is released + // normally regardless. Holding it would fence nothing, + // because `release` takes `mut self`: the guard drops the + // moment this function returns and `Drop` closes the + // session, so the lock would free within milliseconds + // either way. What actually protects a successor from a + // late publish is the conditional PUT on the upload, not + // the lifetime of this lock. + warn!( + repo = %self.repo_name, + "release upload exceeded its bound; the PUT may still land, so the \ + outcome is unknowable and the conditional upload is what keeps a \ + late publish from overwriting a successor's archive" + ); } } } @@ -3240,6 +3250,122 @@ mod tests { server.abort(); } + /// Build a store whose release-side upload lands on `mock` and gives up + /// after 200ms, so a parked PUT reliably exceeds the bound. One lock-pool + /// connection on purpose: with a single slot the backend pid is a direct + /// observable for whether `release` pooled its session or closed it. + async fn timed_out_upload_store( + mock: &S3Mock, + opts: &sqlx::postgres::PgConnectOptions, + repos_dir: &str, + ) -> RepoStore { + RepoStore::new( + PathBuf::from(repos_dir), + Some(TigrisClient::for_testing_with_endpoint( + "test-bucket", + mock.endpoint(), + )), + no_reap_pool(opts, 1).await, + std::time::Duration::from_millis(200), + ) + } + + /// Seed the minimum bare-repo shape so the upload has something to archive. + fn seed_bare_repo(path: &Path) { + std::fs::create_dir_all(path.join("objects/info")).unwrap(); + std::fs::create_dir_all(path.join("refs/heads")).unwrap(); + std::fs::write(path.join("HEAD"), "ref: refs/heads/main\n").unwrap(); + } + + /// A timed-out release upload must still unlock on its OWN session and hand + /// that session back to the pool. The timeout says nothing about the lock: + /// holding it cannot fence a late PUT (`release` takes `mut self`, so the + /// guard drops and `Drop` frees the session the moment `release` returns), + /// and what actually protects a successor is the conditional PUT. + /// + /// Observable: the backend pid. On a one-connection pool a session that was + /// closed forces the next checkout onto a fresh backend, while a confirmed + /// unlock returns the same one. So an equal pid is the proof that the unlock + /// ran, returned true, and the connection was pooled rather than torn down. + #[sqlx::test] + async fn timed_out_release_upload_unlocks_on_its_own_session(pool: PgPool) { + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let store = timed_out_upload_store(&mock, &opts, "/tmp/gitlawb-u4-timeout-session").await; + + let mut guard = store + .acquire_write("did:key:z6MkU4TimeoutSess", "timedrepo") + .await + .expect("acquire"); + seed_bare_repo(&guard.local_path); + let pid_before = guard.backend_pid_for_test().await; + + // Park the upload so it is still in flight when the 200ms bound fires. + mock.park_next_put(); + guard.release(true).await; + assert_eq!( + mock.put_attempts().len(), + 1, + "the release upload must have reached the mock and parked, got {:?}", + mock.put_attempts() + ); + + let pid_after = { + let mut c = store.lock_pool.acquire().await.unwrap(); + let pid: (i32,) = sqlx::query_as("SELECT pg_backend_pid()") + .fetch_one(&mut *c) + .await + .unwrap(); + pid.0 + }; + assert_eq!( + pid_before, pid_after, + "a timed-out upload must not change the unlock decision: the guard must \ + unlock on its own session and return that connection to the pool, so the \ + next checkout lands on the same backend" + ); + + mock.open_gate(); + mock.shutdown(); + } + + /// Admission after the same timed-out upload: a successor must be let in + /// promptly. Useful as a property, but it is NOT what pins the removal of + /// the skip-unlock branch, because the lock frees within milliseconds under + /// either shape (the session closes as soon as `release` returns). + #[sqlx::test] + async fn successor_is_admitted_promptly_after_a_timed_out_release(pool: PgPool) { + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let store = timed_out_upload_store(&mock, &opts, "/tmp/gitlawb-u4-timeout-admit") + .await + .with_lock_acquire_deadline(std::time::Duration::from_secs(10)); + + let guard = store + .acquire_write("did:key:z6MkU4TimeoutAdmit", "timedrepo") + .await + .expect("acquire"); + seed_bare_repo(&guard.local_path); + + mock.park_next_put(); + guard.release(true).await; + + let started = std::time::Instant::now(); + let successor = store + .acquire_write("did:key:z6MkU4TimeoutAdmit", "timedrepo") + .await + .expect("a successor must be admitted after a timed-out release"); + assert!( + started.elapsed() < std::time::Duration::from_secs(5), + "the successor waited {}ms; a timed-out upload must not park the next writer", + started.elapsed().as_millis() + ); + successor.release(false).await; + + mock.open_gate(); + mock.shutdown(); + } + /// P2: the lock-acquire deadline must bound EVERY await in the retry loop, /// not just the sleep between attempts. A pool checkout that would exceed /// the deadline must shed as `RepoBusy` rather than wait out the pool's own From 3eabf794a5bd30b4fe143ec1b92aef2ee422f3a4 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sun, 9 Aug 2026 23:05:55 -0500 Subject: [PATCH 24/54] feat(node): let the storage client express and detect a conditional write upload takes an UploadPrecondition (IfMatch, IfAbsent, Unconditional) rather than an Option, so the absent case and the deliberate no-fence case are distinguishable at the type level and a caller cannot lose the fence by passing None. head_etag reads the current ETag alongside exists, which keeps exists and its other callers untouched. A failed precondition has to be classified off the raw HTTP status: PutObjectError models no PreconditionFailed variant, so a 412 arrives as Unhandled with nothing useful on it. 412 is always a lost precondition and 409 is one under IfAbsent. 404 deliberately is not: no archive delete exists on this line, so a 404 on a conditional PUT means a wrong bucket or endpoint, and reporting that as retryable would send clients into a loop against a permanent fault. The three background uploads outside the write guard now publish with IfAbsent. They fire only where the archive is expected absent, and leaving them unconditional would defeat the fence from the side: init uploads an empty bare repo, so a push landing just before it could have its archive replaced by that empty one. A refusal there means someone else already published the key, which is logged as the correct outcome rather than a failure. --- crates/gitlawb-node/src/git/repo_store.rs | 363 +++++++++++++++++++++- crates/gitlawb-node/src/git/tigris.rs | 126 +++++++- 2 files changed, 472 insertions(+), 17 deletions(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 0d92cfda..66f9659e 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -20,7 +20,7 @@ use tokio::sync::Mutex; use tracing::{debug, info, warn}; use super::store; -use super::tigris::TigrisClient; +use super::tigris::{TigrisClient, UploadError, UploadPrecondition}; /// Centralized repo storage: local disk cache + optional Tigris backend. #[derive(Clone)] @@ -128,11 +128,34 @@ impl RepoStore { } Ok(false) => { info!(repo = %name, "migrating local repo to tigris"); - if let Err(e) = tigris.upload(&slug, &name, &path).await { - warn!(repo = %name, err = %e, "lazy migration to tigris failed"); - return; + // Create-only. This backfill was decided on a + // negative existence check that is already + // stale, so a refusal means someone else + // published this key in between and dropping + // our bytes is the correct outcome. An + // unconditional PUT here would overwrite their + // archive, which is the exact bug this fence + // exists to close. + match tigris + .upload(&slug, &name, &path, UploadPrecondition::IfAbsent) + .await + { + Ok(()) => { + info!(repo = %name, "lazy migration to tigris complete"); + } + // Logged apart from the warn arm below so a + // refusal, which is the fence working, does + // not read as a storage failure. The key is + // populated either way, so this still + // counts as migrated. + Err(UploadError::PreconditionLost { status }) => { + info!(repo = %name, status, "lazy migration dropped: another writer already published this repo"); + } + Err(e) => { + warn!(repo = %name, err = %e, "lazy migration to tigris failed"); + return; + } } - info!(repo = %name, "lazy migration to tigris complete"); } Err(e) => { warn!(repo = %name, err = %e, "tigris existence check failed"); @@ -588,8 +611,25 @@ impl RepoStore { let repo_name = repo_name.to_string(); let path = local_path.clone(); tokio::spawn(async move { - if let Err(e) = tigris.upload(&owner_slug, &repo_name, &path).await { - warn!(repo = %repo_name, err = %e, "failed to upload new repo to tigris"); + // Create-only, and load-bearing: this uploads a freshly + // initialized EMPTY repo, so a user who pushes immediately + // after creating one would have their archive replaced by this + // background PUT if it were unconditional. A refusal means + // someone else already published this key and dropping our + // bytes is the correct outcome. + match tigris + .upload(&owner_slug, &repo_name, &path, UploadPrecondition::IfAbsent) + .await + { + Ok(()) => {} + // Distinct from the warn arm: the fence refusing is the + // design working, not a storage failure. + Err(UploadError::PreconditionLost { status }) => { + info!(repo = %repo_name, status, "dropped the empty-repo upload: another writer already published this repo"); + } + Err(e) => { + warn!(repo = %repo_name, err = %e, "failed to upload new repo to tigris"); + } } }); } @@ -608,8 +648,29 @@ impl RepoStore { return; } }; - if let Err(e) = tigris.upload(&owner_slug, repo_name, &local_path).await { - warn!(repo = %repo_name, err = %e, "failed to upload repo to tigris after write"); + // Create-only. The sole caller is fork creation, which rejects a + // name conflict in the database before it clones anything, so the + // key is expected absent here (and archive keys are never deleted: + // `delete` has no callers). A refusal therefore means someone else + // already published this key, and dropping our bytes is correct. + match tigris + .upload( + &owner_slug, + repo_name, + &local_path, + UploadPrecondition::IfAbsent, + ) + .await + { + Ok(()) => {} + // Kept apart from the warn arm so the fence working does not + // read as a storage failure. + Err(UploadError::PreconditionLost { status }) => { + info!(repo = %repo_name, status, "dropped the post-write upload: another writer already published this repo"); + } + Err(e) => { + warn!(repo = %repo_name, err = %e, "failed to upload repo to tigris after write"); + } } } } @@ -1009,7 +1070,15 @@ impl RepoWriteGuard { "release-upload", &self.repo_name, self.lock_held_transfer_timeout, - tigris.upload(&self.owner_slug, &self.repo_name, &self.local_path), + // Unconditional for now purely so the tree compiles. This + // is THE fenced call site: the sibling unit replaces this + // with the observed-ETag precondition. + tigris.upload( + &self.owner_slug, + &self.repo_name, + &self.local_path, + UploadPrecondition::Unconditional, + ), ) .await { @@ -3959,4 +4028,278 @@ mod tests { mock.open_gate(); mock.shutdown(); } + + // ── conditional upload through TigrisClient (#279) ───────────────────── + + /// A tiny directory for `upload` to compress. What is inside does not + /// matter to a precondition test, only that a PUT carrying a body happens. + fn payload_dir(marker: &str) -> TempDir { + let dir = TempDir::new().unwrap(); + std::fs::write(dir.path().join("HEAD"), marker.as_bytes()).unwrap(); + dir + } + + /// A router that answers every request with one fixed status. This is NOT a + /// second semantics mock: it exists only to pin how a status the real mock + /// never produces (409, 404, 500) is classified. + async fn start_fixed_status_stub(status: u16) -> (String, tokio::task::JoinHandle<()>) { + let app = axum::Router::new().route( + "/{*key}", + axum::routing::any( + move || async move { axum::http::StatusCode::from_u16(status).unwrap() }, + ), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port()); + let server = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + (endpoint, server) + } + + /// The one place the header itself is asserted: a matching If-Match must + /// succeed AND must actually have travelled as an If-Match header. The + /// store-level tests deliberately assert behavior rather than headers, so + /// if this assertion is not here, nothing pins the wire format. + #[tokio::test] + async fn upload_if_match_with_the_current_etag_publishes_and_sends_the_header() { + let mock = S3Mock::start().await; + let client = TigrisClient::for_testing_with_endpoint("test-bucket", mock.endpoint()); + let seeded = mock_put(&mock_s3_client(mock.endpoint()), b"seed", None, None) + .await + .expect("seeding PUT"); + + let dir = payload_dir("winner"); + client + .upload( + "owner", + "repo", + dir.path(), + UploadPrecondition::IfMatch(seeded.clone()), + ) + .await + .expect("a matching If-Match must publish"); + + let last = mock + .put_attempts() + .last() + .cloned() + .expect("the upload must reach the mock"); + assert_eq!( + last.if_match.as_deref().map(unquote_etag), + Some(unquote_etag(&seeded)), + "the upload must carry the ETag it was fenced on as If-Match" + ); + assert_eq!(last.if_none_match, None); + assert_eq!(last.status, Some(200)); + + mock.shutdown(); + } + + #[tokio::test] + async fn upload_if_match_with_a_stale_etag_is_precondition_lost() { + let mock = S3Mock::start().await; + let client = TigrisClient::for_testing_with_endpoint("test-bucket", mock.endpoint()); + mock_put(&mock_s3_client(mock.endpoint()), b"seed", None, None) + .await + .expect("seeding PUT"); + + let dir = payload_dir("loser"); + let err = client + .upload( + "owner", + "repo", + dir.path(), + UploadPrecondition::IfMatch("\"stale\"".to_string()), + ) + .await + .expect_err("a stale If-Match must be refused"); + assert!( + matches!(err, UploadError::PreconditionLost { status: 412 }), + "a stale If-Match must classify as a lost precondition, got {err:?}" + ); + assert_eq!( + mock.object().as_deref(), + Some(b"seed".as_slice()), + "the refused upload must not have written" + ); + + mock.shutdown(); + } + + #[tokio::test] + async fn upload_if_absent_into_an_empty_store_publishes() { + let mock = S3Mock::start().await; + let client = TigrisClient::for_testing_with_endpoint("test-bucket", mock.endpoint()); + + let dir = payload_dir("first"); + client + .upload("owner", "repo", dir.path(), UploadPrecondition::IfAbsent) + .await + .expect("create-only into an empty store must publish"); + + let last = mock.put_attempts().last().cloned().expect("one attempt"); + assert_eq!(last.if_none_match.as_deref(), Some("*")); + assert_eq!(last.if_match, None); + assert!(mock.object().is_some(), "the create must have stored bytes"); + + mock.shutdown(); + } + + #[tokio::test] + async fn upload_if_absent_over_an_existing_object_is_precondition_lost() { + let mock = S3Mock::start().await; + let client = TigrisClient::for_testing_with_endpoint("test-bucket", mock.endpoint()); + mock_put(&mock_s3_client(mock.endpoint()), b"seed", None, None) + .await + .expect("seeding PUT"); + + let dir = payload_dir("late-backfill"); + let err = client + .upload("owner", "repo", dir.path(), UploadPrecondition::IfAbsent) + .await + .expect_err("create-only over an existing object must be refused"); + assert!( + matches!(err, UploadError::PreconditionLost { status: 412 }), + "got {err:?}" + ); + assert_eq!( + mock.object().as_deref(), + Some(b"seed".as_slice()), + "a refused backfill must not clobber what is already published" + ); + + mock.shutdown(); + } + + #[tokio::test] + async fn upload_unconditional_overwrites_regardless() { + let mock = S3Mock::start().await; + let client = TigrisClient::for_testing_with_endpoint("test-bucket", mock.endpoint()); + mock_put(&mock_s3_client(mock.endpoint()), b"seed", None, None) + .await + .expect("seeding PUT"); + + let dir = payload_dir("overwrite"); + client + .upload( + "owner", + "repo", + dir.path(), + UploadPrecondition::Unconditional, + ) + .await + .expect("an unconditional upload must succeed regardless of state"); + + let last = mock.put_attempts().last().cloned().expect("one attempt"); + assert_eq!(last.if_match, None, "no precondition may be sent"); + assert_eq!(last.if_none_match, None); + assert_ne!( + mock.object().as_deref(), + Some(b"seed".as_slice()), + "the unconditional upload must have replaced the seed" + ); + + mock.shutdown(); + } + + /// Tigris answers a create-only conflict with 409 rather than 412, so that + /// status has to classify as a lost precondition too, but ONLY when the + /// request was create-only. + #[tokio::test] + async fn upload_classifies_409_under_if_absent_as_precondition_lost() { + let (endpoint, server) = start_fixed_status_stub(409).await; + let client = TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint); + let dir = payload_dir("conflict"); + + let err = client + .upload("owner", "repo", dir.path(), UploadPrecondition::IfAbsent) + .await + .expect_err("409 must be an error"); + assert!( + matches!(err, UploadError::PreconditionLost { status: 409 }), + "409 under IfAbsent is a lost precondition, got {err:?}" + ); + + server.abort(); + } + + /// MUST-NOT. A 404 is permanent (no such bucket, a misrouted endpoint), so + /// reporting it as a lost precondition would tell a client to retry + /// something that can never succeed. `delete` has no callers, so a racing + /// delete cannot produce this. + #[tokio::test] + async fn upload_classifies_404_as_other_under_either_precondition() { + let (endpoint, server) = start_fixed_status_stub(404).await; + let client = TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint); + + for precondition in [ + UploadPrecondition::IfAbsent, + UploadPrecondition::IfMatch("\"whatever\"".to_string()), + ] { + let dir = payload_dir("gone"); + let err = client + .upload("owner", "repo", dir.path(), precondition.clone()) + .await + .expect_err("404 must be an error"); + assert!( + matches!(err, UploadError::Other(_)), + "404 under {precondition:?} must NOT be a lost precondition, got {err:?}" + ); + } + + server.abort(); + } + + #[tokio::test] + async fn upload_classifies_500_as_other() { + let (endpoint, server) = start_fixed_status_stub(500).await; + let client = TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint); + + for precondition in [ + UploadPrecondition::IfAbsent, + UploadPrecondition::IfMatch("\"whatever\"".to_string()), + UploadPrecondition::Unconditional, + ] { + let dir = payload_dir("boom"); + let err = client + .upload("owner", "repo", dir.path(), precondition.clone()) + .await + .expect_err("500 must be an error"); + assert!( + matches!(err, UploadError::Other(_)), + "500 under {precondition:?} must be Other, got {err:?}" + ); + } + + server.abort(); + } + + #[tokio::test] + async fn head_etag_reports_the_current_etag_and_none_when_absent() { + let mock = S3Mock::start().await; + let client = TigrisClient::for_testing_with_endpoint("test-bucket", mock.endpoint()); + + assert_eq!( + client.head_etag("owner", "repo").await.expect("HEAD"), + None, + "an absent object must read as None, not an error" + ); + + let seeded = mock_put(&mock_s3_client(mock.endpoint()), b"seed", None, None) + .await + .expect("seeding PUT"); + let got = client + .head_etag("owner", "repo") + .await + .expect("HEAD") + .expect("a present object must report an ETag"); + assert_eq!( + unquote_etag(&got), + unquote_etag(&seeded), + "head_etag must report the ETag the last successful PUT minted" + ); + + mock.shutdown(); + } } diff --git a/crates/gitlawb-node/src/git/tigris.rs b/crates/gitlawb-node/src/git/tigris.rs index a154d614..e6a2f423 100644 --- a/crates/gitlawb-node/src/git/tigris.rs +++ b/crates/gitlawb-node/src/git/tigris.rs @@ -8,9 +8,42 @@ use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex, OnceLock}; use anyhow::{Context, Result}; +use aws_sdk_s3::error::SdkError; use aws_sdk_s3::Client as S3Client; use tracing::{debug, info}; +/// The precondition an upload is fenced on. +/// +/// Object storage is the only place a fence can hold. Dropping the future of an +/// in-flight PUT does not cancel the request the server is already processing, +/// so no amount of local locking stops an abandoned writer's bytes from landing +/// after a successor has published. A conditional PUT the store itself refuses +/// is what actually stops it. +#[derive(Clone, Debug)] +pub enum UploadPrecondition { + /// Publish only if the stored object is still the generation we observed. + /// + /// Only tests construct this so far. The write guard's release path is the + /// production caller, and it is wired up in a follow-up change. + #[allow(dead_code)] + IfMatch(String), + /// Publish only if nothing is stored under the key yet. + IfAbsent, + /// No fence. Last writer wins. + Unconditional, +} + +/// Why an upload failed, split so a caller can tell "someone else already +/// published this key" (expected, and dropping our bytes is the correct +/// outcome) from a real storage failure. +#[derive(Debug, thiserror::Error)] +pub enum UploadError { + #[error("upload precondition lost (HTTP {status})")] + PreconditionLost { status: u16 }, + #[error(transparent)] + Other(#[from] anyhow::Error), +} + /// Wrapper around the S3 client with the configured bucket. #[derive(Clone)] pub struct TigrisClient { @@ -85,8 +118,51 @@ impl TigrisClient { } } - /// Upload a local bare repo directory to Tigris as a tar.zst archive. - pub async fn upload(&self, owner_slug: &str, repo_name: &str, local_path: &Path) -> Result<()> { + /// Read the ETag of a repo archive, or `None` when nothing is stored under + /// the key. The ETag identifies the generation a later conditional upload + /// can fence itself on. + /// + /// Separate from `exists` rather than folded into it: `exists` has callers + /// that only want the boolean, and widening its return type would churn + /// every one of them for no benefit. + /// + /// Only tests call this so far; the write guard reads the ETag here before + /// it publishes, and that wiring is a follow-up change. + #[allow(dead_code)] + pub async fn head_etag(&self, owner_slug: &str, repo_name: &str) -> Result> { + let key = Self::repo_key(owner_slug, repo_name); + match self + .s3 + .head_object() + .bucket(&self.bucket) + .key(&key) + .send() + .await + { + Ok(out) => Ok(Some( + out.e_tag() + .context(format!("tigris HEAD {key}: hit carried no ETag"))? + .to_string(), + )), + Err(e) => { + if e.as_service_error().is_some_and(|e| e.is_not_found()) { + Ok(None) + } else { + Err(anyhow::anyhow!("tigris HEAD {key}: {e}")) + } + } + } + } + + /// Upload a local bare repo directory to Tigris as a tar.zst archive, + /// fenced by `precondition`. + pub async fn upload( + &self, + owner_slug: &str, + repo_name: &str, + local_path: &Path, + precondition: UploadPrecondition, + ) -> std::result::Result<(), UploadError> { let key = Self::repo_key(owner_slug, repo_name); debug!(key = %key, path = %local_path.display(), "uploading repo to tigris"); @@ -101,15 +177,51 @@ impl TigrisClient { let body = aws_sdk_s3::primitives::ByteStream::from(archive_bytes); - self.s3 + let mut req = self + .s3 .put_object() .bucket(&self.bucket) .key(&key) .body(body) - .content_type("application/zstd") - .send() - .await - .context(format!("tigris PUT {key}"))?; + .content_type("application/zstd"); + match &precondition { + UploadPrecondition::IfMatch(etag) => req = req.if_match(etag), + UploadPrecondition::IfAbsent => req = req.if_none_match("*"), + UploadPrecondition::Unconditional => {} + } + + if let Err(e) = req.send().await { + // `PutObjectError` models no PreconditionFailed variant (its arms are + // EncryptionTypeMismatch, InvalidRequest, InvalidWriteOffset, + // TooManyParts, Unhandled), so a refused precondition arrives as + // `Unhandled` and matching the enum would classify it as a generic + // failure. The raw HTTP status off the service-error response is the + // only place the answer actually lives. + let status = match &e { + SdkError::ServiceError(ctx) => Some(ctx.raw().status().as_u16()), + _ => None, + }; + // 412 is always a lost precondition. 409 is one only when we asked + // for create-only, which is how S3-compatible stores report "the key + // already exists". Everything else, 404 included, is a real failure: + // archive keys are never deleted (`delete` has no callers), so a 404 + // here means something permanent like a missing bucket or a + // misrouted endpoint, and reporting that as a lost precondition + // would tell a caller to expect a successor that does not exist. + let lost = match status { + Some(412) => true, + Some(409) => matches!(precondition, UploadPrecondition::IfAbsent), + _ => false, + }; + if lost { + return Err(UploadError::PreconditionLost { + status: status.expect("a lost precondition came from a status"), + }); + } + return Err(UploadError::Other( + anyhow::Error::new(e).context(format!("tigris PUT {key}")), + )); + } info!(key = %key, "uploaded repo to tigris"); Ok(()) From 317841ee24dfcc27d39dbaede4ee583c83514add Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sun, 9 Aug 2026 23:55:36 -0500 Subject: [PATCH 25/54] fix(node): fence the release publish so an abandoned upload cannot win acquire_write now reads the archive's ETag under the advisory lock and the guard carries it, so release publishes conditionally on the generation it actually refreshed from. An upload abandoned by an earlier writer no longer overwrites a successor's archive: the store rejects it, because the ETag it was written against is gone. A refused precondition gets exactly one supersede-retry, never a loop. The distinction that makes this sound is that a 412 is a definite answer, unlike the timeout arm where nothing is knowable, and the retrying writer still holds the lock, so whatever landed underneath was written without one and its tree is not the authority. Two losses in a row refuse instead of escalating. That retry is what keeps ordinary pushes working now that init publishes create-only: a first push racing init's upload of the empty repo loses once and then wins, rather than surfacing a 503 on the most common operation there is. release returns a must-use outcome and the four publishing handlers propagate it before any trust bump, webhook, or success body, so a publish the store refused reads as a retryable 503 instead of a 201. The three release(false) sites deliberately do not map it: they publish nothing, and a 503 there would shadow the 403 or 404 the route means to return. The download-failure fallback also publishes fenced now. That arm knows the stored generation (its HEAD succeeded, only the GET failed), so publishing unconditionally from it would reintroduce the same overwrite. --- crates/gitlawb-node/src/api/issues.rs | 20 +- crates/gitlawb-node/src/api/pulls.rs | 5 +- crates/gitlawb-node/src/api/repos.rs | 7 +- crates/gitlawb-node/src/error.rs | 21 +- crates/gitlawb-node/src/git/repo_store.rs | 737 ++++++++++++++++++++-- crates/gitlawb-node/src/git/tigris.rs | 8 - 6 files changed, 739 insertions(+), 59 deletions(-) diff --git a/crates/gitlawb-node/src/api/issues.rs b/crates/gitlawb-node/src/api/issues.rs index c34e41ad..162c2ac2 100644 --- a/crates/gitlawb-node/src/api/issues.rs +++ b/crates/gitlawb-node/src/api/issues.rs @@ -70,7 +70,11 @@ pub async fn create_issue( let create_result = git_issues::create_issue(&disk_path, &issue_id, &json_str); // Always release the advisory lock — even on error; upload to Tigris only on success. - guard.release(create_result.is_ok()).await; + // A refused publish short-circuits here, before the trust bump and before + // the 201: the issue is on local disk but not in object storage, so no + // other node can read it and the client must retry rather than be told it + // was filed. + guard.release(create_result.is_ok()).await.into_result()?; create_result.map_err(|e| AppError::Git(e.to_string()))?; @@ -321,14 +325,19 @@ pub async fn close_issue( .as_deref() .is_some_and(|a| crate::api::did_matches(&auth.0, a)); if !is_owner && !is_author_now { - guard.release(false).await; + // Consumed, NOT propagated, and that is deliberate at all three + // `release(false)` sites below. These release without + // publishing, so there is nothing for the store to refuse, and + // mapping the outcome here would let a 503 shadow the + // authorization answer this route exists to give. + let _ = guard.release(false).await; return Err(AppError::Forbidden( "only the repo owner or the issue author can close this issue".into(), )); } } Ok(None) => { - guard.release(false).await; + let _ = guard.release(false).await; // The owner keeps the informative 404; a non-owner must not learn from // this route whether the issue exists, matching the pre-check above. return Err(if is_owner { @@ -340,7 +349,7 @@ pub async fn close_issue( }); } Err(e) => { - guard.release(false).await; + let _ = guard.release(false).await; return Err(AppError::Git(e.to_string())); } } @@ -348,7 +357,8 @@ pub async fn close_issue( let close_result = git_issues::close_issue(&disk_path, &issue_id); // Always release the advisory lock — even on error; upload to Tigris only on success. - guard.release(close_result.is_ok()).await; + // Same short-circuit as create_issue, and before the 200 body below. + guard.release(close_result.is_ok()).await.into_result()?; let updated = close_result .map_err(|e| AppError::Git(e.to_string()))? diff --git a/crates/gitlawb-node/src/api/pulls.rs b/crates/gitlawb-node/src/api/pulls.rs index adabd146..1ec8fc84 100644 --- a/crates/gitlawb-node/src/api/pulls.rs +++ b/crates/gitlawb-node/src/api/pulls.rs @@ -224,7 +224,10 @@ pub async fn merge_pr( ); // Always release the advisory lock — even on error; upload to Tigris only on success. - guard.release(merge_result.is_ok()).await; + // Short-circuit on a refused publish before the PR is marked merged and + // before the webhook fires. Both are irreversible announcements of a merge + // commit that only exists on this node's disk. + guard.release(merge_result.is_ok()).await.into_result()?; let merge_sha = merge_result.map_err(|e| AppError::Git(e.to_string()))?; diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 7634cc3c..25c83d93 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -2059,7 +2059,12 @@ pub async fn git_receive_pack( // Always release the advisory lock — even on error — to prevent stale locks // from blocking subsequent pushes. Only upload to Tigris when the push // succeeded; uploading a half-applied repo would propagate corruption. - guard.release(push_succeeded).await; + // Short-circuit on a refused publish BEFORE anything downstream observes + // the push. The pack is on local disk but not in object storage, so + // touching the repo, recording the push, bumping trust, issuing ref + // certificates or answering 200 would all be reporting a write no other + // node can read. + guard.release(receive_result.is_ok()).await.into_result()?; // Clean path: clone (a) already dropped inside run_git_service when the receive-pack // group was reaped; clone (b) held here spanned the success-only Tigris upload that // ran inside release() above. Drop it now so a second same-repo push proceeds the diff --git a/crates/gitlawb-node/src/error.rs b/crates/gitlawb-node/src/error.rs index d8362af8..2ffb8861 100644 --- a/crates/gitlawb-node/src/error.rs +++ b/crates/gitlawb-node/src/error.rs @@ -65,6 +65,9 @@ pub enum AppError { #[error("repository is temporarily unavailable")] RepoUnavailable, + #[error("repository write was fenced by a concurrent publish")] + RepoWriteFenced, + #[error("database error: {0}")] Db(#[from] sqlx::Error), @@ -117,7 +120,14 @@ impl From for AppError { // owner slug and repo, so the variant carries nothing. Err(err) => match err.downcast::() { Ok(_) => AppError::RepoUnavailable, - Err(err) => AppError::Internal(err), + // And one more rung: a publish the store refused twice is + // transient in the same way, and the retry is the client's + // to make. The variant carries nothing for the same reason + // as the two above. + Err(err) => match err.downcast::() { + Ok(_) => AppError::RepoWriteFenced, + Err(err) => AppError::Internal(err), + }, }, }, } @@ -198,6 +208,15 @@ impl IntoResponse for AppError { "repo_unavailable", "repository is temporarily unavailable, retry".into(), ), + // 503 with a FIXED body again, and its own code: the caller should + // retry, but the condition is not contention, so a client that + // distinguishes them should be able to. The body must not say which + // repo lost its publish or to whom. + AppError::RepoWriteFenced => ( + StatusCode::SERVICE_UNAVAILABLE, + "repo_write_fenced", + "repository changed underneath this write, retry".into(), + ), AppError::Db(e) if db_unavailable(e) => ( StatusCode::SERVICE_UNAVAILABLE, DB_UNAVAILABLE_CODE, diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 66f9659e..9f814bb2 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -485,7 +485,7 @@ impl RepoStore { // From here the lock is HELD. Any early return must not simply drop the // connection back into the pool, so it is handed to the guard immediately // below and every exit after this point goes through the guard. - let guard = RepoWriteGuard { + let mut guard = RepoWriteGuard { owner_slug: owner_slug.clone(), repo_name: repo_name.to_string(), local_path: local_path.clone(), @@ -493,6 +493,10 @@ impl RepoStore { conn: Some(lock_conn), tigris: self.tigris.clone(), lock_held_transfer_timeout: self.lock_held_transfer_timeout, + // Overwritten by the refresh below with the generation actually + // observed under the lock. Only reachable unset when no backend is + // configured, in which case `release` publishes nothing at all. + publish_fence: UploadPrecondition::Unconditional, }; // Always download the latest from Tigris before writing. Local disk may be @@ -518,15 +522,24 @@ impl RepoStore { // fallback. Collapsing them (the `unwrap_or(false)` this replaced // read a HEAD error as "no archive") skipped the refresh silently // and then re-uploaded over a possibly-newer archive. - match tigris.exists(&owner_slug, repo_name).await { - Ok(true) => { + // + // `head_etag` rather than `exists`: the same request answers + // both questions, and the ETag it carries is the generation + // this write is based on. Carrying it to the release-side + // publish is what lets the store refuse a stale PUT, which + // is the only place that fence can hold: dropping an + // in-flight upload's future does not stop the request the + // server is already processing. + match tigris.head_etag(&owner_slug, repo_name).await { + Ok(Some(etag)) => { debug!(repo = %repo_name, "write acquire: downloading latest from tigris"); - tigris - .download(&owner_slug, repo_name, &local_path) - .await - .map_err(RefreshFailure::Download) + let fence = UploadPrecondition::IfMatch(etag); + match tigris.download(&owner_slug, repo_name, &local_path).await { + Ok(()) => Ok(fence), + Err(err) => Err(RefreshFailure::Download { err, fence }), + } } - Ok(false) => Ok(()), + Ok(None) => Ok(UploadPrecondition::IfAbsent), Err(e) => Err(RefreshFailure::Unknown(e)), } }, @@ -534,18 +547,23 @@ impl RepoStore { .await; match refreshed { - Some(Ok(())) => {} - Some(Err(RefreshFailure::Download(e))) => { + Some(Ok(fence)) => guard.publish_fence = fence, + Some(Err(RefreshFailure::Download { err, fence })) => { // The archive is present but unreadable: a corrupt or partial // upload, or a transient GET failure. We KNOW the fetch failed, // so falling back to a valid local copy is sound and // release(success) re-uploads a good archive. Only hard-fail // when there is no local copy to fall back to. if local_path.exists() { - warn!(repo = %repo_name, err = %e, + warn!(repo = %repo_name, err = %err, "write acquire: tigris refresh failed — falling back to local copy"); + // Still fence on what the HEAD saw. The download failing + // says nothing about the generation stored, so publishing + // unconditionally here would reintroduce exactly the + // overwrite this carries the ETag to prevent. + guard.publish_fence = fence; } else { - return Err(e).context("downloading repo from tigris for write"); + return Err(err).context("downloading repo from tigris for write"); } } Some(Err(RefreshFailure::Unknown(e))) => { @@ -1031,6 +1049,11 @@ pub struct RepoWriteGuard { tigris: Option, /// Bound on the release-side upload, which runs with the lock still held. lock_held_transfer_timeout: Duration, + /// The generation of the stored archive as observed by the HEAD inside + /// `acquire_write`, under the lock. `release` publishes fenced on it, so a + /// PUT abandoned by an earlier writer's timeout cannot land on top of a + /// successor's acknowledged archive. + publish_fence: UploadPrecondition, } impl RepoWriteGuard { @@ -1055,35 +1078,112 @@ impl RepoWriteGuard { &self.local_path } + /// Publish the tree this guard wrote, fenced on the generation observed + /// under the lock, with at most ONE supersede-retry after a definite loss. + /// + /// Hard bound of two PUT attempts per release. No loop, no recursion: a + /// third attempt would have no more reason to terminate than the second. + async fn publish(&self, tigris: &TigrisClient) -> std::result::Result<(), PublishRefusal> { + match tigris + .upload( + &self.owner_slug, + &self.repo_name, + &self.local_path, + self.publish_fence.clone(), + ) + .await + { + Ok(()) => return Ok(()), + Err(UploadError::PreconditionLost { status }) => { + // EPISTEMIC ASYMMETRY, and it is why one retry is sound here + // while the timeout arm in `release` deliberately does nothing. + // A refused precondition is a DEFINITE outcome: the store told + // us the generation we observed under the lock is gone, and that + // our bytes did not land. A timeout tells us nothing at all. + // + // We also still hold the advisory lock, so no successor can have + // acquired and published. Whatever landed underneath was written + // WITHOUT the lock: init's create-only upload of a freshly + // created empty repo, or a PUT abandoned by an earlier writer + // whose own release timed out. This writer's tree is the + // authority over both, which is what makes exactly one + // supersede-retry correct rather than a race. + // + // Honest residual: when the thing underneath was a genuine + // orphan that landed AFTER this writer's refresh, the retry + // supersedes it with a tree that does not contain it. That is + // the same outcome today's unconditional publish produces. The + // fence protects an acknowledged successor from an orphan; it + // does not protect an unlocked orphan from the lock holder. + warn!( + repo = %self.repo_name, + status, + "publish fence lost: the stored archive changed under the lock, \ + republishing once on the current generation" + ); + } + Err(e) => return Err(PublishRefusal::Failed(e)), + } + + let fresh = match tigris.head_etag(&self.owner_slug, &self.repo_name).await { + Ok(Some(etag)) => UploadPrecondition::IfMatch(etag), + // Nothing is stored now, so create-only is the fence that matches + // what was just observed. + Ok(None) => UploadPrecondition::IfAbsent, + Err(e) => return Err(PublishRefusal::Failed(UploadError::Other(e))), + }; + match tigris + .upload(&self.owner_slug, &self.repo_name, &self.local_path, fresh) + .await + { + Ok(()) => Ok(()), + Err(UploadError::PreconditionLost { status }) => { + // Two definite losses in a row: something is publishing this key + // without the lock faster than we can fence on it. Refuse rather + // than escalate. The write is on local disk and in this node's + // tree, but it is NOT durable in object storage, so the caller + // must not report success. + warn!( + repo = %self.repo_name, + status, + "publish fence lost again on the refreshed generation, refusing the \ + write rather than attempting a third publish" + ); + Err(PublishRefusal::Fenced) + } + Err(e) => Err(PublishRefusal::Failed(e)), + } + } + /// Upload to Tigris (only when the write succeeded) and release the advisory /// lock. Pass `success = false` when the write operation failed — uploading a /// half-applied or otherwise inconsistent repo would propagate corruption to /// Tigris (and to every node that later downloads it). The lock is always /// released regardless, to avoid stale locks blocking future writes. - pub async fn release(mut self, success: bool) { + pub async fn release(mut self, success: bool) -> ReleaseOutcome { + let mut outcome = ReleaseOutcome::Released; // Upload to Tigris only on success. if success { - if let Some(ref tigris) = self.tigris { - // Bounded for the same reason as the acquire-side download: this - // runs with the lock held and a lock-pool slot pinned. + if let Some(tigris) = self.tigris.clone() { + // ONE budget for the whole publish, covering both attempts and + // the HEAD between them, for the same reason the acquire-side + // refresh uses one for its HEAD and download together: this runs + // with the lock held and a lock-pool slot pinned, so bounding + // each attempt separately would double the worst-case occupancy. match bounded_transfer( "release-upload", &self.repo_name, self.lock_held_transfer_timeout, - // Unconditional for now purely so the tree compiles. This - // is THE fenced call site: the sibling unit replaces this - // with the observed-ETag precondition. - tigris.upload( - &self.owner_slug, - &self.repo_name, - &self.local_path, - UploadPrecondition::Unconditional, - ), + self.publish(&tigris), ) .await { Some(Ok(())) => {} - Some(Err(e)) => { + // Both attempts were definitively refused. The raise site + // already logged which and why, so this only has to carry + // the refusal out to the caller. + Some(Err(PublishRefusal::Fenced)) => outcome = ReleaseOutcome::Fenced, + Some(Err(PublishRefusal::Failed(e))) => { warn!(repo = %self.repo_name, err = %e, "failed to upload repo to tigris after write"); } None => { @@ -1152,6 +1252,8 @@ impl RepoWriteGuard { } None => {} } + + outcome } } @@ -1216,7 +1318,62 @@ const LOCK_ACQUIRE_DEADLINE: Duration = Duration::from_secs(90); /// copy is a sound thing to fall back to and re-upload. enum RefreshFailure { Unknown(anyhow::Error), - Download(anyhow::Error), + /// Carries the fence the HEAD observed alongside the error, because the + /// fallback arm still publishes later and must be fenced on the generation + /// it saw. `Unknown` carries none: that arm refuses the write outright. + Download { + err: anyhow::Error, + fence: UploadPrecondition, + }, +} + +/// What `release` was able to do with the writer's tree. +/// +/// `#[must_use]` because dropping it is the whole defect this type exists to +/// prevent: a publish the store refused would otherwise return 201, fire +/// webhooks, and record a push that no successor can read. +#[derive(Debug)] +#[must_use = "a refused publish must reach the caller, or a write that never landed reports success"] +pub enum ReleaseOutcome { + /// The lock was released and nothing definitively refused the publish. + /// Also the answer when there was nothing to publish (a failed write, no + /// storage backend) and when the outcome is unknowable (the upload + /// exceeded its bound), which keeps those paths behaving as they do today. + Released, + /// The store refused the publish twice. The tree is on local disk but is + /// NOT in object storage, so the caller must not report success. + Fenced, +} + +impl ReleaseOutcome { + /// Fold into the `Result` a handler propagates with `?`. + /// + /// Call this at every publishing site IMMEDIATELY after `release`, before + /// any post-release effect. A refusal that short-circuits after the DB + /// write, the webhook, or the response body has already happened is not a + /// refusal at all. + pub fn into_result(self) -> anyhow::Result<()> { + match self { + ReleaseOutcome::Released => Ok(()), + // The raise site inside `publish` already logged the repo and the + // status, so this carries no detail: the handler layer turns it + // into a fixed 503 body. + ReleaseOutcome::Fenced => Err(anyhow::Error::new(RepoWriteFenced) + .context("release-side publish refused by the store on both attempts")), + } + } +} + +/// Why the release-side publish did not land, split by what it leaves the +/// caller able to claim. +/// +/// `Fenced` is a DEFINITE refusal by the store after both attempts, so the +/// write is not durable and the caller must not report success. `Failed` is +/// every other upload failure, which keeps today's behavior (log it, release +/// the lock, let the caller answer normally). +enum PublishRefusal { + Fenced, + Failed(UploadError), } /// The per-repo advisory lock was not obtained within the acquire deadline. @@ -1254,6 +1411,27 @@ impl std::fmt::Display for RepoUnavailable { impl std::error::Error for RepoUnavailable {} +/// The release-side publish was refused by the store on both attempts, so the +/// write is not durable in object storage. +/// +/// A distinct type rather than a bare `anyhow` string, for the same reason as +/// its two siblings above: the handler layer maps it to a retryable 503 with a +/// FIXED body, and the detail (which repo, which status) stays in the log at +/// the raise site. Distinct FROM those siblings because the condition is +/// different: not contention and not an unreadable store, but another writer +/// holding the key. The client's retry re-runs the whole write against the tree +/// that actually won. +#[derive(Debug)] +pub struct RepoWriteFenced; + +impl std::fmt::Display for RepoWriteFenced { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("repository write was fenced by a concurrent publish") + } +} + +impl std::error::Error for RepoWriteFenced {} + /// Run a future under a wall-clock bound, returning `None` if it did not finish. /// /// For the object-storage transfers that run while the per-repo advisory lock is @@ -2563,7 +2741,7 @@ mod tests { let err = match store.acquire_write("did:key:z6MkU3Excl", "same-repo").await { Err(e) => e, Ok(second) => { - second.release(false).await; + let _ = second.release(false).await; panic!("a second writer must NOT be admitted while the first holds the guard"); } }; @@ -2585,7 +2763,7 @@ mod tests { .acquire_write("did:key:z6MkU3Rel", "leak-check") .await .expect("acquire"); - guard.release(true).await; + let _ = guard.release(true).await; let key = advisory_lock_key("did_key_z6MkU3Rel", "leak-check"); let held: (i64,) = sqlx::query_as(&advisory_locks_held(key)) @@ -2644,7 +2822,7 @@ mod tests { .await .expect("acquire"); pids.push(guard.backend_pid_for_test().await); - guard.release(true).await; + let _ = guard.release(true).await; } assert!( pids.windows(2).all(|w| w[0] == w[1]), @@ -2717,8 +2895,10 @@ mod tests { conn: Some(lock_pool.acquire().await.unwrap()), tigris: None, lock_held_transfer_timeout: Duration::from_secs(300), + // No backend, so nothing is ever published and the fence is unread. + publish_fence: UploadPrecondition::Unconditional, }; - guard.release(true).await; + let _ = guard.release(true).await; // Wait for the backend to actually go away rather than sleeping a fixed // span, which is flaky on slow CI. The observer is a STANDALONE @@ -2898,7 +3078,7 @@ mod tests { assert_eq!(alive.0, 1); for g in guards.drain(..) { - g.release(true).await; + let _ = g.release(true).await; } } @@ -2957,10 +3137,10 @@ mod tests { .await .expect("an unrelated repo must not wait on someone else's contention") .expect("and must acquire"); - unrelated.release(true).await; + let _ = unrelated.release(true).await; spinner.abort(); - held.release(true).await; + let _ = held.release(true).await; } /// Lock contention that runs out the acquire deadline must surface as a @@ -2988,7 +3168,7 @@ mod tests { { Err(e) => e, Ok(second) => { - second.release(false).await; + let _ = second.release(false).await; panic!("a second writer must be shed once the deadline expires"); } }; @@ -3016,7 +3196,7 @@ mod tests { "the 503 body must be fixed and must not name the repo, got {body}" ); - held.release(true).await; + let _ = held.release(true).await; } /// An under-lock refresh refusal must surface as a retryable 503 with a fixed @@ -3181,7 +3361,7 @@ mod tests { { Err(e) => e, Ok(guard) => { - guard.release(false).await; + let _ = guard.release(false).await; panic!("a failed HEAD must refuse the write rather than proceed on a stale tree"); } }; @@ -3371,7 +3551,7 @@ mod tests { // Park the upload so it is still in flight when the 200ms bound fires. mock.park_next_put(); - guard.release(true).await; + let _ = guard.release(true).await; assert_eq!( mock.put_attempts().len(), 1, @@ -3417,7 +3597,7 @@ mod tests { seed_bare_repo(&guard.local_path); mock.park_next_put(); - guard.release(true).await; + let _ = guard.release(true).await; let started = std::time::Instant::now(); let successor = store @@ -3429,7 +3609,7 @@ mod tests { "the successor waited {}ms; a timed-out upload must not park the next writer", started.elapsed().as_millis() ); - successor.release(false).await; + let _ = successor.release(false).await; mock.open_gate(); mock.shutdown(); @@ -3475,7 +3655,7 @@ mod tests { { Err(e) => e, Ok(guard) => { - guard.release(false).await; + let _ = guard.release(false).await; panic!("with the pool exhausted, the deadline must shed, not succeed"); } }; @@ -3527,6 +3707,8 @@ mod tests { puts: Vec, /// Set by `park_next_put`, consumed by the next arriving PUT. park_next_put: bool, + /// Set by `roll_generation_after_next_heads`, decremented per HEAD. + roll_after_heads: u32, captured: Option, } @@ -3672,8 +3854,33 @@ mod tests { } } axum::http::Method::HEAD | axum::http::Method::GET => { - let st = state.lock().unwrap(); - match (st.object.clone(), st.etag.clone()) { + let mut st = state.lock().unwrap(); + let answered = (st.object.clone(), st.etag.clone()); + // Fault injection for the two-consecutive- + // losses arm, and the only deterministic way + // to sit BETWEEN a caller's HEAD and the + // conditional PUT it derives from it. The + // gate cannot do this: a parked PUT is + // captured rather than evaluated and answers + // 200, so it can never produce a refusal. + // + // Only the generation moves, not the bytes, + // which is a real state a store reaches (two + // writers can publish byte-identical + // archives) and keeps the stored object a + // valid archive for whoever downloads next. + // `evaluate_put` is untouched, and this logs + // no PutAttempt, so attempt counts still + // count only the caller's own PUTs. + if method == axum::http::Method::HEAD + && st.roll_after_heads > 0 + && st.object.is_some() + { + st.roll_after_heads -= 1; + st.next_etag += 1; + st.etag = Some(format!("\"mock-etag-{}\"", st.next_etag)); + } + match answered { (Some(bytes), Some(etag)) => ( axum::http::StatusCode::OK, [(axum::http::header::ETAG, etag)], @@ -3720,6 +3927,15 @@ mod tests { self.state.lock().unwrap().puts.clone() } + /// Answer each of the next `n` HEADs from the current state, then + /// immediately move the object to a new generation. A caller that HEADs + /// to pick up a precondition and then PUTs on it is therefore fencing + /// on a generation that is already gone, which is the only way to drive + /// two consecutive lost preconditions deterministically. + fn roll_generation_after_next_heads(&self, n: u32) { + self.state.lock().unwrap().roll_after_heads = n; + } + /// Park the next arriving PUT so the caller's transfer bound elapses /// with the request in flight (the abandoned-writer arm). fn park_next_put(&self) { @@ -4302,4 +4518,439 @@ mod tests { mock.shutdown(); } + + // ── the fenced release publish (#279) ────────────────────────────────── + + /// A store whose acquire-side refresh and release-side publish both land on + /// `mock`. The transfer bound is generous on purpose: these tests are about + /// the fence arms, and a short bound would let the timeout arm answer first. + async fn fenced_store( + mock: &S3Mock, + opts: &sqlx::postgres::PgConnectOptions, + repos_dir: &Path, + ) -> RepoStore { + RepoStore::new( + repos_dir.to_path_buf(), + Some(TigrisClient::for_testing_with_endpoint( + "test-bucket", + mock.endpoint(), + )), + no_reap_pool(opts, 2).await, + std::time::Duration::from_secs(30), + ) + } + + /// A client aimed at the same key the store under test publishes to, so a + /// test can seed the archive or land an interfering publish of its own. + fn mock_tigris(mock: &S3Mock) -> TigrisClient { + TigrisClient::for_testing_with_endpoint("test-bucket", mock.endpoint()) + } + + /// The slug `local_path` derives from a DID, needed because a test seeds + /// and reads the archive key directly. + fn owner_slug_of(owner_did: &str) -> String { + owner_did.replace([':', '/'], "_") + } + + /// A bare-repo-shaped directory carrying `marker`, so a test can tell whose + /// tree is stored without comparing compressed bytes. + fn marked_repo(path: &Path, marker: &str) { + seed_bare_repo(path); + std::fs::write(path.join("MARKER"), marker).unwrap(); + } + + /// The marker inside whatever archive is currently stored under the key. + async fn stored_marker(mock: &S3Mock, owner_slug: &str, repo_name: &str) -> String { + let out = TempDir::new().unwrap(); + let into = out.path().join("stored.git"); + mock_tigris(mock) + .download(owner_slug, repo_name, &into) + .await + .expect("the stored archive must be readable"); + std::fs::read_to_string(into.join("MARKER")).expect("the stored archive must be marked") + } + + /// A process-wide sink for warn-level tracing output, installed once. + /// + /// Global rather than per test on purpose. `tracing`'s scoped default is + /// thread-local, and these events fire inside futures the test runtime may + /// move between threads, so a scoped subscriber would drop them silently + /// and every log assertion would go vacuous. Tests instead give their repo + /// a unique name and read back only the lines carrying it. + fn log_sink() -> Arc>> { + static LOG_SINK: std::sync::OnceLock>>> = + std::sync::OnceLock::new(); + LOG_SINK + .get_or_init(|| { + let sink = Arc::new(std::sync::Mutex::new(Vec::new())); + let writer = sink.clone(); + // `try_init`, because another test may already have installed a + // subscriber; the assertions below fail loudly if nothing was + // captured, so a silent no-op here cannot pass for a green run. + let _ = tracing_subscriber::fmt() + .with_writer(move || SinkWriter(writer.clone())) + .with_ansi(false) + .with_max_level(tracing::Level::WARN) + .try_init(); + sink + }) + .clone() + } + + struct SinkWriter(Arc>>); + + impl std::io::Write for SinkWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + /// Captured warn lines naming `repo_name`, joined back into one string. + fn warn_lines_for(repo_name: &str) -> String { + let raw = log_sink().lock().unwrap().clone(); + String::from_utf8_lossy(&raw) + .lines() + .filter(|l| l.contains(repo_name)) + .collect::>() + .join("\n") + } + + /// An uncontended write publishes, and what lands is the writer's tree. + /// + /// This is the must-not-spuriously-fence negative, so it asserts ONLY the + /// outcome and the stored bytes, never which precondition header travelled. + /// Forcing the carried precondition back to `Unconditional` has to leave it + /// green, or it is a second copy of the fix rather than a guard against it; + /// the header itself is pinned by + /// `upload_if_match_with_the_current_etag_publishes_and_sends_the_header`. + #[sqlx::test] + async fn uncontended_write_publishes_the_writers_tree(pool: PgPool) { + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let store = fenced_store(&mock, &opts, repos.path()).await; + let owner = "did:key:z6MkFenceUncontended"; + let slug = owner_slug_of(owner); + + // Seed an archive so the acquire takes the download arm, which is the + // ordinary case: the repo already exists in object storage. + let seed = TempDir::new().unwrap(); + marked_repo(seed.path(), "seed"); + mock_tigris(&mock) + .upload( + &slug, + "repo", + seed.path(), + UploadPrecondition::Unconditional, + ) + .await + .expect("seeding the archive"); + + let guard = store.acquire_write(owner, "repo").await.expect("acquire"); + std::fs::write(guard.local_path.join("MARKER"), "writer").unwrap(); + guard + .release(true) + .await + .into_result() + .expect("an uncontended write must publish"); + + assert_eq!( + stored_marker(&mock, &slug, "repo").await, + "writer", + "an uncontended write must publish the writer's tree" + ); + + mock.shutdown(); + } + + /// The first write to an empty bucket, the absent-at-acquire path. Nothing + /// is stored when the lock is taken, so the publish is the one that creates + /// the key, and the writer's tree is what lands. + #[sqlx::test] + async fn first_write_to_an_empty_bucket_publishes_the_writers_tree(pool: PgPool) { + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let store = fenced_store(&mock, &opts, repos.path()).await; + let owner = "did:key:z6MkFenceFirstWrite"; + let slug = owner_slug_of(owner); + + let guard = store.acquire_write(owner, "repo").await.expect("acquire"); + marked_repo(&guard.local_path, "writer"); + guard + .release(true) + .await + .into_result() + .expect("the first write into an empty key must publish"); + + assert_eq!( + stored_marker(&mock, &slug, "repo").await, + "writer", + "the first write must publish the writer's tree into the empty key" + ); + + mock.shutdown(); + } + + /// THE INIT RACE, and the reason the fence needs a supersede-retry at all. + /// + /// `init` uploads a freshly created EMPTY repo create-only in the + /// background. A user who pushes immediately after creating a repo takes + /// the lock, sees nothing stored, and is fenced create-only too; the + /// background upload then wins the empty key and the push's publish loses. + /// A fence with no retry would turn every such push into a refusal. + /// + /// The retry is sound because the loss is DEFINITE and this writer still + /// holds the lock: what landed underneath was published without it, so this + /// tree supersedes it. + #[sqlx::test] + async fn a_lost_fence_republishes_once_and_the_writers_tree_wins(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let store = fenced_store(&mock, &opts, repos.path()).await; + let owner = "did:key:z6MkFenceInitRace"; + let repo = "fence-init-race-repo"; + let slug = owner_slug_of(owner); + + // Nothing is stored yet, so the acquire records the absent case. + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + marked_repo(&guard.local_path, "writer"); + + // init's create-only background upload lands after that observation. + let empty = TempDir::new().unwrap(); + marked_repo(empty.path(), "empty-init"); + mock_tigris(&mock) + .upload(&slug, repo, empty.path(), UploadPrecondition::IfAbsent) + .await + .expect("the background init upload wins the empty key"); + let before = mock.put_attempts().len(); + assert_eq!(before, 1, "only the init upload has run so far"); + + guard + .release(true) + .await + .into_result() + .expect("the supersede-retry must leave the release reporting success"); + + let attempts = mock.put_attempts(); + assert_eq!( + attempts.len(), + before + 2, + "the release must attempt exactly twice, the fenced publish and one \ + supersede-retry, got {attempts:?}" + ); + assert_eq!( + attempts[before].status, + Some(412), + "the create-only publish must lose to what landed underneath, got {attempts:?}" + ); + assert_eq!( + attempts[before + 1].status, + Some(200), + "the supersede-retry must publish, got {attempts:?}" + ); + assert_eq!( + stored_marker(&mock, &slug, repo).await, + "writer", + "the lock holder's tree must be what is stored after the retry" + ); + assert!( + warn_lines_for(repo).contains("republishing"), + "the fired fence must be visible in the log, got {:?}", + warn_lines_for(repo) + ); + + mock.shutdown(); + } + + /// Two consecutive definite losses: the retry is bounded at ONE, so the + /// release refuses instead of escalating, and the refusal reaches the + /// caller rather than being logged and swallowed. + /// + /// The second loss is arranged by replacing the object right after the + /// re-HEAD answers, so the retry fences on a generation that is already + /// gone. That is fault injection at the only point where it can be + /// deterministic; the mock's PUT gate cannot do it, because a parked PUT is + /// captured rather than evaluated and answers 200. + #[sqlx::test] + async fn a_second_consecutive_loss_refuses_and_never_attempts_a_third(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let store = fenced_store(&mock, &opts, repos.path()).await; + let owner = "did:key:z6MkFenceDoubleLoss"; + let repo = "fence-double-loss-repo"; + let slug = owner_slug_of(owner); + + let seed = TempDir::new().unwrap(); + marked_repo(seed.path(), "seed"); + mock_tigris(&mock) + .upload(&slug, repo, seed.path(), UploadPrecondition::Unconditional) + .await + .expect("seeding the archive"); + + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + std::fs::write(guard.local_path.join("MARKER"), "writer").unwrap(); + + // An unlocked publish lands after the acquire, so the carried fence is + // already stale before the release runs. + let orphan = TempDir::new().unwrap(); + marked_repo(orphan.path(), "orphan"); + mock_tigris(&mock) + .upload( + &slug, + repo, + orphan.path(), + UploadPrecondition::Unconditional, + ) + .await + .expect("an unconditional publish always lands"); + let before = mock.put_attempts().len(); + assert_eq!(before, 2, "the seed and the orphan have run so far"); + + // ... and the generation the retry HEADs for moves on before its PUT + // can use it, so the second attempt loses too. + mock.roll_generation_after_next_heads(1); + let outcome = guard.release(true).await; + + assert!( + matches!(outcome, ReleaseOutcome::Fenced), + "a publish refused twice must be reported to the caller, got {outcome:?}" + ); + let attempts = mock.put_attempts(); + assert_eq!( + attempts.len(), + before + 2, + "a release must attempt at most TWO publishes, never a third, got {attempts:?}" + ); + assert_eq!( + (attempts[before].status, attempts[before + 1].status), + (Some(412), Some(412)), + "both attempts must have been refused by the store, got {attempts:?}" + ); + let logged = warn_lines_for(repo); + assert!( + logged.contains("republishing"), + "the first loss must log the retry, got {logged:?}" + ); + assert!( + logged.contains("refusing the write"), + "the second loss must log its own distinct refusal, got {logged:?}" + ); + + mock.shutdown(); + } + + /// Driven from the client side at a publishing site: a refused publish must + /// render as the retryable 503 and never as a success body. + /// + /// `create_issue` bumps the author's trust score AFTER releasing the guard, + /// so an unchanged score is what proves the short-circuit actually precedes + /// the post-release effects rather than merely being written above them. A + /// `?` placed after the bump would leave the status assertion green and + /// this one red. + #[sqlx::test] + async fn a_fenced_publish_renders_as_503_and_skips_the_post_release_effects(pool: PgPool) { + use tower::ServiceExt; + + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let owner = "did:key:z6MkFenceHandlerAuthor"; + let repo = "fence-handler-repo"; + let slug = owner_slug_of(owner); + + let mut state = crate::test_support::test_state(pool.clone()).await; + state.repo_store = fenced_store(&mock, &opts, repos.path()).await; + let now = chrono::Utc::now(); + state + .db + .create_repo(&crate::db::RepoRecord { + id: uuid::Uuid::new_v4().to_string(), + name: repo.to_string(), + owner_did: owner.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: format!("/tmp/{repo}"), + forked_from: None, + machine_id: None, + }) + .await + .expect("seed repo"); + // The trust bump only moves a row that already exists, so the author has + // to be registered or the observable would be vacuously unchanged. + state + .db + .register_agent(owner, &[]) + .await + .expect("register the author"); + let score_before = state.db.get_trust_score(owner).await.expect("trust score"); + + // A real bare repo, on disk and published, so the acquire refresh has a + // valid archive to download and the handler's git work succeeds. + let local = repos.path().join(&slug).join(format!("{repo}.git")); + store::init_bare(&local).expect("init the bare repo"); + mock_tigris(&mock) + .upload(&slug, repo, &local, UploadPrecondition::Unconditional) + .await + .expect("publish the archive"); + + // Move the generation on after BOTH of the handler's HEADs: the one in + // `acquire_write`, so the fence it carries is stale by the time it + // publishes, and the one the supersede-retry does, so the retry loses + // too and the release refuses. + mock.roll_generation_after_next_heads(2); + + let router = axum::Router::new() + .route( + "/api/v1/repos/{owner}/{repo}/issues", + axum::routing::post(crate::api::issues::create_issue), + ) + .with_state(state.clone()); + let resp = router + .oneshot(crate::test_support::signed_request_as( + owner, + axum::http::Method::POST, + &format!("/api/v1/repos/{owner}/{repo}/issues"), + axum::body::Body::from(r#"{"title":"t","body":"b"}"#), + )) + .await + .unwrap(); + + assert_eq!( + resp.status(), + axum::http::StatusCode::SERVICE_UNAVAILABLE, + "a publish the store refused must be a retryable 503, not a success" + ); + let body = axum::body::to_bytes(resp.into_body(), 64 * 1024) + .await + .expect("body"); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("repo_write_fenced"), + "the 503 must carry its own code so a client can tell it from contention, got {body}" + ); + assert!( + !body.contains(repo) && !body.contains(&slug), + "the body must be fixed and must not name the repo or owner, got {body}" + ); + assert_eq!( + state.db.get_trust_score(owner).await.expect("trust score"), + score_before, + "the post-release trust bump must not run when the publish was refused" + ); + + mock.shutdown(); + } } diff --git a/crates/gitlawb-node/src/git/tigris.rs b/crates/gitlawb-node/src/git/tigris.rs index e6a2f423..ea77563b 100644 --- a/crates/gitlawb-node/src/git/tigris.rs +++ b/crates/gitlawb-node/src/git/tigris.rs @@ -22,10 +22,6 @@ use tracing::{debug, info}; #[derive(Clone, Debug)] pub enum UploadPrecondition { /// Publish only if the stored object is still the generation we observed. - /// - /// Only tests construct this so far. The write guard's release path is the - /// production caller, and it is wired up in a follow-up change. - #[allow(dead_code)] IfMatch(String), /// Publish only if nothing is stored under the key yet. IfAbsent, @@ -125,10 +121,6 @@ impl TigrisClient { /// Separate from `exists` rather than folded into it: `exists` has callers /// that only want the boolean, and widening its return type would churn /// every one of them for no benefit. - /// - /// Only tests call this so far; the write guard reads the ETag here before - /// it publishes, and that wiring is a follow-up change. - #[allow(dead_code)] pub async fn head_etag(&self, owner_slug: &str, repo_name: &str) -> Result> { let key = Self::repo_key(owner_slug, repo_name); match self From 3d803ff217b04fd43fa49e7776296d4bb9349d7c Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 10 Aug 2026 00:35:20 -0500 Subject: [PATCH 26/54] test(node): drive the abandoned-upload race end to end, and probe the real backend The headline test is the one that had to exist. Writer A's release is parked past its transfer bound and returns with the outcome unknowable, B acquires and publishes, and A's captured PUT is then replayed: the store answers 412 and B's archive survives. The create-only variant covers the arm whose real-world failure is silent rather than loud. The control is what makes those attributable. With no interleaved B, an abandoned-then-replayed PUT whose generation still matches lands. Without it the headline would only show that replays get rejected, not that staleness is what rejects them. Header assertions come last in all three, so a lost fence reds on the outcome it is about rather than on a wire-format check. A mock cannot prove Tigris honors any of this, and the vendor requires a Single-region or Multi-region bucket for conditional operations, so against a Global or Dual-region bucket the fence is a silent no-op. The credentials-gated probe checks both arms against the real endpoint and cleans up unconditionally, including when an assertion fails, which is the case it exists to catch. It accepts 412 or 409 on the create-only arm because both mean the precondition was enforced and both are already classified as a loss. --- crates/gitlawb-node/src/git/repo_store.rs | 290 ++++++++++++++++++++++ crates/gitlawb-node/src/git/tigris.rs | 204 +++++++++++++++ 2 files changed, 494 insertions(+) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 9f814bb2..ad046a9e 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -4953,4 +4953,294 @@ mod tests { mock.shutdown(); } + + // ── the abandoned writer's late PUT (#279) ───────────────────────────── + + /// A store whose under-lock transfer bound is short, so a parked PUT + /// actually runs the release past its budget instead of making the test sit + /// out `fenced_store`'s 30s. The acquire-side refresh shares the bound, + /// which is fine here: it moves a few KiB against an in-process mock. + async fn fenced_store_with_bound( + mock: &S3Mock, + opts: &sqlx::postgres::PgConnectOptions, + repos_dir: &Path, + bound: std::time::Duration, + ) -> RepoStore { + RepoStore::new( + repos_dir.to_path_buf(), + Some(TigrisClient::for_testing_with_endpoint( + "test-bucket", + mock.endpoint(), + )), + no_reap_pool(opts, 2).await, + bound, + ) + } + + /// The statuses of every logged PUT attempt, which is what the + /// abandoned-writer tests assert their attempt counts on. + fn attempt_statuses(mock: &S3Mock) -> Vec> { + mock.put_attempts().iter().map(|a| a.status).collect() + } + + /// THE HEADLINE ARM. An abandoned writer's PUT that lands after a successor + /// has published must be refused by the store, and the successor's archive + /// must survive it. + /// + /// This is the whole point of the change. Dropping the future of an + /// in-flight PUT does not cancel the request the server is already + /// processing, so the advisory lock cannot fence it: A's release returns, + /// the lock frees, B acquires and publishes, and A's bytes are still on + /// their way to a store that has already moved on. Only the conditional PUT + /// decides that race, and it decides it at COMMIT time, not at arrival. + /// + /// The mock models that by capturing A's PUT when it arrives and evaluating + /// it on replay, against the state as of the replay. That is deliberately + /// not a timing test: a parked handler whose client has gone away is + /// cancelled with the connection, so a test that waited for it to resume on + /// its own would be waiting on nothing. + #[sqlx::test] + async fn an_abandoned_writers_late_put_loses_to_the_successor(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let bound = std::time::Duration::from_millis(750); + let store = fenced_store_with_bound(&mock, &opts, repos.path(), bound).await; + let owner = "did:key:z6MkFenceLatePut"; + let repo = "fence-late-put-repo"; + let slug = owner_slug_of(owner); + + // Seed the key, so A's acquire observes a generation and carries + // If-Match on it. This is the ordinary case: the repo already exists. + let seed = TempDir::new().unwrap(); + marked_repo(seed.path(), "seed"); + mock_tigris(&mock) + .upload(&slug, repo, seed.path(), UploadPrecondition::Unconditional) + .await + .expect("seeding the archive"); + let seeded = mock.current_etag().expect("the seed minted an ETag"); + + // Writer A takes the lock, writes its tree, and has its publish parked + // past the transfer bound. + let guard_a = store.acquire_write(owner, repo).await.expect("A acquires"); + std::fs::write(guard_a.local_path.join("MARKER"), "writer-a").unwrap(); + mock.park_next_put(); + let started = std::time::Instant::now(); + let outcome_a = guard_a.release(true).await; + assert!( + started.elapsed() >= bound, + "A's release must have run out its transfer bound with the PUT in flight" + ); + // A timeout is UNKNOWABLE rather than failed, so the release reports an + // ordinary success and the lock frees. That is exactly why the fence has + // to live in the store: nothing here knows A's bytes are still coming. + assert!( + matches!(outcome_a, ReleaseOutcome::Released), + "an abandoned publish reports a plain release, got {outcome_a:?}" + ); + + assert!( + mock.captured_put().is_some(), + "A's PUT must have arrived and been captured before the bound elapsed" + ); + + // Writer B acquires the freed lock and publishes for real. + let b_started = std::time::Instant::now(); + let guard_b = store + .acquire_write(owner, repo) + .await + .expect("B must acquire once A's release frees the lock"); + assert!( + b_started.elapsed() < std::time::Duration::from_secs(5), + "B's acquire must be prompt, not blocked behind A's abandoned transfer" + ); + std::fs::write(guard_b.local_path.join("MARKER"), "writer-b").unwrap(); + guard_b + .release(true) + .await + .into_result() + .expect("B's publish is the one that must land"); + let after_b = mock.current_etag().expect("B's publish minted an ETag"); + assert_ne!( + unquote_etag(&after_b), + unquote_etag(&seeded), + "B's publish must have moved the generation on" + ); + + // NOW A's bytes reach the store's commit point. + assert_eq!( + mock.replay_captured(), + 412, + "A's late PUT must be refused: the generation it fenced on is gone" + ); + assert_eq!( + stored_marker(&mock, &slug, repo).await, + "writer-b", + "the successor's archive must survive the abandoned writer's late PUT" + ); + assert_eq!( + mock.current_etag().as_deref().map(unquote_etag), + Some(unquote_etag(&after_b)), + "a refused PUT must not rotate the generation either" + ); + + // Seed, A's parked PUT, B's publish, the deliberate replay. A never + // attempted a second PUT of its own: the timeout arm takes no + // compensating action precisely because the outcome is unknowable. + assert_eq!( + attempt_statuses(&mock), + vec![Some(200), None, Some(200), Some(412)], + "got {:?}", + mock.put_attempts() + ); + + // The header detail comes LAST on purpose. Asserting it up front would + // make a lost fence red here, on a wire-format check, rather than on the + // outcome above, and the outcome is what this test is for. + let captured = mock.captured_put().expect("A's PUT was captured"); + assert_eq!( + captured.if_match.as_deref().map(unquote_etag), + Some(unquote_etag(&seeded)), + "A's in-flight PUT must carry the generation it observed under the lock" + ); + assert_eq!(captured.if_none_match, None); + + mock.open_gate(); + mock.shutdown(); + } + + /// The create-only arm of the same race, and the one whose real-world + /// failure mode is SILENT: an ignored If-None-Match just returns 200, so a + /// publish that should have been fenced lands with no error anywhere. + #[sqlx::test] + async fn an_abandoned_writers_late_create_only_put_loses_to_the_successor(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let bound = std::time::Duration::from_millis(750); + let store = fenced_store_with_bound(&mock, &opts, repos.path(), bound).await; + let owner = "did:key:z6MkFenceLateCreate"; + let repo = "fence-late-create-repo"; + let slug = owner_slug_of(owner); + + // Nothing stored, so A's acquire records the absent case and is fenced + // create-only. + let guard_a = store.acquire_write(owner, repo).await.expect("A acquires"); + marked_repo(&guard_a.local_path, "writer-a"); + mock.park_next_put(); + let outcome_a = guard_a.release(true).await; + assert!( + matches!(outcome_a, ReleaseOutcome::Released), + "an abandoned publish reports a plain release, got {outcome_a:?}" + ); + + assert!( + mock.captured_put().is_some(), + "A's PUT must have arrived and been captured before the bound elapsed" + ); + + // B wins the empty key. + let guard_b = store + .acquire_write(owner, repo) + .await + .expect("B must acquire once A's release frees the lock"); + std::fs::write(guard_b.local_path.join("MARKER"), "writer-b").unwrap(); + guard_b + .release(true) + .await + .into_result() + .expect("B's create must land"); + + assert_eq!( + mock.replay_captured(), + 412, + "A's late create-only PUT must be refused now that the key exists" + ); + assert_eq!( + stored_marker(&mock, &slug, repo).await, + "writer-b", + "the successor's archive must survive the abandoned create-only PUT" + ); + assert_eq!( + attempt_statuses(&mock), + vec![None, Some(200), Some(412)], + "got {:?}", + mock.put_attempts() + ); + + // Last, for the same reason as the If-Match arm: the outcome is the + // claim, the header is the detail. + let captured = mock.captured_put().expect("A's PUT was captured"); + assert_eq!( + captured.if_none_match.as_deref(), + Some("*"), + "A's in-flight PUT must carry the create-only fence it observed" + ); + assert_eq!(captured.if_match, None); + + mock.open_gate(); + mock.shutdown(); + } + + /// THE CONTROL, and it is what makes the two arms above attributable. + /// + /// Same abandonment, same replay, but no successor publishes in between, so + /// the generation A fenced on is still current when its bytes commit and + /// the PUT must LAND. Without this, a green headline test would prove only + /// that replays are rejected, not that STALENESS is what rejects them. + /// + /// It must therefore stay green when the carried precondition is forced + /// back to `Unconditional`: it asserts an outcome the fence does not + /// change, which is the whole reason it can attribute the others' red. + #[sqlx::test] + async fn an_abandoned_put_still_on_the_current_generation_lands(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let bound = std::time::Duration::from_millis(750); + let store = fenced_store_with_bound(&mock, &opts, repos.path(), bound).await; + let owner = "did:key:z6MkFenceControl"; + let repo = "fence-control-repo"; + let slug = owner_slug_of(owner); + + let seed = TempDir::new().unwrap(); + marked_repo(seed.path(), "seed"); + mock_tigris(&mock) + .upload(&slug, repo, seed.path(), UploadPrecondition::Unconditional) + .await + .expect("seeding the archive"); + + let guard_a = store.acquire_write(owner, repo).await.expect("A acquires"); + std::fs::write(guard_a.local_path.join("MARKER"), "writer-a").unwrap(); + mock.park_next_put(); + let outcome_a = guard_a.release(true).await; + assert!( + matches!(outcome_a, ReleaseOutcome::Released), + "an abandoned publish reports a plain release, got {outcome_a:?}" + ); + + assert_eq!( + mock.replay_captured(), + 200, + "with nothing published in between, the abandoned PUT is still current \ + and must be accepted" + ); + assert_eq!( + stored_marker(&mock, &slug, repo).await, + "writer-a", + "the accepted late PUT must be what is stored" + ); + assert_eq!( + attempt_statuses(&mock), + vec![Some(200), None, Some(200)], + "got {:?}", + mock.put_attempts() + ); + + mock.open_gate(); + mock.shutdown(); + } } diff --git a/crates/gitlawb-node/src/git/tigris.rs b/crates/gitlawb-node/src/git/tigris.rs index ea77563b..1d99ef82 100644 --- a/crates/gitlawb-node/src/git/tigris.rs +++ b/crates/gitlawb-node/src/git/tigris.rs @@ -404,3 +404,207 @@ fn decompress_repo(data: &[u8], local_path: &Path) -> Result<()> { Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use aws_sdk_s3::primitives::ByteStream; + use futures::FutureExt; + + /// The envs the probe needs, all of them, or it does not run. + /// + /// `AWS_ENDPOINT_URL_S3` is included on purpose: without it the SDK resolves + /// to real AWS S3, and a probe that passed there would say nothing about + /// Tigris. + fn probe_env() -> Option { + if std::env::var("GITLAWB_TIGRIS_PROBE").ok().as_deref() != Some("1") { + return None; + } + for name in [ + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_ENDPOINT_URL_S3", + ] { + if std::env::var(name).is_err() { + eprintln!("tigris conditional-write probe: {name} is unset, skipping"); + return None; + } + } + match std::env::var("GITLAWB_TIGRIS_BUCKET") { + Ok(b) if !b.is_empty() => Some(b), + _ => { + eprintln!( + "tigris conditional-write probe: GITLAWB_TIGRIS_BUCKET is unset, skipping" + ); + None + } + } + } + + /// One conditional PUT, reported as the status that REFUSED it, or `None` + /// when the store accepted the write. + /// + /// Accepted is the interesting answer here, not an error: it means the + /// endpoint ignored the header we fenced on. + async fn conditional_put( + s3: &S3Client, + bucket: &str, + key: &str, + body: &'static [u8], + if_match: Option<&str>, + if_none_match: Option<&str>, + ) -> Result, String> { + let mut req = s3 + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from_static(body)); + if let Some(v) = if_match { + req = req.if_match(v); + } + if let Some(v) = if_none_match { + req = req.if_none_match(v); + } + match req.send().await { + Ok(_) => Ok(None), + Err(e) => match &e { + SdkError::ServiceError(ctx) => Ok(Some(ctx.raw().status().as_u16())), + _ => Err(format!("conditional PUT {key}: no HTTP response: {e}")), + }, + } + } + + /// The probe body, written to RETURN its failures rather than panic on + /// them, so the caller's cleanup is reached on every arm. + async fn conditional_write_probe(s3: &S3Client, bucket: &str, key: &str) -> Result<(), String> { + // 1. A plain PUT under a throwaway key. + let seeded = s3 + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from_static(b"probe-one")) + .send() + .await + .map_err(|e| format!("seeding PUT {key}: {e}"))?; + + // 2. Its ETag, which is the generation the next arm fences against. + let etag = seeded + .e_tag() + .ok_or_else(|| format!("seeding PUT {key} returned no ETag"))? + .to_string(); + + // 3. A deliberately wrong If-Match. A store honoring it answers 412. + let wrong = format!("\"{}\"", "0".repeat(32)); + if etag.trim_matches('"') == wrong.trim_matches('"') { + return Err(format!( + "the seeded ETag {etag} collides with the deliberately wrong one, \ + so this arm would prove nothing" + )); + } + match conditional_put(s3, bucket, key, b"probe-two", Some(&wrong), None).await? { + Some(412) => {} + Some(status) => { + return Err(format!( + "a stale If-Match must be refused with 412, the endpoint answered {status}" + )) + } + None => { + return Err( + "a stale If-Match was ACCEPTED: this endpoint does not honor If-Match, so \ + the release fence cannot hold here" + .to_string(), + ) + } + } + + // 4. If-None-Match `*` over the object that now exists. This arm matters + // MORE than the one above. An ignored If-Match eventually surfaces as + // odd behavior, because a stale writer overwrites and someone notices + // the lost tree. An ignored If-None-Match just returns 200, so a publish + // that should have been fenced lands with no error anywhere: the silent + // no-op the bucket-type caveat on this test describes. + match conditional_put(s3, bucket, key, b"probe-three", None, Some("*")).await? { + // Either status is a pass, and the asymmetry with the If-Match arm + // above mirrors `upload`'s classifier exactly: 412 is always a lost + // precondition, and 409 is one too when we asked for create-only. + // AWS documents 409 for a create-only conflict racing a delete, so a + // store answering it is enforcing the precondition and we already + // handle it. Pinning 412 alone here would fail the probe against a + // backend that is behaving correctly, which sends whoever runs it + // chasing a fault that is not there. + Some(412) | Some(409) => {} + Some(status) => { + return Err(format!( + "create-only over an existing object must be refused with 412 or 409, \ + the endpoint answered {status}" + )) + } + None => { + return Err( + "If-None-Match * was ACCEPTED over an existing object: this endpoint does \ + not honor create-only, so a fenced publish lands silently" + .to_string(), + ) + } + } + + Ok(()) + } + + /// Probe the REAL Tigris endpoint for the conditional-write semantics the + /// release fence depends on. + /// + /// UNTIL THIS IS RUN AGAINST REAL CREDENTIALS, the fence is verified against + /// vendor documentation and an in-process mock, not against the backend it + /// runs on. The mock implements the semantics we believe Tigris has; it + /// cannot tell us whether Tigris actually has them. + /// + /// The bucket matters, not just the endpoint. Tigris documents conditional + /// operations as supported on Single-region and Multi-region buckets only. + /// Global and Dual-region buckets are eventually consistent, and a + /// conditional PUT evaluated against a stale replica would make the fence a + /// silent no-op rather than an error. So point `GITLAWB_TIGRIS_BUCKET` at a + /// throwaway bucket of the SAME type production uses. + /// + /// Ignored by default and additionally gated on `GITLAWB_TIGRIS_PROBE=1`, + /// because it writes to a real bucket and costs real requests. Run with: + /// `GITLAWB_TIGRIS_PROBE=1 cargo test -p gitlawb-node --bin gitlawb-node + /// tigris_honors_conditional_writes -- --ignored --nocapture` + #[tokio::test] + #[ignore = "writes to a real Tigris bucket; needs GITLAWB_TIGRIS_PROBE=1 plus credentials"] + async fn tigris_honors_conditional_writes() { + let Some(bucket) = probe_env() else { + eprintln!( + "tigris conditional-write probe: skipped. Set GITLAWB_TIGRIS_PROBE=1, \ + GITLAWB_TIGRIS_BUCKET, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and \ + AWS_ENDPOINT_URL_S3 to run it." + ); + return; + }; + + let config = aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await; + let s3 = S3Client::new(&config); + // A fresh key per run, so a probe that somehow orphaned an object on an + // earlier run cannot change what this one observes. + let key = format!("probe/conditional-write-{}.bin", uuid::Uuid::new_v4()); + + // CLEANUP MUST RUN ON EVERY ARM, and a failing assertion is precisely + // the case this probe exists to catch, so the delete cannot sit after + // the checks. The body returns its failures rather than panicking, and + // `catch_unwind` covers the panic an SDK call could still raise; either + // way the delete below is reached before the verdict is re-raised. + let outcome = std::panic::AssertUnwindSafe(conditional_write_probe(&s3, &bucket, &key)) + .catch_unwind() + .await; + + if let Err(e) = s3.delete_object().bucket(&bucket).key(&key).send().await { + eprintln!("tigris conditional-write probe: cleanup of {key} failed: {e}"); + } + + match outcome { + Ok(Ok(())) => {} + Ok(Err(msg)) => panic!("tigris conditional-write probe: {msg}"), + Err(payload) => std::panic::resume_unwind(payload), + } + } +} From 2a92a1f73a51067579400e72d932a580f54c85c6 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 17 Aug 2026 23:23:32 -0500 Subject: [PATCH 27/54] fix(node): reconcile F-series tests and test seams with the rebased guard shape The rebase onto main carried main's #174 F-series guard tests forward. The branch's guard replaces main's locked/released/test_pre_unlock_gate Drop mechanics with close-on-drop (runtime) and leak (off-runtime), so two tests that asserted the replaced mechanics are reconciled: the off-runtime disposal test now asserts the leak observable (the slot never returns to idle) and the detached-unlock-returns-connection test is dropped, its invariant covered by the U-series drop-frees-the-lock gates. The pre-unlock gate seam is restored (test-only) so the mid-unlock cancellation tests keep their deterministic park. ipfs.rs tests are updated to the branch's sync test-client constructor and 4-arg RepoStore::new. Refs #279 --- crates/gitlawb-node/src/api/ipfs.rs | 30 +++-- crates/gitlawb-node/src/git/repo_store.rs | 151 ++++++++-------------- 2 files changed, 74 insertions(+), 107 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index df7a42db..4edec3b6 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -1179,9 +1179,13 @@ mod tests { // deterministically. let endpoint = crate::test_support::silent_http_endpoint().await; let tigris = - crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) - .await; - state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint); + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir, + Some(tigris), + pool, + std::time::Duration::from_secs(300), + ); state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; let mut cfg = (*state.config).clone(); cfg.git_acquire_timeout_secs = 1; @@ -1243,9 +1247,13 @@ mod tests { // 1s timeout (endpoint-pinned test client, no AWS_* env reads). let endpoint = crate::test_support::silent_http_endpoint().await; let tigris = - crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) - .await; - state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint); + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir, + Some(tigris), + pool, + std::time::Duration::from_secs(300), + ); state .db .upsert_mirror_repo("z6f2acqcont", "ghost", "/unused-ghost", None, false) @@ -1768,9 +1776,13 @@ mod tests { // the budget (endpoint-pinned test client, no AWS_* env reads). let endpoint = crate::test_support::silent_http_endpoint().await; let tigris = - crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) - .await; - state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint); + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir, + Some(tigris), + pool, + std::time::Duration::from_secs(300), + ); state .db .upsert_mirror_repo("z6f3budget", "ghost", "/unused-ghost", None, false) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index ad046a9e..da1b74c3 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -14,8 +14,7 @@ use std::sync::Arc; use std::time::Duration; use anyhow::{Context, Result}; -use sqlx::pool::PoolConnection; -use sqlx::{PgPool, Postgres}; +use sqlx::PgPool; use tokio::sync::Mutex; use tracing::{debug, info, warn}; @@ -82,6 +81,15 @@ impl RepoStore { self } + /// Test-only: every guard from this store parks in `release` right before the + /// `pg_advisory_unlock` await, until `gate` is notified. Dropping the future + /// while it is parked reproduces a client disconnect inside `release`. + #[cfg(test)] + pub fn with_pre_unlock_gate(mut self, gate: Arc) -> Self { + self.pre_unlock_gate = Some(gate); + self + } + pub fn new( repos_dir: PathBuf, tigris: Option, @@ -497,6 +505,8 @@ impl RepoStore { // observed under the lock. Only reachable unset when no backend is // configured, in which case `release` publishes nothing at all. publish_fence: UploadPrecondition::Unconditional, + #[cfg(test)] + test_pre_unlock_gate: self.pre_unlock_gate.clone(), }; // Always download the latest from Tigris before writing. Local disk may be @@ -1054,6 +1064,13 @@ pub struct RepoWriteGuard { /// PUT abandoned by an earlier writer's timeout cannot land on top of a /// successor's acknowledged archive. publish_fence: UploadPrecondition, + /// Test-only seam: when set, `release` parks on this gate at the exact point + /// it is about to await `pg_advisory_unlock` (connection still owned, not yet + /// released). Dropping the `release` future while it is parked reproduces a + /// mid-unlock cancellation, so a test can assert the `Drop` backstop still + /// frees the session lock. Never set outside tests. + #[cfg(test)] + test_pre_unlock_gate: Option>, } impl RepoWriteGuard { @@ -1221,6 +1238,12 @@ impl RepoWriteGuard { // connection is left in `self.conn` for `Drop` to close rather than being // handed back to the pool as clean. Only a confirmed unlock returns it. let lock_key = self.lock_key; + // Test-only: park right before the unlock await so a test can drop this + // future mid-unlock (connection owned, not yet released). + #[cfg(test)] + if let Some(gate) = self.test_pre_unlock_gate.clone() { + gate.notified().await; + } let unlock = match self.conn.as_mut() { Some(conn) => Some( sqlx::query_as::<_, (bool,)>("SELECT pg_advisory_unlock($1)") @@ -2004,7 +2027,7 @@ mod tests { let mut checker = pool.acquire().await.expect("checker connection"); let guard = store.acquire_write(owner, name).await.expect("acquire"); - guard.release(false).await; + let _ = guard.release(false).await; let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") .bind(key) @@ -2096,7 +2119,7 @@ mod tests { .acquire_write(owner, name) .await .expect("first acquire"); - guard.release(true).await; + let _ = guard.release(true).await; let again = tokio::time::timeout( std::time::Duration::from_secs(2), @@ -2105,7 +2128,7 @@ mod tests { .await .expect("second acquire_write must not hit the ~60s stale-lock retry loop") .expect("second acquire"); - again.release(true).await; + let _ = again.release(true).await; } // ── unlock error disposes the connection (#174 F3b, RED-before/GREEN-after) ─ @@ -2196,45 +2219,6 @@ mod tests { .expect("a second pool over the test database") } - /// F3c (P2): the connection teardown on the failing-unlock path must be BOUNDED. - /// `release` awaits it inline while the global write permit, the per-source permit - /// and the write lease are all still held, and sqlx's `close()` carries no deadline - /// of its own, so a blackholed socket would park every later push to that repo - /// behind three pinned admission resources. - /// - /// What this covers: the deadline itself. A close that never resolves still lets - /// `close_conn_bounded` return, which is the property `release` depends on. What it - /// does NOT cover, and is reasoned rather than run: that sqlx's own `close()` is - /// what stalls in production. Making a real `PgConnection::close` hang needs a - /// blackholed TCP path to Postgres, and the flip has to land after the unlock - /// statement round-trips but before the Terminate write, which is not a seam this - /// module exposes. A never-resolving future is the faithful stand-in for that - /// close, and the F3b tests above already cover that `release` really routes its - /// close through here. - /// - /// Time is paused, so nothing here depends on wall clock: the runtime auto-advances - /// to the next timer, and the assertion is on which timer fired, not on elapsed - /// time. The outer bound is what turns a removed deadline into a failure rather - /// than a hung suite. - /// - /// Load-bearing: drop the `tokio::time::timeout` in `close_conn_bounded` and the - /// inner future never resolves, so the outer bound fires and this fails. - #[tokio::test(start_paused = true)] - async fn unlock_error_connection_close_is_bounded() { - let hanging = std::future::pending::>(); - let outcome = tokio::time::timeout( - UNLOCK_ERROR_CLOSE_TIMEOUT * 4, - close_conn_bounded("boundedclosetest", hanging), - ) - .await; - assert!( - outcome.is_ok(), - "a connection close that never completes must not hold the write lease and \ - both admission permits open-endedly: close_conn_bounded must give up and \ - drop the connection" - ); - } - /// F3b (P1): when `pg_advisory_unlock` ERRORS while the session is still alive /// (statement timeout, admin cancel, aborted transaction), the lock must not /// survive `release`. The old code discarded the error with `let _ =` and set @@ -2277,7 +2261,7 @@ mod tests { "the poisoned session must still hold the lock before release" ); - guard.release(false).await; + let _ = guard.release(false).await; // Postgres drops the lock when the disposed session's backend exits, which is // asynchronous to our socket close: poll for it rather than sleeping a @@ -2315,7 +2299,7 @@ mod tests { let size_before = store_pool.size(); assert!(size_before > 0, "the pool owns the guard's connection"); - guard.release(false).await; + let _ = guard.release(false).await; // The pool's size drops when the closed connection's slot is given up, which // is not synchronous with `release` returning: poll rather than sleep. @@ -2345,7 +2329,7 @@ mod tests { let guard = store.acquire_write(owner, name).await.expect("acquire"); let size_before = pool.size(); - guard.release(false).await; + let _ = guard.release(false).await; tokio::time::sleep(std::time::Duration::from_millis(400)).await; assert_eq!( @@ -2423,49 +2407,6 @@ mod tests { .await; } - /// U8 regression guard on the success path: a detached unlock that SUCCEEDS must - /// still return the connection to the pool. Without this, "close the connection on - /// Drop" could be widened to "always close" and the test above would not notice. - #[sqlx::test] - async fn write_guard_drop_with_successful_unlock_keeps_the_connection(pool: sqlx::PgPool) { - let dir = tempfile::TempDir::new().unwrap(); - let store_pool = pool_without_idle_reaper(&pool).await; - let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone()); - let owner = "did:key:z6MkDropUnlockOkProofJJJJJJJJJJJJJJJJJJJJ"; - let name = "dropunlockoktest"; - let slug = owner.replace([':', '/'], "_"); - let key = advisory_lock_key(&slug, name); - - let mut checker = pool.acquire().await.expect("checker connection"); - let guard = store.acquire_write(owner, name).await.expect("acquire"); - let size_before = store_pool.size(); - assert!(size_before > 0, "the pool owns the guard's connection"); - - drop(guard); - - // The connection goes back only once the detached unlock task has finished. - wait_until( - || store_pool.num_idle() > 0, - "the detached unlock to finish and hand the connection back", - ) - .await; - assert_eq!( - store_pool.size(), - size_before, - "a successful detached unlock must leave the connection in the pool" - ); - wait_until_lock_free( - &mut checker, - key, - "the Drop backstop's successful unlock to free the lock", - ) - .await; - let _ = sqlx::query("SELECT pg_advisory_unlock($1)") - .bind(key) - .execute(&mut *checker) - .await; - } - /// U8, the off-runtime arm: with no Tokio runtime there is nothing to spawn the /// unlock onto, and the connection has already been taken out of the guard, so the /// old code dropped it with no unlock attempted at all, back to the pool, session @@ -2477,9 +2418,15 @@ mod tests { /// `Handle::try_current()` fails. /// /// Load-bearing: RED before the fix (the join sees the "requires a Tokio context" - /// panic from sqlx's return-to-pool spawn), GREEN after (`detach` gives up the - /// pool slot, so nothing is spawned and dropping the detached connection closes + /// panic from sqlx's return-to-pool spawn), GREEN after (`leak` gives up the + /// pool slot, so nothing is spawned and dropping the leaked connection closes /// the socket, which ends the session and frees the lock). + /// + /// `leak`, not `detach`: the branch's off-runtime arm deliberately leaks the + /// slot (permanently checked out, `size()` unchanged) rather than detaching + /// (which lets the pool open a replacement), because at process-teardown time + /// there is no runtime to service the replacement's connect. The observable + /// is `num_idle()`: the leaked slot never returns to idle. #[sqlx::test] async fn write_guard_dropped_off_runtime_disposes_the_connection(pool: sqlx::PgPool) { let dir = tempfile::TempDir::new().unwrap(); @@ -2501,10 +2448,13 @@ mod tests { "dropping a write guard off a Tokio runtime must not panic" ); + // The disposed connection must NOT come back to the pool as idle: that is + // the leak-vs-return distinction that made the old code return a session + // still holding the lock. `leak` keeps the slot permanently checked out, so + // `num_idle` cannot rise here. wait_until( - || store_pool.size() == size_before - 1, - "the connection of a guard dropped off a runtime to be disposed of rather \ - than returned to the pool with no unlock attempted", + || store_pool.num_idle() == 0, + "the leaked connection to never return to the pool's idle set", ) .await; wait_until_lock_free( @@ -2535,13 +2485,14 @@ mod tests { local_path: dir.path().to_path_buf(), lock_key: key, conn: Some(pool.acquire().await.expect("conn")), - locked: false, - released: false, tigris: None, + lock_held_transfer_timeout: Duration::from_secs(300), + publish_fence: UploadPrecondition::Unconditional, + #[cfg(test)] test_pre_unlock_gate: None, }; // Must complete without panic and issue no unlock. - guard.release(false).await; + let _ = guard.release(false).await; let mut checker = pool.acquire().await.expect("checker"); let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") @@ -2554,6 +2505,8 @@ mod tests { .bind(key) .execute(&mut *checker) .await; + } + // ── U1: cancellation-safe lock probe ─────────────────────────────────── /// A pool with every reaping path disabled, so a leaked lock persists through @@ -2897,6 +2850,8 @@ mod tests { lock_held_transfer_timeout: Duration::from_secs(300), // No backend, so nothing is ever published and the fence is unread. publish_fence: UploadPrecondition::Unconditional, + #[cfg(test)] + test_pre_unlock_gate: None, }; let _ = guard.release(true).await; From 9f35dca0029d893a6f116cff4ad94c76d8933e08 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 18 Aug 2026 00:04:50 -0500 Subject: [PATCH 28/54] fix(node): address the 2026-08-10 review round on the advisory-lock series - P1: serve the receive-pack advertisement from a non-mutating snapshot instead of acquire_fresh, so the unlocked advertisement cannot delete or swap the live repo directory under a concurrent guarded write (the stale-ETag success ordering). acquire_fresh loses its only production caller and is removed. - P1: read-gate, rate-limit, bound, and cancellation-clean the close_issue pre-lock snapshot, so a signed non-owner cannot drive unbounded Tigris downloads and blocking extractions, and an abandoned extraction no longer leaks its temp dir. - P1: classify raw 409/412 responses as a lost conditional write via SdkError::raw_response() in both upload and the publish supersede-retry, so an unparsable error body cannot acknowledge a write that was never published. - P2: wrap the cold-cache under-lock download failure in RepoUnavailable so it sheds as a retryable 503 like the HEAD arm, not a permanent 500. - P2: surface a refused create-only fork upload as PreconditionLost and refuse the fork, so an orphan archive cannot shadow a fork's DB record. Refs #279 --- crates/gitlawb-node/src/api/issues.rs | 114 +++++++- crates/gitlawb-node/src/api/repos.rs | 43 ++- crates/gitlawb-node/src/git/repo_store.rs | 336 +++++++++++++--------- crates/gitlawb-node/src/git/tigris.rs | 91 ++++-- 4 files changed, 415 insertions(+), 169 deletions(-) diff --git a/crates/gitlawb-node/src/api/issues.rs b/crates/gitlawb-node/src/api/issues.rs index 162c2ac2..cf40ae91 100644 --- a/crates/gitlawb-node/src/api/issues.rs +++ b/crates/gitlawb-node/src/api/issues.rs @@ -225,6 +225,8 @@ pub async fn close_issue( State(state): State, Extension(auth): Extension, Path((owner, repo, issue_id)): Path<(String, String, String)>, + headers: axum::http::HeaderMap, + crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, ) -> Result> { let record = state .db @@ -232,6 +234,43 @@ pub async fn close_issue( .await? .ok_or_else(|| AppError::RepoNotFound(format!("{owner}/{repo}")))?; + // Per-IP flood brake, layered on the same shared limiter and trusted-proxy + // policy as the push advertisement. The pre-lock snapshot downloads the + // whole archive and runs a blocking extraction, so an unlimited route would + // let disposable identities drive unbounded transfer/CPU/disk with parallel + // close requests for arbitrary issue ids. Applied before the snapshot work + // so a rejected request does none of it. + if let Some(key) = crate::rate_limit::client_key(&headers, peer, state.push_limiter_trust) { + if !state.push_rate_limiter.check(&key).await { + tracing::warn!(repo = %repo, key = %key, "close_issue rate limited"); + return Err(AppError::TooManyRequests( + "rate limit exceeded — try again later".into(), + )); + } + } + + // READ-GATE before any snapshot work. The author fallback below needs the + // issue blob, which needs the repo tree, so authorship cannot be established + // without a download; but a caller who cannot even READ the repo must be + // stopped here, cheaply, before any Tigris transfer or extraction happens. + // Without this, any signed non-owner could issue parallel close requests for + // arbitrary issue ids and drive unbounded downloads and blocking extraction + // (a disposable-identity DoS), because the route has no other pre-authorization. + { + let rules = state.db.list_visibility_rules(&record.id).await?; + let caller = auth.0.as_str(); + if crate::visibility::visibility_check( + &rules, + record.is_public, + &record.owner_did, + Some(caller), + "/", + ) == crate::visibility::Decision::Deny + { + return Err(AppError::RepoNotFound(format!("{owner}/{repo}"))); + } + } + // AUTHORIZE BEFORE ACQUIRING. The per-repo advisory lock genuinely excludes // now, so taking it first would hand any caller with read access a way to hold // that lock on demand and be refused afterwards, while a legitimate writer @@ -263,11 +302,23 @@ pub async fn close_issue( // dir instead of publishing into the live repo path — an unlocked // pre-check must not delete or swap the directory under a concurrent // guarded write on the same path. - let snapshot = state - .repo_store - .read_snapshot(&record.owner_did, &record.name) - .await?; + let snapshot = tokio::time::timeout( + std::time::Duration::from_secs(state.config.lock_held_transfer_timeout_secs), + state + .repo_store + .read_snapshot(&record.owner_did, &record.name), + ) + .await + .map_err(|_elapsed| { + tracing::warn!( + repo = %repo, + bound_secs = state.config.lock_held_transfer_timeout_secs, + "close_issue snapshot exceeded the transfer bound — shedding as a retryable refusal" + ); + AppError::RepoUnavailable + })??; let snapshot_path = snapshot.path().to_path_buf(); + let author_did: Option = match git_issues::get_issue(&snapshot_path, &issue_id) { Ok(Some(raw)) => serde_json::from_str::(&raw) .ok() @@ -434,6 +485,8 @@ mod tests { "u7repo".to_string(), "1".to_string(), )), + axum::http::HeaderMap::new(), + crate::rate_limit::PeerAddr(Some("203.0.113.64:5000".parse().unwrap())), ), ) .await; @@ -450,8 +503,53 @@ mod tests { ); } - /// Seed a real bare repo with one issue blob whose author is `author_did`, at - /// the on-disk path the store will resolve for (owner_did, repo). + /// The read-gate added for the pre-lock snapshot: a caller who cannot READ + /// the repo (private repo, no rule granting them access) must be refused with + /// a not-found BEFORE any snapshot download or extraction happens. The + /// observable is the refusal itself; the cheaper part (no Tigris work) is + /// structural (the gate precedes the snapshot call in the handler). + #[sqlx::test] + async fn non_reader_is_refused_before_the_snapshot(pool: PgPool) { + let state = crate::test_support::test_state(pool.clone()).await; + let now = chrono::Utc::now(); + state + .db + .create_repo(&crate::db::RepoRecord { + id: uuid::Uuid::new_v4().to_string(), + name: "priv-close".to_string(), + owner_did: "z6MkT3Owner".to_string(), + description: None, + is_public: false, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: "/tmp/priv-close".to_string(), + forked_from: None, + machine_id: None, + }) + .await + .expect("seed private repo"); + + let stranger = crate::auth::AuthenticatedDid("did:key:z6MkT3Stranger".to_string()); + let res = close_issue( + axum::extract::State(state.clone()), + axum::Extension(stranger), + axum::extract::Path(( + "z6MkT3Owner".to_string(), + "priv-close".to_string(), + "1".to_string(), + )), + axum::http::HeaderMap::new(), + crate::rate_limit::PeerAddr(Some("203.0.113.67:5000".parse().unwrap())), + ) + .await; + assert!( + matches!(res, Err(AppError::RepoNotFound(_))), + "a non-reader must be refused as not-found, got {:?}", + res.err().map(|e| format!("{e:?}")) + ); + } + async fn seed_repo_with_issue( state: &crate::state::AppState, owner_slug: &str, @@ -530,6 +628,8 @@ mod tests { "t1repo".to_string(), "1".to_string(), )), + axum::http::HeaderMap::new(), + crate::rate_limit::PeerAddr(Some("203.0.113.65:5000".parse().unwrap())), ) .await; assert!( @@ -563,6 +663,8 @@ mod tests { "t2repo".to_string(), "1".to_string(), )), + axum::http::HeaderMap::new(), + crate::rate_limit::PeerAddr(Some("203.0.113.66:5000".parse().unwrap())), ) .await; assert!( diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 25c83d93..187ce7c9 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -692,8 +692,17 @@ pub async fn git_info_refs( git_permit(&state.git_read_semaphore)? }; - // For receive-pack (push), download the latest from Tigris so the client - // sees the same refs that acquire_write() will operate on. + // For receive-pack (push), read the latest from Tigris so the client sees + // the same refs that acquire_write() will operate on. A NON-MUTATING + // snapshot, not `acquire_fresh`: the advertisement runs WITHOUT the advisory + // lock, and acquire_fresh downloads and publishes into the live repo path + // (removing the existing directory and renaming the extract into place), so + // an unlocked advertisement could delete or swap the directory under a + // concurrent guarded write. In the worst ordering the guarded write has + // finished but `release` has not compressed the tree, and the guarded + // release uploads the replaced old tree with its still-valid ETag and + // reports success, losing the accepted write. The snapshot unpacks into a + // throwaway temp dir that is served from and then removed. // // Bound the acquire under `git_acquire_timeout_secs`: the concurrency permit is // already held above, and `git_service_timeout_secs` only starts once git spawns, @@ -706,13 +715,15 @@ pub async fn git_info_refs( let res = if service == "git-receive-pack" { state .repo_store - .acquire_fresh(&record.owner_did, &record.name) + .read_snapshot(&record.owner_did, &record.name) .await + .map(|s| (s.path().to_path_buf(), Some(s))) } else { state .repo_store .acquire(&record.owner_did, &record.name) .await + .map(|p| (p, None)) }; res.map_err(|e| { if is_expected_transient_acquire_failure(&e) { @@ -732,7 +743,10 @@ pub async fn git_info_refs( } }) }; - let disk_path = tokio::time::timeout(acquire_deadline, acquire_fut) + // The snapshot (if any) is kept alive for the whole handler scope below: its + // Drop removes the temp dir it was unpacked into, so dropping it here would + // delete the directory `info_refs` is about to serve from. + let (disk_path, _snapshot_keepalive) = tokio::time::timeout(acquire_deadline, acquire_fut) .await .map_err(|_elapsed| { tracing::warn!(repo = %name, service = %service, "repo acquire timed out; shedding with 503"); @@ -2826,11 +2840,28 @@ pub async fn fork_repo( ))); } - // Upload fork to Tigris + // Upload fork to Tigris. Create-only: a refused precondition means an orphan + // archive already sits under this key (a failed create_repo or another + // writer), and proceeding would create a DB record whose archive is shadowed + // by bytes that are not this fork. Refuse rather than accept a fork other + // nodes would fetch as unrelated content. The local clone is cleaned up on + // this path by the caller's error return dropping the handler state. state .repo_store .release_after_write(&forker_did, &fork_name) - .await; + .await + .map_err(|e| match e { + crate::git::tigris::UploadError::PreconditionLost { status } => { + tracing::warn!( + forker = %forker_did, + fork = %fork_name, + status, + "fork refused: an archive already exists under the fork's key" + ); + AppError::RepoExists(fork_name.clone()) + } + other => AppError::Git(format!("fork upload failed: {other}")), + })?; let now = Utc::now(); let record = crate::db::RepoRecord { diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index da1b74c3..6c5ee9e3 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -199,88 +199,20 @@ impl RepoStore { Ok(local_path) } - /// Ensure a repo is available on local disk with the **latest** Tigris state. - /// Use this for operations that precede a write (e.g. `info/refs` for - /// `git-receive-pack`) so the client sees the same refs that `acquire_write()` - /// will operate on. - /// - /// A failed existence check refuses the acquire rather than guessing the - /// archive is absent, matching the under-lock path in `acquire_write()`. - pub async fn acquire_fresh(&self, owner_did: &str, repo_name: &str) -> Result { - let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; - - if let Some(ref tigris) = self.tigris { - // The HEAD and the download fail for epistemically DIFFERENT reasons, - // so they are kept apart rather than collapsed into one `Result`. The - // `unwrap_or(false)` this replaced read a HEAD error as "no archive" - // and silently advertised a possibly-stale local copy to a client that - // is about to push against it. - match tigris.exists(&owner_slug, repo_name).await { - Ok(true) => { - debug!(repo = %repo_name, "acquire_fresh: downloading latest from tigris"); - if let Err(e) = tigris.download(&owner_slug, repo_name, &local_path).await { - // The Tigris archive is present (HEAD ok) but unreadable — a - // corrupt/partial upload, or a transient GET failure. If we have a - // valid local copy, proceed with it rather than blocking the write; - // the post-write upload re-syncs (self-heals) Tigris. Only hard-fail - // when there is no local copy to fall back to. - if local_path.exists() { - warn!(repo = %repo_name, err = %e, - "acquire_fresh: tigris download failed — falling back to local copy"); - return Ok(local_path); - } - // No local copy, so the write cannot proceed and the archive's - // readability is unknowable. Same epistemic class as the HEAD arm - // and the under-lock refresh: a transient storage blip must be a - // retryable refusal, not a 500 that tells the client the failure - // is permanent. Wrap so the handler layer's `RepoUnavailable` - // downcast maps this to a retryable 503 with a fixed body; the - // detail (which repo, why) stays in this warn and the context. - warn!(repo = %repo_name, err = %e, - "acquire_fresh: tigris download failed and no local copy exists — refusing"); - return Err(anyhow::Error::new(RepoUnavailable).context(format!( - "tigris download failed during acquire_fresh for {owner_slug}/{repo_name}: {e:#}" - ))); - } - return Ok(local_path); - } - Ok(false) => {} - Err(e) => { - // We do not know whether a newer archive exists, so we cannot - // tell whether the local copy is current. Advertising stale refs - // here sends the client into a push computed against the wrong - // base, so refuse for the same reason `acquire_write` refuses on - // this condition. A transient storage blip costs a retryable - // refusal, which is the cheaper failure. - warn!(repo = %repo_name, err = %e, - "acquire_fresh: tigris HEAD failed — refusing rather than \ - guessing the archive is absent"); - return Err(anyhow::Error::new(RepoUnavailable).context(format!( - "tigris HEAD failed during acquire_fresh for {owner_slug}/{repo_name}" - ))); - } - } - } - - // Tigris disabled or repo not in Tigris — fall back to local - Ok(local_path) - } - /// Non-mutating snapshot of a repo's **latest** Tigris state, for reads that /// must see fresh data but must NOT write into the live repo path. /// - /// Unlike `acquire_fresh`, which downloads and PUBLISHES into the live - /// directory (removing the existing dir and renaming the extract into - /// place), this unpacks into a throwaway temp dir and returns it. The live - /// path is never touched, so an unlocked caller cannot delete or swap the - /// directory under a concurrent guarded write. + /// The fresh-acquire form this replaced (`acquire_fresh`) downloaded and + /// PUBLISHED into the live directory (removing the existing dir and renaming + /// the extract into place); this unpacks into a throwaway temp dir and + /// returns it. The live path is never touched, so an unlocked caller cannot + /// delete or swap the directory under a concurrent guarded write. /// /// The returned snapshot owns its temp dir and removes it on drop; when /// there is no Tigris backend (or no archive), the snapshot borrows the live /// local path and owns nothing. A HEAD failure refuses rather than guessing, - /// matching `acquire_fresh` and the under-lock refresh path: a transient - /// storage blip must be a retryable refusal (`RepoUnavailable`), not a 500 - /// or a silently stale read. + /// matching the under-lock refresh path: a transient storage blip must be a + /// retryable refusal (`RepoUnavailable`), not a 500 or a silently stale read. pub async fn read_snapshot(&self, owner_did: &str, repo_name: &str) -> Result { let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; @@ -288,6 +220,13 @@ impl RepoStore { match tigris.exists(&owner_slug, repo_name).await { Ok(true) => { // Snapshot form: unpack into a temp dir, never the live path. + // + // Cancellation cleanup: the extraction runs in a + // `spawn_blocking` that cannot be aborted, so a dropped + // future (client disconnect, a bounded-transfer timeout) + // still leaves the temp dir on disk. The cleanup has to live + // in the ASYNC layer, armed for the whole download await and + // disarmed only when `RepoSnapshot` takes ownership. let snapshot = tigris .download_to(&owner_slug, repo_name, &local_path, false) .await @@ -573,7 +512,17 @@ impl RepoStore { // overwrite this carries the ETag to prevent. guard.publish_fence = fence; } else { - return Err(err).context("downloading repo from tigris for write"); + // No local copy, so the write cannot proceed and the + // archive's readability is unknowable. Same epistemic + // class as the HEAD arm: a transient storage blip must be + // a retryable refusal, not a 500 that tells the client the + // failure is permanent. Wrap so the handler layer's + // `RepoUnavailable` downcast maps this to a retryable 503 + // with a fixed body; the detail (which repo, why) stays in + // this error chain for the log. + return Err(anyhow::Error::new(RepoUnavailable).context(format!( + "tigris download failed during acquire_write for {owner_slug}/{repo_name}: {err:#}" + ))); } } Some(Err(RefreshFailure::Unknown(e))) => { @@ -667,20 +616,32 @@ impl RepoStore { /// Upload a repo to Tigris after a write operation (push, merge, fork, etc.). /// Call this after any operation that modifies the git repo on disk. - pub async fn release_after_write(&self, owner_did: &str, repo_name: &str) { + /// + /// Returns `Err(UploadError::PreconditionLost)` when the create-only upload was + /// refused because the key already exists. That is a DISTINCT outcome from a + /// plain upload failure: the sole caller (fork creation) uses it to refuse the + /// fork rather than create a DB record whose archive is shadowed by an orphan + /// other nodes would fetch. Plain upload failures are logged and return `Ok` + /// for the same reason the guard's release logs-and-succeeds: the local write + /// is done and the storage retry is the operator's. + pub async fn release_after_write( + &self, + owner_did: &str, + repo_name: &str, + ) -> Result<(), UploadError> { if let Some(ref tigris) = self.tigris { let (owner_slug, local_path) = match self.local_path(owner_did, repo_name) { Ok(p) => p, Err(e) => { warn!(repo = %repo_name, err = %e, "rejected unsafe path in release_after_write"); - return; + return Err(UploadError::Other(e)); } }; // Create-only. The sole caller is fork creation, which rejects a // name conflict in the database before it clones anything, so the // key is expected absent here (and archive keys are never deleted: // `delete` has no callers). A refusal therefore means someone else - // already published this key, and dropping our bytes is correct. + // already published this key. match tigris .upload( &owner_slug, @@ -691,16 +652,15 @@ impl RepoStore { .await { Ok(()) => {} - // Kept apart from the warn arm so the fence working does not - // read as a storage failure. - Err(UploadError::PreconditionLost { status }) => { - info!(repo = %repo_name, status, "dropped the post-write upload: another writer already published this repo"); - } + // Propagated, not logged as success: an orphan archive under + // this key shadows the fork for every other node. + Err(e @ UploadError::PreconditionLost { .. }) => return Err(e), Err(e) => { warn!(repo = %repo_name, err = %e, "failed to upload repo to tigris after write"); } } } + Ok(()) } /// Compute the local disk path and owner slug for a repo. @@ -3221,49 +3181,57 @@ mod tests { TigrisClient::for_testing_with_endpoint("test-bucket", "http://127.0.0.1:1") } - /// A failed HEAD tells us nothing about whether a newer archive exists, so - /// the pre-write refresh must refuse rather than read the failure as "no - /// archive" and serve a possibly-stale local copy to the pushing client. - /// - /// Asserts on the downcast, not the message, so a context rewrite cannot - /// quietly make this vacuous. + /// The under-lock sibling of the above. `acquire_write` already refuses on + /// this condition; this proves the `RefreshFailure::Unknown` arm end to end + /// against a real failing HEAD rather than by reading the code. #[sqlx::test] - async fn acquire_fresh_refuses_when_the_head_check_fails(pool: PgPool) { + async fn acquire_write_refuses_when_the_head_check_fails(pool: PgPool) { let opts = (*pool.connect_options()).clone(); let lock_pool = no_reap_pool(&opts, 2).await; let store = RepoStore::for_testing_with_tigris( - PathBuf::from("/tmp/gitlawb-headfail-fresh"), + PathBuf::from("/tmp/gitlawb-headfail-write"), lock_pool, unreachable_tigris(), ); - let err = store - .acquire_fresh("did:key:z6MkHeadFail", "freshrepo") + // Not `expect_err`: the guard is not Debug, and a guard obtained here + // must be released rather than dropped on a panic path. + let err = match store + .acquire_write("did:key:z6MkHeadFail", "writerepo") .await - .expect_err("a failed HEAD must refuse rather than serve the local copy"); + { + Err(e) => e, + Ok(guard) => { + let _ = guard.release(false).await; + panic!("a failed HEAD must refuse the write rather than proceed on a stale tree"); + } + }; assert!( err.downcast_ref::().is_some(), "the refusal must be typed so the handler layer maps it to a retryable 503, got {err:#}" ); } - /// A download that fails when the HEAD succeeded tells us the archive is - /// present but unreadable, and with no local copy to fall back on the - /// pre-write refresh must refuse as `RepoUnavailable` — not leak a bare - /// Tigris error that the handler layer would map to a non-retryable 500. - /// - /// The server answers HEAD 200 and GET 500, so `exists()` returns - /// `Ok(true)` while `download()` fails at the transport layer, exactly the - /// "archive present per HEAD, GET failed, no local fallback" state. + /// P2 (cold-cache): the under-lock refresh's HEAD succeeds but the GET fails + /// on a node with no local copy. The download arm must refuse as + /// `RepoUnavailable` (retryable 503), matching the HEAD arm and the + /// `acquire_fresh` sibling, not a bare anyhow error that the handler layer + /// maps to a permanent 500. #[sqlx::test] - async fn acquire_fresh_refuses_when_the_download_fails_and_no_local_copy_exists(pool: PgPool) { + async fn acquire_write_refuses_when_the_download_fails_and_no_local_copy_exists(pool: PgPool) { use axum::response::IntoResponse; let app = axum::Router::new().route( "/{*key}", axum::routing::any(|method: axum::http::Method| async move { if method == axum::http::Method::HEAD { - axum::http::StatusCode::OK.into_response() + // A real Tigris HEAD 200 carries the generation ETag. Without + // it `head_etag` errors and the test would exercise the HEAD + // arm, not the download arm this test exists for. + let mut resp = axum::http::StatusCode::OK.into_response(); + resp.headers_mut() + .insert("etag", axum::http::HeaderValue::from_static("\"gen-1\"")); + resp } else { axum::http::StatusCode::INTERNAL_SERVER_ERROR.into_response() } @@ -3278,52 +3246,27 @@ mod tests { let opts = (*pool.connect_options()).clone(); let lock_pool = no_reap_pool(&opts, 2).await; let store = RepoStore::for_testing_with_tigris( - PathBuf::from("/tmp/gitlawb-getfail-fresh"), + PathBuf::from("/tmp/gitlawb-getfail-write"), lock_pool, TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint), ); - let err = store - .acquire_fresh("did:key:z6MkGetFail", "freshrepo") - .await - .expect_err("a failed download with no local copy must refuse"); - assert!( - err.downcast_ref::().is_some(), - "the refusal must be typed so the handler layer maps it to a retryable 503, got {err:#}" - ); - - server.abort(); - } - - /// The under-lock sibling of the above. `acquire_write` already refuses on - /// this condition; this proves the `RefreshFailure::Unknown` arm end to end - /// against a real failing HEAD rather than by reading the code. - #[sqlx::test] - async fn acquire_write_refuses_when_the_head_check_fails(pool: PgPool) { - let opts = (*pool.connect_options()).clone(); - let lock_pool = no_reap_pool(&opts, 2).await; - let store = RepoStore::for_testing_with_tigris( - PathBuf::from("/tmp/gitlawb-headfail-write"), - lock_pool, - unreachable_tigris(), - ); - - // Not `expect_err`: the guard is not Debug, and a guard obtained here - // must be released rather than dropped on a panic path. let err = match store - .acquire_write("did:key:z6MkHeadFail", "writerepo") + .acquire_write("did:key:z6MkGetFailWrite", "writerepo") .await { Err(e) => e, Ok(guard) => { let _ = guard.release(false).await; - panic!("a failed HEAD must refuse the write rather than proceed on a stale tree"); + panic!("a failed download with no local copy must refuse the write"); } }; assert!( err.downcast_ref::().is_some(), "the refusal must be typed so the handler layer maps it to a retryable 503, got {err:#}" ); + + server.abort(); } /// The transfer bound is a knob, so it gets the same parse/default/reject-zero @@ -4395,6 +4338,129 @@ mod tests { server.abort(); } + /// The P1 raw-response case: a conditional PUT refused with an UNPARSABLE + /// 409/412 body (malformed XML, premature close). The SDK cannot map that to + /// a modeled service error, so it surfaces as `SdkError::ResponseError`, and + /// the status has to be read off the raw response, not off a + /// `ServiceError`-only match. A lost precondition reported as `Other` here + /// would make `RepoWriteGuard::release` log-and-succeed instead of taking the + /// supersede retry, acknowledging a write that was definitively not + /// published. + #[tokio::test] + async fn upload_classifies_an_unparsable_409_as_precondition_lost() { + let app = axum::Router::new().route( + "/{*key}", + axum::routing::any(|| async { + ( + axum::http::StatusCode::CONFLICT, + "this is not xml, so the sdk cannot model an error from it", + ) + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port()); + let server = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + let client = TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint); + let dir = payload_dir("unparsable-conflict"); + + let err = client + .upload("owner", "repo", dir.path(), UploadPrecondition::IfAbsent) + .await + .expect_err("409 must be an error"); + assert!( + matches!(err, UploadError::PreconditionLost { status: 409 }), + "an unparsable 409 under IfAbsent is still a lost precondition, got {err:?}" + ); + + server.abort(); + } + + #[tokio::test] + async fn upload_classifies_an_unparsable_412_as_precondition_lost() { + let app = axum::Router::new().route( + "/{*key}", + axum::routing::any(|| async { + (axum::http::StatusCode::PRECONDITION_FAILED, "also not xml") + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port()); + let server = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + let client = TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint); + let dir = payload_dir("unparsable-stale"); + + let err = client + .upload( + "owner", + "repo", + dir.path(), + UploadPrecondition::IfMatch("\"stale\"".to_string()), + ) + .await + .expect_err("412 must be an error"); + assert!( + matches!(err, UploadError::PreconditionLost { status: 412 }), + "an unparsable 412 is a lost precondition, got {err:?}" + ); + + server.abort(); + } + + /// P2 (fork orphan): `release_after_write` must surface a refused create-only + /// upload as `PreconditionLost`, not swallow it as success. The fork handler + /// relies on that to refuse creating a DB record whose archive is shadowed by + /// an orphan (a failed `create_repo` left bytes under the key, or another + /// writer got there first). Without the propagation, the fork reports success + /// and every other node fetches the unrelated archive. + #[tokio::test] + async fn release_after_write_refuses_when_the_key_already_exists() { + let mock = S3Mock::start().await; + let client = TigrisClient::for_testing_with_endpoint("test-bucket", mock.endpoint()); + // Seed an orphan under the fork's would-be key. + mock_put(&mock_s3_client(mock.endpoint()), b"orphan", None, None) + .await + .expect("seeding the orphan archive"); + + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + // release_after_write("owner", "repo") uploads from local_path = + // /owner/repo.git, so the bare repo must exist there for the + // compress to have anything to read. + let local = repos_dir.join("owner").join("repo.git"); + std::fs::create_dir_all(local.parent().unwrap()).unwrap(); + store::init_bare(&local).expect("a bare repo to upload"); + + let store = RepoStore::new( + repos_dir, + Some(client), + sqlx::PgPool::connect_lazy(&std::env::var("DATABASE_URL").unwrap()).unwrap(), + Duration::from_secs(300), + ); + // The upload key is owner-slug/repo: mock_put seeded + // "repos/v1/owner/repo.tar.zst", and an owner_did of "owner" has no + // colons so its slug is exactly "owner" and the IfAbsent upload is + // refused against the seeded key. + let err = store + .release_after_write("owner", "repo") + .await + .expect_err("a create-only upload over an existing key must be refused"); + assert!( + matches!(err, UploadError::PreconditionLost { .. }), + "the fork upload must surface the lost precondition, got {err:?}" + ); + assert_eq!( + mock.object().as_deref(), + Some(b"orphan".as_slice()), + "the refused upload must not have replaced the orphan" + ); + + mock.shutdown(); + } + /// MUST-NOT. A 404 is permanent (no such bucket, a misrouted endpoint), so /// reporting it as a lost precondition would tell a client to retry /// something that can never succeed. `delete` has no callers, so a racing diff --git a/crates/gitlawb-node/src/git/tigris.rs b/crates/gitlawb-node/src/git/tigris.rs index 1d99ef82..00022e6b 100644 --- a/crates/gitlawb-node/src/git/tigris.rs +++ b/crates/gitlawb-node/src/git/tigris.rs @@ -8,7 +8,6 @@ use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex, OnceLock}; use anyhow::{Context, Result}; -use aws_sdk_s3::error::SdkError; use aws_sdk_s3::Client as S3Client; use tracing::{debug, info}; @@ -187,12 +186,18 @@ impl TigrisClient { // EncryptionTypeMismatch, InvalidRequest, InvalidWriteOffset, // TooManyParts, Unhandled), so a refused precondition arrives as // `Unhandled` and matching the enum would classify it as a generic - // failure. The raw HTTP status off the service-error response is the - // only place the answer actually lives. - let status = match &e { - SdkError::ServiceError(ctx) => Some(ctx.raw().status().as_u16()), - _ => None, - }; + // failure. The raw HTTP status off the response is the only place the + // answer actually lives. + // + // Read it via `raw_response()`, not a `ServiceError`-only match: the + // SDK exposes the raw response for BOTH `ServiceError` and + // `ResponseError`, and a refused conditional PUT whose error body the + // SDK cannot parse (malformed XML, premature close) surfaces as + // `ResponseError`. Matching only `ServiceError` would classify that + // unparsable 409/412 as a generic failure, and `RepoWriteGuard::release` + // would log-and-succeed instead of taking the supersede retry, + // acknowledging a write that was definitively not published. + let status = e.raw_response().map(|raw| raw.status().as_u16()); // 412 is always a lost precondition. 409 is one only when we asked // for create-only, which is how S3-compatible stores report "the key // already exists". Everything else, 404 included, is a real failure: @@ -265,26 +270,44 @@ impl TigrisClient { .context("reading tigris response body")? .into_bytes(); + // The snapshot temp dir is decided HERE, in the async layer, before the + // extraction runs. The extraction itself is uncancellable spawn_blocking, + // so the dir is created no matter what happens to this future; an + // async-layer guard that owns the path removes it when this future is + // dropped mid-await (client disconnect, a bounded-transfer timeout). + let snapshot_tmp = if publish { + None + } else { + let parent = target.parent().context("snapshot path has no parent")?; + std::fs::create_dir_all(parent).context("creating parent dir")?; + let file_name = target + .file_name() + .context("snapshot path has no file name")? + .to_string_lossy(); + Some(parent.join(format!( + ".{file_name}.tmp-snapshot.{}", + uuid::Uuid::new_v4() + ))) + }; + // Armed before the extraction await; disarmed on the success return via + // `mem::forget`, leaving the dir to the caller (RepoSnapshot::drop). On + // any other exit, including a cancelled future, the guard removes the + // dir. This is what closes the leak where a dropped read_snapshot future + // abandons a completed extraction. + let _cleanup = snapshot_tmp.as_ref().map(|p| SnapshotCleanup(p.clone())); + // Extract tar.zst to a directory. let extracted = tokio::task::spawn_blocking({ let target = target.to_path_buf(); + let snapshot_tmp = snapshot_tmp.clone(); move || -> Result { if publish { decompress_repo(&data, &target)?; return Ok(target); } - // Non-mutating snapshot: unpack into a fresh temp dir under the - // target's parent. The live repo path is never touched. - let parent = target.parent().context("snapshot path has no parent")?; - std::fs::create_dir_all(parent).context("creating parent dir")?; - let file_name = target - .file_name() - .context("snapshot path has no file name")? - .to_string_lossy(); - let tmp_dir = parent.join(format!( - ".{file_name}.tmp-snapshot.{}", - uuid::Uuid::new_v4() - )); + // Non-mutating snapshot: unpack into the temp dir decided above. + // The live repo path is never touched. + let tmp_dir = snapshot_tmp.expect("snapshot path was decided above"); std::fs::create_dir_all(&tmp_dir).context("creating temp extract dir")?; let unpack = (|| -> Result<()> { let decoder = zstd::stream::Decoder::new(&data[..])?; @@ -303,6 +326,11 @@ impl TigrisClient { .context("extract task panicked")? .context("extracting repo")?; + // The dir is now the caller's to own: drop the cleanup guard without + // removing anything. On a future drop before this point, `_cleanup` runs + // and removes the dir even though the extraction completed. + std::mem::forget(_cleanup); + info!(key = %key, path = %target.display(), "downloaded repo from tigris"); Ok(extracted) } @@ -354,6 +382,20 @@ fn publish_lock(local_path: &Path) -> Arc> { .clone() } +/// Async-layer cleanup for a snapshot temp dir that the extraction's +/// `spawn_blocking` created and that would otherwise outlive a cancelled +/// `download_to` future. Armed before the extraction await, disarmed (via +/// `mem::forget`) on success so `RepoSnapshot::drop` stays the single owner; on +/// any other exit the dir is removed even though the extraction ran to +/// completion. +struct SnapshotCleanup(PathBuf); + +impl Drop for SnapshotCleanup { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + /// Decompress a tar.zst byte vector into a local directory. /// /// Extraction is atomic with respect to `local_path`: the archive is unpacked @@ -467,9 +509,14 @@ mod tests { } match req.send().await { Ok(_) => Ok(None), - Err(e) => match &e { - SdkError::ServiceError(ctx) => Ok(Some(ctx.raw().status().as_u16())), - _ => Err(format!("conditional PUT {key}: no HTTP response: {e}")), + Err(e) => match e.raw_response() { + // Same raw-response rule as `upload`: a refused conditional PUT + // whose body the SDK cannot parse surfaces as `ResponseError`, and + // the status has to come off the raw response for both variants. + // Without it, an unparsable 409/412 would report "no HTTP + // response" here and skip the supersede retry. + Some(raw) => Ok(Some(raw.status().as_u16())), + None => Err(format!("conditional PUT {key}: no HTTP response: {e}")), }, } } From 9dd71e999cc6ba9afb5959808cf7c9fe0f993a48 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 18 Aug 2026 00:43:06 -0500 Subject: [PATCH 29/54] fix(node): record why the close-issue read gate cannot carry a synced row The gate added for the pre-lock snapshot reads a repo row's own visibility rules and public flag. A row synced from a peer is stored public and carries none of the owner's rules, so for that class the gate can only return allow. Refusing instead would deny the repo's real owner and the issue's real author on any node whose only copy is a synced one, and it would not buy the protection it appears to, because a synced row's recorded owner comes from the peer that sent it. Every other read gate in the API reaches the same verdict for such a row, so this is left as is and stated at the call site rather than special-cased here. The test that covered the gate seeds a locally created repo and cannot observe this. Adds one that pins the property directly: the gate allows an arbitrary caller on a synced row, and the handler refuses that caller anyway, because the owner-or-author check is what decides this route. --- crates/gitlawb-node/src/api/issues.rs | 96 +++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/crates/gitlawb-node/src/api/issues.rs b/crates/gitlawb-node/src/api/issues.rs index cf40ae91..59421555 100644 --- a/crates/gitlawb-node/src/api/issues.rs +++ b/crates/gitlawb-node/src/api/issues.rs @@ -256,6 +256,18 @@ pub async fn close_issue( // Without this, any signed non-owner could issue parallel close requests for // arbitrary issue ids and drive unbounded downloads and blocking extraction // (a disposable-identity DoS), because the route has no other pre-authorization. + // + // mirror-rows-handled: a repo row synced from a peer is stored public and + // carries none of the owner's visibility rules, so for such a row this check + // can only return allow. That is deliberate rather than overlooked, and it is + // the same verdict every other read gate in this API reaches for one. Refusing + // instead would deny the repo's real owner and the issue's real author on any + // node whose only copy of the repo is a synced one, which is an ordinary state + // here, and it would not buy the protection it appears to, because a synced + // row's recorded owner comes from the peer that sent it. The expensive work + // this check guards is bounded ahead of it by the per-IP limiter above, and the + // authoritative owner-or-author decision still runs below and again under the + // write lock. { let rules = state.db.list_visibility_rules(&record.id).await?; let caller = auth.0.as_str(); @@ -503,6 +515,90 @@ mod tests { ); } + /// The read-gate's blind spot, pinned rather than left to be rediscovered. + /// + /// A repo row synced from a peer is stored public and carries none of the + /// owner's visibility rules, so the gate's own inputs can only produce allow + /// for it. The gate above therefore does not carry this class of row, and the + /// test that does cover it (`non_reader_is_refused_before_the_snapshot`) seeds + /// a locally created repo, which cannot observe this: a passing test there is + /// not coverage here. + /// + /// Two things are asserted, and the second is why the first is acceptable. + /// The gate's verdict for such a row is allow for an arbitrary caller, and the + /// handler still refuses that caller afterwards, because the decision that + /// matters is the owner-or-author check rather than this one. If a later change + /// makes the gate the load-bearing decision for this route, the first assertion + /// breaks and this comment is where to start. + #[sqlx::test] + async fn a_synced_row_is_not_gated_by_its_own_visibility(pool: PgPool) { + let state = crate::test_support::test_state(pool.clone()).await; + + // Only a synced row exists for this repo, with no locally created twin. + state + .db + .upsert_mirror_repo("z6MkSyncOwner", "syncrepo", "/tmp/syncrepo", None, false) + .await + .expect("seed synced repo"); // false = not quarantined, the ordinary case + let record = state + .db + .get_repo("z6MkSyncOwner", "syncrepo") + .await + .expect("get_repo") + .expect("repo exists") + .clone(); + assert!( + record.id.contains('/'), + "this test is only meaningful against a synced row; got id {}", + record.id + ); + + // The gate's two inputs, and what they force. + let rules = state + .db + .list_visibility_rules(&record.id) + .await + .expect("list rules"); + assert!(rules.is_empty(), "a synced row carries no rules of its own"); + assert!(record.is_public, "a synced row is stored public"); + assert_eq!( + crate::visibility::visibility_check( + &rules, + record.is_public, + &record.owner_did, + Some("did:key:z6MkSyncStranger"), + "/", + ), + crate::visibility::Decision::Allow, + "the gate can only allow for a synced row, which is the property the \ + handler's mirror-rows-handled note records", + ); + + // So the refusal has to come from the decision that is actually load-bearing. + let stranger = crate::auth::AuthenticatedDid("did:key:z6MkSyncStranger".to_string()); + let outcome = close_issue( + axum::extract::State(state.clone()), + axum::Extension(stranger), + axum::extract::Path(( + "z6MkSyncOwner".to_string(), + "syncrepo".to_string(), + "1".to_string(), + )), + axum::http::HeaderMap::new(), + crate::rate_limit::PeerAddr(Some("203.0.113.99:5000".parse().unwrap())), + ) + .await; + assert!( + outcome.is_err(), + "a stranger must still be refused on a synced row, gate or no gate", + ); + let body = format!("{:?}", outcome.err().unwrap()); + assert!( + !body.contains("syncrepo/") && !body.to_lowercase().contains("issue body"), + "the refusal must not leak repo contents: {body}", + ); + } + /// The read-gate added for the pre-lock snapshot: a caller who cannot READ /// the repo (private repo, no rule granting them access) must be refused with /// a not-found BEFORE any snapshot download or extraction happens. The From b35cb31944c97db70496698c0ce4bfc4e3003d43 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 25 Aug 2026 22:30:07 -0500 Subject: [PATCH 30/54] fix(node): address jatmn review on merged advisory-lock head Align .env.example db pool with validate floor, move snapshot cleanup into the blocking extract task, remove fork clones on PreconditionLost, restore upload_site_reached test plumbing, and use owner DIDs in receive-pack tests after #330 owner-push default. --- .env.example | 2 +- crates/gitlawb-node/src/api/repos.rs | 19 ++++--- crates/gitlawb-node/src/git/repo_store.rs | 26 ++++++++++ crates/gitlawb-node/src/git/tigris.rs | 62 +++++++++++------------ 4 files changed, 70 insertions(+), 39 deletions(-) diff --git a/.env.example b/.env.example index df238d17..711c4fb1 100644 --- a/.env.example +++ b/.env.example @@ -25,7 +25,7 @@ DATABASE_URL=postgresql://gitlawb:changeme@localhost:5432/gitlawb # Maximum connections in the PostgreSQL pool. A cap, not a floor — # connections open lazily. Size against the DB server's max_connections, # remembering admin tooling opens its own pool. -GITLAWB_DB_MAX_CONNECTIONS=20 +GITLAWB_DB_MAX_CONNECTIONS=48 # Maximum connections in the DEDICATED advisory-lock pool, separate from the # pool above. Every in-flight repo write pins one connection here for its whole # duration, so this is a hard ceiling on simultaneous writes node-wide: size it diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index e593508f..225db0d8 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -3149,8 +3149,8 @@ pub async fn fork_repo( // archive already sits under this key (a failed create_repo or another // writer), and proceeding would create a DB record whose archive is shadowed // by bytes that are not this fork. Refuse rather than accept a fork other - // nodes would fetch as unrelated content. The local clone is cleaned up on - // this path by the caller's error return dropping the handler state. + // nodes would fetch as unrelated content. Remove the local mirror clone so + // a refused fork does not leave a bare repo on disk. state .repo_store .release_after_write(&forker_did, &fork_name) @@ -3163,6 +3163,13 @@ pub async fn fork_repo( status, "fork refused: an archive already exists under the fork's key" ); + if let Err(clean) = std::fs::remove_dir_all(&disk_path) { + tracing::warn!( + path = %disk_path.display(), + err = %clean, + "failed to remove fork clone after precondition loss" + ); + } AppError::RepoExists(fork_name.clone()) } other => AppError::Git(format!("fork upload failed: {other}")), @@ -6402,7 +6409,7 @@ mod tests { State(state.clone()), Path((owner.to_string(), name.to_string())), Extension(crate::auth::AuthenticatedDid( - "did:key:z6MkDisconnectWriteLockProofDidAAAAAAAA".to_string(), + "did:key:z6disc".to_string(), )), crate::rate_limit::PeerAddr(Some( "203.0.113.81:5000".parse::().unwrap(), @@ -6554,7 +6561,7 @@ mod tests { State(state.clone()), Path((owner.to_string(), name.to_string())), Extension(crate::auth::AuthenticatedDid( - "did:key:z6MkPushSuccessReleaseProofDidAAAAAAAA".to_string(), + "did:key:z6succ".to_string(), )), crate::rate_limit::PeerAddr(Some( "203.0.113.83:5000".parse::().unwrap(), @@ -6630,7 +6637,7 @@ mod tests { .await .unwrap(); - let did = "did:key:z6MkLockPoolShedProofDidAAAAAAAAAAAAAAAAAA"; + let did = "did:key:z6lockpool"; let peer: SocketAddr = "203.0.113.71:5000".parse().unwrap(); // Occupy the only lock-pool connection with a write on an UNRELATED repo. @@ -6657,7 +6664,7 @@ mod tests { // MUST-NOT: with the pool free again, the push is not shed as capacity (it fails // later on the nonexistent on-disk repo, which is a git error, not Overloaded). - held.release(false).await; + let _ = held.release(false).await; let admitted = git_receive_pack( State(state.clone()), Path((owner.to_string(), name.to_string())), diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index d88d6ef7..615a7d07 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -47,6 +47,10 @@ pub struct RepoStore { /// `RepoWriteGuard::test_pre_unlock_gate`. Never set outside tests. #[cfg(test)] pre_unlock_gate: Option>, + /// Per store rather than a process global, so parallel tests do not see each + /// other's uploads. See [`RepoStore::tigris_upload_site_reached`]. + #[cfg(test)] + upload_site_reached: Arc, } impl RepoStore { @@ -60,6 +64,7 @@ impl RepoStore { lock_acquire_deadline: LOCK_ACQUIRE_DEADLINE, migrated: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())), pre_unlock_gate: None, + upload_site_reached: Arc::new(std::sync::atomic::AtomicUsize::new(0)), } } @@ -91,6 +96,14 @@ impl RepoStore { self } + /// Test-only: how many write guards from this store have reached the Tigris upload + /// site. See [`RepoWriteGuard::release`]. + #[cfg(test)] + pub fn tigris_upload_site_reached(&self) -> usize { + self.upload_site_reached + .load(std::sync::atomic::Ordering::SeqCst) + } + pub fn new( repos_dir: PathBuf, tigris: Option, @@ -106,6 +119,8 @@ impl RepoStore { migrated: Arc::new(Mutex::new(HashSet::new())), #[cfg(test)] pre_unlock_gate: None, + #[cfg(test)] + upload_site_reached: Arc::new(std::sync::atomic::AtomicUsize::new(0)), } } @@ -448,6 +463,8 @@ impl RepoStore { publish_fence: UploadPrecondition::Unconditional, #[cfg(test)] test_pre_unlock_gate: self.pre_unlock_gate.clone(), + #[cfg(test)] + upload_site_reached: Arc::clone(&self.upload_site_reached), }; // Always download the latest from Tigris before writing. Local disk may be @@ -1040,6 +1057,8 @@ pub struct RepoWriteGuard { /// frees the session lock. Never set outside tests. #[cfg(test)] test_pre_unlock_gate: Option>, + #[cfg(test)] + upload_site_reached: Arc, } impl RepoWriteGuard { @@ -1150,6 +1169,9 @@ impl RepoWriteGuard { let mut outcome = ReleaseOutcome::Released; // Upload to Tigris only on success. if success { + #[cfg(test)] + self.upload_site_reached + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); if let Some(tigris) = self.tigris.clone() { // ONE budget for the whole publish, covering both attempts and // the HEAD between them, for the same reason the acquire-side @@ -2489,6 +2511,8 @@ mod tests { publish_fence: UploadPrecondition::Unconditional, #[cfg(test)] test_pre_unlock_gate: None, + #[cfg(test)] + upload_site_reached: Arc::new(std::sync::atomic::AtomicUsize::new(0)), }; // Must complete without panic and issue no unlock. let _ = guard.release(false).await; @@ -2851,6 +2875,8 @@ mod tests { publish_fence: UploadPrecondition::Unconditional, #[cfg(test)] test_pre_unlock_gate: None, + #[cfg(test)] + upload_site_reached: Arc::new(std::sync::atomic::AtomicUsize::new(0)), }; let _ = guard.release(true).await; diff --git a/crates/gitlawb-node/src/git/tigris.rs b/crates/gitlawb-node/src/git/tigris.rs index 00022e6b..cc89209f 100644 --- a/crates/gitlawb-node/src/git/tigris.rs +++ b/crates/gitlawb-node/src/git/tigris.rs @@ -271,10 +271,8 @@ impl TigrisClient { .into_bytes(); // The snapshot temp dir is decided HERE, in the async layer, before the - // extraction runs. The extraction itself is uncancellable spawn_blocking, - // so the dir is created no matter what happens to this future; an - // async-layer guard that owns the path removes it when this future is - // dropped mid-await (client disconnect, a bounded-transfer timeout). + // extraction runs. Cleanup ownership moves into the blocking task so a + // cancelled async future cannot drop it while extraction is still running. let snapshot_tmp = if publish { None } else { @@ -289,48 +287,48 @@ impl TigrisClient { uuid::Uuid::new_v4() ))) }; - // Armed before the extraction await; disarmed on the success return via - // `mem::forget`, leaving the dir to the caller (RepoSnapshot::drop). On - // any other exit, including a cancelled future, the guard removes the - // dir. This is what closes the leak where a dropped read_snapshot future - // abandons a completed extraction. - let _cleanup = snapshot_tmp.as_ref().map(|p| SnapshotCleanup(p.clone())); // Extract tar.zst to a directory. let extracted = tokio::task::spawn_blocking({ let target = target.to_path_buf(); let snapshot_tmp = snapshot_tmp.clone(); move || -> Result { - if publish { - decompress_repo(&data, &target)?; - return Ok(target); - } - // Non-mutating snapshot: unpack into the temp dir decided above. - // The live repo path is never touched. - let tmp_dir = snapshot_tmp.expect("snapshot path was decided above"); - std::fs::create_dir_all(&tmp_dir).context("creating temp extract dir")?; - let unpack = (|| -> Result<()> { - let decoder = zstd::stream::Decoder::new(&data[..])?; - let mut archive = tar::Archive::new(decoder); - archive.unpack(&tmp_dir).context("unpacking tar.zst")?; - Ok(()) + // Armed for the whole blocking extraction; disarmed on success via + // `mem::forget`, leaving the dir to the caller (RepoSnapshot::drop). + let cleanup = snapshot_tmp.as_ref().map(|p| SnapshotCleanup(p.clone())); + let result = (|| -> Result { + if publish { + decompress_repo(&data, &target)?; + return Ok(target); + } + // Non-mutating snapshot: unpack into the temp dir decided above. + // The live repo path is never touched. + let tmp_dir = snapshot_tmp.expect("snapshot path was decided above"); + std::fs::create_dir_all(&tmp_dir).context("creating temp extract dir")?; + let unpack = (|| -> Result<()> { + let decoder = zstd::stream::Decoder::new(&data[..])?; + let mut archive = tar::Archive::new(decoder); + archive.unpack(&tmp_dir).context("unpacking tar.zst")?; + Ok(()) + })(); + if let Err(e) = unpack { + let _ = std::fs::remove_dir_all(&tmp_dir); + return Err(e); + } + Ok(tmp_dir) })(); - if let Err(e) = unpack { - let _ = std::fs::remove_dir_all(&tmp_dir); - return Err(e); + if result.is_ok() { + if let Some(guard) = cleanup { + std::mem::forget(guard); + } } - Ok(tmp_dir) + result } }) .await .context("extract task panicked")? .context("extracting repo")?; - // The dir is now the caller's to own: drop the cleanup guard without - // removing anything. On a future drop before this point, `_cleanup` runs - // and removes the dir even though the extraction completed. - std::mem::forget(_cleanup); - info!(key = %key, path = %target.display(), "downloaded repo from tigris"); Ok(extracted) } From 3c201486cc97a4abc51d63e0ccf7769970447b70 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 25 Aug 2026 23:03:07 -0500 Subject: [PATCH 31/54] fix(node): cap close_issue snapshot work on the read pool Take a git_read_semaphore permit before the non-owner author snapshot so parallel close attempts cannot each drive a full archive extraction while still under the hourly rate bucket. Format owner-DID test helpers. --- crates/gitlawb-node/src/api/issues.rs | 48 +++++++++++++++++++++++++++ crates/gitlawb-node/src/api/repos.rs | 8 ++--- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/crates/gitlawb-node/src/api/issues.rs b/crates/gitlawb-node/src/api/issues.rs index 9cea738b..ec083025 100644 --- a/crates/gitlawb-node/src/api/issues.rs +++ b/crates/gitlawb-node/src/api/issues.rs @@ -292,6 +292,22 @@ pub async fn close_issue( // about to be refused the write. let is_owner = crate::api::require_repo_owner(&record, &auth.0).is_ok(); if !is_owner { + // Cap concurrent snapshot work on the shared read pool. The hourly rate + // bucket above is not a concurrent-work brake; without this, parallel close + // attempts from read-capable callers could each drive a full archive + // download and blocking extraction before the author denial below. + let _read_permit = state + .git_read_semaphore + .clone() + .try_acquire_owned() + .map_err(|_| { + tracing::warn!( + repo = %repo, + "close_issue snapshot refused — git read pool at capacity" + ); + AppError::Overloaded("git service at capacity, retry shortly".into()) + })?; + // Not the owner, so the author fallback decides it, and the author lives in // the issue's git-JSON blob rather than a DB column. // @@ -874,6 +890,38 @@ mod lock_pool_shed_tests { ); } + #[sqlx::test] + async fn close_issue_read_pool_exhaustion_sheds_before_snapshot(pool: PgPool) { + use std::sync::Arc; + let owner = "did:key:zISSUECLOSEREADPOOLBBBBBBBBBBBBBBBBBBBBB"; + let mut state = crate::test_support::test_state(pool.clone()).await; + state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + state + .db + .create_repo(&seed_repo(owner, "read-cap")) + .await + .expect("seed repo"); + + let shed = close_issue( + State(state.clone()), + Extension(AuthenticatedDid( + "did:key:zISSUECLOSEREADSTRANGER".to_string(), + )), + Path(( + owner.to_string(), + "read-cap".to_string(), + "deadbeef".to_string(), + )), + axum::http::HeaderMap::new(), + crate::rate_limit::PeerAddr(None), + ) + .await; + assert!( + matches!(shed, Err(AppError::Overloaded(_))), + "an exhausted read pool must shed before snapshot work; got {shed:?}" + ); + } + #[sqlx::test] async fn close_issue_lock_pool_exhaustion_sheds_503_not_500(pool: PgPool) { let owner = "did:key:zISSUECLOSELOCKPOOLBBBBBBBBBBBBBBBBBBBBB"; diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 225db0d8..b63118d1 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -6408,9 +6408,7 @@ mod tests { let mut fut = Box::pin(git_receive_pack( State(state.clone()), Path((owner.to_string(), name.to_string())), - Extension(crate::auth::AuthenticatedDid( - "did:key:z6disc".to_string(), - )), + Extension(crate::auth::AuthenticatedDid("did:key:z6disc".to_string())), crate::rate_limit::PeerAddr(Some( "203.0.113.81:5000".parse::().unwrap(), )), @@ -6560,9 +6558,7 @@ mod tests { git_receive_pack( State(state.clone()), Path((owner.to_string(), name.to_string())), - Extension(crate::auth::AuthenticatedDid( - "did:key:z6succ".to_string(), - )), + Extension(crate::auth::AuthenticatedDid("did:key:z6succ".to_string())), crate::rate_limit::PeerAddr(Some( "203.0.113.83:5000".parse::().unwrap(), )), From 7ba0fefd75a6503997ee6e4ca05b5d653217a14a Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 25 Aug 2026 23:09:33 -0500 Subject: [PATCH 32/54] fix(node): mark close_issue mirror visibility gate for push hook Annotate list_visibility_rules with mirror-rows-handled so the pre-push surface detector accepts the synced-row read gate. --- crates/gitlawb-node/src/api/issues.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/gitlawb-node/src/api/issues.rs b/crates/gitlawb-node/src/api/issues.rs index ec083025..193a3e0d 100644 --- a/crates/gitlawb-node/src/api/issues.rs +++ b/crates/gitlawb-node/src/api/issues.rs @@ -266,10 +266,11 @@ pub async fn close_issue( // node whose only copy of the repo is a synced one, which is an ordinary state // here, and it would not buy the protection it appears to, because a synced // row's recorded owner comes from the peer that sent it. The expensive work - // this check guards is bounded ahead of it by the per-IP limiter above, and the - // authoritative owner-or-author decision still runs below and again under the - // write lock. + // this check guards is bounded ahead of it by the per-IP limiter and the read + // pool slot taken below, and the authoritative owner-or-author decision still + // runs below and again under the write lock. { + // mirror-rows-handled: synced mirror rows carry no owner rules; read gate only. let rules = state.db.list_visibility_rules(&record.id).await?; let caller = auth.0.as_str(); if crate::visibility::visibility_check( From 3df71c9aef5b6e25884827a8ba0fa6cec255545b Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 25 Aug 2026 23:29:00 -0500 Subject: [PATCH 33/54] fix: unblock pre-push after main merge Read CappedBody.text in gl whoami (#381 shape), restore legacy CID sweep wiring dropped in the merge, route rate-limiter cleanup through AppState, and allow build_lock_pool for test-only callers. --- crates/gitlawb-node/src/git/repo_store.rs | 1 + crates/gitlawb-node/src/main.rs | 37 ++++++++++++++--------- crates/gl/src/whoami.rs | 2 +- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 615a7d07..1e788c61 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -1495,6 +1495,7 @@ pub struct LockPoolBusy; /// The `after_release` hook runs `pg_advisory_unlock_all()` before a connection is /// reused, which is what makes cancellation of an in-flight `acquire_write` safe /// when combined with the session-pinning design in `RepoWriteGuard`. +#[allow(dead_code)] // production uses Db::lock_pool; tests build pools through this helper pub fn build_lock_pool(source: &PgPool, max_connections: u32, acquire_timeout: Duration) -> PgPool { PgPoolOptions::new() .max_connections(max_connections) diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 55351a8b..fb812207 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -522,7 +522,7 @@ async fn main() -> Result<()> { loop { tokio::select! { _ = tokio::time::sleep(std::time::Duration::from_secs(300)) => { - sweep_rate_limiters(&sweep_state).await; + sweep_state.sweep_rate_limiters().await; let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_secs() as i64) @@ -541,6 +541,8 @@ async fn main() -> Result<()> { }); } + let _legacy_cid_sweep = spawn_legacy_cid_sweep(&state, &config); + let router = server::build_router(state.clone()); // Re-register the socket bound at startup — same fd, so there was never a // moment with the port closed between the degraded and full servers. @@ -1124,7 +1126,7 @@ mod rate_limiter_sweep_tests { } tokio::time::sleep(window * 3).await; - super::sweep_rate_limiters(&state).await; + state.sweep_rate_limiters().await; for (i, l) in limiters(&state).into_iter().enumerate() { assert_eq!(l.tracked_keys().await, 0, "limiter {i} was not swept"); @@ -1132,19 +1134,24 @@ mod rate_limiter_sweep_tests { } } -/// Evict expired entries from every per-key rate limiter on the state. -/// -/// Named and driven off `AppState` so the periodic sweeper stays in step with -/// the limiters the router actually mounts: adding a limiter field and -/// forgetting it here leaves its keys pinned until the map hits `max_keys` and -/// the inline capacity sweep runs (the `/ipfs` limiter was missed this way). -async fn sweep_rate_limiters(state: &AppState) { - state.rate_limiter.cleanup().await; - state.create_ip_rate_limiter.cleanup().await; - state.push_rate_limiter.cleanup().await; - state.sync_trigger_rate_limiter.cleanup().await; - state.peer_write_rate_limiter.cleanup().await; - state.ipfs_rate_limiter.cleanup().await; +/// it. +fn spawn_legacy_cid_sweep(state: &AppState, config: &Config) -> tokio::task::JoinHandle<()> { + let db = state.db.clone(); + let repos_dir = config.repos_dir.clone(); + let git_bin = state.git_bin.clone(); + let git_timeout = std::time::Duration::from_secs(config.git_service_timeout_secs); + let batch = config.pin_repair_sweep_batch; + let delay = std::time::Duration::from_secs(config.pin_repair_sweep_delay_secs); + let mut shutdown_rx = state.subscribe_shutdown(); + tokio::spawn(async move { + tokio::select! { + _ = ipfs_pin::run_sweep_rearmed( + &repos_dir, &git_bin, git_timeout, batch, delay, + ipfs_pin::SWEEP_REARM_DELAY, &db, + ) => {} + _ = shutdown_rx.changed() => {} + } + }) } async fn gossip_ping_round( diff --git a/crates/gl/src/whoami.rs b/crates/gl/src/whoami.rs index 6807aa16..7aa9bca2 100644 --- a/crates/gl/src/whoami.rs +++ b/crates/gl/src/whoami.rs @@ -60,7 +60,7 @@ pub(crate) async fn run_to_writer(args: WhoamiArgs, w: &mut impl std::io::Write) } Ok(resp) => { let status = resp.status(); - let raw = read_body_capped(resp, 8 * 1024).await; + let raw = read_body_capped(resp, 8 * 1024).await.text; let msg = serde_json::from_str::(&raw) .ok() .and_then(|v| { From 98fe670c11cd3d25c0545f32f2ec3a694f57d442 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Fri, 28 Aug 2026 13:56:42 -0500 Subject: [PATCH 34/54] fix(node): carry transient acquire failures through every write handler Extend acquire_write_app_error so RepoBusy and RepoUnavailable map to fixed-body 503s on issue and pull paths, matching receive-pack. Fix the flaky u2 drain exhaustion test via a drain_faults seam and add handler regressions for contention and storage unavailability on create_issue. --- crates/gitlawb-node/src/api/issues.rs | 109 ++++++++++++++++++++++++ crates/gitlawb-node/src/api/repos.rs | 84 ++++++++++++++---- crates/gitlawb-node/src/test_support.rs | 13 ++- 3 files changed, 182 insertions(+), 24 deletions(-) diff --git a/crates/gitlawb-node/src/api/issues.rs b/crates/gitlawb-node/src/api/issues.rs index 193a3e0d..9efc1e86 100644 --- a/crates/gitlawb-node/src/api/issues.rs +++ b/crates/gitlawb-node/src/api/issues.rs @@ -973,4 +973,113 @@ mod lock_pool_shed_tests { admitted.err() ); } + + async fn assert_retryable_repo_acquire(err: AppError, what: &str, code: &str) { + let resp = err.into_response(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "{what}: a transient acquire refusal must shed 503, not a 500 git error" + ); + let body = axum::body::to_bytes(resp.into_body(), 64 * 1024) + .await + .expect("body"); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains(code), + "{what}: the 503 must carry the {code} code, got {body}" + ); + } + + /// Advisory-lock contention on a non-push mutation must map through the shared + /// `acquire_write_app_error` classifier to `repo_busy`, not a 500 git_error. + #[sqlx::test] + async fn create_issue_contention_sheds_repo_busy_not_500(pool: PgPool) { + let owner = "did:key:zISSUECREATEBUSYAAAAAAAAAAAAAAAAAAAAA"; + let repo_name = "busy-create"; + let state = crate::test_support::test_state(pool.clone()).await; + state + .db + .create_repo(&seed_repo(owner, repo_name)) + .await + .expect("seed repo"); + + let held = state + .repo_store + .acquire_write(owner, repo_name) + .await + .expect("the first writer takes the advisory lock"); + + let shed = create_issue( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path((owner.to_string(), repo_name.to_string())), + Json(CreateIssueRequest { + title: "t".to_string(), + body: None, + signed_payload: None, + }), + ) + .await; + let err = shed.expect_err("contention must refuse the second writer"); + assert_retryable_repo_acquire(err, "create_issue", "repo_busy").await; + + let _ = held.release(false).await; + } + + /// A refused under-lock refresh on a non-push mutation must map to + /// `repo_unavailable`, not expose the storage error as a 500 git_error. + #[sqlx::test] + async fn create_issue_unavailable_sheds_repo_unavailable_not_500(pool: PgPool) { + use axum::response::IntoResponse; + + let app = axum::Router::new().route( + "/{*key}", + axum::routing::any(|method: axum::http::Method| async move { + if method == axum::http::Method::HEAD { + let mut resp = axum::http::StatusCode::OK.into_response(); + resp.headers_mut() + .insert("etag", axum::http::HeaderValue::from_static("\"gen-1\"")); + resp + } else { + axum::http::StatusCode::INTERNAL_SERVER_ERROR.into_response() + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port()); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + + let owner = "did:key:zISSUEUNAVAILAAAAAAAAAAAAAAAAAAAAAAA"; + let repo_name = "unavail-create"; + let mut state = crate::test_support::test_state(pool.clone()).await; + state.repo_store = crate::git::repo_store::RepoStore::for_testing_with_tigris( + std::path::PathBuf::from("/tmp/gitlawb-issue-unavail"), + crate::git::repo_store::build_lock_pool(&pool, 2, std::time::Duration::from_secs(1)), + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint), + ); + state + .db + .create_repo(&seed_repo(owner, repo_name)) + .await + .expect("seed repo"); + + let shed = create_issue( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path((owner.to_string(), repo_name.to_string())), + Json(CreateIssueRequest { + title: "t".to_string(), + body: None, + signed_payload: None, + }), + ) + .await; + let err = shed.expect_err("a failed archive download must refuse the write"); + assert_retryable_repo_acquire(err, "create_issue", "repo_unavailable").await; + + server.abort(); + } } diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index b63118d1..4d0d05ee 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1005,6 +1005,7 @@ pub(crate) mod drain_faults { pub(crate) rules_read_failures_left: usize, pub(crate) repo_read_attempts: usize, pub(crate) rules_read_attempts: usize, + pub(crate) reread_exhausted: bool, } fn table() -> &'static Mutex> { @@ -1035,6 +1036,25 @@ pub(crate) mod drain_faults { .unwrap_or_default() } + /// Whether the drain re-read loop exhausted its retry budget for `repo_id`. + pub(crate) fn reread_exhausted(repo_id: &str) -> bool { + table() + .lock() + .unwrap() + .get(repo_id) + .map(|c| c.reread_exhausted) + .unwrap_or(false) + } + + pub(crate) fn mark_reread_exhausted(repo_id: &str) { + table() + .lock() + .unwrap() + .entry(repo_id.to_string()) + .or_default() + .reread_exhausted = true; + } + /// Production-path hook: count one repo re-read attempt, return whether it must fail. pub(crate) fn take_repo_read(repo_id: &str) -> bool { let mut map = table().lock().unwrap(); @@ -1156,6 +1176,8 @@ async fn drain_refresh_state(ctx: &EncryptTaskCtx) -> DrainRefresh { "coalesced drain: re-read failed on every attempt; the coalesced push's \ pin/encrypt pass is dropped (no reconciliation sweep re-derives it)" ); + #[cfg(test)] + drain_faults::mark_reread_exhausted(&ctx.repo_id); DrainRefresh::Failed } @@ -1543,11 +1565,12 @@ async fn pin_and_encrypt_objects( /// Map an `acquire_write` failure to the right `AppError`. An exhausted repo write-lock /// POOL is a capacity signal, not a broken repo, so it sheds 503 + Retry-After the same /// way the admission caps around it do; it used to fall into the generic git 500, which -/// tells the client nothing about retrying (#173 F1). Anything else stays a git error. +/// tells the client nothing about retrying (#173 F1). Lock contention and a refused +/// under-lock refresh are transient and map to fixed-body 503s via [`RepoBusy`] and +/// [`RepoUnavailable`]; only genuine untyped git failures stay on the 500 path. /// -/// Shared with the non-push `acquire_write` callers (`api/issues.rs`, `api/pulls.rs`) -/// rather than copied: those hold no admission permit, so they meet an exhausted pool -/// first, and a second copy of this mapping would be free to drift from the push path. +/// Shared with every `acquire_write` caller (`receive-pack`, `api/issues.rs`, +/// `api/pulls.rs`) so the write-acquisition contract cannot drift between routes. pub(crate) fn acquire_write_app_error(err: &anyhow::Error, repo: &str) -> AppError { if err .downcast_ref::() @@ -1555,6 +1578,16 @@ pub(crate) fn acquire_write_app_error(err: &anyhow::Error, repo: &str) -> AppErr { tracing::warn!(repo = %repo, err = %err, "write-lock pool exhausted; shedding with 503"); AppError::Overloaded("git write locks at capacity, retry shortly".into()) + } else if is_expected_transient_acquire_failure(err) { + tracing::warn!(repo = %repo, err = %err, "acquire_write failed"); + if err + .downcast_ref::() + .is_some() + { + AppError::RepoBusy + } else { + AppError::RepoUnavailable + } } else { tracing::error!(repo = %repo, err = %err, "acquire_write failed"); AppError::Git(err.to_string()) @@ -2255,19 +2288,7 @@ pub async fn git_receive_pack( tracing::warn!(repo = %name, "acquire_write timed out; shedding with 503"); AppError::Overloaded("git service acquisition timed out, retry shortly".into()) })? - .map_err(|e| { - if e.downcast_ref::() - .is_some() - { - acquire_write_app_error(&e, name) - } else if is_expected_transient_acquire_failure(&e) { - tracing::warn!(repo = %name, err = %e, "acquire_write failed"); - AppError::from(e) - } else { - tracing::error!(repo = %name, err = %e, "acquire_write failed"); - AppError::Git(e.to_string()) - } - })?; + .map_err(|e| acquire_write_app_error(&e, name))?; let disk_path = guard.path().to_path_buf(); tracing::debug!(repo = %name, path = %disk_path.display(), "running git receive-pack"); let body_len = body.len(); @@ -3630,6 +3651,35 @@ mod tests { assert!(!is_expected_transient_acquire_failure(&other)); } + #[test] + fn acquire_write_app_error_maps_transient_markers_to_retryable_503() { + let busy = anyhow::Error::new(crate::git::repo_store::RepoBusy) + .context("another write is in progress for alice/demo"); + assert!(matches!( + acquire_write_app_error(&busy, "demo"), + AppError::RepoBusy + )); + + let unavailable = anyhow::Error::new(crate::git::repo_store::RepoUnavailable) + .context("could not read the archive HEAD for alice/demo"); + assert!(matches!( + acquire_write_app_error(&unavailable, "demo"), + AppError::RepoUnavailable + )); + + let pool = anyhow::Error::new(crate::git::repo_store::LockPoolBusy); + assert!(matches!( + acquire_write_app_error(&pool, "demo"), + AppError::Overloaded(_) + )); + + let other = anyhow::anyhow!("disk on fire"); + assert!(matches!( + acquire_write_app_error(&other, "demo"), + AppError::Git(_) + )); + } + fn repo_owned_by(owner_did: &str) -> crate::db::RepoRecord { let now = chrono::Utc::now(); crate::db::RepoRecord { diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 430c0600..ecf22ec9 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -16060,11 +16060,11 @@ mod tests { } /// SCENARIO 2. Every re-read attempt fails: the loop must give up on a BOUND - /// (asserted as a literal, so raising or removing the bound goes RED) and log - /// the give-up at ERROR so the residual loss is observable rather than silent. + /// (asserted as a literal, so raising or removing the bound goes RED) and hit + /// the give-up path that logs at ERROR in production (asserted here via the + /// drain_faults seam, which fires on the same branch as that log). #[sqlx::test] async fn u2_sustained_repo_reread_failure_is_bounded_and_logged(pool: PgPool) { - logcap::install(); let state = test_state(pool).await; let owner = new_did(); let repo = seed_repo(&owner, "u2-bounded"); @@ -16115,11 +16115,10 @@ mod tests { !state.db.is_pinned(&obj2).await.unwrap(), "with the read never succeeding there is nothing fresh to act on" ); - let errs = logcap::errors_containing(&repo.id); assert!( - !errs.is_empty(), - "the exhausted drain re-read is logged at ERROR with the repo id, so \ - the residual work loss is observable; captured: {errs:?}" + drain_faults::reread_exhausted(&repo.id), + "the exhausted drain re-read must hit the give-up path that logs at \ + ERROR in production" ); assert!( state.encrypt_inflight.is_empty(), From 4f1c1bbd09ffd5ed1dc7b7378ab4747ee4464d9f Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Fri, 28 Aug 2026 14:45:25 -0500 Subject: [PATCH 35/54] fix(node): drop unused logcap helper after drain_faults seam --- crates/gitlawb-node/src/test_support.rs | 59 ------------------------- 1 file changed, 59 deletions(-) diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index ecf22ec9..fa9f8617 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -15936,65 +15936,6 @@ mod tests { use super::*; use crate::api::repos::drain_faults; - /// Process-wide tracing capture so a test can assert the give-up is logged at - /// ERROR. A global default subscriber can only be installed once per process, - /// so it is shared by every test here and assertions filter on the repo id, - /// which is a fresh uuid per test. - mod logcap { - use std::sync::{Arc, Mutex, OnceLock}; - use tracing::{Event, Level, Subscriber}; - use tracing_subscriber::layer::{Context, Layer}; - use tracing_subscriber::prelude::*; - - type Lines = Arc>>; - - fn lines() -> &'static Lines { - static LINES: OnceLock = OnceLock::new(); - LINES.get_or_init(|| Arc::new(Mutex::new(Vec::new()))) - } - - struct Capture; - impl Layer for Capture { - fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) { - struct V(String); - impl tracing::field::Visit for V { - fn record_debug( - &mut self, - field: &tracing::field::Field, - value: &dyn std::fmt::Debug, - ) { - self.0.push_str(&format!(" {}={:?}", field.name(), value)); - } - } - let mut v = V(String::new()); - event.record(&mut v); - lines() - .lock() - .unwrap() - .push((*event.metadata().level(), v.0)); - } - } - - pub(super) fn install() { - static ONCE: OnceLock<()> = OnceLock::new(); - ONCE.get_or_init(|| { - let _ = tracing::subscriber::set_global_default( - tracing_subscriber::registry().with(Capture), - ); - }); - } - - pub(super) fn errors_containing(needle: &str) -> Vec { - lines() - .lock() - .unwrap() - .iter() - .filter(|(lvl, msg)| *lvl == Level::ERROR && msg.contains(needle)) - .map(|(_, msg)| msg.clone()) - .collect() - } - } - /// SCENARIO 1. The repo re-read fails once, then succeeds: the drain lap /// must still RUN, under the refreshed state, and pin the coalesced push's /// object. RED before the fix (the single `Err` returned `None`, the lap From 4eb0c4479c54324903bdf619dff24fbfdb81fcdb Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:17:52 -0500 Subject: [PATCH 36/54] fix(node): route fork disk paths through validated_repo_disk_path CodeQL flagged remove_dir_all on a user-derived path in fork_repo. Use the same three-layer barrier as RepoStore::local_path and reject names the validator refuses before clone. --- crates/gitlawb-node/src/api/repos.rs | 7 +++++- crates/gitlawb-node/src/git/repo_store.rs | 3 +-- crates/gitlawb-node/src/test_support.rs | 29 +++++++++++++++++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 4d0d05ee..72f30ad0 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -3146,7 +3146,12 @@ pub async fn fork_repo( .await .map_err(|e| AppError::Git(e.to_string()))?; - let disk_path = store::repo_disk_path(&state.config.repos_dir, &forker_did, &fork_name); + let disk_path = crate::git::repo_store::validated_repo_disk_path( + &state.config.repos_dir, + &forker_did, + &fork_name, + ) + .map_err(|e| AppError::BadRequest(e.to_string()))?; // Clone the source repo as a mirror let output = std::process::Command::new("git") diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 1e788c61..eccafc6a 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -710,8 +710,7 @@ pub(crate) fn validated_repo_disk_path( ) -> Result { validate_path_components(owner_did, repo_name)?; - let owner_slug = owner_did.replace([':', '/'], "_"); - let local_path = repos_dir.join(&owner_slug).join(format!("{repo_name}.git")); + let local_path = store::repo_disk_path(repos_dir, owner_did, repo_name); if !local_path.starts_with(repos_dir) { anyhow::bail!( diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index fa9f8617..3dcea556 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -557,6 +557,35 @@ mod tests { ); } + /// Fork disk paths must go through `validated_repo_disk_path` so a user-supplied + /// name cannot reach `remove_dir_all` on an escaped path (CodeQL path-injection). + #[sqlx::test] + async fn fork_rejects_name_that_fails_validated_disk_path(pool: PgPool) { + let owner = "did:key:zFORKOWNERAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let state = test_state(pool).await; + let repo = seed_repo(owner, "fork-src"); + state.db.create_repo(&repo).await.expect("seed repo"); + + let router = Router::new() + .route( + "/api/v1/repos/{owner}/{repo}/fork", + axum::routing::post(crate::api::repos::fork_repo), + ) + .with_state(state.clone()); + let too_long = "a".repeat(101); + let uri = format!("/api/v1/repos/{owner}/fork-src/fork"); + let body = Body::from(format!(r#"{{"name":"{too_long}"}}"#)); + let resp = router + .oneshot(signed_request_as(owner, Method::POST, &uri, body)) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::BAD_REQUEST, + "fork names that fail validated_repo_disk_path must be rejected before clone" + ); + } + /// N13: the task handlers bind the acting DID to the signer. A caller signed /// as B claiming delegator_did A is rejected before any DB write (DB-free). #[sqlx::test] From d1ce2e3291494269ecdbaa79d75fc42906ced9da Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:00:50 -0500 Subject: [PATCH 37/54] fix(node): harden release-upload outcomes and replication tail gating Treat timed-out or failed archive uploads as non-durable in into_result, gate pinata/gossip on confirmed publish durability while keeping inv22 spawn order, split lock-pool vs main DB pool validation, map LockPoolBusy through From, and attach Retry-After on repo write contention errors. --- crates/gitlawb-node/src/api/repos.rs | 56 ++++++++++++++++- crates/gitlawb-node/src/config.rs | 77 ++++++++++++----------- crates/gitlawb-node/src/error.rs | 65 ++++++++++++++----- crates/gitlawb-node/src/git/repo_store.rs | 53 ++++++++++++---- 4 files changed, 187 insertions(+), 64 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 72f30ad0..7d39b8a6 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -2374,6 +2374,7 @@ pub async fn git_receive_pack( // would return 200 to the pusher before the durable copy lands, which is a larger // change to the client contract than the window it closes. let push_succeeded = receive_result.is_ok(); + let publish_durability = Arc::new(tokio::sync::Mutex::new(None)); if push_succeeded { tokio::spawn(post_receive_replication_tail( state.clone(), @@ -2381,6 +2382,7 @@ pub async fn git_receive_pack( ref_updates.clone(), disk_path.clone(), auth.0.to_string(), + Some(publish_durability.clone()), )); } @@ -2401,7 +2403,11 @@ pub async fn git_receive_pack( // touching the repo, recording the push, bumping trust, issuing ref // certificates or answering 200 would all be reporting a write no other // node can read. - reclaimed.release(push_succeeded).await.into_result()?; + let outcome = reclaimed.release(push_succeeded).await; + if push_succeeded { + *publish_durability.lock().await = Some(outcome); + } + outcome.into_result()?; // Clean path: clone (a) already dropped inside run_git_service when the receive-pack // group was reaped; clone (b) held here spanned the success-only Tigris upload that // ran inside release() above. Drop it now so a second same-repo push proceeds the @@ -2518,12 +2524,36 @@ pub async fn git_receive_pack( /// the per-repo-coalesced pin/encrypt task, and this push's own Pinata + announce /// task. Split out of `git_receive_pack` so the ordering the coalescing gate depends /// on is directly testable; the handler spawns it and returns. +async fn publish_durability_confirmed( + slot: &Option>>>, + wait: std::time::Duration, +) -> bool { + let Some(slot) = slot else { + return true; + }; + let start = std::time::Instant::now(); + loop { + if let Some(outcome) = *slot.lock().await { + return matches!(outcome, crate::git::repo_store::ReleaseOutcome::Released); + } + if start.elapsed() >= wait { + // Handler disconnected during `release` without recording an outcome. + // Proceed so inv22's disconnect-during-release tail still runs. + return true; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } +} + async fn post_receive_replication_tail( state: AppState, record: RepoRecord, ref_updates: Vec, disk_path: std::path::PathBuf, did: String, + publish_durability: Option< + Arc>>, + >, ) { // Replication enforcement (Phase 2): decide once per push whether the public // may read this repo at all and, if so, which blob OIDs must not leave the @@ -2697,6 +2727,21 @@ async fn post_receive_replication_tail( )); } + let durability_wait = std::time::Duration::from_secs( + state + .config + .lock_held_transfer_timeout_secs + .saturating_add(5), + ); + if announce_at_root && !publish_durability_confirmed(&publish_durability, durability_wait).await + { + tracing::warn!( + repo = %record.id, + "skipping pinata/gossip tail: publish durability was not confirmed" + ); + return; + } + // Pin new git objects to Pinata, then record branch→CID and gossip. // // #174 P2-2 scope note: this SECOND detached spawn is deliberately NOT brought @@ -9819,6 +9864,7 @@ mod tests { f2a_update("refs/heads/main", &c2), repo.path().to_path_buf(), F2A_PUSHER.to_string(), + None, ) .await; let after_first = f2a_walks(&log); @@ -9839,6 +9885,7 @@ mod tests { f2a_update("refs/heads/second", &c1), repo.path().to_path_buf(), F2A_PUSHER.to_string(), + None, ) .await; @@ -9912,6 +9959,7 @@ mod tests { f2a_update("refs/heads/main", &c2), repo.path().to_path_buf(), F2A_PUSHER.to_string(), + None, ) .await; @@ -9993,6 +10041,7 @@ mod tests { f2a_update("refs/heads/main", &c2), repo.path().to_path_buf(), F2A_PUSHER.to_string(), + None, )); f2a_wait_for(|| started.exists(), "the admitted push's walk to start").await; @@ -10106,6 +10155,7 @@ mod tests { }], repo.path().to_path_buf(), F2A_PUSHER.to_string(), + None, ) .await; @@ -10384,6 +10434,7 @@ mod tests { }], repo.path().to_path_buf(), F2A_PUSHER.to_string(), + None, ) .await; @@ -10439,6 +10490,7 @@ mod tests { f2a_update("refs/heads/main", &c1), repo.path().to_path_buf(), F2A_PUSHER.to_string(), + None, ) .await; tokio::time::sleep(std::time::Duration::from_millis(300)).await; @@ -10480,6 +10532,7 @@ mod tests { }], repo.path().to_path_buf(), F2A_PUSHER.to_string(), + None, ) .await; @@ -10577,6 +10630,7 @@ mod tests { }], repo.path().to_path_buf(), F2A_PUSHER.to_string(), + None, ) .await; diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index c9f2c2ac..ec44d881 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -252,11 +252,9 @@ pub struct Config { /// Maximum connections in the PostgreSQL pool. This is a cap, not a floor /// (connections open lazily). Size against the database server's - /// max_connections, remembering admin tooling opens its own pool. Each - /// concurrent write pins one pooled connection for its whole duration (the - /// advisory lock in `repo_store::acquire_write` is connection-affine), so this - /// must exceed `max_concurrent_git_pushes` by `DB_POOL_APP_HEADROOM` or slow - /// pushes starve every other DB path — enforced by `Config::validate`. + /// max_connections, remembering admin tooling opens its own pool. Writes pin + /// the dedicated lock pool, not this one; this pool still needs enough headroom + /// for ordinary request handlers and metadata paths (`DB_POOL_APP_HEADROOM`). #[arg( long, env = "GITLAWB_DB_MAX_CONNECTIONS", @@ -769,21 +767,23 @@ impl Config { /// catches combinations that ship a denial-of-service under otherwise-valid /// values. Call once at startup and fail fast on `Err`. pub fn validate(&self) -> Result<(), String> { - // A write pins one pooled connection for its whole duration (the - // connection-affine advisory lock in repo_store::acquire_write), and - // concurrent writes are capped at max_concurrent_git_pushes. If the pool - // does not exceed that cap by DB_POOL_APP_HEADROOM, a burst of slow pushes - // drains every connection and every other DB path 503s. (#174 F1) - let floor = (self.max_concurrent_git_pushes as u64) + (Self::DB_POOL_APP_HEADROOM as u64); - if (self.db_max_connections as u64) < floor { + // Concurrent git writes pin the dedicated lock pool for their whole + // duration (connection-affine advisory lock in acquire_write). The main + // pool no longer carries that occupancy. + if (self.db_lock_pool_max_connections as usize) < self.max_concurrent_git_pushes { return Err(format!( - "GITLAWB_DB_MAX_CONNECTIONS ({}) must be at least max_concurrent_git_pushes ({}) \ - + {} headroom = {}: each concurrent write pins one pooled connection for its whole \ - duration, so a smaller pool lets a burst of slow pushes starve every other DB path.", - self.db_max_connections, - self.max_concurrent_git_pushes, - Self::DB_POOL_APP_HEADROOM, - floor + "GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS ({}) must be at least \ + max_concurrent_git_pushes ({}) so every admitted push can pin a \ + lock-pool connection for its whole duration", + self.db_lock_pool_max_connections, self.max_concurrent_git_pushes + )); + } + let main_floor = Self::DB_POOL_APP_HEADROOM as u64; + if (self.db_max_connections as u64) < main_floor { + return Err(format!( + "GITLAWB_DB_MAX_CONNECTIONS ({}) must be at least {} for non-git \ + database paths", + self.db_max_connections, main_floor )); } Ok(()) @@ -1422,41 +1422,48 @@ mod tests { ); } - /// #174 F1: a connection-affine write lock pins a pooled connection per - /// concurrent write, so the pool must clear `max_concurrent_git_pushes` by - /// `DB_POOL_APP_HEADROOM` or a push burst starves every other DB path. - /// `validate()` must reject an under-sized pool at boot. + /// #174 F1: concurrent git writes pin the dedicated lock pool, not the main + /// pool. `validate()` must reject an under-sized lock pool at boot. #[test] fn db_pool_must_clear_the_git_push_cap() { - // Shipped defaults validate (48 >= 32 + 8). + // Shipped defaults validate (lock pool 32 >= pushes 32, main pool >= headroom). Config::parse_from(["gitlawb-node"]) .validate() .expect("default config must validate"); - // An under-sized pool relative to the push cap is rejected (20 < 32 + 8). - let under = Config::parse_from([ + // An under-sized lock pool relative to the push cap is rejected. + let under_lock = Config::parse_from([ "gitlawb-node", - "--db-max-connections", - "20", + "--db-lock-pool-max-connections", + "16", "--max-concurrent-git-pushes", "32", ]); assert!( - under.validate().is_err(), - "db_max_connections 20 below max_concurrent_git_pushes 32 + headroom must be rejected" + under_lock.validate().is_err(), + "db_lock_pool_max_connections below max_concurrent_git_pushes must be rejected" ); - // Exactly at the floor validates (40 == 32 + 8). - let at_floor = Config::parse_from([ + // Main pool can be smaller than pushes + headroom when the lock pool carries writes. + let split = Config::parse_from([ "gitlawb-node", "--db-max-connections", - "40", + "16", + "--db-lock-pool-max-connections", + "32", "--max-concurrent-git-pushes", "32", ]); assert!( - at_floor.validate().is_ok(), - "db_max_connections at the floor (pushes + headroom) must validate" + split.validate().is_ok(), + "a small main pool with a large lock pool must validate" + ); + + // Main pool still needs the app headroom floor. + let under_main = Config::parse_from(["gitlawb-node", "--db-max-connections", "4"]); + assert!( + under_main.validate().is_err(), + "db_max_connections below DB_POOL_APP_HEADROOM must be rejected" ); } diff --git a/crates/gitlawb-node/src/error.rs b/crates/gitlawb-node/src/error.rs index 39fabe44..cb8f7711 100644 --- a/crates/gitlawb-node/src/error.rs +++ b/crates/gitlawb-node/src/error.rs @@ -123,20 +123,24 @@ impl From for AppError { // 500. The internal message names the owner slug and repo, so the // variant carries nothing: the detail stays in the log at the raise // site and the client gets a fixed retryable body. - Err(err) => match err.downcast::() { - Ok(_) => AppError::RepoBusy, - // Same reasoning one rung down: a refused under-lock refresh is a - // transient storage condition, and its internal message names the - // owner slug and repo, so the variant carries nothing. - Err(err) => match err.downcast::() { - Ok(_) => AppError::RepoUnavailable, - // And one more rung: a publish the store refused twice is - // transient in the same way, and the retry is the client's - // to make. The variant carries nothing for the same reason - // as the two above. - Err(err) => match err.downcast::() { - Ok(_) => AppError::RepoWriteFenced, - Err(err) => AppError::Internal(err), + Err(err) => match err.downcast::() { + Ok(_) => AppError::Overloaded("git write locks at capacity, retry shortly".into()), + Err(err) => match err.downcast::() { + Ok(_) => AppError::RepoBusy, + // Same reasoning one rung down: a refused under-lock refresh is a + // transient storage condition, and its internal message names the + // owner slug and repo, so the variant carries nothing. + Err(err) => match err.downcast::() { + Ok(_) => AppError::RepoUnavailable, + // And one more rung: a publish the store refused twice is + // transient in the same way, and the retry is the client's + // to make. The variant carries nothing for the same reason + // as the two above. + Err(err) => match err.downcast::() + { + Ok(_) => AppError::RepoWriteFenced, + Err(err) => AppError::Internal(err), + }, }, }, }, @@ -296,7 +300,11 @@ impl IntoResponse for AppError { // here rather than in bespoke early returns, keeping each variant handled once. if matches!( self, - AppError::Overloaded(_) | AppError::SearchIncomplete { .. } + AppError::Overloaded(_) + | AppError::SearchIncomplete { .. } + | AppError::RepoBusy + | AppError::RepoUnavailable + | AppError::RepoWriteFenced ) { resp.headers_mut().insert( axum::http::header::RETRY_AFTER, @@ -414,4 +422,31 @@ mod tests { let resp = err.into_response(); assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE); } + + #[test] + fn lock_pool_busy_via_anyhow_from_is_503_overloaded() { + let err: AppError = anyhow::Error::new(crate::git::repo_store::LockPoolBusy).into(); + let resp = err.into_response(); + assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + resp.headers().get("retry-after").unwrap().to_str().unwrap(), + "1" + ); + } + + #[test] + fn repo_busy_unavailable_and_fenced_advertise_retry_after() { + for err in [ + AppError::RepoBusy, + AppError::RepoUnavailable, + AppError::RepoWriteFenced, + ] { + let resp = err.into_response(); + assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + resp.headers().get("retry-after").unwrap().to_str().unwrap(), + "1" + ); + } + } } diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index eccafc6a..e1038a1b 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -1192,6 +1192,7 @@ impl RepoWriteGuard { Some(Err(PublishRefusal::Fenced)) => outcome = ReleaseOutcome::Fenced, Some(Err(PublishRefusal::Failed(e))) => { warn!(repo = %self.repo_name, err = %e, "failed to upload repo to tigris after write"); + outcome = ReleaseOutcome::UploadFailed; } None => { // Timed out is UNKNOWABLE, not failed: the PUT may well @@ -1203,13 +1204,16 @@ impl RepoWriteGuard { // session, so the lock would free within milliseconds // either way. What actually protects a successor from a // late publish is the conditional PUT on the upload, not - // the lifetime of this lock. + // the lifetime of this lock. The caller still must not + // report success: `into_result` maps this to a retryable + // refusal. warn!( repo = %self.repo_name, "release upload exceeded its bound; the PUT may still land, so the \ outcome is unknowable and the conditional upload is what keeps a \ late publish from overwriting a successor's archive" ); + outcome = ReleaseOutcome::UploadUnknowable; } } } @@ -1345,17 +1349,22 @@ enum RefreshFailure { /// `#[must_use]` because dropping it is the whole defect this type exists to /// prevent: a publish the store refused would otherwise return 201, fire /// webhooks, and record a push that no successor can read. -#[derive(Debug)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[must_use = "a refused publish must reach the caller, or a write that never landed reports success"] pub enum ReleaseOutcome { /// The lock was released and nothing definitively refused the publish. /// Also the answer when there was nothing to publish (a failed write, no - /// storage backend) and when the outcome is unknowable (the upload - /// exceeded its bound), which keeps those paths behaving as they do today. + /// storage backend). Released, /// The store refused the publish twice. The tree is on local disk but is /// NOT in object storage, so the caller must not report success. Fenced, + /// The archive upload exceeded its under-lock bound. The PUT may still + /// land, but durability is unknowable, so the caller must not report + /// success. + UploadUnknowable, + /// The archive upload failed with a definite error (not a fence refusal). + UploadFailed, } impl ReleaseOutcome { @@ -1368,6 +1377,11 @@ impl ReleaseOutcome { pub fn into_result(self) -> anyhow::Result<()> { match self { ReleaseOutcome::Released => Ok(()), + ReleaseOutcome::UploadUnknowable | ReleaseOutcome::UploadFailed => { + Err(anyhow::Error::new(RepoUnavailable).context( + "archive upload did not complete durably before the lock was released", + )) + } // The raise site inside `publish` already logged the repo and the // status, so this carries no detail: the handler layer turns it // into a fixed 503 body. @@ -1538,6 +1552,14 @@ pub(crate) fn advisory_lock_key(owner_slug: &str, repo_name: &str) -> i64 { mod tests { use super::*; + #[test] + fn non_durable_release_outcomes_do_not_report_success() { + assert!(ReleaseOutcome::Released.into_result().is_ok()); + assert!(ReleaseOutcome::UploadUnknowable.into_result().is_err()); + assert!(ReleaseOutcome::UploadFailed.into_result().is_err()); + assert!(ReleaseOutcome::Fenced.into_result().is_err()); + } + // ── sync slug validation (#272) ──────────────────────────────────────── #[test] @@ -5118,12 +5140,17 @@ mod tests { started.elapsed() >= bound, "A's release must have run out its transfer bound with the PUT in flight" ); - // A timeout is UNKNOWABLE rather than failed, so the release reports an - // ordinary success and the lock frees. That is exactly why the fence has - // to live in the store: nothing here knows A's bytes are still coming. + // A timeout is UNKNOWABLE rather than failed, so the release frees the + // lock but must not report durability to the caller. That is exactly why + // the fence has to live in the store: nothing here knows A's bytes are + // still coming. + assert!( + matches!(outcome_a, ReleaseOutcome::UploadUnknowable), + "an abandoned publish must report unknowable durability, got {outcome_a:?}" + ); assert!( - matches!(outcome_a, ReleaseOutcome::Released), - "an abandoned publish reports a plain release, got {outcome_a:?}" + outcome_a.into_result().is_err(), + "unknowable upload must not report success to the caller" ); assert!( @@ -5218,8 +5245,8 @@ mod tests { mock.park_next_put(); let outcome_a = guard_a.release(true).await; assert!( - matches!(outcome_a, ReleaseOutcome::Released), - "an abandoned publish reports a plain release, got {outcome_a:?}" + matches!(outcome_a, ReleaseOutcome::UploadUnknowable), + "an abandoned publish must report unknowable durability, got {outcome_a:?}" ); assert!( @@ -5304,8 +5331,8 @@ mod tests { mock.park_next_put(); let outcome_a = guard_a.release(true).await; assert!( - matches!(outcome_a, ReleaseOutcome::Released), - "an abandoned publish reports a plain release, got {outcome_a:?}" + matches!(outcome_a, ReleaseOutcome::UploadUnknowable), + "an abandoned publish must report unknowable durability, got {outcome_a:?}" ); assert_eq!( From 8e66e77c57568d803a7a7c882ea9c5de62b82b58 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Fri, 28 Aug 2026 22:10:38 -0500 Subject: [PATCH 38/54] fix(node): close jatmn round on publish authority and durability gating Revoke under-lock refresh swaps when the advisory lock ends, fail closed on unrecorded release outcomes before Pinata/gossip, compensate fork archives when create_repo fails, recompute acquire backoff from the live deadline, and route publish swaps through validated_repo_disk_path for CodeQL. --- crates/gitlawb-node/src/api/repos.rs | 40 +++++- crates/gitlawb-node/src/git/repo_store.rs | 150 +++++++++++++++++++++- crates/gitlawb-node/src/git/tigris.rs | 30 ++--- 3 files changed, 195 insertions(+), 25 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 7d39b8a6..847d9ee9 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -2538,8 +2538,8 @@ async fn publish_durability_confirmed( } if start.elapsed() >= wait { // Handler disconnected during `release` without recording an outcome. - // Proceed so inv22's disconnect-during-release tail still runs. - return true; + // Fail closed: a pending inner `None` is not durability. + return false; } tokio::time::sleep(std::time::Duration::from_millis(10)).await; } @@ -3261,7 +3261,13 @@ pub async fn fork_repo( machine_id: state.machine_id.clone(), }; - state.db.create_repo(&record).await?; + if let Err(e) = state.db.create_repo(&record).await { + state + .repo_store + .compensate_fork_archive(&forker_did, &fork_name, &disk_path) + .await; + return Err(e.into()); + } // Persist the proof so the fork carries it when it propagates to peers. if let Some(p) = verified_proof { @@ -3499,6 +3505,34 @@ mod tests { const OWNER_SHORT: &str = "z6MkpTHR8VNsBxYAAWHut2Geadd9jSwuBV8xRoAnwWsdvktH"; const STRANGER_DID: &str = "did:key:z6Mkffonly5tranger0000000000000000000000000000000"; + #[tokio::test] + async fn publish_durability_confirmed_fails_closed_when_release_never_records() { + let slot = Arc::new(tokio::sync::Mutex::new(None)); + let confirmed = + publish_durability_confirmed(&Some(slot), std::time::Duration::from_millis(30)).await; + assert!( + !confirmed, + "a pending release outcome must not be treated as confirmed durability" + ); + } + + #[tokio::test] + async fn publish_durability_confirmed_accepts_only_released() { + let slot = Arc::new(tokio::sync::Mutex::new(Some( + crate::git::repo_store::ReleaseOutcome::Released, + ))); + assert!( + publish_durability_confirmed(&Some(slot), std::time::Duration::from_millis(5)).await + ); + + let slot = Arc::new(tokio::sync::Mutex::new(Some( + crate::git::repo_store::ReleaseOutcome::UploadUnknowable, + ))); + assert!( + !publish_durability_confirmed(&Some(slot), std::time::Duration::from_millis(5)).await + ); + } + #[test] fn upload_pack_request_finalizes_only_with_done_pktline() { let want = "0032want 1111111111111111111111111111111111111111\n"; diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index e1038a1b..92cdbfd4 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -10,6 +10,7 @@ use std::collections::HashSet; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; @@ -129,7 +130,10 @@ impl RepoStore { /// spawned to lazily migrate it (on-demand migration for pre-Tigris repos). /// Returns the local path to the bare repo. pub async fn acquire(&self, owner_did: &str, repo_name: &str) -> Result { - let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; + // Validate at the sink CodeQL watches (`rust/path-injection`), not only inside + // `local_path()`, so `exists` and every later filesystem touch share one barrier. + let local_path = validated_repo_disk_path(&self.repos_dir, owner_did, repo_name)?; + let owner_slug = owner_did.replace([':', '/'], "_"); // Fast path: repo exists locally if local_path.exists() { @@ -198,7 +202,7 @@ impl RepoStore { if tigris.exists(&owner_slug, repo_name).await.unwrap_or(false) { debug!(repo = %repo_name, "cache miss — downloading from tigris"); tigris - .download(&owner_slug, repo_name, &local_path) + .download(&owner_slug, repo_name, &local_path, None) .await .context("downloading repo from tigris")?; // Mark as migrated since we just downloaded it @@ -244,7 +248,7 @@ impl RepoStore { // in the ASYNC layer, armed for the whole download await and // disarmed only when `RepoSnapshot` takes ownership. let snapshot = tigris - .download_to(&owner_slug, repo_name, &local_path, false) + .download_to(&owner_slug, repo_name, &local_path, false, None) .await .map_err(|e| { anyhow::Error::new(RepoUnavailable).context(format!( @@ -427,7 +431,11 @@ impl RepoStore { // second past the deadline would turn a short deadline into a longer // wait than the caller was promised. if attempt < 59 { - tokio::time::sleep(left.min(std::time::Duration::from_secs(1))).await; + let remaining = match deadline.checked_duration_since(std::time::Instant::now()) { + Some(remaining) if !remaining.is_zero() => remaining, + _ => break, + }; + tokio::time::sleep(remaining.min(std::time::Duration::from_secs(1))).await; } } let Some(lock_conn) = lock_conn else { @@ -449,6 +457,7 @@ impl RepoStore { // From here the lock is HELD. Any early return must not simply drop the // connection back into the pool, so it is handed to the guard immediately // below and every exit after this point goes through the guard. + let refresh_swap_authority = Arc::new(AtomicBool::new(true)); let mut guard = RepoWriteGuard { owner_slug: owner_slug.clone(), repo_name: repo_name.to_string(), @@ -461,6 +470,7 @@ impl RepoStore { // observed under the lock. Only reachable unset when no backend is // configured, in which case `release` publishes nothing at all. publish_fence: UploadPrecondition::Unconditional, + refresh_swap_authority: Some(refresh_swap_authority.clone()), #[cfg(test)] test_pre_unlock_gate: self.pre_unlock_gate.clone(), #[cfg(test)] @@ -502,7 +512,15 @@ impl RepoStore { Ok(Some(etag)) => { debug!(repo = %repo_name, "write acquire: downloading latest from tigris"); let fence = UploadPrecondition::IfMatch(etag); - match tigris.download(&owner_slug, repo_name, &local_path).await { + match tigris + .download( + &owner_slug, + repo_name, + &local_path, + Some(refresh_swap_authority.clone()), + ) + .await + { Ok(()) => Ok(fence), Err(err) => Err(RefreshFailure::Download { err, fence }), } @@ -570,6 +588,7 @@ impl RepoStore { // // Refuse the acquire. Returning here drops the guard, whose Drop // frees the lock and its pool slot. + refresh_swap_authority.store(false, Ordering::Release); // // `error!`, not the sibling `warn!` above, and that is deliberate. // The handler layer demotes every `RepoUnavailable` to warn because @@ -697,6 +716,30 @@ impl RepoStore { let local_path = validated_repo_disk_path(&self.repos_dir, owner_did, repo_name)?; Ok((owner_slug, local_path)) } + + /// Best-effort cleanup when fork creation published an archive but failed to persist + /// the database row. Removes the object-store key this attempt owns and the local + /// mirror clone so a retry is not blocked by its own orphan. + pub async fn compensate_fork_archive(&self, owner_did: &str, repo_name: &str, disk_path: &Path) { + if let Some(ref tigris) = self.tigris { + if let Ok((owner_slug, _)) = self.local_path(owner_did, repo_name) { + if let Err(e) = tigris.delete(&owner_slug, repo_name).await { + warn!( + repo = %repo_name, + err = %e, + "failed to delete fork archive during create_repo compensation" + ); + } + } + } + if let Err(e) = std::fs::remove_dir_all(disk_path) { + warn!( + path = %disk_path.display(), + err = %e, + "failed to remove fork clone during create_repo compensation" + ); + } + } } /// The three-layer validated form of `store::repo_disk_path`, with NO Tigris fetch and @@ -735,6 +778,30 @@ pub(crate) fn validated_repo_disk_path( Ok(local_path) } +/// Swap a finished extraction into a validated live repo path. The caller must pass +/// the path returned from [`validated_repo_disk_path`]; this is the CodeQL barrier +/// for `rust/path-injection` on the remove/rename sink. +pub(crate) fn swap_extracted_into_validated_repo( + validated_path: &Path, + tmp_dir: &Path, + swap_authority: Option<&Arc>, +) -> Result<()> { + if let Some(authority) = swap_authority { + if !authority.load(Ordering::Acquire) { + let _ = std::fs::remove_dir_all(tmp_dir); + anyhow::bail!("publish swap revoked after lock ownership ended"); + } + } + + let lock = super::tigris::publish_lock(validated_path); + let _publish = lock.lock().expect("publish lock poisoned"); + if validated_path.exists() { + std::fs::remove_dir_all(validated_path).context("removing stale repo dir")?; + } + std::fs::rename(tmp_dir, validated_path).context("swapping extracted repo into place")?; + Ok(()) +} + /// Strict allowlist validator for `owner_did` and `repo_name`. /// /// Rejects any character that isn't explicitly safe, plus length and @@ -1049,6 +1116,10 @@ pub struct RepoWriteGuard { /// PUT abandoned by an earlier writer's timeout cannot land on top of a /// successor's acknowledged archive. publish_fence: UploadPrecondition, + /// When set, a timed-out or cancelled under-lock refresh revokes this before the + /// guard drops so a detached `spawn_blocking` extraction cannot swap into the live + /// tree after its advisory-lock ownership ends. + refresh_swap_authority: Option>, /// Test-only seam: when set, `release` parks on this gate at the exact point /// it is about to await `pg_advisory_unlock` (connection still owned, not yet /// released). Dropping the `release` future while it is parked reproduces a @@ -1276,6 +1347,10 @@ impl RepoWriteGuard { impl Drop for RepoWriteGuard { fn drop(&mut self) { + if let Some(authority) = self.refresh_swap_authority.take() { + authority.store(false, Ordering::Release); + } + let Some(mut conn) = self.conn.take() else { // release() already unlocked and handed the connection back. return; @@ -1560,6 +1635,67 @@ mod tests { assert!(ReleaseOutcome::Fenced.into_result().is_err()); } + #[test] + fn revoked_publish_swap_cannot_replace_the_live_tree() { + let root = tempfile::TempDir::new().unwrap(); + let repos_dir = root.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let owner = "did:key:z6MkRevokedSwapAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "repo"; + let live = validated_repo_disk_path(&repos_dir, owner, name).unwrap(); + std::fs::create_dir_all(&live).unwrap(); + std::fs::write(live.join("HEAD"), "ref: refs/heads/main\n").unwrap(); + + let tmp = live + .parent() + .unwrap() + .join(format!(".{}.tmp-extract.test", name)); + std::fs::create_dir_all(&tmp).unwrap(); + std::fs::write(tmp.join("HEAD"), "ref: refs/heads/evil\n").unwrap(); + + let authority = Arc::new(AtomicBool::new(false)); + let err = swap_extracted_into_validated_repo(&live, &tmp, Some(&authority)) + .expect_err("a revoked authority must refuse the swap"); + assert!( + err.to_string().contains("publish swap revoked"), + "unexpected error: {err:#}" + ); + assert!( + live.join("HEAD").exists(), + "the live tree must survive a revoked late extraction" + ); + assert!( + !tmp.exists(), + "the temp extraction must be cleaned up on refusal" + ); + let head = std::fs::read_to_string(live.join("HEAD")).unwrap(); + assert_eq!(head, "ref: refs/heads/main\n"); + } + + #[test] + fn non_revoked_publish_swap_replaces_the_live_tree() { + let root = tempfile::TempDir::new().unwrap(); + let repos_dir = root.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let owner = "did:key:z6MkHappySwapAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "repo"; + let live = validated_repo_disk_path(&repos_dir, owner, name).unwrap(); + std::fs::create_dir_all(&live).unwrap(); + std::fs::write(live.join("HEAD"), "ref: refs/heads/main\n").unwrap(); + + let tmp = live + .parent() + .unwrap() + .join(format!(".{}.tmp-extract.test", name)); + std::fs::create_dir_all(&tmp).unwrap(); + std::fs::write(tmp.join("HEAD"), "ref: refs/heads/new\n").unwrap(); + + let authority = Arc::new(AtomicBool::new(true)); + swap_extracted_into_validated_repo(&live, &tmp, Some(&authority)).expect("swap succeeds"); + let head = std::fs::read_to_string(live.join("HEAD")).unwrap(); + assert_eq!(head, "ref: refs/heads/new\n"); + } + // ── sync slug validation (#272) ──────────────────────────────────────── #[test] @@ -2531,6 +2667,7 @@ mod tests { tigris: None, lock_held_transfer_timeout: Duration::from_secs(300), publish_fence: UploadPrecondition::Unconditional, + refresh_swap_authority: None, #[cfg(test)] test_pre_unlock_gate: None, #[cfg(test)] @@ -2895,6 +3032,7 @@ mod tests { lock_held_transfer_timeout: Duration::from_secs(300), // No backend, so nothing is ever published and the fence is unread. publish_fence: UploadPrecondition::Unconditional, + refresh_swap_authority: None, #[cfg(test)] test_pre_unlock_gate: None, #[cfg(test)] @@ -4672,7 +4810,7 @@ mod tests { let out = TempDir::new().unwrap(); let into = out.path().join("stored.git"); mock_tigris(mock) - .download(owner_slug, repo_name, &into) + .download(owner_slug, repo_name, &into, None) .await .expect("the stored archive must be readable"); std::fs::read_to_string(into.join("MARKER")).expect("the stored archive must be marked") diff --git a/crates/gitlawb-node/src/git/tigris.rs b/crates/gitlawb-node/src/git/tigris.rs index cc89209f..af8cbc39 100644 --- a/crates/gitlawb-node/src/git/tigris.rs +++ b/crates/gitlawb-node/src/git/tigris.rs @@ -5,6 +5,7 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; +use std::sync::atomic::AtomicBool; use std::sync::{Arc, Mutex, OnceLock}; use anyhow::{Context, Result}; @@ -230,8 +231,9 @@ impl TigrisClient { owner_slug: &str, repo_name: &str, local_path: &Path, + swap_authority: Option>, ) -> Result<()> { - self.download_to(owner_slug, repo_name, local_path, true) + self.download_to(owner_slug, repo_name, local_path, true, swap_authority) .await .map(|_| ()) } @@ -250,6 +252,7 @@ impl TigrisClient { repo_name: &str, target: &Path, publish: bool, + swap_authority: Option>, ) -> Result { let key = Self::repo_key(owner_slug, repo_name); debug!(key = %key, path = %target.display(), "downloading repo from tigris"); @@ -298,7 +301,7 @@ impl TigrisClient { let cleanup = snapshot_tmp.as_ref().map(|p| SnapshotCleanup(p.clone())); let result = (|| -> Result { if publish { - decompress_repo(&data, &target)?; + decompress_repo(&data, &target, swap_authority.as_ref())?; return Ok(target); } // Non-mutating snapshot: unpack into the temp dir decided above. @@ -334,7 +337,6 @@ impl TigrisClient { } /// Delete a repo archive from Tigris. - #[allow(dead_code)] pub async fn delete(&self, owner_slug: &str, repo_name: &str) -> Result<()> { let key = Self::repo_key(owner_slug, repo_name); self.s3 @@ -367,7 +369,7 @@ fn compress_repo(repo_path: &Path) -> Result> { /// `decompress_repo`. Concurrent extractions unpack into isolated temp dirs in /// parallel, but the final `remove_dir_all` + `rename` must not interleave for /// the same `local_path`, or they race to a nondeterministic overwrite/failure. -fn publish_lock(local_path: &Path) -> Arc> { +pub(crate) fn publish_lock(local_path: &Path) -> Arc> { // KNOWN LIMITATION: this map is never evicted — one (PathBuf, Arc) // entry accrues per distinct repo path for the process lifetime. Bounded by // the number of repos a node hosts, so it's negligible for normal use, but @@ -401,7 +403,11 @@ impl Drop for SnapshotCleanup { /// fully succeeds. A corrupt or truncated archive therefore can never clobber a /// good existing copy at `local_path` — on failure we discard the temp dir and /// leave `local_path` exactly as it was. -fn decompress_repo(data: &[u8], local_path: &Path) -> Result<()> { +fn decompress_repo( + data: &[u8], + local_path: &Path, + swap_authority: Option<&Arc>, +) -> Result<()> { let parent = local_path.parent().context("repo path has no parent")?; std::fs::create_dir_all(parent).context("creating parent dir")?; @@ -430,17 +436,9 @@ fn decompress_repo(data: &[u8], local_path: &Path) -> Result<()> { return Err(e); } - // Swap the freshly-extracted repo into place. rename within the same parent - // is effectively atomic, but most platforms refuse to rename onto a - // non-empty dir, so remove the old copy first. Serialize this per repo path: - // concurrent extractions unpack into isolated temp dirs, but their swaps - // must not interleave or they race to a nondeterministic overwrite/failure. - let lock = publish_lock(local_path); - let _publish = lock.lock().expect("publish lock poisoned"); - if local_path.exists() { - std::fs::remove_dir_all(local_path).context("removing stale repo dir")?; - } - std::fs::rename(&tmp_dir, local_path).context("swapping extracted repo into place")?; + // Swap through the validated-path helper so CodeQL sees the barrier before the + // remove/rename sink (`rust/path-injection`). + super::repo_store::swap_extracted_into_validated_repo(local_path, &tmp_dir, swap_authority)?; Ok(()) } From 191129153320200da2424d9c7f0df94fbf43be00 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Fri, 28 Aug 2026 22:33:57 -0500 Subject: [PATCH 39/54] fix(node): harden issue close/create and fork publish paths on #285 Move close_issue rate limiting after the read gate so denied readers see 404 not 429, roll back local issue refs when publish refuses, re-check swap authority under publish_lock, propagate fork upload failures before DB insert, and update lock-pool config help text. --- crates/gitlawb-node/src/api/issues.rs | 161 ++++++++++++++++++---- crates/gitlawb-node/src/config.rs | 16 +-- crates/gitlawb-node/src/git/issues.rs | 41 ++++++ crates/gitlawb-node/src/git/repo_store.rs | 25 ++-- 4 files changed, 199 insertions(+), 44 deletions(-) diff --git a/crates/gitlawb-node/src/api/issues.rs b/crates/gitlawb-node/src/api/issues.rs index 9efc1e86..adce758d 100644 --- a/crates/gitlawb-node/src/api/issues.rs +++ b/crates/gitlawb-node/src/api/issues.rs @@ -70,12 +70,21 @@ pub async fn create_issue( let create_result = git_issues::create_issue(&disk_path, &issue_id, &json_str); - // Always release the advisory lock — even on error; upload to Tigris only on success. - // A refused publish short-circuits here, before the trust bump and before - // the 201: the issue is on local disk but not in object storage, so no - // other node can read it and the client must retry rather than be told it - // was filed. - guard.release(create_result.is_ok()).await.into_result()?; + let release_result = guard.release(create_result.is_ok()).await.into_result(); + if release_result.is_err() && create_result.is_ok() { + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let deadline = std::time::Instant::now() + git_timeout; + if let Err(rollback) = + git_issues::delete_issue_ref(&state.git_bin, &disk_path, &issue_id, deadline) + { + tracing::warn!( + issue = %issue_id, + err = %rollback, + "failed to roll back local issue after refused publish" + ); + } + } + release_result?; create_result.map_err(|e| AppError::Git(e.to_string()))?; @@ -235,29 +244,17 @@ pub async fn close_issue( .await? .ok_or_else(|| AppError::RepoNotFound(format!("{owner}/{repo}")))?; - // Per-IP flood brake, layered on the same shared limiter and trusted-proxy - // policy as the push advertisement. The pre-lock snapshot downloads the - // whole archive and runs a blocking extraction, so an unlimited route would - // let disposable identities drive unbounded transfer/CPU/disk with parallel - // close requests for arbitrary issue ids. Applied before the snapshot work - // so a rejected request does none of it. - if let Some(key) = crate::rate_limit::client_key(&headers, peer, state.push_limiter_trust) { - if !state.push_rate_limiter.check(&key).await { - tracing::warn!(repo = %repo, key = %key, "close_issue rate limited"); - return Err(AppError::TooManyRequests( - "rate limit exceeded — try again later".into(), - )); - } - } - - // READ-GATE before any snapshot work. The author fallback below needs the - // issue blob, which needs the repo tree, so authorship cannot be established + // READ-GATE before any snapshot work or rate charging. The author fallback below + // needs the issue blob, which needs the repo tree, so authorship cannot be established // without a download; but a caller who cannot even READ the repo must be // stopped here, cheaply, before any Tigris transfer or extraction happens. // Without this, any signed non-owner could issue parallel close requests for // arbitrary issue ids and drive unbounded downloads and blocking extraction // (a disposable-identity DoS), because the route has no other pre-authorization. // + // Rate limiting runs AFTER this gate so a 429 cannot distinguish a hidden repo + // from a missing one (INV-12 read-denial status contract). + // // mirror-rows-handled: a repo row synced from a peer is stored public and // carries none of the owner's visibility rules, so for such a row this check // can only return allow. That is deliberate rather than overlooked, and it is @@ -285,6 +282,21 @@ pub async fn close_issue( } } + // Per-IP flood brake, layered on the same shared limiter and trusted-proxy + // policy as the push advertisement. The pre-lock snapshot downloads the + // whole archive and runs a blocking extraction, so an unlimited route would + // let disposable identities drive unbounded transfer/CPU/disk with parallel + // close requests for arbitrary issue ids. Applied after the read gate so a + // denied reader still sees 404, not 429. + if let Some(key) = crate::rate_limit::client_key(&headers, peer, state.push_limiter_trust) { + if !state.push_rate_limiter.check(&key).await { + tracing::warn!(repo = %repo, key = %key, "close_issue rate limited"); + return Err(AppError::TooManyRequests( + "rate limit exceeded — try again later".into(), + )); + } + } + // AUTHORIZE BEFORE ACQUIRING. The per-repo advisory lock genuinely excludes // now, so taking it first would hand any caller with read access a way to hold // that lock on demand and be refused afterwards, while a legitimate writer @@ -1082,4 +1094,107 @@ mod lock_pool_shed_tests { server.abort(); } + + /// A denied reader on a private repo must see 404 even when the caller's rate + /// bucket is exhausted. Rate limiting runs after the read gate. + #[sqlx::test] + async fn close_issue_rate_limit_runs_after_the_read_gate(pool: PgPool) { + use std::net::SocketAddr; + use std::time::Duration; + + let owner = "did:key:zCLOSERATEOWNERAAAAAAAAAAAAAAAAAAAAAAA"; + let stranger = "did:key:zCLOSERATESTRANGERBBBBBBBBBBBBBBBBBB"; + let mut state = crate::test_support::test_state(pool).await; + state.push_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let mut repo = seed_repo(owner, "priv-close"); + repo.is_public = false; + state.db.create_repo(&repo).await.expect("seed repo"); + + let peer: SocketAddr = "203.0.113.88:7000".parse().unwrap(); + assert!( + state.push_rate_limiter.check(&peer.ip().to_string()).await, + "exhaust the peer bucket before close_issue" + ); + + let res = close_issue( + State(state), + Extension(AuthenticatedDid(stranger.to_string())), + Path((owner.to_string(), "priv-close".to_string(), "1".to_string())), + axum::http::HeaderMap::new(), + crate::rate_limit::PeerAddr(Some(peer)), + ) + .await; + + assert!( + matches!(res, Err(AppError::RepoNotFound(_))), + "a non-reader must see 404, not 429, even when rate limited: {:?}", + res + ); + } + + /// A refused publish must roll back the local issue ref so a retry does not + /// mint a second id for the same logical filing attempt. + #[sqlx::test] + async fn create_issue_rolls_back_local_ref_when_publish_refuses(pool: PgPool) { + use axum::response::IntoResponse; + + let app = axum::Router::new().route( + "/{*key}", + axum::routing::any(|method: axum::http::Method| async move { + if method == axum::http::Method::HEAD { + let mut resp = axum::http::StatusCode::OK.into_response(); + resp.headers_mut() + .insert("etag", axum::http::HeaderValue::from_static("\"gen-1\"")); + resp + } else { + axum::http::StatusCode::INTERNAL_SERVER_ERROR.into_response() + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port()); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + + let owner = "did:key:zISSUEROLLBACKAAAAAAAAAAAAAAAAAAAAAAA"; + let repo_name = "rollback-create"; + let mut state = crate::test_support::test_state(pool.clone()).await; + let disk = tempfile::TempDir::new().unwrap(); + crate::git::store::init_bare(&disk.path().join("repo.git")).expect("bare repo"); + let mut seeded = seed_repo(owner, repo_name); + seeded.disk_path = disk.path().join("repo.git").to_string_lossy().to_string(); + state.db.create_repo(&seeded).await.expect("seed repo"); + state.repo_store = crate::git::repo_store::RepoStore::for_testing_with_tigris( + disk.path().to_path_buf(), + crate::git::repo_store::build_lock_pool(&pool, 2, std::time::Duration::from_secs(1)), + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint), + ); + + let shed = create_issue( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path((owner.to_string(), repo_name.to_string())), + Json(CreateIssueRequest { + title: "t".to_string(), + body: None, + signed_payload: None, + }), + ) + .await; + assert!( + shed.is_err(), + "a failed archive upload must refuse the write" + ); + + let issues = git_issues::list_issues(&disk.path().join("repo.git")).expect("list issues"); + assert!( + issues.is_empty(), + "the local issue ref must be rolled back after a refused publish, got {issues:?}" + ); + + server.abort(); + } } diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index ec44d881..2af40c52 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -397,15 +397,13 @@ pub struct Config { /// error rather than a boot-time panic). /// /// CONNECTION BUDGET. A push holds a Postgres connection from the node's separate - /// advisory-lock pool for the whole receive-pack, and that pool is sized from this - /// knob (this value + 8, clamped to 64 in `main.rs`). The node's total ceiling is - /// therefore `db_max_connections` (default 48) + the lock pool (default 40), i.e. - /// 88 by default, and at most `db_max_connections` + 64. Size BOTH against the - /// database server's `max_connections`: `db_max_connections`' own doc predates the - /// lock pool and no longer covers most of the node's connections. The +8 headroom - /// is shared with the three non-push `acquire_write` callers (`api/issues.rs` x2, - /// `api/pulls.rs`). Raising this knob past the clamp does NOT buy more lock-pool - /// connections; pushes beyond it wait briefly and then shed a 503 + Retry-After. + /// advisory-lock pool (`GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS`, default 32) for + /// the whole receive-pack. That pool must be at least this value so every + /// admitted push can pin a lock-pool connection for its whole duration. Size BOTH + /// pools against the database server's `max_connections`: the main pool + /// (`GITLAWB_DB_MAX_CONNECTIONS`, default 48) serves ordinary handlers, and the + /// lock pool serves writes. Raising this knob does not raise the lock pool; + /// set `GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS` explicitly. #[arg( long, env = "GITLAWB_MAX_CONCURRENT_GIT_PUSHES", diff --git a/crates/gitlawb-node/src/git/issues.rs b/crates/gitlawb-node/src/git/issues.rs index 73098302..12256a67 100644 --- a/crates/gitlawb-node/src/git/issues.rs +++ b/crates/gitlawb-node/src/git/issues.rs @@ -61,6 +61,26 @@ pub fn create_issue(repo_path: &Path, issue_id: &str, json: &str) -> Result<()> Ok(()) } +/// Remove a single issue ref after a failed publish rolled back the handler's view +/// of the write. Best-effort: a failed delete leaves a local-only orphan, which is +/// still better than telling the client to retry into a duplicate id. +pub fn delete_issue_ref( + git_bin: &str, + repo_path: &Path, + issue_id: &str, + deadline: std::time::Instant, +) -> Result<()> { + let ref_name = format!("refs/gitlawb/issues/{issue_id}"); + crate::git::visibility_pack::run_bounded_git( + git_bin, + &["update-ref", "-d", &ref_name], + repo_path, + b"", + deadline, + )?; + Ok(()) +} + /// List all issue refs and return their JSON content. pub fn list_issues(repo_path: &Path) -> Result> { // List all refs under refs/gitlawb/issues/ @@ -278,6 +298,27 @@ mod tests { assert!(result.unwrap_err().to_string().contains("ambiguous")); } + #[test] + fn delete_issue_ref_removes_a_created_ref() { + let dir = TempDir::new().unwrap(); + init_repo(&dir); + let full_id = "eee88888-0000-0000-0000-000000000000"; + create_issue( + dir.path(), + full_id, + r#"{"id":"eee88888-0000-0000-0000-000000000000","status":"open"}"#, + ) + .unwrap(); + delete_issue_ref( + "git", + dir.path(), + full_id, + std::time::Instant::now() + std::time::Duration::from_secs(30), + ) + .unwrap(); + assert_eq!(resolve_issue_id(dir.path(), full_id).unwrap(), None); + } + #[test] fn test_close_issue_via_prefix() { let dir = TempDir::new().unwrap(); diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 92cdbfd4..48332661 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -657,11 +657,10 @@ impl RepoStore { /// /// Returns `Err(UploadError::PreconditionLost)` when the create-only upload was /// refused because the key already exists. That is a DISTINCT outcome from a - /// plain upload failure: the sole caller (fork creation) uses it to refuse the - /// fork rather than create a DB record whose archive is shadowed by an orphan - /// other nodes would fetch. Plain upload failures are logged and return `Ok` - /// for the same reason the guard's release logs-and-succeeds: the local write - /// is done and the storage retry is the operator's. + /// plain upload failure. The sole caller is fork creation, which uses the + /// former to refuse the fork rather than create a DB record shadowed by an + /// orphan other nodes would fetch. Plain upload failures are propagated so fork + /// creation does not insert a DB row when the archive never landed. pub async fn release_after_write( &self, owner_did: &str, @@ -693,9 +692,7 @@ impl RepoStore { // Propagated, not logged as success: an orphan archive under // this key shadows the fork for every other node. Err(e @ UploadError::PreconditionLost { .. }) => return Err(e), - Err(e) => { - warn!(repo = %repo_name, err = %e, "failed to upload repo to tigris after write"); - } + Err(e) => return Err(e), } } Ok(()) @@ -720,7 +717,12 @@ impl RepoStore { /// Best-effort cleanup when fork creation published an archive but failed to persist /// the database row. Removes the object-store key this attempt owns and the local /// mirror clone so a retry is not blocked by its own orphan. - pub async fn compensate_fork_archive(&self, owner_did: &str, repo_name: &str, disk_path: &Path) { + pub async fn compensate_fork_archive( + &self, + owner_did: &str, + repo_name: &str, + disk_path: &Path, + ) { if let Some(ref tigris) = self.tigris { if let Ok((owner_slug, _)) = self.local_path(owner_did, repo_name) { if let Err(e) = tigris.delete(&owner_slug, repo_name).await { @@ -786,15 +788,14 @@ pub(crate) fn swap_extracted_into_validated_repo( tmp_dir: &Path, swap_authority: Option<&Arc>, ) -> Result<()> { + let lock = super::tigris::publish_lock(validated_path); + let _publish = lock.lock().expect("publish lock poisoned"); if let Some(authority) = swap_authority { if !authority.load(Ordering::Acquire) { let _ = std::fs::remove_dir_all(tmp_dir); anyhow::bail!("publish swap revoked after lock ownership ended"); } } - - let lock = super::tigris::publish_lock(validated_path); - let _publish = lock.lock().expect("publish lock poisoned"); if validated_path.exists() { std::fs::remove_dir_all(validated_path).context("removing stale repo dir")?; } From 5d3a4ea462631c5b641f5d32faeb3713801965ec Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sat, 29 Aug 2026 12:01:06 -0500 Subject: [PATCH 40/54] fix(node): close write-attempt ownership gaps on advisory-lock release Make publish swaps claim-exclusive via CAS, invalidate the local read cache on definite publish refusals, harden fork create compensation and clone cleanup, and keep snapshot temp dirs owned through the async handoff. --- crates/gitlawb-node/src/api/repos.rs | 65 ++++++++-- crates/gitlawb-node/src/git/repo_store.rs | 142 +++++++++++++++++++--- crates/gitlawb-node/src/git/tigris.rs | 55 +++++---- 3 files changed, 215 insertions(+), 47 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 847d9ee9..a27040e5 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -3,6 +3,7 @@ use axum::http::StatusCode; use axum::response::Response; use axum::Json; use bytes::Bytes; +use std::path::PathBuf; use std::sync::Arc; use crate::auth::{caller_authorized_to_push, AuthenticatedDid}; @@ -3121,6 +3122,40 @@ pub async fn list_federated_repos( // ── Fork ────────────────────────────────────────────────────────────────── +/// Removes a fork's local mirror clone on every exit until disarmed after the +/// database row commits. +struct ForkCloneGuard(Option); + +impl ForkCloneGuard { + fn new(path: PathBuf) -> Self { + Self(Some(path)) + } + + fn path(&self) -> &std::path::Path { + self.0 + .as_deref() + .expect("fork clone guard disarmed while still in use") + } + + fn disarm(&mut self) { + self.0 = None; + } +} + +impl Drop for ForkCloneGuard { + fn drop(&mut self) { + if let Some(path) = self.0.take() { + if let Err(e) = std::fs::remove_dir_all(&path) { + tracing::warn!( + path = %path.display(), + err = %e, + "failed to remove fork clone during attempt cleanup" + ); + } + } + } +} + #[derive(Debug, Deserialize)] pub struct ForkRepoRequest { pub name: Option, // defaults to source repo name @@ -3216,12 +3251,14 @@ pub async fn fork_repo( ))); } + let mut clone_guard = ForkCloneGuard::new(disk_path.clone()); + // Upload fork to Tigris. Create-only: a refused precondition means an orphan // archive already sits under this key (a failed create_repo or another // writer), and proceeding would create a DB record whose archive is shadowed // by bytes that are not this fork. Refuse rather than accept a fork other - // nodes would fetch as unrelated content. Remove the local mirror clone so - // a refused fork does not leave a bare repo on disk. + // nodes would fetch as unrelated content. The clone guard removes the local + // mirror on every upload failure path. state .repo_store .release_after_write(&forker_did, &fork_name) @@ -3234,13 +3271,6 @@ pub async fn fork_repo( status, "fork refused: an archive already exists under the fork's key" ); - if let Err(clean) = std::fs::remove_dir_all(&disk_path) { - tracing::warn!( - path = %disk_path.display(), - err = %clean, - "failed to remove fork clone after precondition loss" - ); - } AppError::RepoExists(fork_name.clone()) } other => AppError::Git(format!("fork upload failed: {other}")), @@ -3256,19 +3286,32 @@ pub async fn fork_repo( default_branch: source.default_branch.clone(), created_at: now, updated_at: now, - disk_path: disk_path.to_string_lossy().to_string(), + disk_path: clone_guard.path().to_string_lossy().to_string(), forked_from: Some(source.id.clone()), machine_id: state.machine_id.clone(), }; if let Err(e) = state.db.create_repo(&record).await { + if let Some(committed) = state.db.get_repo(&forker_short, &fork_name).await? { + clone_guard.disarm(); + tracing::warn!( + fork = %fork_name, + forker = %forker_did, + "fork create_repo returned an error but the row is present — treating as success" + ); + return Ok((StatusCode::CREATED, Json(to_response(&committed, &state, 0)))); + } + clone_guard.disarm(); + let disk_path_for_compensate = disk_path.clone(); state .repo_store - .compensate_fork_archive(&forker_did, &fork_name, &disk_path) + .compensate_fork_archive(&forker_did, &fork_name, &disk_path_for_compensate) .await; return Err(e.into()); } + clone_guard.disarm(); + // Persist the proof so the fork carries it when it propagates to peers. if let Some(p) = verified_proof { if let Err(e) = p.record_for_repo(&state.db, &record.id).await { diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 48332661..da5935e6 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -255,10 +255,12 @@ impl RepoStore { "tigris snapshot download failed during read_snapshot for {owner_slug}/{repo_name}: {e:#}" )) })?; - return Ok(RepoSnapshot { - path: snapshot.clone(), - owned: true, - }); + return match snapshot { + super::tigris::DownloadExtract::Snapshot(dir) => Ok(dir.into_repo_snapshot()), + super::tigris::DownloadExtract::Published(_) => { + unreachable!("snapshot downloads never publish into the live path") + } + }; } Ok(false) => {} Err(e) => { @@ -588,7 +590,7 @@ impl RepoStore { // // Refuse the acquire. Returning here drops the guard, whose Drop // frees the lock and its pool slot. - refresh_swap_authority.store(false, Ordering::Release); + revoke_swap_authority(&refresh_swap_authority); // // `error!`, not the sibling `warn!` above, and that is deliberate. // The handler layer demotes every `RepoUnavailable` to warn because @@ -716,7 +718,9 @@ impl RepoStore { /// Best-effort cleanup when fork creation published an archive but failed to persist /// the database row. Removes the object-store key this attempt owns and the local - /// mirror clone so a retry is not blocked by its own orphan. + /// mirror clone so a retry is not blocked by its own orphan. Object-store deletion + /// is retried asynchronously when the first attempt fails so a transient DELETE does + /// not tombstone the fork name. pub async fn compensate_fork_archive( &self, owner_did: &str, @@ -725,12 +729,21 @@ impl RepoStore { ) { if let Some(ref tigris) = self.tigris { if let Ok((owner_slug, _)) = self.local_path(owner_did, repo_name) { - if let Err(e) = tigris.delete(&owner_slug, repo_name).await { - warn!( - repo = %repo_name, - err = %e, - "failed to delete fork archive during create_repo compensation" - ); + match tigris.delete(&owner_slug, repo_name).await { + Ok(()) => {} + Err(e) => { + warn!( + repo = %repo_name, + err = %e, + "failed to delete fork archive during create_repo compensation — scheduling retry" + ); + let tigris = tigris.clone(); + let slug = owner_slug.clone(); + let name = repo_name.to_string(); + tokio::spawn(async move { + retry_fork_archive_delete(&tigris, &slug, &name).await; + }); + } } } } @@ -744,6 +757,38 @@ impl RepoStore { } } +async fn retry_fork_archive_delete(tigris: &TigrisClient, owner_slug: &str, repo_name: &str) { + const MAX_ATTEMPTS: u32 = 6; + for attempt in 0..MAX_ATTEMPTS { + match tigris.delete(owner_slug, repo_name).await { + Ok(()) => { + info!( + repo = %repo_name, + attempt, + "fork archive compensation delete succeeded on retry" + ); + return; + } + Err(e) if attempt + 1 < MAX_ATTEMPTS => { + warn!( + repo = %repo_name, + attempt, + err = %e, + "fork archive compensation delete failed — retrying" + ); + tokio::time::sleep(Duration::from_secs(1u64 << attempt.min(4))).await; + } + Err(e) => { + warn!( + repo = %repo_name, + err = %e, + "fork archive compensation delete failed after all retries — operator cleanup required" + ); + } + } + } +} + /// The three-layer validated form of `store::repo_disk_path`, with NO Tigris fetch and /// no `RepoStore` (#173 round 11, F3). Extracted from `RepoStore::local_path` so a /// second caller that must not pull a cold repo gets the same barrier instead of the @@ -780,6 +825,21 @@ pub(crate) fn validated_repo_disk_path( Ok(local_path) } +/// Revoke an in-flight publish swap. Any worker that has not yet claimed the +/// commit token will refuse before touching the live tree. +pub(crate) fn revoke_swap_authority(authority: &AtomicBool) { + authority.store(false, Ordering::Release); +} + +/// Claim exclusive rights to perform the destructive publish swap. Revocation and +/// commit are mutually exclusive: only one caller can win the `true -> false` +/// transition, and that caller alone may remove/rename the live directory. +pub(crate) fn try_claim_swap_commit(authority: &AtomicBool) -> bool { + authority + .compare_exchange(true, false, Ordering::AcqRel, Ordering::Acquire) + .is_ok() +} + /// Swap a finished extraction into a validated live repo path. The caller must pass /// the path returned from [`validated_repo_disk_path`]; this is the CodeQL barrier /// for `rust/path-injection` on the remove/rename sink. @@ -791,7 +851,7 @@ pub(crate) fn swap_extracted_into_validated_repo( let lock = super::tigris::publish_lock(validated_path); let _publish = lock.lock().expect("publish lock poisoned"); if let Some(authority) = swap_authority { - if !authority.load(Ordering::Acquire) { + if !try_claim_swap_commit(authority) { let _ = std::fs::remove_dir_all(tmp_dir); anyhow::bail!("publish swap revoked after lock ownership ended"); } @@ -803,6 +863,29 @@ pub(crate) fn swap_extracted_into_validated_repo( Ok(()) } +/// Remove a refused write from the unlocked read cache so `acquire` cannot serve +/// a tree that never landed in object storage. +fn invalidate_local_write_cache(local_path: &Path, repo_name: &str, reason: &str) { + if !local_path.exists() { + return; + } + if let Err(e) = std::fs::remove_dir_all(local_path) { + warn!( + repo = %repo_name, + path = %local_path.display(), + err = %e, + reason, + "failed to invalidate local write cache after a refused publish" + ); + } else { + debug!( + repo = %repo_name, + reason, + "invalidated local write cache after a refused publish" + ); + } +} + /// Strict allowlist validator for `owner_did` and `repo_name`. /// /// Rejects any character that isn't explicitly safe, plus length and @@ -1087,6 +1170,10 @@ impl RepoSnapshot { pub fn path(&self) -> &Path { &self.path } + + pub(crate) fn from_owned_path(path: PathBuf) -> Self { + Self { path, owned: true } + } } impl Drop for RepoSnapshot { @@ -1293,6 +1380,19 @@ impl RepoWriteGuard { warn!(repo = %self.repo_name, "write failed — skipping tigris upload to avoid propagating an inconsistent repo"); } + if success { + match outcome { + ReleaseOutcome::Fenced | ReleaseOutcome::UploadFailed => { + invalidate_local_write_cache( + &self.local_path, + &self.repo_name, + "definite publish refusal", + ); + } + ReleaseOutcome::Released | ReleaseOutcome::UploadUnknowable => {} + } + } + // Release the advisory lock on the SAME session that took it. Unlocking // through the pool would land on an arbitrary backend, where the call is a // silent no-op. @@ -1349,7 +1449,7 @@ impl RepoWriteGuard { impl Drop for RepoWriteGuard { fn drop(&mut self) { if let Some(authority) = self.refresh_swap_authority.take() { - authority.store(false, Ordering::Release); + revoke_swap_authority(&authority); } let Some(mut conn) = self.conn.take() else { @@ -1636,6 +1736,15 @@ mod tests { assert!(ReleaseOutcome::Fenced.into_result().is_err()); } + #[test] + fn swap_commit_token_is_exclusive() { + let authority = Arc::new(AtomicBool::new(true)); + assert!(try_claim_swap_commit(&authority)); + assert!(!try_claim_swap_commit(&authority)); + revoke_swap_authority(&authority); + assert!(!try_claim_swap_commit(&authority)); + } + #[test] fn revoked_publish_swap_cannot_replace_the_live_tree() { let root = tempfile::TempDir::new().unwrap(); @@ -5065,12 +5174,17 @@ mod tests { // ... and the generation the retry HEADs for moves on before its PUT // can use it, so the second attempt loses too. mock.roll_generation_after_next_heads(1); + let local_path = guard.local_path.clone(); let outcome = guard.release(true).await; assert!( matches!(outcome, ReleaseOutcome::Fenced), "a publish refused twice must be reported to the caller, got {outcome:?}" ); + assert!( + !local_path.exists(), + "a fenced publish must invalidate the local read cache" + ); let attempts = mock.put_attempts(); assert_eq!( attempts.len(), diff --git a/crates/gitlawb-node/src/git/tigris.rs b/crates/gitlawb-node/src/git/tigris.rs index af8cbc39..dc262f7a 100644 --- a/crates/gitlawb-node/src/git/tigris.rs +++ b/crates/gitlawb-node/src/git/tigris.rs @@ -253,7 +253,7 @@ impl TigrisClient { target: &Path, publish: bool, swap_authority: Option>, - ) -> Result { + ) -> Result { let key = Self::repo_key(owner_slug, repo_name); debug!(key = %key, path = %target.display(), "downloading repo from tigris"); @@ -295,14 +295,11 @@ impl TigrisClient { let extracted = tokio::task::spawn_blocking({ let target = target.to_path_buf(); let snapshot_tmp = snapshot_tmp.clone(); - move || -> Result { - // Armed for the whole blocking extraction; disarmed on success via - // `mem::forget`, leaving the dir to the caller (RepoSnapshot::drop). - let cleanup = snapshot_tmp.as_ref().map(|p| SnapshotCleanup(p.clone())); - let result = (|| -> Result { + move || -> Result { + let result = (|| -> Result { if publish { decompress_repo(&data, &target, swap_authority.as_ref())?; - return Ok(target); + return Ok(DownloadExtract::Published(target)); } // Non-mutating snapshot: unpack into the temp dir decided above. // The live repo path is never touched. @@ -318,13 +315,8 @@ impl TigrisClient { let _ = std::fs::remove_dir_all(&tmp_dir); return Err(e); } - Ok(tmp_dir) + Ok(DownloadExtract::Snapshot(TempSnapshotDir { path: tmp_dir })) })(); - if result.is_ok() { - if let Some(guard) = cleanup { - std::mem::forget(guard); - } - } result } }) @@ -382,20 +374,39 @@ pub(crate) fn publish_lock(local_path: &Path) -> Arc> { .clone() } -/// Async-layer cleanup for a snapshot temp dir that the extraction's -/// `spawn_blocking` created and that would otherwise outlive a cancelled -/// `download_to` future. Armed before the extraction await, disarmed (via -/// `mem::forget`) on success so `RepoSnapshot::drop` stays the single owner; on -/// any other exit the dir is removed even though the extraction ran to -/// completion. -struct SnapshotCleanup(PathBuf); +/// Owns a snapshot temp dir from extraction through handoff to [`RepoSnapshot`]. +/// Dropped when the `spawn_blocking` join result is abandoned, so cancellation +/// before the outer future resumes still removes the directory. +pub(crate) struct TempSnapshotDir { + path: PathBuf, +} + +impl TempSnapshotDir { + pub(crate) fn path(&self) -> &Path { + &self.path + } + + pub(crate) fn into_repo_snapshot(self) -> super::repo_store::RepoSnapshot { + let path = self.path.clone(); + std::mem::forget(self); + super::repo_store::RepoSnapshot::from_owned_path(path) + } +} -impl Drop for SnapshotCleanup { +impl Drop for TempSnapshotDir { fn drop(&mut self) { - let _ = std::fs::remove_dir_all(&self.0); + let _ = std::fs::remove_dir_all(&self.path); } } +/// What [`TigrisClient::download_to`] produced. +pub enum DownloadExtract { + /// Published into the validated live repo path. + Published(PathBuf), + /// Unpacked into a throwaway temp dir that cleans up on drop until adopted. + Snapshot(TempSnapshotDir), +} + /// Decompress a tar.zst byte vector into a local directory. /// /// Extraction is atomic with respect to `local_path`: the archive is unpacked From 2ad2f12860be3ddeb94b82afd99b34abe66ff590 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sat, 29 Aug 2026 12:29:36 -0500 Subject: [PATCH 41/54] fix(node): gate replication tail on publish durability and split close_issue limits Wait for a confirmed release outcome before post_receive_replication_tail runs any walks or coalescing work. Give close_issue its own per-IP rate bucket so it cannot drain receive-pack quota. Add regression tests for both paths. --- crates/gitlawb-node/src/api/issues.rs | 31 ++++++++-- crates/gitlawb-node/src/api/repos.rs | 81 ++++++++++++++++++++----- crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/main.rs | 20 ++++++ crates/gitlawb-node/src/state.rs | 5 ++ crates/gitlawb-node/src/test_support.rs | 1 + 6 files changed, 120 insertions(+), 19 deletions(-) diff --git a/crates/gitlawb-node/src/api/issues.rs b/crates/gitlawb-node/src/api/issues.rs index adce758d..276e5e1b 100644 --- a/crates/gitlawb-node/src/api/issues.rs +++ b/crates/gitlawb-node/src/api/issues.rs @@ -289,7 +289,7 @@ pub async fn close_issue( // close requests for arbitrary issue ids. Applied after the read gate so a // denied reader still sees 404, not 429. if let Some(key) = crate::rate_limit::client_key(&headers, peer, state.push_limiter_trust) { - if !state.push_rate_limiter.check(&key).await { + if !state.close_issue_rate_limiter.check(&key).await { tracing::warn!(repo = %repo, key = %key, "close_issue rate limited"); return Err(AppError::TooManyRequests( "rate limit exceeded — try again later".into(), @@ -1105,7 +1105,7 @@ mod lock_pool_shed_tests { let owner = "did:key:zCLOSERATEOWNERAAAAAAAAAAAAAAAAAAAAAAA"; let stranger = "did:key:zCLOSERATESTRANGERBBBBBBBBBBBBBBBBBB"; let mut state = crate::test_support::test_state(pool).await; - state.push_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); + state.close_issue_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; let mut repo = seed_repo(owner, "priv-close"); @@ -1114,8 +1114,8 @@ mod lock_pool_shed_tests { let peer: SocketAddr = "203.0.113.88:7000".parse().unwrap(); assert!( - state.push_rate_limiter.check(&peer.ip().to_string()).await, - "exhaust the peer bucket before close_issue" + state.close_issue_rate_limiter.check(&peer.ip().to_string()).await, + "exhaust the close_issue bucket before close_issue" ); let res = close_issue( @@ -1134,6 +1134,29 @@ mod lock_pool_shed_tests { ); } + /// close_issue and receive-pack must not share one per-IP bucket. + #[sqlx::test] + async fn close_issue_rate_limit_does_not_drain_push_bucket(pool: PgPool) { + use std::net::SocketAddr; + use std::time::Duration; + + let mut state = crate::test_support::test_state(pool).await; + state.close_issue_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); + state.push_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let peer: SocketAddr = "203.0.113.99:7000".parse().unwrap(); + let key = peer.ip().to_string(); + assert!( + state.close_issue_rate_limiter.check(&key).await, + "exhaust only the close_issue bucket" + ); + assert!( + state.push_rate_limiter.check(&key).await, + "push traffic must keep its own bucket after close_issue is exhausted" + ); + } + /// A refused publish must roll back the local issue ref so a retry does not /// mint a second id for the same logical filing attempt. #[sqlx::test] diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index a27040e5..a9d242e0 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -2556,6 +2556,20 @@ async fn post_receive_replication_tail( Arc>>, >, ) { + let durability_wait = std::time::Duration::from_secs( + state + .config + .lock_held_transfer_timeout_secs + .saturating_add(5), + ); + if !publish_durability_confirmed(&publish_durability, durability_wait).await { + tracing::warn!( + repo = %record.id, + "skipping post-receive tail: publish durability was not confirmed" + ); + return; + } + // Replication enforcement (Phase 2): decide once per push whether the public // may read this repo at all and, if so, which blob OIDs must not leave the // node. `withheld == None` means this push pins nothing (private / mode A / @@ -2728,21 +2742,6 @@ async fn post_receive_replication_tail( )); } - let durability_wait = std::time::Duration::from_secs( - state - .config - .lock_held_transfer_timeout_secs - .saturating_add(5), - ); - if announce_at_root && !publish_durability_confirmed(&publish_durability, durability_wait).await - { - tracing::warn!( - repo = %record.id, - "skipping pinata/gossip tail: publish durability was not confirmed" - ); - return; - } - // Pin new git objects to Pinata, then record branch→CID and gossip. // // #174 P2-2 scope note: this SECOND detached spawn is deliberately NOT brought @@ -3576,6 +3575,21 @@ mod tests { ); } + #[test] + fn fork_clone_guard_removes_mirror_on_drop() { + let root = tempfile::TempDir::new().unwrap(); + let mirror = root.path().join("fork.git"); + std::fs::create_dir_all(&mirror).unwrap(); + { + let _guard = ForkCloneGuard::new(mirror.clone()); + assert!(mirror.exists()); + } + assert!( + !mirror.exists(), + "dropping the fork clone guard must remove the mirror directory" + ); + } + #[test] fn upload_pack_request_finalizes_only_with_done_pktline() { let want = "0032want 1111111111111111111111111111111111111111\n"; @@ -9912,6 +9926,43 @@ mod tests { const F2A_PUSHER: &str = "did:key:z6MkF2aPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; + /// When publish durability is fenced, the tail must not run walks or take + /// coalescing keys before returning. + #[cfg(unix)] + #[sqlx::test] + async fn post_receive_tail_skips_all_work_when_publish_fenced(pool: sqlx::PgPool) { + let repo = tempfile::TempDir::new().unwrap(); + let bin = tempfile::TempDir::new().unwrap(); + let log = bin.path().join("git.log"); + let git_bin = f2a_logging_git(bin.path(), &log); + u5_init_repo(repo.path()); + let c1 = u5_commit_file(repo.path(), "a.txt", "one\n"); + let (state, rec) = f2a_state(pool, &git_bin, "z6f2afence", "fence-repo", false).await; + let slot = Arc::new(tokio::sync::Mutex::new(Some( + crate::git::repo_store::ReleaseOutcome::Fenced, + ))); + post_receive_replication_tail( + state.clone(), + rec, + f2a_update("refs/heads/main", &c1), + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + Some(slot), + ) + .await; + assert_eq!( + f2a_walks(&log), + 0, + "a fenced publish must not run the replication walk; log:\n{}", + f2a_log(&log) + ); + assert_eq!( + state.encrypt_inflight.len(), + 0, + "a fenced publish must not take the per-repo coalescing key" + ); + } + /// Scenario 1 (the finding). A second rapid push to the same repo coalesces /// WITHOUT running the withheld walk. Asserted on the walk's git children, not /// on the `Coalesced` outcome: with `try_begin` below the walk (the pre-fix diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 27b67786..6d81dbd4 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -517,6 +517,7 @@ mod tests { rate_limiter: RateLimiter::new(100, Duration::from_secs(60)), create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + close_issue_rate_limiter: RateLimiter::new(120, Duration::from_secs(3600)), ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), ipfs_work_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index fb812207..e6b3d401 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -346,6 +346,23 @@ async fn main() -> Result<()> { tracing::warn!("GITLAWB_PUSH_RATE_LIMIT=0 — per-IP push rate limiting disabled"); } + // close_issue drives a full archive download for non-owner author pre-checks. + // Keep it on its own bucket so it cannot drain the receive-pack limit. + let close_issue_limit = std::env::var("GITLAWB_CLOSE_ISSUE_RATE_LIMIT") + .ok() + .and_then(|v| v.trim().parse::().ok()) + .unwrap_or(120); + let close_issue_rate_limiter = rate_limit::RateLimiter::new_bounded( + close_issue_limit, + std::time::Duration::from_secs(3600), + 200_000, + ); + if close_issue_limit == 0 { + tracing::warn!( + "GITLAWB_CLOSE_ISSUE_RATE_LIMIT=0 — per-IP close_issue rate limiting disabled" + ); + } + // Which forwarded header the edge is trusted to set. Default None (trust // nothing, key on the socket peer). Fly nodes set GITLAWB_TRUSTED_PROXY=fly; // a node behind Caddy/NGINX sets it to x-forwarded-for. @@ -395,6 +412,7 @@ async fn main() -> Result<()> { rate_limiter, create_ip_rate_limiter, push_rate_limiter, + close_issue_rate_limiter, ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, ipfs_max_legacy_probes: AppState::ipfs_legacy_probe_budget(&config), ipfs_legacy_scan_page_rows: crate::api::ipfs::LEGACY_SCAN_PAGE_ROWS, @@ -1106,6 +1124,7 @@ mod rate_limiter_sweep_tests { state.rate_limiter = RateLimiter::new(10, window); state.create_ip_rate_limiter = RateLimiter::new(10, window); state.push_rate_limiter = RateLimiter::new(10, window); + state.close_issue_rate_limiter = RateLimiter::new(10, window); state.sync_trigger_rate_limiter = RateLimiter::new(10, window); state.peer_write_rate_limiter = RateLimiter::new(10, window); state.ipfs_rate_limiter = RateLimiter::new(10, window); @@ -1115,6 +1134,7 @@ mod rate_limiter_sweep_tests { s.rate_limiter.clone(), s.create_ip_rate_limiter.clone(), s.push_rate_limiter.clone(), + s.close_issue_rate_limiter.clone(), s.sync_trigger_rate_limiter.clone(), s.peer_write_rate_limiter.clone(), s.ipfs_rate_limiter.clone(), diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 24607e5a..7fcd1f5d 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -79,6 +79,10 @@ pub struct AppState { /// brake a push flood from a DID farm (one throwaway DID per repo), so the /// push path throttles on the resolved client IP instead. pub push_rate_limiter: RateLimiter, + /// Per-client-IP rate limiter for `close_issue`'s pre-lock snapshot path. + /// Distinct from `push_rate_limiter` so a flood of close attempts cannot + /// drain the receive-pack budget for the same source IP. + pub close_issue_rate_limiter: RateLimiter, /// Per-client-IP ROUTE brake for `GET /ipfs/{cid}`: charged ONCE per request by the /// `rate_limit_by_ip` middleware (server.rs), never inside the handler. It bounds /// request RATE (the "requests per hour" contract of `GITLAWB_IPFS_RATE_LIMIT`) on @@ -342,6 +346,7 @@ impl AppState { self.rate_limiter.cleanup().await; self.create_ip_rate_limiter.cleanup().await; self.push_rate_limiter.cleanup().await; + self.close_issue_rate_limiter.cleanup().await; self.ipfs_rate_limiter.cleanup().await; self.ipfs_work_rate_limiter.cleanup().await; self.sync_trigger_rate_limiter.cleanup().await; diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 3dcea556..e5c9b0c5 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -103,6 +103,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { rate_limiter: RateLimiter::new(100, Duration::from_secs(60)), create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + close_issue_rate_limiter: RateLimiter::new(120, Duration::from_secs(3600)), ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), ipfs_work_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, From 487687c5fd67ecc4e0389bff177911a73fd06e0e Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sat, 29 Aug 2026 15:29:01 -0500 Subject: [PATCH 42/54] fix(node): add ValidatedRepoDiskPath for CodeQL and reconcile tail durability Introduce a validated-path newtype with the same inline join and component walk main uses, so path-injection sinks only accept sanitised repo paths. Reconcile the publish-durability gate with the disconnect-safe tail spawn via PublishDurabilitySlot and explicit UploadUnknowable handling. --- crates/gitlawb-node/src/api/issues.rs | 11 ++- crates/gitlawb-node/src/api/repos.rs | 105 +++++++++++++++++----- crates/gitlawb-node/src/git/repo_store.rs | 95 +++++++++++++++----- crates/gitlawb-node/src/git/store.rs | 1 + crates/gitlawb-node/src/git/tigris.rs | 25 +++--- crates/gitlawb-node/src/ipfs_pin.rs | 2 +- 6 files changed, 177 insertions(+), 62 deletions(-) diff --git a/crates/gitlawb-node/src/api/issues.rs b/crates/gitlawb-node/src/api/issues.rs index 276e5e1b..42152fd4 100644 --- a/crates/gitlawb-node/src/api/issues.rs +++ b/crates/gitlawb-node/src/api/issues.rs @@ -1105,7 +1105,8 @@ mod lock_pool_shed_tests { let owner = "did:key:zCLOSERATEOWNERAAAAAAAAAAAAAAAAAAAAAAA"; let stranger = "did:key:zCLOSERATESTRANGERBBBBBBBBBBBBBBBBBB"; let mut state = crate::test_support::test_state(pool).await; - state.close_issue_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); + state.close_issue_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; let mut repo = seed_repo(owner, "priv-close"); @@ -1114,7 +1115,10 @@ mod lock_pool_shed_tests { let peer: SocketAddr = "203.0.113.88:7000".parse().unwrap(); assert!( - state.close_issue_rate_limiter.check(&peer.ip().to_string()).await, + state + .close_issue_rate_limiter + .check(&peer.ip().to_string()) + .await, "exhaust the close_issue bucket before close_issue" ); @@ -1141,7 +1145,8 @@ mod lock_pool_shed_tests { use std::time::Duration; let mut state = crate::test_support::test_state(pool).await; - state.close_issue_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); + state.close_issue_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); state.push_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index a9d242e0..95b23c67 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -2375,7 +2375,7 @@ pub async fn git_receive_pack( // would return 200 to the pusher before the durable copy lands, which is a larger // change to the client contract than the window it closes. let push_succeeded = receive_result.is_ok(); - let publish_durability = Arc::new(tokio::sync::Mutex::new(None)); + let publish_durability = PublishDurabilitySlot::new(); if push_succeeded { tokio::spawn(post_receive_replication_tail( state.clone(), @@ -2383,7 +2383,7 @@ pub async fn git_receive_pack( ref_updates.clone(), disk_path.clone(), auth.0.to_string(), - Some(publish_durability.clone()), + Some(publish_durability.arc()), )); } @@ -2406,7 +2406,7 @@ pub async fn git_receive_pack( // node can read. let outcome = reclaimed.release(push_succeeded).await; if push_succeeded { - *publish_durability.lock().await = Some(outcome); + publish_durability.record(outcome).await; } outcome.into_result()?; // Clean path: clone (a) already dropped inside run_git_service when the receive-pack @@ -2525,6 +2525,47 @@ pub async fn git_receive_pack( /// the per-repo-coalesced pin/encrypt task, and this push's own Pinata + announce /// task. Split out of `git_receive_pack` so the ordering the coalescing gate depends /// on is directly testable; the handler spawns it and returns. +/// Records the release-side publish outcome for the detached post-receive tail. +/// If the handler future is dropped before [`Self::record`], the drop sets +/// [`ReleaseOutcome::UploadUnknowable`] so the tail can proceed on local disk +/// without waiting out the full transfer bound. +struct PublishDurabilitySlot { + inner: Arc>>, + recorded: std::sync::atomic::AtomicBool, +} + +impl PublishDurabilitySlot { + fn new() -> Self { + Self { + inner: Arc::new(tokio::sync::Mutex::new(None)), + recorded: std::sync::atomic::AtomicBool::new(false), + } + } + + fn arc(&self) -> Arc>> { + Arc::clone(&self.inner) + } + + async fn record(&self, outcome: crate::git::repo_store::ReleaseOutcome) { + self.recorded + .store(true, std::sync::atomic::Ordering::SeqCst); + *self.inner.lock().await = Some(outcome); + } +} + +impl Drop for PublishDurabilitySlot { + fn drop(&mut self) { + if self.recorded.load(std::sync::atomic::Ordering::SeqCst) { + return; + } + if let Ok(mut slot) = self.inner.try_lock() { + if slot.is_none() { + *slot = Some(crate::git::repo_store::ReleaseOutcome::UploadUnknowable); + } + } + } +} + async fn publish_durability_confirmed( slot: &Option>>>, wait: std::time::Duration, @@ -2535,12 +2576,19 @@ async fn publish_durability_confirmed( let start = std::time::Instant::now(); loop { if let Some(outcome) = *slot.lock().await { - return matches!(outcome, crate::git::repo_store::ReleaseOutcome::Released); + return matches!( + outcome, + crate::git::repo_store::ReleaseOutcome::Released + | crate::git::repo_store::ReleaseOutcome::UploadUnknowable + ); } if start.elapsed() >= wait { - // Handler disconnected during `release` without recording an outcome. - // Fail closed: a pending inner `None` is not durability. - return false; + // No outcome within the release-side transfer bound plus slack: the + // handler was dropped mid-release after a successful receive-pack, or + // release is stuck past its own deadline. The tail is only spawned on + // push success, so local-disk replication work may proceed. Explicit + // non-Released outcomes are handled above. + return true; } tokio::time::sleep(std::time::Duration::from_millis(10)).await; } @@ -3126,8 +3174,8 @@ pub async fn list_federated_repos( struct ForkCloneGuard(Option); impl ForkCloneGuard { - fn new(path: PathBuf) -> Self { - Self(Some(path)) + fn new(path: crate::git::repo_store::ValidatedRepoDiskPath) -> Self { + Self(Some(path.into_path_buf())) } fn path(&self) -> &std::path::Path { @@ -3291,20 +3339,22 @@ pub async fn fork_repo( }; if let Err(e) = state.db.create_repo(&record).await { - if let Some(committed) = state.db.get_repo(&forker_short, &fork_name).await? { + if let Some(committed) = state.db.get_repo(forker_short, &fork_name).await? { clone_guard.disarm(); tracing::warn!( fork = %fork_name, forker = %forker_did, "fork create_repo returned an error but the row is present — treating as success" ); - return Ok((StatusCode::CREATED, Json(to_response(&committed, &state, 0)))); + return Ok(( + StatusCode::CREATED, + Json(to_response(&committed, &state, 0)), + )); } clone_guard.disarm(); - let disk_path_for_compensate = disk_path.clone(); state .repo_store - .compensate_fork_archive(&forker_did, &fork_name, &disk_path_for_compensate) + .compensate_fork_archive(&forker_did, &fork_name, disk_path.as_path()) .await; return Err(e.into()); } @@ -3548,13 +3598,13 @@ mod tests { const STRANGER_DID: &str = "did:key:z6Mkffonly5tranger0000000000000000000000000000000"; #[tokio::test] - async fn publish_durability_confirmed_fails_closed_when_release_never_records() { + async fn publish_durability_confirmed_proceeds_when_release_never_records() { let slot = Arc::new(tokio::sync::Mutex::new(None)); let confirmed = publish_durability_confirmed(&Some(slot), std::time::Duration::from_millis(30)).await; assert!( - !confirmed, - "a pending release outcome must not be treated as confirmed durability" + confirmed, + "an abandoned handler leaves the slot pending; after the bounded wait the tail may proceed on local disk" ); } @@ -3570,6 +3620,14 @@ mod tests { let slot = Arc::new(tokio::sync::Mutex::new(Some( crate::git::repo_store::ReleaseOutcome::UploadUnknowable, ))); + assert!( + publish_durability_confirmed(&Some(slot), std::time::Duration::from_millis(5)).await, + "an unknowable upload still landed on local disk, so the tail may proceed" + ); + + let slot = Arc::new(tokio::sync::Mutex::new(Some( + crate::git::repo_store::ReleaseOutcome::UploadFailed, + ))); assert!( !publish_durability_confirmed(&Some(slot), std::time::Duration::from_millis(5)).await ); @@ -3578,14 +3636,19 @@ mod tests { #[test] fn fork_clone_guard_removes_mirror_on_drop() { let root = tempfile::TempDir::new().unwrap(); - let mirror = root.path().join("fork.git"); - std::fs::create_dir_all(&mirror).unwrap(); + let validated = crate::git::repo_store::validated_repo_disk_path( + root.path(), + "did:key:testfork", + "fork", + ) + .expect("test fork path must validate"); + std::fs::create_dir_all(validated.as_path()).unwrap(); { - let _guard = ForkCloneGuard::new(mirror.clone()); - assert!(mirror.exists()); + let _guard = ForkCloneGuard::new(validated.clone()); + assert!(validated.exists()); } assert!( - !mirror.exists(), + !validated.exists(), "dropping the fork clone guard must remove the mirror directory" ); } diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index da5935e6..f4d623d1 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -194,7 +194,7 @@ impl RepoStore { }); } } - return Ok(local_path); + return Ok(local_path.into_path_buf()); } // Try downloading from Tigris @@ -210,13 +210,13 @@ impl RepoStore { .lock() .await .insert(format!("{owner_slug}/{repo_name}")); - return Ok(local_path); + return Ok(local_path.into_path_buf()); } } // Not found anywhere — return path anyway; caller will get a meaningful // error from git when the path doesn't exist. - Ok(local_path) + Ok(local_path.into_path_buf()) } /// Non-mutating snapshot of a repo's **latest** Tigris state, for reads that @@ -256,7 +256,9 @@ impl RepoStore { )) })?; return match snapshot { - super::tigris::DownloadExtract::Snapshot(dir) => Ok(dir.into_repo_snapshot()), + super::tigris::DownloadExtract::Snapshot(dir) => { + Ok(dir.into_repo_snapshot()) + } super::tigris::DownloadExtract::Published(_) => { unreachable!("snapshot downloads never publish into the live path") } @@ -275,7 +277,7 @@ impl RepoStore { // Tigris disabled or repo not in Tigris — fall back to local. Ok(RepoSnapshot { - path: local_path, + path: local_path.into_path_buf(), owned: false, }) } @@ -651,7 +653,7 @@ impl RepoStore { }); } - Ok(local_path) + Ok(local_path.into_path_buf()) } /// Upload a repo to Tigris after a write operation (push, merge, fork, etc.). @@ -710,7 +712,11 @@ impl RepoStore { /// (or the prefix/root from `repos_dir`); any `ParentDir`/`CurDir` /// segment is rejected. This is the CodeQL-recognised barrier /// pattern for `rust/path-injection`. - fn local_path(&self, owner_did: &str, repo_name: &str) -> Result<(String, PathBuf)> { + fn local_path( + &self, + owner_did: &str, + repo_name: &str, + ) -> Result<(String, ValidatedRepoDiskPath)> { let owner_slug = owner_did.replace([':', '/'], "_"); let local_path = validated_repo_disk_path(&self.repos_dir, owner_did, repo_name)?; Ok((owner_slug, local_path)) @@ -789,6 +795,38 @@ async fn retry_fork_archive_delete(tigris: &TigrisClient, owner_slug: &str, repo } } +/// A repository disk path that has passed the three-layer validation barrier. +/// +/// Only [`validated_repo_disk_path`] may construct this type, so sinks such as +/// `remove_dir_all` / `rename` can take it and static analysers can treat it as +/// sanitised input (CodeQL `rust/path-injection`). +#[derive(Debug, Clone)] +pub(crate) struct ValidatedRepoDiskPath(PathBuf); + +impl ValidatedRepoDiskPath { + pub(crate) fn as_path(&self) -> &Path { + &self.0 + } + + pub(crate) fn into_path_buf(self) -> PathBuf { + self.0 + } +} + +impl std::ops::Deref for ValidatedRepoDiskPath { + type Target = Path; + + fn deref(&self) -> &Path { + &self.0 + } +} + +impl AsRef for ValidatedRepoDiskPath { + fn as_ref(&self) -> &Path { + &self.0 + } +} + /// The three-layer validated form of `store::repo_disk_path`, with NO Tigris fetch and /// no `RepoStore` (#173 round 11, F3). Extracted from `RepoStore::local_path` so a /// second caller that must not pull a cold repo gets the same barrier instead of the @@ -797,10 +835,11 @@ pub(crate) fn validated_repo_disk_path( repos_dir: &Path, owner_did: &str, repo_name: &str, -) -> Result { +) -> Result { validate_path_components(owner_did, repo_name)?; - let local_path = store::repo_disk_path(repos_dir, owner_did, repo_name); + let owner_slug = owner_did.replace([':', '/'], "_"); + let local_path = repos_dir.join(&owner_slug).join(format!("{repo_name}.git")); if !local_path.starts_with(repos_dir) { anyhow::bail!( @@ -809,6 +848,10 @@ pub(crate) fn validated_repo_disk_path( ); } + // Explicit component walk, sanitisation barrier that static analysers + // (CodeQL `rust/path-injection`) recognise. The path must be composed entirely + // of Normal segments after the root prefix; any ParentDir or CurDir component + // is a traversal attempt. for component in local_path.components() { use std::path::Component; match component { @@ -822,7 +865,7 @@ pub(crate) fn validated_repo_disk_path( } } - Ok(local_path) + Ok(ValidatedRepoDiskPath(local_path)) } /// Revoke an in-flight publish swap. Any worker that has not yet claimed the @@ -844,11 +887,12 @@ pub(crate) fn try_claim_swap_commit(authority: &AtomicBool) -> bool { /// the path returned from [`validated_repo_disk_path`]; this is the CodeQL barrier /// for `rust/path-injection` on the remove/rename sink. pub(crate) fn swap_extracted_into_validated_repo( - validated_path: &Path, + validated_path: &ValidatedRepoDiskPath, tmp_dir: &Path, swap_authority: Option<&Arc>, ) -> Result<()> { - let lock = super::tigris::publish_lock(validated_path); + let live = validated_path.as_path(); + let lock = super::tigris::publish_lock(live); let _publish = lock.lock().expect("publish lock poisoned"); if let Some(authority) = swap_authority { if !try_claim_swap_commit(authority) { @@ -856,10 +900,10 @@ pub(crate) fn swap_extracted_into_validated_repo( anyhow::bail!("publish swap revoked after lock ownership ended"); } } - if validated_path.exists() { - std::fs::remove_dir_all(validated_path).context("removing stale repo dir")?; + if live.exists() { + std::fs::remove_dir_all(live).context("removing stale repo dir")?; } - std::fs::rename(tmp_dir, validated_path).context("swapping extracted repo into place")?; + std::fs::rename(tmp_dir, live).context("swapping extracted repo into place")?; Ok(()) } @@ -1189,7 +1233,7 @@ impl Drop for RepoSnapshot { pub struct RepoWriteGuard { owner_slug: String, repo_name: String, - pub local_path: PathBuf, + pub local_path: ValidatedRepoDiskPath, lock_key: i64, /// The connection that TOOK the lock. Postgres advisory locks are /// session-scoped, so only this session can release it; holding it here is @@ -1238,7 +1282,7 @@ impl RepoWriteGuard { /// Path to the bare repo on local disk. pub fn path(&self) -> &Path { - &self.local_path + self.local_path.as_path() } /// Publish the tree this guard wrote, fenced on the generation observed @@ -2771,7 +2815,7 @@ mod tests { let guard = RepoWriteGuard { owner_slug: slug, repo_name: name.to_string(), - local_path: dir.path().to_path_buf(), + local_path: validated_repo_disk_path(dir.path(), owner, name).expect("test path"), lock_key: key, conn: Some(pool.acquire().await.expect("conn")), tigris: None, @@ -3131,11 +3175,13 @@ mod tests { pid.0 }; + let dir = tempfile::TempDir::new().unwrap(); // A guard whose key was never locked: release()'s unlock returns false. let guard = RepoWriteGuard { owner_slug: "did_key_z6MkU5".to_string(), repo_name: "never-locked".to_string(), - local_path: PathBuf::from("/tmp/gitlawb-u5"), + local_path: validated_repo_disk_path(dir.path(), "did:key:z6MkU5", "never-locked") + .expect("test path"), lock_key: 995_001, conn: Some(lock_pool.acquire().await.unwrap()), tigris: None, @@ -3705,8 +3751,9 @@ mod tests { .await .expect("snapshot reads the archive"); let snap_path = snap.path().to_path_buf(); + let live_path_buf = live_path.as_path().to_path_buf(); assert_ne!( - snap_path, live_path, + snap_path, live_path_buf, "the snapshot must unpack into a temp dir, not the live path" ); assert!( @@ -4918,12 +4965,14 @@ mod tests { /// The marker inside whatever archive is currently stored under the key. async fn stored_marker(mock: &S3Mock, owner_slug: &str, repo_name: &str) -> String { let out = TempDir::new().unwrap(); - let into = out.path().join("stored.git"); + let validated = super::validated_repo_disk_path(out.path(), "did:key:stored", "stored") + .expect("test repo path must validate"); mock_tigris(mock) - .download(owner_slug, repo_name, &into, None) + .download(owner_slug, repo_name, &validated, None) .await .expect("the stored archive must be readable"); - std::fs::read_to_string(into.join("MARKER")).expect("the stored archive must be marked") + std::fs::read_to_string(validated.join("MARKER")) + .expect("the stored archive must be marked") } /// A process-wide sink for warn-level tracing output, installed once. diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 5617b419..9ca16e88 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -870,6 +870,7 @@ pub fn merge_branch( } /// Resolve a repo disk path: {repos_dir}/{owner_slug}/{repo_name}.git +#[allow(dead_code)] // exercised from test_support and state tests; production uses validated_repo_disk_path pub fn repo_disk_path(repos_dir: &Path, owner_did: &str, repo_name: &str) -> PathBuf { // Sanitize the DID for use as a directory name let owner_slug = owner_did.replace([':', '/'], "_"); diff --git a/crates/gitlawb-node/src/git/tigris.rs b/crates/gitlawb-node/src/git/tigris.rs index dc262f7a..2adf1c2a 100644 --- a/crates/gitlawb-node/src/git/tigris.rs +++ b/crates/gitlawb-node/src/git/tigris.rs @@ -230,7 +230,7 @@ impl TigrisClient { &self, owner_slug: &str, repo_name: &str, - local_path: &Path, + local_path: &super::repo_store::ValidatedRepoDiskPath, swap_authority: Option>, ) -> Result<()> { self.download_to(owner_slug, repo_name, local_path, true, swap_authority) @@ -250,12 +250,12 @@ impl TigrisClient { &self, owner_slug: &str, repo_name: &str, - target: &Path, + target: &super::repo_store::ValidatedRepoDiskPath, publish: bool, swap_authority: Option>, ) -> Result { let key = Self::repo_key(owner_slug, repo_name); - debug!(key = %key, path = %target.display(), "downloading repo from tigris"); + debug!(key = %key, path = %target.as_path().display(), "downloading repo from tigris"); let resp = self .s3 @@ -293,13 +293,13 @@ impl TigrisClient { // Extract tar.zst to a directory. let extracted = tokio::task::spawn_blocking({ - let target = target.to_path_buf(); + let target = target.clone(); let snapshot_tmp = snapshot_tmp.clone(); move || -> Result { let result = (|| -> Result { if publish { decompress_repo(&data, &target, swap_authority.as_ref())?; - return Ok(DownloadExtract::Published(target)); + return Ok(DownloadExtract::Published(())); } // Non-mutating snapshot: unpack into the temp dir decided above. // The live repo path is never touched. @@ -324,7 +324,7 @@ impl TigrisClient { .context("extract task panicked")? .context("extracting repo")?; - info!(key = %key, path = %target.display(), "downloaded repo from tigris"); + info!(key = %key, path = %target.as_path().display(), "downloaded repo from tigris"); Ok(extracted) } @@ -382,10 +382,6 @@ pub(crate) struct TempSnapshotDir { } impl TempSnapshotDir { - pub(crate) fn path(&self) -> &Path { - &self.path - } - pub(crate) fn into_repo_snapshot(self) -> super::repo_store::RepoSnapshot { let path = self.path.clone(); std::mem::forget(self); @@ -402,7 +398,7 @@ impl Drop for TempSnapshotDir { /// What [`TigrisClient::download_to`] produced. pub enum DownloadExtract { /// Published into the validated live repo path. - Published(PathBuf), + Published(()), /// Unpacked into a throwaway temp dir that cleans up on drop until adopted. Snapshot(TempSnapshotDir), } @@ -416,13 +412,14 @@ pub enum DownloadExtract { /// leave `local_path` exactly as it was. fn decompress_repo( data: &[u8], - local_path: &Path, + local_path: &super::repo_store::ValidatedRepoDiskPath, swap_authority: Option<&Arc>, ) -> Result<()> { - let parent = local_path.parent().context("repo path has no parent")?; + let live = local_path.as_path(); + let parent = live.parent().context("repo path has no parent")?; std::fs::create_dir_all(parent).context("creating parent dir")?; - let file_name = local_path + let file_name = live .file_name() .context("repo path has no file name")? .to_string_lossy(); diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 5d4579a3..5ee09aa5 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -676,7 +676,7 @@ async fn warm_candidates( &repo.owner_did, &repo.name, ) { - Ok(p) if p.is_dir() => out.push((repo, created_at_key, p)), + Ok(p) if p.is_dir() => out.push((repo, created_at_key, p.into_path_buf())), Ok(_) => {} Err(e) => { tracing::warn!(repo_id = %repo.id, err = %e, "sweep discovery: rejected unsafe repo path"); From 9d02139dad4c66ab924b21d33337e0ab2ea8e473 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sun, 30 Aug 2026 13:04:54 -0500 Subject: [PATCH 43/54] fix(node): close three jatmn findings on advisory-lock PR Retry fork create confirmation before compensating or returning, schedule background recovery when the lookup stays unavailable, install publish durability outcomes before the recorded flag, and size the lock pool for non-push mutations (default 40). --- .env.example | 2 +- crates/gitlawb-node/src/api/repos.rs | 184 ++++++++++++++++++++++++--- crates/gitlawb-node/src/config.rs | 50 ++++++-- 3 files changed, 204 insertions(+), 32 deletions(-) diff --git a/.env.example b/.env.example index 711c4fb1..75fce06c 100644 --- a/.env.example +++ b/.env.example @@ -33,7 +33,7 @@ GITLAWB_DB_MAX_CONNECTIONS=48 # stops a push burst from starving ordinary request handlers. Budget # (GITLAWB_DB_MAX_CONNECTIONS + this) per node against the server's # max_connections, times node count, plus admin tooling. -GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS=32 +GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS=40 # Upper bound, in seconds, on any object-storage transfer that runs while a # per-repo write lock is HELD (the archive download inside acquire_write and the # upload inside release). These were free before the lock's connection was diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 95b23c67..0acfa3ef 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -2547,9 +2547,9 @@ impl PublishDurabilitySlot { } async fn record(&self, outcome: crate::git::repo_store::ReleaseOutcome) { + *self.inner.lock().await = Some(outcome); self.recorded .store(true, std::sync::atomic::Ordering::SeqCst); - *self.inner.lock().await = Some(outcome); } } @@ -3173,6 +3173,95 @@ pub async fn list_federated_repos( /// database row commits. struct ForkCloneGuard(Option); +const FORK_CREATE_CONFIRM_ATTEMPTS: u32 = 5; + +/// After `create_repo` fails, re-read the row with bounded retries so a +/// transport-level error does not skip archive compensation when the insert never +/// landed, and so a successful insert is not mistaken for a failure. +async fn confirm_fork_repo_row( + db: &crate::db::Db, + owner_short: &str, + fork_name: &str, +) -> anyhow::Result> { + let mut delay = std::time::Duration::from_millis(25); + for attempt in 0..FORK_CREATE_CONFIRM_ATTEMPTS { + match db.get_repo(owner_short, fork_name).await { + Ok(row) => return Ok(row), + Err(e) if attempt + 1 < FORK_CREATE_CONFIRM_ATTEMPTS => { + tracing::warn!( + fork = %fork_name, + attempt, + err = %e, + "fork create_repo confirmation lookup failed — retrying" + ); + tokio::time::sleep(delay).await; + delay = delay + .saturating_mul(2) + .min(std::time::Duration::from_secs(1)); + } + Err(e) => return Err(e), + } + } + unreachable!("loop returns on every attempt") +} + +/// When the confirmation lookup stays unavailable after bounded retries, keep +/// retrying in the background and compensate only after establishing that no row +/// exists for this fork name. +async fn schedule_fork_create_recovery( + db: std::sync::Arc, + repo_store: crate::git::repo_store::RepoStore, + owner_short: String, + fork_name: String, + owner_did: String, + disk_path: std::path::PathBuf, +) { + let mut delay = std::time::Duration::from_millis(250); + for attempt in 0..12 { + match db.get_repo(&owner_short, &fork_name).await { + Ok(Some(_)) => { + tracing::info!( + fork = %fork_name, + attempt, + "fork create_repo recovery found a committed row — no archive compensation" + ); + return; + } + Ok(None) => { + tracing::warn!( + fork = %fork_name, + attempt, + "fork create_repo recovery confirmed no row — compensating orphan archive" + ); + repo_store + .compensate_fork_archive(&owner_did, &fork_name, disk_path.as_path()) + .await; + return; + } + Err(e) if attempt + 1 < 12 => { + tracing::warn!( + fork = %fork_name, + attempt, + err = %e, + "fork create_repo recovery lookup failed — retrying" + ); + tokio::time::sleep(delay).await; + delay = delay + .saturating_mul(2) + .min(std::time::Duration::from_secs(30)); + } + Err(e) => { + tracing::error!( + fork = %fork_name, + err = %e, + "fork create_repo recovery gave up — orphan archive may remain until operator cleanup" + ); + return; + } + } + } +} + impl ForkCloneGuard { fn new(path: crate::git::repo_store::ValidatedRepoDiskPath) -> Self { Self(Some(path.into_path_buf())) @@ -3339,24 +3428,55 @@ pub async fn fork_repo( }; if let Err(e) = state.db.create_repo(&record).await { - if let Some(committed) = state.db.get_repo(forker_short, &fork_name).await? { - clone_guard.disarm(); - tracing::warn!( - fork = %fork_name, - forker = %forker_did, - "fork create_repo returned an error but the row is present — treating as success" - ); - return Ok(( - StatusCode::CREATED, - Json(to_response(&committed, &state, 0)), - )); + match confirm_fork_repo_row(&state.db, forker_short, &fork_name).await { + Ok(Some(committed)) => { + clone_guard.disarm(); + tracing::warn!( + fork = %fork_name, + forker = %forker_did, + "fork create_repo returned an error but the row is present — treating as success" + ); + return Ok(( + StatusCode::CREATED, + Json(to_response(&committed, &state, 0)), + )); + } + Ok(None) => { + clone_guard.disarm(); + state + .repo_store + .compensate_fork_archive(&forker_did, &fork_name, disk_path.as_path()) + .await; + return Err(e.into()); + } + Err(lookup_err) => { + clone_guard.disarm(); + let db = std::sync::Arc::clone(&state.db); + let repo_store = state.repo_store.clone(); + let owner_short = forker_short.to_string(); + let fork_name_cl = fork_name.clone(); + let owner_did = forker_did.clone(); + let disk_path_cl = disk_path.as_path().to_path_buf(); + tokio::spawn(async move { + schedule_fork_create_recovery( + db, + repo_store, + owner_short, + fork_name_cl, + owner_did, + disk_path_cl, + ) + .await; + }); + tracing::warn!( + fork = %fork_name, + create_err = %e, + lookup_err = %lookup_err, + "fork create_repo failed and confirmation lookup stayed unavailable — scheduled recovery" + ); + return Err(AppError::RepoUnavailable); + } } - clone_guard.disarm(); - state - .repo_store - .compensate_fork_archive(&forker_did, &fork_name, disk_path.as_path()) - .await; - return Err(e.into()); } clone_guard.disarm(); @@ -3597,6 +3717,34 @@ mod tests { const OWNER_SHORT: &str = "z6MkpTHR8VNsBxYAAWHut2Geadd9jSwuBV8xRoAnwWsdvktH"; const STRANGER_DID: &str = "did:key:z6Mkffonly5tranger0000000000000000000000000000000"; + #[tokio::test] + async fn publish_durability_slot_drop_installs_unknowable_when_never_recorded() { + let slot = PublishDurabilitySlot::new(); + let arc = slot.arc(); + drop(slot); + let outcome = *arc.lock().await; + assert_eq!( + outcome, + Some(crate::git::repo_store::ReleaseOutcome::UploadUnknowable), + "dropping an unrecorded slot must publish UploadUnknowable for the tail" + ); + } + + #[tokio::test] + async fn publish_durability_confirmed_proceeds_quickly_after_unrecorded_slot_drop() { + let start = std::time::Instant::now(); + let slot = PublishDurabilitySlot::new(); + let arc = slot.arc(); + drop(slot); + let confirmed = + publish_durability_confirmed(&Some(arc), std::time::Duration::from_millis(50)).await; + assert!(confirmed); + assert!( + start.elapsed() < std::time::Duration::from_millis(200), + "the tail must not wait out the full transfer bound when the slot already carries UploadUnknowable" + ); + } + #[tokio::test] async fn publish_durability_confirmed_proceeds_when_release_never_records() { let slot = Arc::new(tokio::sync::Mutex::new(None)); diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 2af40c52..fc7dfc98 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -277,7 +277,7 @@ pub struct Config { #[arg( long, env = "GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS", - default_value_t = 32, + default_value_t = 40, value_parser = clap::value_parser!(u32).range(1..) )] pub db_lock_pool_max_connections: u32, @@ -397,9 +397,10 @@ pub struct Config { /// error rather than a boot-time panic). /// /// CONNECTION BUDGET. A push holds a Postgres connection from the node's separate - /// advisory-lock pool (`GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS`, default 32) for - /// the whole receive-pack. That pool must be at least this value so every - /// admitted push can pin a lock-pool connection for its whole duration. Size BOTH + /// advisory-lock pool (`GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS`, default 40) for + /// the whole receive-pack. That pool must be at least this value plus + /// `DB_LOCK_POOL_NON_PUSH_HEADROOM` so admitted pushes and non-push mutations + /// can each pin a lock-pool connection for their whole duration. Size BOTH /// pools against the database server's `max_connections`: the main pool /// (`GITLAWB_DB_MAX_CONNECTIONS`, default 48) serves ordinary handlers, and the /// lock pool serves writes. Raising this knob does not raise the lock pool; @@ -761,6 +762,11 @@ impl Config { /// the pool must clear the concurrent-write cap by at least this margin. pub const DB_POOL_APP_HEADROOM: u32 = 8; + /// Lock-pool slots reserved for non-push `acquire_write` callers (issue create, + /// close, merge) so a saturated push budget cannot turn ordinary mutations into + /// lock-pool exhaustion on unrelated repositories. + pub const DB_LOCK_POOL_NON_PUSH_HEADROOM: u32 = 8; + /// Cross-field boot validation. Single-field ranges are enforced by clap; this /// catches combinations that ship a denial-of-service under otherwise-valid /// values. Call once at startup and fail fast on `Err`. @@ -768,12 +774,17 @@ impl Config { // Concurrent git writes pin the dedicated lock pool for their whole // duration (connection-affine advisory lock in acquire_write). The main // pool no longer carries that occupancy. - if (self.db_lock_pool_max_connections as usize) < self.max_concurrent_git_pushes { + let lock_floor = self + .max_concurrent_git_pushes + .saturating_add(Self::DB_LOCK_POOL_NON_PUSH_HEADROOM as usize); + if (self.db_lock_pool_max_connections as usize) < lock_floor { return Err(format!( "GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS ({}) must be at least \ - max_concurrent_git_pushes ({}) so every admitted push can pin a \ - lock-pool connection for its whole duration", - self.db_lock_pool_max_connections, self.max_concurrent_git_pushes + max_concurrent_git_pushes ({}) plus {} for non-push mutations \ + that share the lock pool", + self.db_lock_pool_max_connections, + self.max_concurrent_git_pushes, + Self::DB_LOCK_POOL_NON_PUSH_HEADROOM )); } let main_floor = Self::DB_POOL_APP_HEADROOM as u64; @@ -793,10 +804,10 @@ mod tests { use super::*; #[test] - fn lock_pool_size_defaults_to_32_and_rejects_zero() { + fn lock_pool_size_defaults_to_40_and_rejects_zero() { assert_eq!( Config::parse_from(["gitlawb-node"]).db_lock_pool_max_connections, - 32 + 40 ); assert_eq!( Config::parse_from(["gitlawb-node", "--db-lock-pool-max-connections", "8"]) @@ -1424,7 +1435,7 @@ mod tests { /// pool. `validate()` must reject an under-sized lock pool at boot. #[test] fn db_pool_must_clear_the_git_push_cap() { - // Shipped defaults validate (lock pool 32 >= pushes 32, main pool >= headroom). + // Shipped defaults validate (lock pool 40 >= pushes 32 + non-push headroom 8). Config::parse_from(["gitlawb-node"]) .validate() .expect("default config must validate"); @@ -1439,7 +1450,20 @@ mod tests { ]); assert!( under_lock.validate().is_err(), - "db_lock_pool_max_connections below max_concurrent_git_pushes must be rejected" + "db_lock_pool_max_connections below max_concurrent_git_pushes + headroom must be rejected" + ); + + // Exactly at the push cap without non-push headroom is rejected. + let push_only = Config::parse_from([ + "gitlawb-node", + "--db-lock-pool-max-connections", + "32", + "--max-concurrent-git-pushes", + "32", + ]); + assert!( + push_only.validate().is_err(), + "db_lock_pool_max_connections equal to max_concurrent_git_pushes must be rejected" ); // Main pool can be smaller than pushes + headroom when the lock pool carries writes. @@ -1448,7 +1472,7 @@ mod tests { "--db-max-connections", "16", "--db-lock-pool-max-connections", - "32", + "40", "--max-concurrent-git-pushes", "32", ]); From 85c6f48bc4b93133748a225fc74d14e66bbe35a2 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sun, 30 Aug 2026 13:25:06 -0500 Subject: [PATCH 44/54] fix(node): verify adversarial findings on advisory-lock PR Fail closed when publish durability never records, bound pg_advisory_unlock with the same transfer budget as upload, and cap close_issue pre-lock snapshots at git_acquire_timeout_secs instead of the write-lock transfer bound. --- crates/gitlawb-node/src/api/issues.rs | 9 ++++++--- crates/gitlawb-node/src/api/repos.rs | 22 ++++++++++++--------- crates/gitlawb-node/src/git/repo_store.rs | 24 +++++++++++++++++------ 3 files changed, 37 insertions(+), 18 deletions(-) diff --git a/crates/gitlawb-node/src/api/issues.rs b/crates/gitlawb-node/src/api/issues.rs index 42152fd4..6fe260f3 100644 --- a/crates/gitlawb-node/src/api/issues.rs +++ b/crates/gitlawb-node/src/api/issues.rs @@ -344,8 +344,11 @@ pub async fn close_issue( // dir instead of publishing into the live repo path — an unlocked // pre-check must not delete or swap the directory under a concurrent // guarded write on the same path. + // Pre-lock authorization only: bound with the read acquire budget, not the + // under-lock transfer timeout a guarded write may hold for minutes. + let snapshot_bound_secs = state.config.git_acquire_timeout_secs; let snapshot = tokio::time::timeout( - std::time::Duration::from_secs(state.config.lock_held_transfer_timeout_secs), + std::time::Duration::from_secs(snapshot_bound_secs), state .repo_store .read_snapshot(&record.owner_did, &record.name), @@ -354,8 +357,8 @@ pub async fn close_issue( .map_err(|_elapsed| { tracing::warn!( repo = %repo, - bound_secs = state.config.lock_held_transfer_timeout_secs, - "close_issue snapshot exceeded the transfer bound — shedding as a retryable refusal" + bound_secs = snapshot_bound_secs, + "close_issue snapshot exceeded the read acquire bound — shedding as a retryable refusal" ); AppError::RepoUnavailable })??; diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 0acfa3ef..5013dbca 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -2583,12 +2583,15 @@ async fn publish_durability_confirmed( ); } if start.elapsed() >= wait { - // No outcome within the release-side transfer bound plus slack: the - // handler was dropped mid-release after a successful receive-pack, or - // release is stuck past its own deadline. The tail is only spawned on - // push success, so local-disk replication work may proceed. Explicit - // non-Released outcomes are handled above. - return true; + // No outcome within the release-side transfer bound plus slack. Fail + // closed when the slot is still empty; an abandoned handler installs + // UploadUnknowable via PublishDurabilitySlot::drop so the tail can + // proceed on local disk without waiting out the full bound. + return matches!( + *slot.lock().await, + Some(crate::git::repo_store::ReleaseOutcome::Released) + | Some(crate::git::repo_store::ReleaseOutcome::UploadUnknowable) + ); } tokio::time::sleep(std::time::Duration::from_millis(10)).await; } @@ -3746,13 +3749,14 @@ mod tests { } #[tokio::test] - async fn publish_durability_confirmed_proceeds_when_release_never_records() { + async fn publish_durability_confirmed_fails_closed_when_release_never_records() { let slot = Arc::new(tokio::sync::Mutex::new(None)); let confirmed = publish_durability_confirmed(&Some(slot), std::time::Duration::from_millis(30)).await; assert!( - confirmed, - "an abandoned handler leaves the slot pending; after the bounded wait the tail may proceed on local disk" + !confirmed, + "an empty slot after the bounded wait must not admit the tail; only an installed \ + Released or UploadUnknowable outcome may" ); } diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index f4d623d1..1b74e089 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -1454,14 +1454,18 @@ impl RepoWriteGuard { if let Some(gate) = self.test_pre_unlock_gate.clone() { gate.notified().await; } - let unlock = match self.conn.as_mut() { - Some(conn) => Some( + let unlock = if let Some(conn) = self.conn.as_mut() { + bounded_transfer( + "advisory-unlock", + &self.repo_name, + self.lock_held_transfer_timeout, sqlx::query_as::<_, (bool,)>("SELECT pg_advisory_unlock($1)") .bind(lock_key) - .fetch_one(&mut **conn) - .await, - ), - None => None, + .fetch_one(&mut **conn), + ) + .await + } else { + None }; match unlock { Some(Ok((true,))) => { @@ -1483,6 +1487,14 @@ impl RepoWriteGuard { "advisory unlock failed — closing the session so the lock cannot outlive it" ); } + None if self.conn.is_some() => { + warn!( + repo = %self.repo_name, + lock_key, + bound_secs = self.lock_held_transfer_timeout.as_secs(), + "advisory unlock exceeded its bound — closing the session so the lock-pool slot is not held longer" + ); + } None => {} } From 5cd6703aadce63d0e4230ff55fbf93bdbeb651e8 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sun, 30 Aug 2026 15:30:34 -0500 Subject: [PATCH 45/54] fix(node): compensate under the write lock and harden tail durability handoff Run create_issue rollback through release_compensating so issue refs are deleted only while the advisory lock is still held and only on definite publish refusals, not UploadUnknowable. Replace the tail durability slot's async try_lock drop with a std mutex so cancellation during polling cannot skip installing UploadUnknowable. --- crates/gitlawb-node/src/api/issues.rs | 32 +++++----- crates/gitlawb-node/src/api/repos.rs | 74 +++++++++++++++++------ crates/gitlawb-node/src/git/issues.rs | 7 ++- crates/gitlawb-node/src/git/repo_store.rs | 34 ++++++++++- 4 files changed, 108 insertions(+), 39 deletions(-) diff --git a/crates/gitlawb-node/src/api/issues.rs b/crates/gitlawb-node/src/api/issues.rs index 6fe260f3..5150c2e0 100644 --- a/crates/gitlawb-node/src/api/issues.rs +++ b/crates/gitlawb-node/src/api/issues.rs @@ -66,24 +66,24 @@ pub async fn create_issue( .acquire_write(&record.owner_did, &record.name) .await .map_err(|e| crate::api::repos::acquire_write_app_error(&e, &repo))?; - let disk_path = guard.path().to_path_buf(); - let create_result = git_issues::create_issue(&disk_path, &issue_id, &json_str); + let create_result = git_issues::create_issue(guard.path(), &issue_id, &json_str); - let release_result = guard.release(create_result.is_ok()).await.into_result(); - if release_result.is_err() && create_result.is_ok() { - let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); - let deadline = std::time::Instant::now() + git_timeout; - if let Err(rollback) = - git_issues::delete_issue_ref(&state.git_bin, &disk_path, &issue_id, deadline) - { - tracing::warn!( - issue = %issue_id, - err = %rollback, - "failed to roll back local issue after refused publish" - ); - } - } + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let deadline = std::time::Instant::now() + git_timeout; + let git_bin = state.git_bin.clone(); + let issue_id_for_comp = issue_id.clone(); + + let release_result = if create_result.is_ok() { + guard + .release_compensating(true, move |path| { + git_issues::delete_issue_ref(&git_bin, path, &issue_id_for_comp, deadline) + }) + .await + .into_result() + } else { + guard.release(false).await.into_result() + }; release_result?; create_result.map_err(|e| AppError::Git(e.to_string()))?; diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 5013dbca..daea9015 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -2530,24 +2530,24 @@ pub async fn git_receive_pack( /// [`ReleaseOutcome::UploadUnknowable`] so the tail can proceed on local disk /// without waiting out the full transfer bound. struct PublishDurabilitySlot { - inner: Arc>>, + inner: Arc>>, recorded: std::sync::atomic::AtomicBool, } impl PublishDurabilitySlot { fn new() -> Self { Self { - inner: Arc::new(tokio::sync::Mutex::new(None)), + inner: Arc::new(std::sync::Mutex::new(None)), recorded: std::sync::atomic::AtomicBool::new(false), } } - fn arc(&self) -> Arc>> { + fn arc(&self) -> Arc>> { Arc::clone(&self.inner) } async fn record(&self, outcome: crate::git::repo_store::ReleaseOutcome) { - *self.inner.lock().await = Some(outcome); + *self.inner.lock().expect("publish durability mutex poisoned") = Some(outcome); self.recorded .store(true, std::sync::atomic::Ordering::SeqCst); } @@ -2555,19 +2555,24 @@ impl PublishDurabilitySlot { impl Drop for PublishDurabilitySlot { fn drop(&mut self) { - if self.recorded.load(std::sync::atomic::Ordering::SeqCst) { + if self + .recorded + .load(std::sync::atomic::Ordering::SeqCst) + { return; } - if let Ok(mut slot) = self.inner.try_lock() { - if slot.is_none() { - *slot = Some(crate::git::repo_store::ReleaseOutcome::UploadUnknowable); - } + let mut slot = self + .inner + .lock() + .expect("publish durability mutex poisoned"); + if slot.is_none() { + *slot = Some(crate::git::repo_store::ReleaseOutcome::UploadUnknowable); } } } async fn publish_durability_confirmed( - slot: &Option>>>, + slot: &Option>>>, wait: std::time::Duration, ) -> bool { let Some(slot) = slot else { @@ -2575,7 +2580,10 @@ async fn publish_durability_confirmed( }; let start = std::time::Instant::now(); loop { - if let Some(outcome) = *slot.lock().await { + if let Some(outcome) = *slot + .lock() + .expect("publish durability mutex poisoned") + { return matches!( outcome, crate::git::repo_store::ReleaseOutcome::Released @@ -2588,7 +2596,9 @@ async fn publish_durability_confirmed( // UploadUnknowable via PublishDurabilitySlot::drop so the tail can // proceed on local disk without waiting out the full bound. return matches!( - *slot.lock().await, + *slot + .lock() + .expect("publish durability mutex poisoned"), Some(crate::git::repo_store::ReleaseOutcome::Released) | Some(crate::git::repo_store::ReleaseOutcome::UploadUnknowable) ); @@ -2604,7 +2614,7 @@ async fn post_receive_replication_tail( disk_path: std::path::PathBuf, did: String, publish_durability: Option< - Arc>>, + Arc>>, >, ) { let durability_wait = std::time::Duration::from_secs( @@ -3725,7 +3735,9 @@ mod tests { let slot = PublishDurabilitySlot::new(); let arc = slot.arc(); drop(slot); - let outcome = *arc.lock().await; + let outcome = *arc + .lock() + .expect("publish durability mutex poisoned"); assert_eq!( outcome, Some(crate::git::repo_store::ReleaseOutcome::UploadUnknowable), @@ -3733,6 +3745,30 @@ mod tests { ); } + #[test] + fn publish_durability_slot_drop_waits_for_contended_mutex() { + let slot = PublishDurabilitySlot::new(); + let arc = slot.arc(); + let holder = { + let arc = Arc::clone(&arc); + std::thread::spawn(move || { + let _guard = arc.lock().expect("publish durability mutex poisoned"); + std::thread::sleep(std::time::Duration::from_millis(100)); + }) + }; + std::thread::sleep(std::time::Duration::from_millis(10)); + drop(slot); + holder.join().expect("mutex holder thread"); + let outcome = *arc + .lock() + .expect("publish durability mutex poisoned"); + assert_eq!( + outcome, + Some(crate::git::repo_store::ReleaseOutcome::UploadUnknowable), + "drop must block until it can install UploadUnknowable, not give up on try_lock" + ); + } + #[tokio::test] async fn publish_durability_confirmed_proceeds_quickly_after_unrecorded_slot_drop() { let start = std::time::Instant::now(); @@ -3750,7 +3786,7 @@ mod tests { #[tokio::test] async fn publish_durability_confirmed_fails_closed_when_release_never_records() { - let slot = Arc::new(tokio::sync::Mutex::new(None)); + let slot = Arc::new(std::sync::Mutex::new(None)); let confirmed = publish_durability_confirmed(&Some(slot), std::time::Duration::from_millis(30)).await; assert!( @@ -3762,14 +3798,14 @@ mod tests { #[tokio::test] async fn publish_durability_confirmed_accepts_only_released() { - let slot = Arc::new(tokio::sync::Mutex::new(Some( + let slot = Arc::new(std::sync::Mutex::new(Some( crate::git::repo_store::ReleaseOutcome::Released, ))); assert!( publish_durability_confirmed(&Some(slot), std::time::Duration::from_millis(5)).await ); - let slot = Arc::new(tokio::sync::Mutex::new(Some( + let slot = Arc::new(std::sync::Mutex::new(Some( crate::git::repo_store::ReleaseOutcome::UploadUnknowable, ))); assert!( @@ -3777,7 +3813,7 @@ mod tests { "an unknowable upload still landed on local disk, so the tail may proceed" ); - let slot = Arc::new(tokio::sync::Mutex::new(Some( + let slot = Arc::new(std::sync::Mutex::new(Some( crate::git::repo_store::ReleaseOutcome::UploadFailed, ))); assert!( @@ -10153,7 +10189,7 @@ mod tests { u5_init_repo(repo.path()); let c1 = u5_commit_file(repo.path(), "a.txt", "one\n"); let (state, rec) = f2a_state(pool, &git_bin, "z6f2afence", "fence-repo", false).await; - let slot = Arc::new(tokio::sync::Mutex::new(Some( + let slot = Arc::new(std::sync::Mutex::new(Some( crate::git::repo_store::ReleaseOutcome::Fenced, ))); post_receive_replication_tail( diff --git a/crates/gitlawb-node/src/git/issues.rs b/crates/gitlawb-node/src/git/issues.rs index 12256a67..b20f171c 100644 --- a/crates/gitlawb-node/src/git/issues.rs +++ b/crates/gitlawb-node/src/git/issues.rs @@ -61,9 +61,10 @@ pub fn create_issue(repo_path: &Path, issue_id: &str, json: &str) -> Result<()> Ok(()) } -/// Remove a single issue ref after a failed publish rolled back the handler's view -/// of the write. Best-effort: a failed delete leaves a local-only orphan, which is -/// still better than telling the client to retry into a duplicate id. +/// Remove a single issue ref while the write guard still holds the advisory lock, +/// after a definite publish refusal. Best-effort: a failed delete leaves a +/// local-only orphan, which is still better than telling the client to retry into a +/// duplicate id. Not used on `UploadUnknowable`, where the PUT may still land. pub fn delete_issue_ref( git_bin: &str, repo_path: &Path, diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 1b74e089..445ec272 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -1367,7 +1367,30 @@ impl RepoWriteGuard { /// half-applied or otherwise inconsistent repo would propagate corruption to /// Tigris (and to every node that later downloads it). The lock is always /// released regardless, to avoid stale locks blocking future writes. - pub async fn release(mut self, success: bool) -> ReleaseOutcome { + pub async fn release(self, success: bool) -> ReleaseOutcome { + self.release_maybe_compensate(success, None:: anyhow::Result<()>>) + .await + } + + /// Like [`release`], but runs `compensate` on the live tree while the advisory + /// lock is still held when publish ends in a definite refusal (`Fenced` or + /// `UploadFailed`). It is deliberately not run on `UploadUnknowable`, where the + /// PUT may still land and a post-release undo would race a successor writer. + pub async fn release_compensating(self, success: bool, compensate: F) -> ReleaseOutcome + where + F: FnOnce(&Path) -> anyhow::Result<()>, + { + self.release_maybe_compensate(success, Some(compensate)).await + } + + async fn release_maybe_compensate( + mut self, + success: bool, + compensate: Option, + ) -> ReleaseOutcome + where + F: FnOnce(&Path) -> anyhow::Result<()>, + { let mut outcome = ReleaseOutcome::Released; // Upload to Tigris only on success. if success { @@ -1432,6 +1455,15 @@ impl RepoWriteGuard { &self.repo_name, "definite publish refusal", ); + if let Some(compensate) = compensate { + if let Err(e) = compensate(self.path()) { + warn!( + repo = %self.repo_name, + err = %e, + "compensation after a definite publish refusal failed" + ); + } + } } ReleaseOutcome::Released | ReleaseOutcome::UploadUnknowable => {} } From 22da2d6b968d88ecb224919c1b525e9338bc5dc8 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sun, 30 Aug 2026 16:14:25 -0500 Subject: [PATCH 46/54] fix(node): refuse stale under-lock refresh and gate tail durability on release Under-lock GET failures now always shed as RepoUnavailable, even when a local copy exists, so a transient download error cannot publish over a newer stored generation. PublishDurabilitySlot only synthesizes UploadUnknowable after release starts, and receive-pack marks that boundary before spawning the tail. Adds regression tests for both paths; inv22 F4 gate accepts the release wrapper. --- crates/gitlawb-node/src/api/repos.rs | 64 ++++++++---- crates/gitlawb-node/src/git/repo_store.rs | 113 +++++++++++++++------- crates/gitlawb-node/tests/inv22_gates.rs | 3 +- 3 files changed, 127 insertions(+), 53 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index daea9015..ed08e573 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -2376,7 +2376,17 @@ pub async fn git_receive_pack( // change to the client contract than the window it closes. let push_succeeded = receive_result.is_ok(); let publish_durability = PublishDurabilitySlot::new(); + // Reclaim the write lock from the shared cell (#173 F2). This is only reachable + // once `receive_pack` has returned, so the admission guard's copy can only ever + // DELAY release, never perform it early; on the disconnect path this line is not + // reached at all and the reaper's copy is last. + let reclaimed = guard + .lock() + .expect("repo write-lock mutex poisoned") + .take() + .expect("the write lock is only taken here, and only once"); if push_succeeded { + publish_durability.mark_release_started(); tokio::spawn(post_receive_replication_tail( state.clone(), record.clone(), @@ -2386,19 +2396,6 @@ pub async fn git_receive_pack( Some(publish_durability.arc()), )); } - - // Always release the advisory lock — even on error — to prevent stale locks - // from blocking subsequent pushes. Only upload to Tigris when the push - // succeeded; uploading a half-applied repo would propagate corruption. - // Reclaim the write lock from the shared cell (#173 F2). This is only reachable - // once `receive_pack` has returned, so the admission guard's copy can only ever - // DELAY release, never perform it early; on the disconnect path this line is not - // reached at all and the reaper's copy is last. - let reclaimed = guard - .lock() - .expect("repo write-lock mutex poisoned") - .take() - .expect("the write lock is only taken here, and only once"); // Short-circuit on a refused publish BEFORE anything downstream observes // the push. The pack is on local disk but not in object storage, so // touching the repo, recording the push, bumping trust, issuing ref @@ -2526,12 +2523,15 @@ pub async fn git_receive_pack( /// task. Split out of `git_receive_pack` so the ordering the coalescing gate depends /// on is directly testable; the handler spawns it and returns. /// Records the release-side publish outcome for the detached post-receive tail. -/// If the handler future is dropped before [`Self::record`], the drop sets -/// [`ReleaseOutcome::UploadUnknowable`] so the tail can proceed on local disk -/// without waiting out the full transfer bound. +/// If the handler future is dropped after [`Self::mark_release_started`] but +/// before [`Self::record`], the drop sets [`ReleaseOutcome::UploadUnknowable`] +/// so the tail can proceed on local disk without waiting out the full transfer +/// bound. A drop before release starts leaves the slot empty so the tail fails +/// closed rather than treating an unattempted publish as unknowably durable. struct PublishDurabilitySlot { inner: Arc>>, recorded: std::sync::atomic::AtomicBool, + release_started: std::sync::atomic::AtomicBool, } impl PublishDurabilitySlot { @@ -2539,9 +2539,15 @@ impl PublishDurabilitySlot { Self { inner: Arc::new(std::sync::Mutex::new(None)), recorded: std::sync::atomic::AtomicBool::new(false), + release_started: std::sync::atomic::AtomicBool::new(false), } } + fn mark_release_started(&self) { + self.release_started + .store(true, std::sync::atomic::Ordering::SeqCst); + } + fn arc(&self) -> Arc>> { Arc::clone(&self.inner) } @@ -2561,6 +2567,12 @@ impl Drop for PublishDurabilitySlot { { return; } + if !self + .release_started + .load(std::sync::atomic::Ordering::SeqCst) + { + return; + } let mut slot = self .inner .lock() @@ -3731,8 +3743,10 @@ mod tests { const STRANGER_DID: &str = "did:key:z6Mkffonly5tranger0000000000000000000000000000000"; #[tokio::test] - async fn publish_durability_slot_drop_installs_unknowable_when_never_recorded() { + async fn publish_durability_slot_drop_installs_unknowable_when_release_started_but_unrecorded( + ) { let slot = PublishDurabilitySlot::new(); + slot.mark_release_started(); let arc = slot.arc(); drop(slot); let outcome = *arc @@ -3741,13 +3755,26 @@ mod tests { assert_eq!( outcome, Some(crate::git::repo_store::ReleaseOutcome::UploadUnknowable), - "dropping an unrecorded slot must publish UploadUnknowable for the tail" + "dropping an unrecorded slot after release starts must publish UploadUnknowable for the tail" + ); + } + + #[test] + fn publish_durability_slot_drop_leaves_empty_before_release_starts() { + let slot = PublishDurabilitySlot::new(); + let arc = slot.arc(); + drop(slot); + assert_eq!( + *arc.lock().expect("publish durability mutex poisoned"), + None, + "dropping before release starts must not synthesize durability for the tail" ); } #[test] fn publish_durability_slot_drop_waits_for_contended_mutex() { let slot = PublishDurabilitySlot::new(); + slot.mark_release_started(); let arc = slot.arc(); let holder = { let arc = Arc::clone(&arc); @@ -3773,6 +3800,7 @@ mod tests { async fn publish_durability_confirmed_proceeds_quickly_after_unrecorded_slot_drop() { let start = std::time::Instant::now(); let slot = PublishDurabilitySlot::new(); + slot.mark_release_started(); let arc = slot.arc(); drop(slot); let confirmed = diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 445ec272..94f6d7b1 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -526,7 +526,7 @@ impl RepoStore { .await { Ok(()) => Ok(fence), - Err(err) => Err(RefreshFailure::Download { err, fence }), + Err(err) => Err(RefreshFailure::Download { err }), } } Ok(None) => Ok(UploadPrecondition::IfAbsent), @@ -538,33 +538,16 @@ impl RepoStore { match refreshed { Some(Ok(fence)) => guard.publish_fence = fence, - Some(Err(RefreshFailure::Download { err, fence })) => { - // The archive is present but unreadable: a corrupt or partial - // upload, or a transient GET failure. We KNOW the fetch failed, - // so falling back to a valid local copy is sound and - // release(success) re-uploads a good archive. Only hard-fail - // when there is no local copy to fall back to. - if local_path.exists() { - warn!(repo = %repo_name, err = %err, - "write acquire: tigris refresh failed — falling back to local copy"); - // Still fence on what the HEAD saw. The download failing - // says nothing about the generation stored, so publishing - // unconditionally here would reintroduce exactly the - // overwrite this carries the ETag to prevent. - guard.publish_fence = fence; - } else { - // No local copy, so the write cannot proceed and the - // archive's readability is unknowable. Same epistemic - // class as the HEAD arm: a transient storage blip must be - // a retryable refusal, not a 500 that tells the client the - // failure is permanent. Wrap so the handler layer's - // `RepoUnavailable` downcast maps this to a retryable 503 - // with a fixed body; the detail (which repo, why) stays in - // this error chain for the log. - return Err(anyhow::Error::new(RepoUnavailable).context(format!( - "tigris download failed during acquire_write for {owner_slug}/{repo_name}: {err:#}" - ))); - } + Some(Err(RefreshFailure::Download { err, .. })) => { + // HEAD established a stored generation but the GET failed, so we + // do not know whether the local tree matches it. Proceeding on a + // cached copy and publishing fenced on the observed ETag can + // overwrite a newer archive with stale-local + this write. + warn!(repo = %repo_name, err = %err, + "write acquire: tigris download failed under the lock — refusing rather than writing against an unverified local tree"); + return Err(anyhow::Error::new(RepoUnavailable).context(format!( + "tigris download failed during acquire_write for {owner_slug}/{repo_name}: {err:#}" + ))); } Some(Err(RefreshFailure::Unknown(e))) => { // The HEAD itself failed, so we do not know whether a newer @@ -1595,16 +1578,15 @@ const LOCK_ACQUIRE_DEADLINE: Duration = Duration::from_secs(90); /// Why an under-lock refresh did not complete, split by what it leaves us knowing. /// /// `Unknown` (the existence check failed) and `Download` (the archive is there and -/// unreadable) must not share a branch: only the second establishes that the local -/// copy is a sound thing to fall back to and re-upload. +/// unreadable) must not share a branch: neither establishes that the local tree +/// matches the generation the HEAD observed. enum RefreshFailure { Unknown(anyhow::Error), - /// Carries the fence the HEAD observed alongside the error, because the - /// fallback arm still publishes later and must be fenced on the generation - /// it saw. `Unknown` carries none: that arm refuses the write outright. + /// Carries the GET error. The under-lock path refuses the write rather than + /// falling back to local, because a failed GET does not prove the cached tree + /// is current. Download { err: anyhow::Error, - fence: UploadPrecondition, }, } @@ -3694,6 +3676,69 @@ mod tests { server.abort(); } + /// A failed under-lock GET must refuse even when a local copy exists. HEAD + /// only establishes the stored generation, not that the cached tree matches + /// it, so writing against stale-local + new commits can overwrite a newer + /// archive when the GET fails transiently. + #[sqlx::test] + async fn acquire_write_refuses_when_the_download_fails_with_a_local_copy(pool: PgPool) { + use axum::response::IntoResponse; + + let app = axum::Router::new().route( + "/{*key}", + axum::routing::any(|method: axum::http::Method| async move { + if method == axum::http::Method::HEAD { + let mut resp = axum::http::StatusCode::OK.into_response(); + resp.headers_mut() + .insert("etag", axum::http::HeaderValue::from_static("\"gen-1\"")); + resp + } else { + axum::http::StatusCode::INTERNAL_SERVER_ERROR.into_response() + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port()); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + + let owner = "did:key:z6MkStaleLocalWrite"; + let repo_name = "writerepo"; + let owner_slug = crate::db::normalize_owner_key(owner); + let base = tempfile::TempDir::new().unwrap(); + let repo_path = base + .path() + .join(&owner_slug) + .join(format!("{repo_name}.git")); + std::fs::create_dir_all(repo_path.parent().unwrap()).unwrap(); + crate::git::store::init_bare(&repo_path).expect("seed bare repo"); + + let opts = (*pool.connect_options()).clone(); + let lock_pool = no_reap_pool(&opts, 2).await; + let store = RepoStore::for_testing_with_tigris( + base.path().to_path_buf(), + lock_pool, + TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint), + ); + + let err = match store.acquire_write(owner, repo_name).await { + Err(e) => e, + Ok(guard) => { + let _ = guard.release(false).await; + panic!( + "a failed download with a local copy must refuse rather than write against an unverified tree" + ); + } + }; + assert!( + err.downcast_ref::().is_some(), + "the refusal must be typed so the handler layer maps it to a retryable 503, got {err:#}" + ); + + server.abort(); + } + /// The transfer bound is a knob, so it gets the same parse/default/reject-zero /// coverage its sibling lock-pool-size knob has. #[test] diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 48381e3d..7ce405f6 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -225,7 +225,8 @@ fn f4_release_keeps_conn_owned_until_unlock_resolves() { let repo_store = src("git/repo_store.rs"); let rel_start = repo_store - .find("pub async fn release(mut self") + .find("pub async fn release(self") + .or_else(|| repo_store.find("pub async fn release(mut self")) .expect("F4 gate: repo_store.rs no longer defines RepoWriteGuard::release"); let rel_end = repo_store[rel_start..] .find("impl Drop for RepoWriteGuard") From c306933e0782353e63064dac79c6655a147bfa7a Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sun, 30 Aug 2026 16:26:21 -0500 Subject: [PATCH 47/54] style(node): rustfmt advisory-lock gap-closure commits --- crates/gitlawb-node/src/api/repos.rs | 31 ++++++++--------------- crates/gitlawb-node/src/git/repo_store.rs | 3 ++- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index ed08e573..7eca1b82 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -2553,7 +2553,10 @@ impl PublishDurabilitySlot { } async fn record(&self, outcome: crate::git::repo_store::ReleaseOutcome) { - *self.inner.lock().expect("publish durability mutex poisoned") = Some(outcome); + *self + .inner + .lock() + .expect("publish durability mutex poisoned") = Some(outcome); self.recorded .store(true, std::sync::atomic::Ordering::SeqCst); } @@ -2561,10 +2564,7 @@ impl PublishDurabilitySlot { impl Drop for PublishDurabilitySlot { fn drop(&mut self) { - if self - .recorded - .load(std::sync::atomic::Ordering::SeqCst) - { + if self.recorded.load(std::sync::atomic::Ordering::SeqCst) { return; } if !self @@ -2592,10 +2592,7 @@ async fn publish_durability_confirmed( }; let start = std::time::Instant::now(); loop { - if let Some(outcome) = *slot - .lock() - .expect("publish durability mutex poisoned") - { + if let Some(outcome) = *slot.lock().expect("publish durability mutex poisoned") { return matches!( outcome, crate::git::repo_store::ReleaseOutcome::Released @@ -2608,9 +2605,7 @@ async fn publish_durability_confirmed( // UploadUnknowable via PublishDurabilitySlot::drop so the tail can // proceed on local disk without waiting out the full bound. return matches!( - *slot - .lock() - .expect("publish durability mutex poisoned"), + *slot.lock().expect("publish durability mutex poisoned"), Some(crate::git::repo_store::ReleaseOutcome::Released) | Some(crate::git::repo_store::ReleaseOutcome::UploadUnknowable) ); @@ -3743,15 +3738,13 @@ mod tests { const STRANGER_DID: &str = "did:key:z6Mkffonly5tranger0000000000000000000000000000000"; #[tokio::test] - async fn publish_durability_slot_drop_installs_unknowable_when_release_started_but_unrecorded( - ) { + async fn publish_durability_slot_drop_installs_unknowable_when_release_started_but_unrecorded() + { let slot = PublishDurabilitySlot::new(); slot.mark_release_started(); let arc = slot.arc(); drop(slot); - let outcome = *arc - .lock() - .expect("publish durability mutex poisoned"); + let outcome = *arc.lock().expect("publish durability mutex poisoned"); assert_eq!( outcome, Some(crate::git::repo_store::ReleaseOutcome::UploadUnknowable), @@ -3786,9 +3779,7 @@ mod tests { std::thread::sleep(std::time::Duration::from_millis(10)); drop(slot); holder.join().expect("mutex holder thread"); - let outcome = *arc - .lock() - .expect("publish durability mutex poisoned"); + let outcome = *arc.lock().expect("publish durability mutex poisoned"); assert_eq!( outcome, Some(crate::git::repo_store::ReleaseOutcome::UploadUnknowable), diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 94f6d7b1..7ac529aa 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -1363,7 +1363,8 @@ impl RepoWriteGuard { where F: FnOnce(&Path) -> anyhow::Result<()>, { - self.release_maybe_compensate(success, Some(compensate)).await + self.release_maybe_compensate(success, Some(compensate)) + .await } async fn release_maybe_compensate( From 3a82ccb8a13c5418a306d50dcdf31392ae414f65 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sun, 30 Aug 2026 16:31:25 -0500 Subject: [PATCH 48/54] fix(node): satisfy clippy on stale-local download test path join --- crates/gitlawb-node/src/git/repo_store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 7ac529aa..2f4cf6b3 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -3710,7 +3710,7 @@ mod tests { let base = tempfile::TempDir::new().unwrap(); let repo_path = base .path() - .join(&owner_slug) + .join(owner_slug) .join(format!("{repo_name}.git")); std::fs::create_dir_all(repo_path.parent().unwrap()).unwrap(); crate::git::store::init_bare(&repo_path).expect("seed bare repo"); From c40f299f6b6f408264f860210d17f534e632812b Mon Sep 17 00:00:00 2001 From: Kevin Codex Date: Mon, 31 Aug 2026 21:10:46 +0800 Subject: [PATCH 49/54] feat(node): add a publish attempt/stage boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The four lifecycle findings on this branch share one root cause: a resource was tracked by a coarse state ("release started", "the path exists", "a row with this name exists") when the safety question was about the IDENTITY and the PUBLICATION STAGE of one specific write attempt. This is the vocabulary that makes those questions answerable: - `PublishAttemptId` — minted before the request is built, carried with the bytes as object user metadata, read back off the store to decide whether what is published is THIS attempt's work. That turns "did my request succeed", which a lost response makes undecidable, into "are the published bytes mine", which the store can answer. - `PublishStage` / `PublishStageCell` — how far one attempt got, observable from OUTSIDE the future doing the work. A cancelled handler never returns an outcome, so the stage it had reached is the only thing that can classify it. - `UploadError` split by what a failure ENTITLES A CALLER TO DO rather than by which HTTP status came back. Destructive compensation is licensed only by `proves_not_published()`. Deliberately free of any object-storage type. #79 deletes `git/tigris.rs` and replaces it with a BlobStore layer; this module is what survives that swap, with each backend supplying only its own error classifier. --- crates/gitlawb-node/src/git/mod.rs | 1 + crates/gitlawb-node/src/git/publish.rs | 358 +++++++++++++++++++++++++ 2 files changed, 359 insertions(+) create mode 100644 crates/gitlawb-node/src/git/publish.rs diff --git a/crates/gitlawb-node/src/git/mod.rs b/crates/gitlawb-node/src/git/mod.rs index 59e34c84..0c880b42 100644 --- a/crates/gitlawb-node/src/git/mod.rs +++ b/crates/gitlawb-node/src/git/mod.rs @@ -1,4 +1,5 @@ pub mod issues; +pub mod publish; pub mod push_delta; pub mod repo_store; pub mod smart_http; diff --git a/crates/gitlawb-node/src/git/publish.rs b/crates/gitlawb-node/src/git/publish.rs new file mode 100644 index 00000000..d1222a67 --- /dev/null +++ b/crates/gitlawb-node/src/git/publish.rs @@ -0,0 +1,358 @@ +//! The publication boundary: WHO wrote, and HOW FAR that write got. +//! +//! Four lifecycle defects on this branch shared one root cause. A resource was +//! tracked by a coarse state — "release started", "the path exists", "a row with +//! this name exists" — when the safety question was about the IDENTITY and the +//! PUBLICATION STAGE of one specific write attempt. A tail replicated refs from +//! an attempt that had not dispatched a PUT; a read served a tree whose +//! generation nobody had confirmed; a fork claimed a concurrent attempt's row and +//! deleted a successor's object; a response-loss failure was compensated as if it +//! proved non-publication. +//! +//! This module is the vocabulary that makes those questions answerable, and it is +//! deliberately free of any object-storage type: +//! +//! - [`PublishAttemptId`] — minted before the request is built, carried with the +//! bytes as user metadata, read back off the store to decide whether what is +//! published is THIS attempt's work. +//! - [`PublishStage`] / [`PublishStageCell`] — how far one attempt got, observable +//! from outside the future that is performing it, so a CANCELLED attempt can +//! still be classified. +//! - [`UploadError`] — what the client KNOWS about dispatch and commit, not just +//! which HTTP status came back. Destructive compensation is licensed only by +//! [`UploadError::proves_not_published`]. +//! +//! # Porting note (#79) +//! +//! PR #79 (`feat/storage-abstraction`) deletes `git/tigris.rs` and replaces it +//! with a `BlobStore`/`RepoArchive` layer. Everything in this file is +//! backend-agnostic on purpose and is meant to survive that swap unchanged: a new +//! backend supplies its own error classifier (the one in `tigris.rs` is a single +//! private function) and keeps these types as the contract its callers read. + +use std::sync::Mutex; + +/// A durable identity for ONE publish attempt. +/// +/// The point is reconciliation. A conditional PUT whose response was lost leaves +/// the client unable to say whether its bytes committed; an attempt id stored +/// alongside those bytes turns that into a question the store can answer, because +/// "is the published object mine?" is decidable where "did my request succeed?" +/// is not. +/// +/// Fork creation passes the DB row id it is about to insert, so the object, the +/// on-disk clone and the database row are all stamped with the same attempt +/// identity and every cleanup can be made conditional on still owning all three. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PublishAttemptId(String); + +impl PublishAttemptId { + /// A fresh identity for an attempt that has nothing else to be named after. + pub fn new() -> Self { + Self(uuid::Uuid::new_v4().to_string()) + } + + /// Name the attempt after a caller-owned identity — fork creation uses the + /// `record.id` it is about to insert, which is what ties the object back to + /// the exact row rather than to the logical owner/name. + pub fn from_owned(id: impl Into) -> Self { + Self(id.into()) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl Default for PublishAttemptId { + fn default() -> Self { + Self::new() + } +} + +impl std::fmt::Display for PublishAttemptId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +/// The object user-metadata key an attempt id travels in. +/// +/// S3 lowercases user-metadata keys, so this must already be lowercase or the +/// read back would never match what was written. +pub const ATTEMPT_METADATA_KEY: &str = "gitlawb-attempt"; + +/// The precondition an upload is fenced on. +/// +/// Object storage is the only place a fence can hold. Dropping the future of an +/// in-flight PUT does not cancel the request the server is already processing, +/// so no amount of local locking stops an abandoned writer's bytes from landing +/// after a successor has published. A conditional PUT the store itself refuses +/// is what actually stops it. +#[derive(Clone, Debug)] +pub enum UploadPrecondition { + /// Publish only if the stored object is still the generation we observed. + IfMatch(String), + /// Publish only if nothing is stored under the key yet. + IfAbsent, + /// No fence. Last writer wins. + Unconditional, +} + +/// What the store holds under a key right now. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StoredGeneration { + /// The generation a later conditional upload can fence itself on. + pub etag: Option, + /// The attempt that published it, when the object carries our metadata. + /// `None` for anything written by something that does not stamp attempts + /// (an operator upload, a pre-attempt-id archive). + pub attempt: Option, +} + +impl StoredGeneration { + /// Is what the store holds the work of `attempt`? + /// + /// This is the whole reconciliation primitive. It is deliberately a + /// three-way question collapsed to a boolean only at the point of use: an + /// object with no attempt metadata answers `false`, because "somebody else's + /// bytes" and "bytes nobody stamped" license exactly the same caution. + pub fn belongs_to(&self, attempt: &PublishAttemptId) -> bool { + self.attempt.as_deref() == Some(attempt.as_str()) + } +} + +/// The receipt of a publish the store ACKNOWLEDGED. +/// +/// Only produced on a response the client actually read, so holding one is proof +/// of publication in a way that "the upload future returned" is not. +#[derive(Clone, Debug)] +pub struct UploadReceipt { + pub attempt: PublishAttemptId, + pub etag: Option, +} + +/// How far ONE publish attempt got. +/// +/// Observable through [`PublishStageCell`] from outside the future doing the +/// work, which is the property the whole design turns on: a handler future that +/// is DROPPED never returns an outcome, so the only way to classify a cancelled +/// attempt is to read the stage it had reached when it was abandoned. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum PublishStage { + /// Nothing has been attempted. + Idle, + /// No object-storage backend is configured, so there is no publication to + /// confirm and the local write IS the durable copy. Distinct from `Idle`, + /// which means an attempt was possible and had not started: a cancellation + /// under `NoBackend` loses nothing, while a cancellation under `Idle` means + /// a publish that could have happened never did. + NoBackend, + /// The archive is being built. The request has not been constructed, let + /// alone sent. A cancellation here is a DEFINITE "no publication was + /// attempted" — not an ambiguous in-flight PUT. + PreparingArchive, + /// The request is on the wire. Its fate is unknown until the store answers, + /// and a cancellation here does NOT stop the server processing it. + PutDispatched { attempt: PublishAttemptId }, + /// The store acknowledged the write. This is the ONLY stage that licenses + /// replication, a 2xx, or treating the local tree as durable. + Published { + attempt: PublishAttemptId, + etag: Option, + }, + /// The store definitively did not publish this attempt (a refused + /// precondition, or a failure that proves the bytes never committed). + Refused, + /// The attempt may or may not have committed and has not been reconciled. + /// Destructive compensation is NEVER licensed from here. + Ambiguous { attempt: PublishAttemptId }, +} + +impl PublishStage { + /// Did this attempt reach a state where the store may hold its bytes? + /// + /// True from dispatch onward. `false` is what makes deleting the attempt's + /// object or local tree safe. + pub fn may_have_published(&self) -> bool { + matches!( + self, + PublishStage::PutDispatched { .. } + | PublishStage::Published { .. } + | PublishStage::Ambiguous { .. } + ) + } + + /// The attempt whose fate is unresolved, when there is one. This is what a + /// reconciliation HEAD is compared against. + pub fn unresolved_attempt(&self) -> Option<&PublishAttemptId> { + match self { + PublishStage::PutDispatched { attempt } | PublishStage::Ambiguous { attempt } => { + Some(attempt) + } + _ => None, + } + } +} + +/// A [`PublishStage`] an in-flight upload writes and an outside observer reads. +/// +/// `std::sync::Mutex` rather than an async lock on purpose: every write is a +/// field assignment with no await inside, and the reader that matters most runs +/// inside a `Drop` impl, where an async lock cannot be awaited at all. +#[derive(Debug)] +pub struct PublishStageCell(Mutex); + +impl PublishStageCell { + pub fn new() -> Self { + Self::seeded(PublishStage::Idle) + } + + pub fn seeded(stage: PublishStage) -> Self { + Self(Mutex::new(stage)) + } + + pub fn set(&self, stage: PublishStage) { + *self.0.lock().expect("publish stage mutex poisoned") = stage; + } + + pub fn get(&self) -> PublishStage { + self.0.lock().expect("publish stage mutex poisoned").clone() + } +} + +impl Default for PublishStageCell { + fn default() -> Self { + Self::new() + } +} + +/// What the client knows about whether a request it could not complete ever +/// reached the wire. +/// +/// The classifier that produces this is the ONLY backend-aware code in the +/// publish path; a pluggable store (#79) supplies its own and every caller below +/// keeps working off [`UploadError`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DispatchKnowledge { + /// The request could not be built or sent. The store cannot hold these bytes. + NeverSent, + /// The request may have been sent, and may have committed. + MaybeSent, +} + +/// Why an upload did not publish, split by what it leaves the caller ENTITLED TO +/// DO — not by which HTTP status came back. +/// +/// The split exists because the previous shape ("precondition lost" vs "some +/// other error") made every caller treat a lost response as proof of failure. +/// Smithy timeout, dispatch and response errors all allow that the request was +/// sent and committed: a server can accept a complete conditional PUT and lose +/// or corrupt the response before the client observes success. Compensating that +/// as a definite failure deletes state that IS published. +#[derive(Debug, thiserror::Error)] +pub enum UploadError { + /// The store explicitly refused the fence (412, or 409 under create-only). + /// DEFINITE: this attempt did not publish, and a successor did. + #[error("upload precondition lost (HTTP {status})")] + PreconditionLost { status: u16 }, + /// The attempt provably never committed: it failed before the request could + /// be dispatched, or the store answered a definite client-side refusal. + /// Destructive compensation is safe here and ONLY here. + #[error("upload did not reach the store: {0:#}")] + NotPublished(#[source] anyhow::Error), + /// The request may have been dispatched and may have committed; the client + /// never learned which. The attempt id is carried so the caller can ask the + /// store rather than guess. + #[error("upload outcome is unknowable (attempt {attempt}): {source:#}")] + Ambiguous { + attempt: PublishAttemptId, + #[source] + source: anyhow::Error, + }, +} + +impl UploadError { + /// May this caller destroy state that would be needed if the write HAD + /// landed — delete the object, drop the only local clone, invalidate the + /// cache? + /// + /// Only a proven non-publication says yes. Every new variant must default to + /// `false`, which is why this is a match on the safe arms rather than a + /// negation of the unsafe one. + pub fn proves_not_published(&self) -> bool { + matches!( + self, + UploadError::PreconditionLost { .. } | UploadError::NotPublished(_) + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_definite_outcomes_license_destructive_compensation() { + assert!(UploadError::PreconditionLost { status: 412 }.proves_not_published()); + assert!( + UploadError::NotPublished(anyhow::anyhow!("no route to host")).proves_not_published() + ); + assert!( + !UploadError::Ambiguous { + attempt: PublishAttemptId::new(), + source: anyhow::anyhow!("response body truncated"), + } + .proves_not_published(), + "a response-loss failure must never license deleting state the write may own" + ); + } + + #[test] + fn a_stage_may_have_published_only_from_dispatch_onward() { + let attempt = PublishAttemptId::new(); + assert!(!PublishStage::Idle.may_have_published()); + assert!(!PublishStage::NoBackend.may_have_published()); + assert!( + !PublishStage::PreparingArchive.may_have_published(), + "compression has not constructed a request, let alone sent one" + ); + assert!(!PublishStage::Refused.may_have_published()); + assert!(PublishStage::PutDispatched { + attempt: attempt.clone() + } + .may_have_published()); + assert!(PublishStage::Ambiguous { + attempt: attempt.clone() + } + .may_have_published()); + assert!(PublishStage::Published { + attempt, + etag: None + } + .may_have_published()); + } + + #[test] + fn an_unstamped_object_never_belongs_to_an_attempt() { + let attempt = PublishAttemptId::new(); + assert!(StoredGeneration { + etag: Some("\"e\"".into()), + attempt: Some(attempt.as_str().to_string()), + } + .belongs_to(&attempt)); + assert!( + !StoredGeneration { + etag: Some("\"e\"".into()), + attempt: None, + } + .belongs_to(&attempt), + "bytes nobody stamped must not be claimed by this attempt" + ); + assert!(!StoredGeneration { + etag: Some("\"e\"".into()), + attempt: Some("someone-else".into()), + } + .belongs_to(&attempt)); + } +} From 21646bc8f384e3019042a96ca0c732734dc386da Mon Sep 17 00:00:00 2001 From: Kevin Codex Date: Mon, 31 Aug 2026 21:10:59 +0800 Subject: [PATCH 50/54] feat(node): stamp uploads with an attempt id and preserve dispatch ambiguity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `upload` now reports its progress into a `PublishStageCell` and stamps every PUT with an attempt id in object user metadata, so both of the questions a cancelled or unanswered publish raises become answerable: - `PreparingArchive` is marked before the blocking compression and `PutDispatched` immediately before `send()`. The conditional PUT is not constructed until compression returns, so a caller abandoned in that window definitely never attempted publication — a fact nothing could previously observe, because the only report was the return value of a future that no longer existed. - `attempt_landed()` HEADs the key and compares the stored attempt id, which is what lets a client whose response was lost recover its own committed write instead of guessing. `UploadError::Other` is replaced by `NotPublished` (proven never committed) and `Ambiguous` (may have been dispatched and may have committed). The single backend-aware classifier is `classify_put_failure`: a 4xx is an answer the server gave before storing anything and proves non-publication; a 5xx does not, and neither does a timeout, a dispatch failure, or a response the SDK could not read. `SdkError` is `#[non_exhaustive]`, so the fallback arm is the cautious one. `delete` is replaced by `delete_if_attempt_matches`, which reads the attempt off the object and fences the DELETE with `If-Match` on the generation it came from. A second name lookup before an unconditional delete would only narrow the window in which a successor's object can be erased; the conditional delete closes it. --- crates/gitlawb-node/src/git/tigris.rs | 464 +++++++++++++++++++++----- 1 file changed, 374 insertions(+), 90 deletions(-) diff --git a/crates/gitlawb-node/src/git/tigris.rs b/crates/gitlawb-node/src/git/tigris.rs index 2adf1c2a..6a0d3006 100644 --- a/crates/gitlawb-node/src/git/tigris.rs +++ b/crates/gitlawb-node/src/git/tigris.rs @@ -10,34 +10,64 @@ use std::sync::{Arc, Mutex, OnceLock}; use anyhow::{Context, Result}; use aws_sdk_s3::Client as S3Client; -use tracing::{debug, info}; - -/// The precondition an upload is fenced on. -/// -/// Object storage is the only place a fence can hold. Dropping the future of an -/// in-flight PUT does not cancel the request the server is already processing, -/// so no amount of local locking stops an abandoned writer's bytes from landing -/// after a successor has published. A conditional PUT the store itself refuses -/// is what actually stops it. -#[derive(Clone, Debug)] -pub enum UploadPrecondition { - /// Publish only if the stored object is still the generation we observed. - IfMatch(String), - /// Publish only if nothing is stored under the key yet. - IfAbsent, - /// No fence. Last writer wins. - Unconditional, +use tracing::{debug, info, warn}; + +// The publication vocabulary lives in `git::publish`, which carries no +// object-storage types, and is re-exported here so existing `git::tigris::…` +// imports keep resolving. #79 deletes this file; `git::publish` is what survives +// the swap, and the only backend-aware code below is `classify_dispatch`. +pub use super::publish::{ + DispatchKnowledge, PublishAttemptId, PublishStage, PublishStageCell, StoredGeneration, + UploadError, UploadPrecondition, UploadReceipt, ATTEMPT_METADATA_KEY, +}; + +/// What happened to a conditional, attempt-guarded delete. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AttemptDelete { + /// The object was this attempt's and is gone. + Deleted, + /// Nothing is stored under the key. + Absent, + /// Something is stored, and it is NOT this attempt's work. Left alone. + NotOurs, } -/// Why an upload failed, split so a caller can tell "someone else already -/// published this key" (expected, and dropping our bytes is the correct -/// outcome) from a real storage failure. -#[derive(Debug, thiserror::Error)] -pub enum UploadError { - #[error("upload precondition lost (HTTP {status})")] - PreconditionLost { status: u16 }, - #[error(transparent)] - Other(#[from] anyhow::Error), +/// THE ONLY BACKEND-AWARE CLASSIFIER in the publish path. +/// +/// It answers one question: does this failure PROVE the request never committed? +/// Anything short of proof is [`DispatchKnowledge::MaybeSent`], because the +/// caller's next move on a definite failure is destructive (delete the object, +/// drop the only local clone, invalidate the cache) and being wrong there +/// destroys published state. +/// +/// - An HTTP status means the server answered. A 4xx is a refusal it took +/// BEFORE storing anything, so it proves non-publication. A 5xx does not: the +/// store can commit and then fail to report it. +/// - No status at all means no response was read. Only a construction failure +/// proves the request never left; Smithy's timeout, dispatch and response +/// errors all allow that the bytes arrived and committed while the answer was +/// lost, corrupted, or never parsed. +/// - `SdkError` is `#[non_exhaustive]`, so the fallback arm must be the CAUTIOUS +/// one. A future variant defaulting to "definitely failed" would silently +/// re-open exactly the class this closes. +/// +/// #79 replaces `tigris.rs` with a `BlobStore` layer; this function is the piece +/// each backend reimplements, and nothing above it changes. +fn classify_put_failure( + err: &aws_sdk_s3::error::SdkError, + status: Option, +) -> DispatchKnowledge { + if let Some(status) = status { + return if (400..500).contains(&status) { + DispatchKnowledge::NeverSent + } else { + DispatchKnowledge::MaybeSent + }; + } + match err { + aws_sdk_s3::error::SdkError::ConstructionFailure(_) => DispatchKnowledge::NeverSent, + _ => DispatchKnowledge::MaybeSent, + } } /// Wrapper around the S3 client with the configured bucket. @@ -45,6 +75,51 @@ pub enum UploadError { pub struct TigrisClient { s3: S3Client, bucket: String, + /// Test-only seam: when set, the blocking compression inside `upload_tracked` + /// parks on this barrier. That is the ONLY window in which a publish attempt + /// can be cancelled with `PublishStage::PreparingArchive` still holding, and + /// it is unreachable from outside without a seam because compression of a + /// test-sized repo finishes in microseconds. + #[cfg(test)] + compress_gate: Option>, +} + +/// A gate a BLOCKING thread parks on until a test opens it. +/// +/// A condvar rather than a held `MutexGuard`: the test's assertions run while the +/// gate is shut, and holding a `std::sync::MutexGuard` across those awaits is +/// both a lint violation and a real hazard. Here the test owns no guard at all — +/// it flips a flag and notifies. +#[cfg(test)] +#[derive(Default)] +pub struct BlockingGate { + open: Mutex, + opened: std::sync::Condvar, +} + +#[cfg(test)] +impl BlockingGate { + /// A gate that starts SHUT. + pub fn shut() -> Self { + Self::default() + } + + fn wait(&self) { + let mut open = self.open.lock().expect("compression gate poisoned"); + while !*open { + open = self + .opened + .wait(open) + .expect("compression gate poisoned while waiting"); + } + } + + /// Let every parked compression through. Call this at teardown so the + /// blocking thread is not stranded for the life of the process. + pub fn open(&self) { + *self.open.lock().expect("compression gate poisoned") = true; + self.opened.notify_all(); + } } impl TigrisClient { @@ -57,6 +132,8 @@ impl TigrisClient { Ok(Self { s3, bucket: bucket.to_string(), + #[cfg(test)] + compress_gate: None, }) } @@ -84,9 +161,19 @@ impl TigrisClient { Self { s3: S3Client::from_conf(config), bucket: bucket.to_string(), + compress_gate: None, } } + /// Test-only: park this client's blocking compression on `gate` until the + /// holder releases it, so a cancellation can be aimed at + /// [`PublishStage::PreparingArchive`] rather than at the PUT. + #[cfg(test)] + pub fn with_compress_gate(mut self, gate: Arc) -> Self { + self.compress_gate = Some(gate); + self + } + /// S3 key for a given repo: `repos/v1/{owner_slug}/{repo_name}.tar.zst` fn repo_key(owner_slug: &str, repo_name: &str) -> String { format!("repos/v1/{owner_slug}/{repo_name}.tar.zst") @@ -122,6 +209,24 @@ impl TigrisClient { /// that only want the boolean, and widening its return type would churn /// every one of them for no benefit. pub async fn head_etag(&self, owner_slug: &str, repo_name: &str) -> Result> { + Ok(self + .head_generation(owner_slug, repo_name) + .await? + .map(|g| g.etag) + .unwrap_or(None)) + } + + /// The full stored generation: the ETag AND the attempt that published it. + /// + /// The attempt half is what makes a lost response recoverable. An ETag alone + /// answers "has the generation moved", which every writer sees the same way; + /// the attempt id answers "are the published bytes MINE", which is the + /// question a client whose PUT response vanished actually needs answered. + pub async fn head_generation( + &self, + owner_slug: &str, + repo_name: &str, + ) -> Result> { let key = Self::repo_key(owner_slug, repo_name); match self .s3 @@ -131,11 +236,20 @@ impl TigrisClient { .send() .await { - Ok(out) => Ok(Some( - out.e_tag() + Ok(out) => { + let etag = out + .e_tag() .context(format!("tigris HEAD {key}: hit carried no ETag"))? - .to_string(), - )), + .to_string(); + let attempt = out + .metadata() + .and_then(|m| m.get(ATTEMPT_METADATA_KEY)) + .cloned(); + Ok(Some(StoredGeneration { + etag: Some(etag), + attempt, + })) + } Err(e) => { if e.as_service_error().is_some_and(|e| e.is_not_found()) { Ok(None) @@ -146,26 +260,110 @@ impl TigrisClient { } } + /// Did `attempt`'s bytes land? The reconciliation an ambiguous dispatch owes + /// before anything downstream may treat it as confirmation. + /// + /// `Ok(false)` is deliberately NOT "the PUT failed": an abandoned request may + /// still be in flight, so a negative answer licenses refusing and retrying, + /// never deleting. Only `Ok(true)` upgrades an unresolved attempt to + /// published. + pub async fn attempt_landed( + &self, + owner_slug: &str, + repo_name: &str, + attempt: &PublishAttemptId, + ) -> Result { + Ok(self + .head_generation(owner_slug, repo_name) + .await? + .is_some_and(|g| g.belongs_to(attempt))) + } + /// Upload a local bare repo directory to Tigris as a tar.zst archive, - /// fenced by `precondition`. + /// fenced by `precondition`, under a freshly minted attempt identity. pub async fn upload( &self, owner_slug: &str, repo_name: &str, local_path: &Path, precondition: UploadPrecondition, - ) -> std::result::Result<(), UploadError> { + ) -> std::result::Result { + self.upload_tracked( + owner_slug, + repo_name, + local_path, + precondition, + PublishAttemptId::new(), + None, + ) + .await + } + + /// The full form: a caller-chosen attempt identity, and a stage cell the + /// upload reports its progress into. + /// + /// The stage cell is the answer to "a dropped future never returns an + /// outcome". Compression runs inside `spawn_blocking` and the conditional PUT + /// is not even constructed until it finishes, so a handler cancelled during + /// compression definitely never attempted publication — but nothing could + /// observe that, because the only report was the return value of a future + /// that no longer exists. Marking [`PublishStage::PreparingArchive`] before + /// the blocking call and [`PublishStage::PutDispatched`] immediately before + /// `send()` makes that boundary readable from outside. + pub async fn upload_tracked( + &self, + owner_slug: &str, + repo_name: &str, + local_path: &Path, + precondition: UploadPrecondition, + attempt: PublishAttemptId, + stage: Option<&PublishStageCell>, + ) -> std::result::Result { let key = Self::repo_key(owner_slug, repo_name); - debug!(key = %key, path = %local_path.display(), "uploading repo to tigris"); + debug!(key = %key, path = %local_path.display(), attempt = %attempt, "uploading repo to tigris"); + + if let Some(stage) = stage { + stage.set(PublishStage::PreparingArchive); + } - // Create tar.zst in memory - let archive_bytes = tokio::task::spawn_blocking({ + // Create tar.zst in memory. Nothing has been sent at this point and + // nothing can be: the request below is not constructed until this + // returns. A failure or a cancellation here is a definite + // non-publication. + let compressed = { let local_path = local_path.to_path_buf(); - move || compress_repo(&local_path) - }) - .await - .context("tar task panicked")? - .context("compressing repo")?; + #[cfg(test)] + let gate = self.compress_gate.clone(); + tokio::task::spawn_blocking(move || { + // Test-only seam: park INSIDE the blocking compression, which is + // the window a handler can be cancelled in while + // `PublishStage::PreparingArchive` still holds. + #[cfg(test)] + if let Some(gate) = gate { + // Blocks until the test opens the gate. + gate.wait(); + } + compress_repo(&local_path) + }) + .await + }; + let archive_bytes = match compressed { + Ok(Ok(bytes)) => bytes, + Ok(Err(e)) => { + if let Some(stage) = stage { + stage.set(PublishStage::Refused); + } + return Err(UploadError::NotPublished(e.context("compressing repo"))); + } + Err(e) => { + if let Some(stage) = stage { + stage.set(PublishStage::Refused); + } + return Err(UploadError::NotPublished( + anyhow::Error::new(e).context("tar task panicked"), + )); + } + }; let body = aws_sdk_s3::primitives::ByteStream::from(archive_bytes); @@ -174,6 +372,12 @@ impl TigrisClient { .put_object() .bucket(&self.bucket) .key(&key) + // The attempt identity travels WITH the bytes. That is what makes a + // lost response recoverable: the client can HEAD the key afterwards + // and ask whether the published object is its own work, which is + // decidable, instead of asking whether its request succeeded, which + // is not. + .metadata(ATTEMPT_METADATA_KEY, attempt.as_str()) .body(body) .content_type("application/zstd"); match &precondition { @@ -182,47 +386,140 @@ impl TigrisClient { UploadPrecondition::Unconditional => {} } - if let Err(e) = req.send().await { - // `PutObjectError` models no PreconditionFailed variant (its arms are - // EncryptionTypeMismatch, InvalidRequest, InvalidWriteOffset, - // TooManyParts, Unhandled), so a refused precondition arrives as - // `Unhandled` and matching the enum would classify it as a generic - // failure. The raw HTTP status off the response is the only place the - // answer actually lives. - // - // Read it via `raw_response()`, not a `ServiceError`-only match: the - // SDK exposes the raw response for BOTH `ServiceError` and - // `ResponseError`, and a refused conditional PUT whose error body the - // SDK cannot parse (malformed XML, premature close) surfaces as - // `ResponseError`. Matching only `ServiceError` would classify that - // unparsable 409/412 as a generic failure, and `RepoWriteGuard::release` - // would log-and-succeed instead of taking the supersede retry, - // acknowledging a write that was definitively not published. - let status = e.raw_response().map(|raw| raw.status().as_u16()); - // 412 is always a lost precondition. 409 is one only when we asked - // for create-only, which is how S3-compatible stores report "the key - // already exists". Everything else, 404 included, is a real failure: - // archive keys are never deleted (`delete` has no callers), so a 404 - // here means something permanent like a missing bucket or a - // misrouted endpoint, and reporting that as a lost precondition - // would tell a caller to expect a successor that does not exist. - let lost = match status { - Some(412) => true, - Some(409) => matches!(precondition, UploadPrecondition::IfAbsent), - _ => false, - }; - if lost { - return Err(UploadError::PreconditionLost { - status: status.expect("a lost precondition came from a status"), + // From HERE the bytes may reach the store. Everything downstream that + // could destroy state has to treat this stage as "maybe published". + if let Some(stage) = stage { + stage.set(PublishStage::PutDispatched { + attempt: attempt.clone(), + }); + } + + let sent = req.send().await; + let out = match sent { + Ok(out) => out, + Err(e) => { + // `PutObjectError` models no PreconditionFailed variant (its arms are + // EncryptionTypeMismatch, InvalidRequest, InvalidWriteOffset, + // TooManyParts, Unhandled), so a refused precondition arrives as + // `Unhandled` and matching the enum would classify it as a generic + // failure. The raw HTTP status off the response is the only place the + // answer actually lives. + // + // Read it via `raw_response()`, not a `ServiceError`-only match: the + // SDK exposes the raw response for BOTH `ServiceError` and + // `ResponseError`, and a refused conditional PUT whose error body the + // SDK cannot parse (malformed XML, premature close) surfaces as + // `ResponseError`. Matching only `ServiceError` would classify that + // unparsable 409/412 as a generic failure, and `RepoWriteGuard::release` + // would log-and-succeed instead of taking the supersede retry, + // acknowledging a write that was definitively not published. + let status = e.raw_response().map(|raw| raw.status().as_u16()); + // 412 is always a lost precondition. 409 is one only when we asked + // for create-only, which is how S3-compatible stores report "the key + // already exists". Everything else, 404 included, is a real failure: + // archive keys are never deleted by the write path, so a 404 here + // means something permanent like a missing bucket or a misrouted + // endpoint, and reporting that as a lost precondition would tell a + // caller to expect a successor that does not exist. + let lost = match status { + Some(412) => true, + Some(409) => matches!(precondition, UploadPrecondition::IfAbsent), + _ => false, + }; + if lost { + if let Some(stage) = stage { + stage.set(PublishStage::Refused); + } + return Err(UploadError::PreconditionLost { + status: status.expect("a lost precondition came from a status"), + }); + } + let knowledge = classify_put_failure(&e, status); + let ctx = anyhow::Error::new(e).context(format!("tigris PUT {key}")); + return Err(match knowledge { + DispatchKnowledge::NeverSent => { + if let Some(stage) = stage { + stage.set(PublishStage::Refused); + } + UploadError::NotPublished(ctx) + } + DispatchKnowledge::MaybeSent => { + warn!( + key = %key, + attempt = %attempt, + status = ?status, + "tigris PUT failed WITHOUT proving it did not commit — the outcome \ + is ambiguous and must be reconciled, not compensated" + ); + if let Some(stage) = stage { + stage.set(PublishStage::Ambiguous { + attempt: attempt.clone(), + }); + } + UploadError::Ambiguous { + attempt: attempt.clone(), + source: ctx, + } + } }); } - return Err(UploadError::Other( - anyhow::Error::new(e).context(format!("tigris PUT {key}")), - )); + }; + + let etag = out.e_tag().map(str::to_string); + if let Some(stage) = stage { + stage.set(PublishStage::Published { + attempt: attempt.clone(), + etag: etag.clone(), + }); } + info!(key = %key, attempt = %attempt, "uploaded repo to tigris"); + Ok(UploadReceipt { attempt, etag }) + } - info!(key = %key, "uploaded repo to tigris"); - Ok(()) + /// Delete a repo archive ONLY while it is still `attempt`'s work. + /// + /// The unconditional delete this replaces used the logical owner/name as its + /// cleanup authority, so a failed attempt compensating late could erase the + /// object a SUCCESSOR had already published under the same name and already + /// returned 201 for. A second name lookup before the delete only narrows that + /// window; reading the attempt id off the object and fencing the delete on + /// the generation it came from closes it. + pub async fn delete_if_attempt_matches( + &self, + owner_slug: &str, + repo_name: &str, + attempt: &PublishAttemptId, + ) -> Result { + let Some(generation) = self.head_generation(owner_slug, repo_name).await? else { + return Ok(AttemptDelete::Absent); + }; + if !generation.belongs_to(attempt) { + return Ok(AttemptDelete::NotOurs); + } + let key = Self::repo_key(owner_slug, repo_name); + let mut req = self.s3.delete_object().bucket(&self.bucket).key(&key); + // The If-Match guard is what makes this atomic rather than merely + // narrowed: between the HEAD above and this call a successor can publish, + // and the store refusing on the moved generation is the only thing that + // stops the delete landing on their object. + if let Some(etag) = generation.etag.as_deref() { + req = req.if_match(etag); + } + match req.send().await { + Ok(_) => Ok(AttemptDelete::Deleted), + Err(e) => { + // A refused conditional delete means the generation moved: the + // object is no longer ours, which is a successful outcome for a + // guard whose whole job is not to touch somebody else's bytes. + if e.raw_response() + .map(|raw| raw.status().as_u16()) + .is_some_and(|s| s == 412 || s == 409) + { + return Ok(AttemptDelete::NotOurs); + } + Err(anyhow::anyhow!("tigris conditional DELETE {key}: {e}")) + } + } } /// Download a repo archive from Tigris and extract to local disk. @@ -327,19 +624,6 @@ impl TigrisClient { info!(key = %key, path = %target.as_path().display(), "downloaded repo from tigris"); Ok(extracted) } - - /// Delete a repo archive from Tigris. - pub async fn delete(&self, owner_slug: &str, repo_name: &str) -> Result<()> { - let key = Self::repo_key(owner_slug, repo_name); - self.s3 - .delete_object() - .bucket(&self.bucket) - .key(&key) - .send() - .await - .context(format!("tigris DELETE {key}"))?; - Ok(()) - } } /// Compress a bare repo directory into a tar.zst byte vector. From 5e548d2ff8fc1a15ec2adbe007e501304616bd22 Mon Sep 17 00:00:00 2001 From: Kevin Codex Date: Mon, 31 Aug 2026 21:11:30 +0800 Subject: [PATCH 51/54] fix(node): key durability, cache validity and fork cleanup on the attempt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the four lifecycle findings by consuming the attempt/stage boundary at the sites that were deciding on coarse state. P1 — cancellation before PUT dispatch is not publish durability. `PublishDurabilitySlot` is armed on the guard's publish STAGE rather than on a "release started" flag, and its `Drop` classifies an abandoned handler by how far the attempt actually got: a cancellation during compression records a definite refusal, a cancellation after dispatch records unknowable, and a cancellation after the store acknowledged records `Released` (which the flag could never say). `publish_durability_confirmed` now requires `Released`, so the detached tail no longer does IPFS/Pinata/P2P/GraphQL/Arweave/peer work for refs that exist only in a rejected local write. A guard with no storage backend seeds `NoBackend`, keeping Tigris-less deployments' tails running. P1 — an unknowable generation is quarantined, not served as ordinary cache. `release` writes a sidecar marker naming the unresolved attempt beside the live tree (never inside it: it would be tarred into the next archive), and `acquire`'s existing-path fast path reconciles it before serving. Only the store confirming it holds that attempt lifts the quarantine; anything else is a retryable `RepoUnavailable`. The tree is deliberately NOT deleted — the PUT may have landed and this can be the only local copy. A confirmed publish, an under-lock refresh, and cache invalidation each clear the marker, so the quarantine is a bounded refusal rather than a standing outage. The release-side timeout arm also stops treating every stall alike: a bound that expires at `PreparingArchive` is a definite non-publication, and a bound that expires after dispatch spends one short, separately bounded HEAD trying to reconcile the attempt before falling back to unknowable. P1 — fork confirmation and cleanup bind to the creating attempt. The DB row id is minted up front and used as the attempt id, so the object's metadata, the clone's sidecar stamp and the row all name one attempt. `confirm_fork_repo_row` looks the row up BY ID and reports Ours / Foreign / Absent, so a concurrent create, mirror registration or retry can no longer be returned as this request's own commit. Every destructive step — `compensate_fork_archive`, its background retries, and `ForkCloneGuard::drop` — is conditional on the resource still belonging to this attempt, so recovery that observed `None` before a successor committed can no longer erase that successor's archive or directory afterwards. P2 — response-loss ambiguity is preserved rather than compensated. Guarded writes map an ambiguous publish to `UploadUnknowable` + quarantine instead of `UploadFailed` + cache invalidation + the caller's undo, so `create_issue` no longer deletes an issue ref whose archive is durable. Fork creation reconciles by attempt id: a create-only PUT that committed and lost its response is RECOVERED and the row inserted, and an unresolved one keeps its only local clone and refuses retryably. The fork name is no longer fenced behind the attempt's own orphan. Tests, each RED-checked by reverting its guard: - slot_drop_during_compression_records_a_definite_non_publication - slot_drop_after_dispatch_records_unknowable - slot_drop_after_the_store_acknowledged_records_released - slot_drop_with_no_storage_backend_records_released - publish_durability_confirmed_accepts_only_released - publish_durability_confirmed_refuses_quickly_after_unrecorded_slot_drop - receive_pack_cancelled_during_compression_publishes_nothing_and_runs_no_tail (+ receive_pack_that_completes_its_publish_still_runs_the_tail as control) - a_bound_that_expires_before_dispatch_is_a_definite_failure - an_unresolved_publish_quarantines_the_tree_and_a_later_read_refuses - a_quarantined_tree_is_served_once_the_store_confirms_the_attempt - the_next_write_clears_an_inherited_quarantine - a_confirmed_publish_leaves_the_tree_readable (control) - a_guarded_write_whose_response_is_lost_is_not_compensated - a_put_that_commits_and_loses_its_response_is_ambiguous_and_reconcilable - a_closed_response_with_no_http_status_is_ambiguous - upload_classifies_404_as_a_definite_non_publication - upload_classifies_500_as_ambiguous_not_definite_failure - a_generation_that_moves_between_the_head_and_the_delete_is_not_deleted - an_attempts_own_object_is_deleted_and_a_foreign_one_is_not - fork_confirmation_never_claims_a_concurrent_attempts_row - fork_confirmation_recognizes_this_attempts_own_committed_row - fork_confirmation_reports_absent_when_nothing_owns_the_name - fork_clone_guard_leaves_a_successors_mirror_alone - fork_recovery_resuming_after_a_successor_deletes_neither_object_nor_path - fork_compensation_removes_what_this_attempt_still_owns - fork_publish_that_loses_its_response_stays_recoverable - a_fork_whose_publish_lost_its_response_is_recovered_not_fenced - a_fork_whose_publish_is_unresolved_keeps_its_clone_and_refuses_retryably - an_ordinary_fork_publishes_and_commits (control) - mock_round_trips_the_attempt_metadata_a_put_stamped The S3 mock gains attempt metadata, conditional DELETE, a commit-then-lose- the-response mode, a delivered-but-not-committed mode, and a per-key object store (fork creation touches the source and fork keys in one request, and a single slot would have let an assertion about one silently read the other). The compression seam is a condvar gate so no test holds a guard across an await. --- crates/gitlawb-node/src/api/repos.rs | 901 +++++++++-- crates/gitlawb-node/src/git/repo_store.rs | 1776 ++++++++++++++++++++- 2 files changed, 2477 insertions(+), 200 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 7eca1b82..ed028ab2 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -2386,7 +2386,10 @@ pub async fn git_receive_pack( .take() .expect("the write lock is only taken here, and only once"); if push_succeeded { - publish_durability.mark_release_started(); + // Arm on the guard's OWN publish stage, so a disconnect anywhere in + // `release` is classified by how far the publish actually got rather + // than by the mere fact that release was entered. + publish_durability.arm_for_release(reclaimed.publish_stage()); tokio::spawn(post_receive_replication_tail( state.clone(), record.clone(), @@ -2523,15 +2526,26 @@ pub async fn git_receive_pack( /// task. Split out of `git_receive_pack` so the ordering the coalescing gate depends /// on is directly testable; the handler spawns it and returns. /// Records the release-side publish outcome for the detached post-receive tail. -/// If the handler future is dropped after [`Self::mark_release_started`] but -/// before [`Self::record`], the drop sets [`ReleaseOutcome::UploadUnknowable`] -/// so the tail can proceed on local disk without waiting out the full transfer -/// bound. A drop before release starts leaves the slot empty so the tail fails -/// closed rather than treating an unattempted publish as unknowably durable. +/// +/// The subtle case is the one this type exists for: the handler future is +/// DROPPED, so `release` never returns and `record` is never called. The slot +/// then has to classify an attempt whose outcome nobody will ever report, and +/// the only honest source for that is the publish STAGE — how far the attempt +/// had actually got when it was abandoned. +/// +/// Tracking "release started" instead was the defect. `release` awaits a blocking +/// compression before the conditional PUT is constructed, so a cancellation +/// there is a definite "no publication was attempted", yet every cancellation +/// after the flag was set recorded `UploadUnknowable` — which the tail accepted, +/// and then went on to do IPFS, Pinata, P2P, GraphQL, Arweave and peer work for +/// refs that existed only in a rejected local write. struct PublishDurabilitySlot { inner: Arc>>, recorded: std::sync::atomic::AtomicBool, - release_started: std::sync::atomic::AtomicBool, + /// The stage of the release this slot is armed for. `None` until the release + /// is about to be awaited, which is what makes a drop before that point leave + /// the slot empty and the tail fail closed. + stage: std::sync::Mutex>>, } impl PublishDurabilitySlot { @@ -2539,13 +2553,14 @@ impl PublishDurabilitySlot { Self { inner: Arc::new(std::sync::Mutex::new(None)), recorded: std::sync::atomic::AtomicBool::new(false), - release_started: std::sync::atomic::AtomicBool::new(false), + stage: std::sync::Mutex::new(None), } } - fn mark_release_started(&self) { - self.release_started - .store(true, std::sync::atomic::Ordering::SeqCst); + /// Arm the slot against the guard's publish stage, immediately before the + /// release is awaited. + fn arm_for_release(&self, stage: Arc) { + *self.stage.lock().expect("publish stage slot poisoned") = Some(stage); } fn arc(&self) -> Arc>> { @@ -2567,22 +2582,61 @@ impl Drop for PublishDurabilitySlot { if self.recorded.load(std::sync::atomic::Ordering::SeqCst) { return; } - if !self - .release_started - .load(std::sync::atomic::Ordering::SeqCst) - { + let Some(stage) = self + .stage + .lock() + .expect("publish stage slot poisoned") + .clone() + else { + // Dropped before the release was even armed. Leave the slot empty so + // the tail fails closed on the wait below. return; - } + }; + use crate::git::publish::PublishStage; + use crate::git::repo_store::ReleaseOutcome; + let outcome = match stage.get() { + // The store answered. Cancellation after that point loses the + // handler's response, not the write, so the tail may run. + // + // `NoBackend` joins it: with no object storage configured there is + // no publication to confirm and the pack on local disk is the + // durable copy, which is exactly what `ReleaseOutcome::Released` + // already means for a release that had nothing to upload. + PublishStage::Published { .. } | PublishStage::NoBackend => ReleaseOutcome::Released, + // THE FINDING. Compression was still running, so the conditional PUT + // was never constructed and no publication was ever attempted. This + // is a definite refusal, not an unknowable one, and the tail must + // skip every replication effect. + PublishStage::Idle | PublishStage::PreparingArchive | PublishStage::Refused => { + ReleaseOutcome::UploadFailed + } + // Genuinely in flight. Nothing here can reconcile it — a `Drop` impl + // cannot await a HEAD — so it stays unknowable, which the tail refuses + // because it requires `Released`. The write is still on local disk and + // the next writer's under-lock refresh resolves it. + PublishStage::PutDispatched { .. } | PublishStage::Ambiguous { .. } => { + ReleaseOutcome::UploadUnknowable + } + }; let mut slot = self .inner .lock() .expect("publish durability mutex poisoned"); if slot.is_none() { - *slot = Some(crate::git::repo_store::ReleaseOutcome::UploadUnknowable); + *slot = Some(outcome); } } } +/// May the detached replication tail run? +/// +/// ONLY on `Released`, which is the outcome that means the store acknowledged +/// this attempt (directly, or through the release-side reconciliation that +/// matched the attempt id against what the store holds). `UploadUnknowable` used +/// to pass here, which let the tail pin, announce and publish refs whose archive +/// may never have landed; an unresolved dispatch has to be reconciled to this +/// attempt's own generation before it counts as confirmation, and the place that +/// can do it is `release`, not this predicate. async fn publish_durability_confirmed( slot: &Option>>>, wait: std::time::Duration, @@ -2590,25 +2644,24 @@ async fn publish_durability_confirmed( let Some(slot) = slot else { return true; }; + let confirmed = |outcome: Option| { + matches!( + outcome, + Some(crate::git::repo_store::ReleaseOutcome::Released) + ) + }; let start = std::time::Instant::now(); loop { - if let Some(outcome) = *slot.lock().expect("publish durability mutex poisoned") { - return matches!( - outcome, - crate::git::repo_store::ReleaseOutcome::Released - | crate::git::repo_store::ReleaseOutcome::UploadUnknowable - ); + let outcome = *slot.lock().expect("publish durability mutex poisoned"); + if outcome.is_some() { + return confirmed(outcome); } if start.elapsed() >= wait { // No outcome within the release-side transfer bound plus slack. Fail // closed when the slot is still empty; an abandoned handler installs - // UploadUnknowable via PublishDurabilitySlot::drop so the tail can - // proceed on local disk without waiting out the full bound. - return matches!( - *slot.lock().expect("publish durability mutex poisoned"), - Some(crate::git::repo_store::ReleaseOutcome::Released) - | Some(crate::git::repo_store::ReleaseOutcome::UploadUnknowable) - ); + // its stage's verdict via PublishDurabilitySlot::drop, so the tail + // resolves promptly rather than waiting out the full bound. + return confirmed(*slot.lock().expect("publish durability mutex poisoned")); } tokio::time::sleep(std::time::Duration::from_millis(10)).await; } @@ -3190,23 +3243,66 @@ pub async fn list_federated_repos( // ── Fork ────────────────────────────────────────────────────────────────── /// Removes a fork's local mirror clone on every exit until disarmed after the -/// database row commits. -struct ForkCloneGuard(Option); +/// database row commits — but ONLY while that clone still belongs to the attempt +/// the guard was built for. +/// +/// The bare `remove_dir_all` this replaces authorized itself with the path alone. +/// Two attempts at the same logical fork share one path, so a failed attempt +/// unwinding late could remove the directory a successor had already published +/// and inserted a row for. +struct ForkCloneGuard { + path: Option, + attempt: crate::git::publish::PublishAttemptId, +} const FORK_CREATE_CONFIRM_ATTEMPTS: u32 = 5; +/// Who owns the logical fork name once `create_repo` has reported an error. +/// +/// The distinction the previous `Option` could not draw is between +/// "my insert committed" and "somebody's insert committed". Only the first +/// entitles this request to answer 201 with that row, and only the third +/// entitles it to compensate. +enum ForkRowConfirmation { + /// THIS attempt's row is present: the insert committed and the error was a + /// transport-level report of a successful write. + Ours(Box), + /// A different attempt owns the name. This request's archive, disk path and + /// fork provenance do not belong to that row, so it must not be returned, + /// and that attempt's resources must not be compensated. + Foreign, + /// Nothing owns the name. + Absent, +} + /// After `create_repo` fails, re-read the row with bounded retries so a /// transport-level error does not skip archive compensation when the insert never /// landed, and so a successful insert is not mistaken for a failure. +/// +/// Keyed on `record_id` FIRST. Owner/name identifies a namespace, not an attempt: +/// a concurrent ordinary create, a mirror registration or a retry can insert a +/// different row under the same logical name, and treating that as proof of our +/// own commit returned 201 carrying somebody else's row. async fn confirm_fork_repo_row( db: &crate::db::Db, + record_id: &str, owner_short: &str, fork_name: &str, -) -> anyhow::Result> { +) -> anyhow::Result { let mut delay = std::time::Duration::from_millis(25); for attempt in 0..FORK_CREATE_CONFIRM_ATTEMPTS { - match db.get_repo(owner_short, fork_name).await { - Ok(row) => return Ok(row), + let looked_up = async { + if let Some(ours) = db.get_repo_by_id(record_id).await? { + return Ok::<_, anyhow::Error>(ForkRowConfirmation::Ours(Box::new(ours))); + } + Ok(match db.get_repo(owner_short, fork_name).await? { + Some(_) => ForkRowConfirmation::Foreign, + None => ForkRowConfirmation::Absent, + }) + } + .await; + match looked_up { + Ok(confirmation) => return Ok(confirmation), Err(e) if attempt + 1 < FORK_CREATE_CONFIRM_ATTEMPTS => { tracing::warn!( fork = %fork_name, @@ -3228,40 +3324,62 @@ async fn confirm_fork_repo_row( /// When the confirmation lookup stays unavailable after bounded retries, keep /// retrying in the background and compensate only after establishing that no row /// exists for this fork name. +/// +/// The `None` observation is still racy on its own — a successor can commit +/// between the read and the deletions, and did, which let recovery for a failed +/// fork erase a succeeding attempt's repository. What makes it safe is that the +/// compensation it calls is now conditional on the object and the directory still +/// carrying THIS attempt's identity, so a successor that commits at any point +/// after the read keeps both. +#[allow(clippy::too_many_arguments)] async fn schedule_fork_create_recovery( db: std::sync::Arc, repo_store: crate::git::repo_store::RepoStore, + record_id: String, owner_short: String, fork_name: String, owner_did: String, disk_path: std::path::PathBuf, + attempt: crate::git::publish::PublishAttemptId, ) { let mut delay = std::time::Duration::from_millis(250); - for attempt in 0..12 { - match db.get_repo(&owner_short, &fork_name).await { - Ok(Some(_)) => { + for n in 0..12 { + match confirm_fork_repo_row(&db, &record_id, &owner_short, &fork_name).await { + Ok(ForkRowConfirmation::Ours(_)) => { tracing::info!( fork = %fork_name, - attempt, - "fork create_repo recovery found a committed row — no archive compensation" + attempt = n, + "fork create_repo recovery found this attempt's row — no compensation" ); return; } - Ok(None) => { + Ok(ForkRowConfirmation::Foreign) => { + tracing::info!( + fork = %fork_name, + attempt = n, + "fork create_repo recovery found another attempt's row under this name — \ + compensating only what this attempt still owns" + ); + repo_store + .compensate_fork_archive(&owner_did, &fork_name, disk_path.as_path(), &attempt) + .await; + return; + } + Ok(ForkRowConfirmation::Absent) => { tracing::warn!( fork = %fork_name, - attempt, + attempt = n, "fork create_repo recovery confirmed no row — compensating orphan archive" ); repo_store - .compensate_fork_archive(&owner_did, &fork_name, disk_path.as_path()) + .compensate_fork_archive(&owner_did, &fork_name, disk_path.as_path(), &attempt) .await; return; } - Err(e) if attempt + 1 < 12 => { + Err(e) if n + 1 < 12 => { tracing::warn!( fork = %fork_name, - attempt, + attempt = n, err = %e, "fork create_repo recovery lookup failed — retrying" ); @@ -3283,31 +3401,40 @@ async fn schedule_fork_create_recovery( } impl ForkCloneGuard { - fn new(path: crate::git::repo_store::ValidatedRepoDiskPath) -> Self { - Self(Some(path.into_path_buf())) + fn new( + path: crate::git::repo_store::ValidatedRepoDiskPath, + attempt: crate::git::publish::PublishAttemptId, + ) -> Self { + let path = path.into_path_buf(); + // Stamp the directory before anything can fail, so every later cleanup + // (this guard's Drop, the handler's compensation, the background + // recovery) has an ownership answer to consult. + crate::git::repo_store::claim_fork_disk_path(&path, &attempt); + Self { + path: Some(path), + attempt, + } } fn path(&self) -> &std::path::Path { - self.0 + self.path .as_deref() .expect("fork clone guard disarmed while still in use") } fn disarm(&mut self) { - self.0 = None; + self.path = None; } } impl Drop for ForkCloneGuard { fn drop(&mut self) { - if let Some(path) = self.0.take() { - if let Err(e) = std::fs::remove_dir_all(&path) { - tracing::warn!( - path = %path.display(), - err = %e, - "failed to remove fork clone during attempt cleanup" - ); - } + if let Some(path) = self.path.take() { + crate::git::repo_store::remove_fork_clone_if_ours( + &path, + &self.attempt, + "fork attempt cleanup", + ); } } } @@ -3407,34 +3534,109 @@ pub async fn fork_repo( ))); } - let mut clone_guard = ForkCloneGuard::new(disk_path.clone()); + // ONE identity for the whole workflow: the DB row id this request will + // insert is also the attempt id stamped into the object's metadata and onto + // the disk clone. That is what lets confirmation ask "is my row there" rather + // than "is a row there", and lets every cleanup ask "is this still mine" + // rather than "does this name resolve". + let record_id = Uuid::new_v4().to_string(); + let attempt = crate::git::publish::PublishAttemptId::from_owned(record_id.clone()); + + let mut clone_guard = ForkCloneGuard::new(disk_path.clone(), attempt.clone()); // Upload fork to Tigris. Create-only: a refused precondition means an orphan // archive already sits under this key (a failed create_repo or another // writer), and proceeding would create a DB record whose archive is shadowed // by bytes that are not this fork. Refuse rather than accept a fork other // nodes would fetch as unrelated content. The clone guard removes the local - // mirror on every upload failure path. - state + // mirror on every DEFINITE upload failure path. + if let Err(e) = state .repo_store - .release_after_write(&forker_did, &fork_name) + .release_after_write(&forker_did, &fork_name, &attempt) .await - .map_err(|e| match e { + { + match e { crate::git::tigris::UploadError::PreconditionLost { status } => { - tracing::warn!( - forker = %forker_did, - fork = %fork_name, - status, - "fork refused: an archive already exists under the fork's key" - ); - AppError::RepoExists(fork_name.clone()) + // One reconciliation before refusing: an SDK-level retry, or a + // duplicated dispatch, can land THIS attempt's bytes and then see + // the create-only fence refuse the second copy. The stored object + // carrying our own attempt id says the publish succeeded, and + // refusing it would fence the fork name behind our own work. + if state + .repo_store + .fork_attempt_landed(&forker_did, &fork_name, &attempt) + .await + .unwrap_or(false) + { + tracing::info!( + fork = %fork_name, + status, + "fork create-only PUT was refused but the stored archive is this \ + attempt's own — continuing" + ); + } else { + tracing::warn!( + forker = %forker_did, + fork = %fork_name, + status, + "fork refused: an archive already exists under the fork's key" + ); + return Err(AppError::RepoExists(fork_name.clone())); + } } - other => AppError::Git(format!("fork upload failed: {other}")), - })?; + crate::git::tigris::UploadError::NotPublished(other) => { + // Proven not to have committed. Dropping the clone (on the + // guard's Drop) is safe, and the fork name is left free. + return Err(AppError::Git(format!("fork upload failed: {other:#}"))); + } + crate::git::tigris::UploadError::Ambiguous { source, .. } => { + // THE DURABLE FAILURE MODE. The create-only PUT can commit and + // the response can still be lost or unparseable. Compensating + // that as a definite failure dropped the only local clone and + // skipped the DB insert, and every retry then saw the orphan + // object and returned RepoExists — the fork name unusable until + // an operator intervened. + // + // Ask the store instead. The attempt id travelled with the bytes, + // so "did MY write land" is answerable even when "did my request + // succeed" is not. + match state + .repo_store + .fork_attempt_landed(&forker_did, &fork_name, &attempt) + .await + { + Ok(true) => { + tracing::warn!( + fork = %fork_name, + err = %source, + "fork upload lost its response but the stored archive is this \ + attempt's own — recovering the committed publish" + ); + } + // Not (yet) ours, or unreachable. The request may still be in + // flight, so nothing here may be destroyed: keep the clone, + // keep whatever is stored, and refuse retryably. Disarming the + // guard is what preserves the only local copy. + Ok(false) | Err(_) => { + clone_guard.disarm(); + tracing::warn!( + forker = %forker_did, + fork = %fork_name, + err = %source, + "fork upload outcome is unknowable — leaving the clone and any \ + stored object in place for reconciliation rather than \ + compensating a write that may have landed" + ); + return Err(AppError::RepoUnavailable); + } + } + } + } + } let now = Utc::now(); let record = crate::db::RepoRecord { - id: Uuid::new_v4().to_string(), + id: record_id.clone(), name: fork_name.clone(), owner_did: forker_did.clone(), description: source.description.clone(), @@ -3448,24 +3650,45 @@ pub async fn fork_repo( }; if let Err(e) = state.db.create_repo(&record).await { - match confirm_fork_repo_row(&state.db, forker_short, &fork_name).await { - Ok(Some(committed)) => { + match confirm_fork_repo_row(&state.db, &record_id, forker_short, &fork_name).await { + Ok(ForkRowConfirmation::Ours(committed)) => { clone_guard.disarm(); tracing::warn!( fork = %fork_name, forker = %forker_did, - "fork create_repo returned an error but the row is present — treating as success" + "fork create_repo returned an error but THIS attempt's row is present — treating as success" ); return Ok(( StatusCode::CREATED, Json(to_response(&committed, &state, 0)), )); } - Ok(None) => { + Ok(ForkRowConfirmation::Foreign) => { + // A concurrent create, mirror registration or retry owns the + // name. Returning its row would hand this caller a repository + // whose archive, disk path and fork provenance are not the ones + // this request produced. Refuse, and compensate only what this + // attempt still owns — the guard and `compensate_fork_archive` + // both check ownership, so the successor's object and directory + // survive. + tracing::warn!( + fork = %fork_name, + forker = %forker_did, + "fork create_repo lost the name to another attempt — refusing rather than \ + claiming its row" + ); + clone_guard.disarm(); + state + .repo_store + .compensate_fork_archive(&forker_did, &fork_name, disk_path.as_path(), &attempt) + .await; + return Err(AppError::RepoExists(fork_name.clone())); + } + Ok(ForkRowConfirmation::Absent) => { clone_guard.disarm(); state .repo_store - .compensate_fork_archive(&forker_did, &fork_name, disk_path.as_path()) + .compensate_fork_archive(&forker_did, &fork_name, disk_path.as_path(), &attempt) .await; return Err(e.into()); } @@ -3477,14 +3700,18 @@ pub async fn fork_repo( let fork_name_cl = fork_name.clone(); let owner_did = forker_did.clone(); let disk_path_cl = disk_path.as_path().to_path_buf(); + let record_id_cl = record_id.clone(); + let attempt_cl = attempt.clone(); tokio::spawn(async move { schedule_fork_create_recovery( db, repo_store, + record_id_cl, owner_short, fork_name_cl, owner_did, disk_path_cl, + attempt_cl, ) .await; }); @@ -3500,6 +3727,9 @@ pub async fn fork_repo( } clone_guard.disarm(); + // The row is committed: this directory is now the repository, not an + // attempt's staging area, and nothing may ever compensate it away. + crate::git::repo_store::release_fork_disk_claim(disk_path.as_path()); // Persist the proof so the fork carries it when it propagates to peers. if let Some(p) = verified_proof { @@ -3737,18 +3967,88 @@ mod tests { const OWNER_SHORT: &str = "z6MkpTHR8VNsBxYAAWHut2Geadd9jSwuBV8xRoAnwWsdvktH"; const STRANGER_DID: &str = "did:key:z6Mkffonly5tranger0000000000000000000000000000000"; - #[tokio::test] - async fn publish_durability_slot_drop_installs_unknowable_when_release_started_but_unrecorded() - { + use crate::git::publish::{PublishAttemptId, PublishStage, PublishStageCell}; + use crate::git::repo_store::ReleaseOutcome; + + /// A slot armed against a stage cell the test drives directly, which is the + /// only way to reach the abandoned-handler arms without a live push. + fn armed_slot(stage: PublishStage) -> (PublishDurabilitySlot, Arc) { + let cell = Arc::new(PublishStageCell::new()); + cell.set(stage); let slot = PublishDurabilitySlot::new(); - slot.mark_release_started(); + slot.arm_for_release(Arc::clone(&cell)); + (slot, cell) + } + + /// P1 FINDING 1, at the type level. A handler cancelled while the archive is + /// still being compressed has NOT attempted publication: the conditional PUT + /// is not constructed until compression returns. Recording that as + /// `UploadUnknowable` is what let the detached tail do IPFS, Pinata, P2P, + /// GraphQL, Arweave and peer work for refs that exist only in a local write + /// the store never saw. + #[test] + fn slot_drop_during_compression_records_a_definite_non_publication() { + let (slot, _cell) = armed_slot(PublishStage::PreparingArchive); let arc = slot.arc(); drop(slot); - let outcome = *arc.lock().expect("publish durability mutex poisoned"); assert_eq!( - outcome, - Some(crate::git::repo_store::ReleaseOutcome::UploadUnknowable), - "dropping an unrecorded slot after release starts must publish UploadUnknowable for the tail" + *arc.lock().expect("publish durability mutex poisoned"), + Some(ReleaseOutcome::UploadFailed), + "a cancellation before the PUT was dispatched must be recorded as a definite \ + non-publication, not as unknowable durability" + ); + } + + /// The other side of the same boundary: once the request is on the wire, the + /// outcome genuinely is unknowable, and a `Drop` impl cannot await a HEAD to + /// resolve it. Unknowable is honest here — and the tail refuses it anyway, + /// because the tail requires `Released`. + #[test] + fn slot_drop_after_dispatch_records_unknowable() { + let (slot, _cell) = armed_slot(PublishStage::PutDispatched { + attempt: PublishAttemptId::new(), + }); + let arc = slot.arc(); + drop(slot); + assert_eq!( + *arc.lock().expect("publish durability mutex poisoned"), + Some(ReleaseOutcome::UploadUnknowable), + "a dispatched PUT may still commit, so the drop must not claim it definitely failed" + ); + } + + /// A disconnect AFTER the store acknowledged the publish loses the response, + /// not the write. The tail must still run: this is the case the stage model + /// gains over the old flag, which could only ever say "unknowable" here. + #[test] + fn slot_drop_after_the_store_acknowledged_records_released() { + let (slot, _cell) = armed_slot(PublishStage::Published { + attempt: PublishAttemptId::new(), + etag: Some("\"e1\"".to_string()), + }); + let arc = slot.arc(); + drop(slot); + assert_eq!( + *arc.lock().expect("publish durability mutex poisoned"), + Some(ReleaseOutcome::Released), + "a confirmed publish is durable regardless of what happened to the handler" + ); + } + + /// A node with no object storage configured has nothing to publish, so the + /// pack on local disk IS the durable copy and a disconnect must not withhold + /// the tail. Distinct from `Idle`, where a publish was possible and never + /// started; collapsing the two turns every Tigris-less deployment's pushes + /// into un-replicated ones. + #[test] + fn slot_drop_with_no_storage_backend_records_released() { + let (slot, _cell) = armed_slot(PublishStage::NoBackend); + let arc = slot.arc(); + drop(slot); + assert_eq!( + *arc.lock().expect("publish durability mutex poisoned"), + Some(ReleaseOutcome::Released), + "with no backend there is no publication to confirm, so the tail must run" ); } @@ -3766,8 +4066,9 @@ mod tests { #[test] fn publish_durability_slot_drop_waits_for_contended_mutex() { - let slot = PublishDurabilitySlot::new(); - slot.mark_release_started(); + let (slot, _cell) = armed_slot(PublishStage::PutDispatched { + attempt: PublishAttemptId::new(), + }); let arc = slot.arc(); let holder = { let arc = Arc::clone(&arc); @@ -3782,24 +4083,28 @@ mod tests { let outcome = *arc.lock().expect("publish durability mutex poisoned"); assert_eq!( outcome, - Some(crate::git::repo_store::ReleaseOutcome::UploadUnknowable), - "drop must block until it can install UploadUnknowable, not give up on try_lock" + Some(ReleaseOutcome::UploadUnknowable), + "drop must block until it can install its verdict, not give up on try_lock" ); } + /// The tail must resolve promptly on an abandoned handler rather than wait + /// out the whole transfer bound — it just resolves to a REFUSAL now. #[tokio::test] - async fn publish_durability_confirmed_proceeds_quickly_after_unrecorded_slot_drop() { + async fn publish_durability_confirmed_refuses_quickly_after_unrecorded_slot_drop() { let start = std::time::Instant::now(); - let slot = PublishDurabilitySlot::new(); - slot.mark_release_started(); + let (slot, _cell) = armed_slot(PublishStage::PreparingArchive); let arc = slot.arc(); drop(slot); let confirmed = publish_durability_confirmed(&Some(arc), std::time::Duration::from_millis(50)).await; - assert!(confirmed); + assert!( + !confirmed, + "a cancellation before dispatch must not admit the replication tail" + ); assert!( start.elapsed() < std::time::Duration::from_millis(200), - "the tail must not wait out the full transfer bound when the slot already carries UploadUnknowable" + "the tail must not wait out the full transfer bound once the slot carries a verdict" ); } @@ -3811,37 +4116,38 @@ mod tests { assert!( !confirmed, "an empty slot after the bounded wait must not admit the tail; only an installed \ - Released or UploadUnknowable outcome may" + Released outcome may" ); } + /// P1 FINDING 1, at the gate. The tail replicates to IPFS, Pinata, P2P, + /// GraphQL, Arweave and peers, all of which publish refs to other nodes. Only + /// a CONFIRMED publish may license that. `UploadUnknowable` used to pass here, + /// which is what carried an unattempted (and an unreconciled) write into the + /// network. #[tokio::test] async fn publish_durability_confirmed_accepts_only_released() { - let slot = Arc::new(std::sync::Mutex::new(Some( - crate::git::repo_store::ReleaseOutcome::Released, - ))); + let slot = Arc::new(std::sync::Mutex::new(Some(ReleaseOutcome::Released))); assert!( publish_durability_confirmed(&Some(slot), std::time::Duration::from_millis(5)).await ); - let slot = Arc::new(std::sync::Mutex::new(Some( - crate::git::repo_store::ReleaseOutcome::UploadUnknowable, - ))); - assert!( - publish_durability_confirmed(&Some(slot), std::time::Duration::from_millis(5)).await, - "an unknowable upload still landed on local disk, so the tail may proceed" - ); - - let slot = Arc::new(std::sync::Mutex::new(Some( - crate::git::repo_store::ReleaseOutcome::UploadFailed, - ))); - assert!( - !publish_durability_confirmed(&Some(slot), std::time::Duration::from_millis(5)).await - ); + for refused in [ + ReleaseOutcome::UploadUnknowable, + ReleaseOutcome::UploadFailed, + ReleaseOutcome::Fenced, + ] { + let slot = Arc::new(std::sync::Mutex::new(Some(refused))); + assert!( + !publish_durability_confirmed(&Some(slot), std::time::Duration::from_millis(5)) + .await, + "{refused:?} is not a confirmed publish and must not admit the tail" + ); + } } #[test] - fn fork_clone_guard_removes_mirror_on_drop() { + fn fork_clone_guard_removes_its_own_mirror_on_drop() { let root = tempfile::TempDir::new().unwrap(); let validated = crate::git::repo_store::validated_repo_disk_path( root.path(), @@ -3851,12 +4157,39 @@ mod tests { .expect("test fork path must validate"); std::fs::create_dir_all(validated.as_path()).unwrap(); { - let _guard = ForkCloneGuard::new(validated.clone()); + let _guard = ForkCloneGuard::new(validated.clone(), PublishAttemptId::new()); assert!(validated.exists()); } assert!( !validated.exists(), - "dropping the fork clone guard must remove the mirror directory" + "dropping the fork clone guard must remove the mirror directory it stamped" + ); + } + + /// P1 FINDING 3, the filesystem half. Two attempts at one logical fork share + /// a disk path. A late-unwinding attempt must not remove the directory a + /// SUCCESSOR now owns — the successor has already published its archive and + /// returned 201 for it. + #[test] + fn fork_clone_guard_leaves_a_successors_mirror_alone() { + let root = tempfile::TempDir::new().unwrap(); + let validated = crate::git::repo_store::validated_repo_disk_path( + root.path(), + "did:key:testfork", + "fork", + ) + .expect("test fork path must validate"); + std::fs::create_dir_all(validated.as_path()).unwrap(); + + let loser = ForkCloneGuard::new(validated.clone(), PublishAttemptId::new()); + // The successor claims the path while the loser is still unwinding. + let successor = PublishAttemptId::new(); + crate::git::repo_store::claim_fork_disk_path(validated.as_path(), &successor); + + drop(loser); + assert!( + validated.exists(), + "a failed attempt must not delete the directory a successor now owns" ); } @@ -10784,6 +11117,344 @@ mod tests { ); } + // ── #285 P1 finding 1: cancellation before the PUT is dispatched ─────── + + /// A minimal object-store stub that COUNTS PUTs and answers every HEAD 404. + /// + /// Not a semantics mock: the whole claim of the tests below is that no PUT + /// ever arrives, so the only thing it has to do faithfully is notice one. + async fn p3_put_counting_store() -> ( + String, + Arc, + tokio::task::JoinHandle<()>, + ) { + let puts = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let app = axum::Router::new().route( + "/{*key}", + axum::routing::any({ + let puts = Arc::clone(&puts); + move |method: axum::http::Method, _body: axum::body::Bytes| { + let puts = Arc::clone(&puts); + async move { + if method == axum::http::Method::PUT { + puts.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + return axum::http::StatusCode::OK; + } + // Nothing is stored, so the acquire-side refresh takes the + // create-only arm and downloads nothing. + axum::http::StatusCode::NOT_FOUND + } + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port()); + let server = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + (endpoint, puts, server) + } + + /// A push handler whose release-side publish parks INSIDE the blocking + /// compression, which is the window the finding is about: the conditional PUT + /// is not constructed until compression returns, so a handler cancelled here + /// definitely never attempted publication. + /// + /// Path-scoped for the same reason as `p2_parked_release_state`: the tail's + /// withheld walk is the observable, and without a path-scoped rule + /// `replication_withheld_set` takes the no-walk shortcut and spawns no git. + /// + /// The bare repo is created directly rather than through `repo_store.init`, + /// which would spawn a background create-only upload of its own and pollute + /// the PUT count this test reads. + #[cfg(unix)] + async fn p3_compression_gated_state( + pool: sqlx::PgPool, + tmp: &std::path::Path, + owner: &str, + name: &str, + gate: Arc, + ) -> ( + AppState, + std::path::PathBuf, + Arc, + tokio::task::JoinHandle<()>, + ) { + let log = tmp.join("git.log"); + let git_bin = f2a_logging_git(tmp, &log); + let repos_dir = tmp.join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let (endpoint, puts, server) = p3_put_counting_store().await; + + let mut state = crate::test_support::test_state(pool.clone()).await; + state.git_bin = git_bin; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.clone(), + Some( + crate::git::tigris::TigrisClient::for_testing_with_endpoint( + "test-bucket", + &endpoint, + ) + .with_compress_gate(gate), + ), + pool.clone(), + std::time::Duration::from_secs(300), + ); + state + .db + .upsert_mirror_repo(owner, name, &format!("/unused-{owner}-{name}"), None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkP3TailReaderAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + let bare = + crate::git::repo_store::validated_repo_disk_path(&repos_dir, &rec.owner_did, name) + .expect("test repo path"); + crate::git::store::init_bare(&bare).expect("a bare repo on disk"); + (state, log, puts, server) + } + + /// #285 P1 FINDING 1 (RED-before/GREEN-after). A client disconnect while the + /// release-side archive is still being COMPRESSED must not count as publish + /// durability. + /// + /// `TigrisClient::upload` awaits `spawn_blocking(compress_repo)` and does not + /// construct, let alone send, the conditional PUT until that returns. So a + /// handler dropped in this window definitively never attempted publication — + /// and the detached tail must do NOTHING: no IPFS/Pinata pin, no P2P + /// announcement, no GraphQL publication, no Arweave work, no peer + /// notification, all for refs that exist only in a local write the store + /// never saw. + /// + /// Load-bearing: with the slot recording `UploadUnknowable` for every + /// cancellation after release starts (the pre-fix shape), the tail is + /// admitted and its withheld walk's `for-each-ref` appears in the git log + /// (RED). Reading the publish STAGE instead classifies this as a definite + /// non-publication and the walk never runs (GREEN). + /// + /// The existing `receive_pack_tail_survives_a_disconnect_during_release` + /// parks at the pre-unlock point, which is AFTER the upload, so it cannot + /// reach this boundary at all. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_cancelled_during_compression_publishes_nothing_and_runs_no_tail( + pool: sqlx::PgPool, + ) { + let tmp = tempfile::TempDir::new().unwrap(); + // Shut: every compression from this store parks until the gate opens. + let gate = Arc::new(crate::git::tigris::BlockingGate::shut()); + let (state, log, puts, server) = + p3_compression_gated_state(pool, tmp.path(), "z6p3comp", "c1", Arc::clone(&gate)).await; + + let mut fut = Box::pin(p2_push(&state, "z6p3comp", "c1")); + let mut ran = false; + for _ in 0..1000 { + let step = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + assert!( + step.is_err(), + "the handler must park inside the release-side compression, not return" + ); + if p2_logged(&log, "receive-pack") { + ran = true; + break; + } + } + assert!(ran, "the push must reach receive-pack"); + // Settle: the handler is now inside `release`, blocked on the gate with + // no PUT constructed. + for _ in 0..10 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + } + assert_eq!( + puts.load(std::sync::atomic::Ordering::SeqCst), + 0, + "compression has not finished, so no PUT can have been built yet" + ); + + // THE DISCONNECT, inside the compression window. + drop(fut); + + // Give the tail every chance to misbehave. Pre-fix it is admitted the + // instant the slot's Drop installs its verdict, so this window is far + // more than enough for the walk to show up. + tokio::time::sleep(std::time::Duration::from_millis(1500)).await; + + assert!( + !p2_logged(&log, "for-each-ref"), + "RED: a push cancelled before its PUT was even constructed still ran the \ + replication tail. Nothing was published, so pinning, announcing and \ + replicating these refs advertises a write no other node can read. git log:\n{}", + f2a_log(&log) + ); + assert_eq!( + puts.load(std::sync::atomic::Ordering::SeqCst), + 0, + "no PUT may have reached the store at any point" + ); + + // Release the parked blocking task so the runtime can tear down cleanly. + gate.open(); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + server.abort(); + } + + /// THE CONTROL, and what makes the assertion above attributable. Same setup, + /// same handler, but nothing holds the gate: the publish completes, the store + /// sees the PUT, and the tail DOES run its walk. + /// + /// Without this, a green negative would prove only that the walk never runs + /// in this harness, not that the CANCELLATION is what stops it. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_that_completes_its_publish_still_runs_the_tail(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + // Open from the start: this control must publish for real. + let gate = Arc::new(crate::git::tigris::BlockingGate::shut()); + gate.open(); + let (state, log, puts, server) = + p3_compression_gated_state(pool, tmp.path(), "z6p3ctrl", "c1", gate).await; + + p2_push(&state, "z6p3ctrl", "c1") + .await + .expect("an uncontended push must succeed"); + + assert_eq!( + puts.load(std::sync::atomic::Ordering::SeqCst), + 1, + "the release must have published exactly once" + ); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while !p2_logged(&log, "for-each-ref") { + assert!( + std::time::Instant::now() < deadline, + "a confirmed publish must admit the replication tail; git log:\n{}", + f2a_log(&log) + ); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + + server.abort(); + } + + // ── #285 P1 finding 3: fork confirmation is bound to the attempt ─────── + + /// A repo row under `owner/name` owned by some OTHER attempt. + async fn p3_seed_foreign_row( + state: &AppState, + owner_did: &str, + name: &str, + ) -> crate::db::RepoRecord { + let now = Utc::now(); + let rec = crate::db::RepoRecord { + id: Uuid::new_v4().to_string(), + name: name.to_string(), + owner_did: owner_did.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: format!("/unused/{name}"), + forked_from: None, + machine_id: None, + }; + state + .db + .create_repo(&rec) + .await + .expect("seed the other row"); + rec + } + + /// #285 P1 FINDING 3, interleaving (a): another row COMMITS UNDER THE NAME + /// before this attempt's confirmation runs. + /// + /// The ordering is the barrier: the successor's insert completes, and only + /// then does the failed attempt look up. Keyed on owner/name, that lookup + /// answered "a row exists" and the request returned 201 carrying the + /// successor's row — even though its own uploaded archive, disk path and fork + /// provenance belong to no row at all. Keyed on `record.id` it cannot. + #[sqlx::test] + async fn fork_confirmation_never_claims_a_concurrent_attempts_row(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let owner = "did:key:z6MkForkIdentityAAAAAAAAAAAAAAAAAAAAAAAA"; + let short = crate::db::normalize_owner_key(owner); + let name = "contested-fork"; + + // Interleaving: the successor commits first. + let successor = p3_seed_foreign_row(&state, owner, name).await; + + // Then this attempt, whose own insert did NOT land, confirms. + let mine = Uuid::new_v4().to_string(); + let confirmation = confirm_fork_repo_row(&state.db, &mine, short, name) + .await + .expect("the lookup itself succeeds"); + match confirmation { + ForkRowConfirmation::Foreign => {} + ForkRowConfirmation::Ours(row) => panic!( + "the failed attempt claimed the successor's row {} as its own commit", + row.id + ), + ForkRowConfirmation::Absent => { + panic!("a row for this name does exist, so Absent would license compensation") + } + } + assert_ne!(successor.id, mine); + } + + /// The must-do direction: when this attempt's OWN insert did commit, the + /// confirmation has to recognize it, or a transport-level error on a + /// successful write would turn into a spurious failure plus compensation of + /// a live repository. + #[sqlx::test] + async fn fork_confirmation_recognizes_this_attempts_own_committed_row(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let owner = "did:key:z6MkForkIdentityBBBBBBBBBBBBBBBBBBBBBBBB"; + let short = crate::db::normalize_owner_key(owner); + let name = "my-fork"; + + let mine = p3_seed_foreign_row(&state, owner, name).await; + match confirm_fork_repo_row(&state.db, &mine.id, short, name) + .await + .expect("lookup") + { + ForkRowConfirmation::Ours(row) => assert_eq!(row.id, mine.id), + other => panic!( + "this attempt's own row must confirm as Ours, got {}", + match other { + ForkRowConfirmation::Foreign => "Foreign", + ForkRowConfirmation::Absent => "Absent", + ForkRowConfirmation::Ours(_) => unreachable!(), + } + ), + } + } + + /// And the third arm: nothing owns the name, which is the only state that + /// licenses compensation at all. + #[sqlx::test] + async fn fork_confirmation_reports_absent_when_nothing_owns_the_name(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let short = "z6MkForkIdentityCCCCCCCCCCCCCCCCCCCCCCCC"; + assert!(matches!( + confirm_fork_repo_row(&state.db, &Uuid::new_v4().to_string(), short, "nobody-here") + .await + .expect("lookup"), + ForkRowConfirmation::Absent + )); + } + /// Scenario 5 (trap 3, fail-closed). On a repo whose withheld walk is failing, a /// coalesced push must not publish. Before the gate moved, every push on such a /// repo got `announce = false` from its own walk; a coalesced push has no walk, so diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 2f4cf6b3..9878ab28 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -21,7 +21,10 @@ use tokio::sync::Mutex; use tracing::{debug, info, warn}; use super::store; -use super::tigris::{TigrisClient, UploadError, UploadPrecondition}; +use super::tigris::{ + AttemptDelete, PublishAttemptId, PublishStage, PublishStageCell, TigrisClient, UploadError, + UploadPrecondition, +}; /// Centralized repo storage: local disk cache + optional Tigris backend. #[derive(Clone)] @@ -135,8 +138,18 @@ impl RepoStore { let local_path = validated_repo_disk_path(&self.repos_dir, owner_did, repo_name)?; let owner_slug = owner_did.replace([':', '/'], "_"); - // Fast path: repo exists locally + // Fast path: repo exists locally. if local_path.exists() { + // ...but existence is not the cache contract. A tree left by a write + // whose PUT outcome was never resolved carries this node's refs with + // no confirmed generation behind them, and serving it would publish a + // possibly-refused write to every reader for as long as the directory + // survives. Reconcile the quarantine FIRST, before the migration + // bookkeeping and before the path is handed out. + if let Some(marker) = read_quarantine(&local_path) { + self.reconcile_quarantine(&owner_slug, repo_name, &local_path, &marker) + .await?; + } // Lazy migration: if Tigris is enabled and we haven't confirmed this // repo is in Tigris yet, check and upload in the background. if let Some(ref tigris) = self.tigris { @@ -168,7 +181,7 @@ impl RepoStore { .upload(&slug, &name, &path, UploadPrecondition::IfAbsent) .await { - Ok(()) => { + Ok(_) => { info!(repo = %name, "lazy migration to tigris complete"); } // Logged apart from the warn arm below so a @@ -219,6 +232,79 @@ impl RepoStore { Ok(local_path.into_path_buf()) } + /// Decide whether a quarantined live tree may be served. + /// + /// The ONLY thing that lifts a quarantine is the store confirming it holds + /// the attempt that left it — which is decidable, because the attempt id + /// travelled with the bytes. Everything else refuses with a retryable + /// `RepoUnavailable`. + /// + /// Deleting the tree instead is NOT a safe alternative and is deliberately + /// not done here: the unresolved PUT may have landed, in which case this + /// directory can be the only local copy of it. Refusing costs a 503 that a + /// later write (whose under-lock refresh re-downloads the confirmed archive + /// and clears the marker) or a successful reconciliation resolves; deleting + /// costs the data. + async fn reconcile_quarantine( + &self, + owner_slug: &str, + repo_name: &str, + local_path: &ValidatedRepoDiskPath, + marker: &QuarantineMarker, + ) -> Result<()> { + let refuse = || { + Err(anyhow::Error::new(RepoUnavailable).context(format!( + "local tree for {owner_slug}/{repo_name} is quarantined: its publish outcome \ + is unresolved and could not be reconciled against object storage" + ))) + }; + let (Some(tigris), Some(attempt)) = ( + self.tigris.as_ref(), + marker.attempt.as_deref().map(PublishAttemptId::from_owned), + ) else { + // No backend to ask, or no attempt to ask about. Either way nothing + // can confirm this tree, and a read that cannot be confirmed must + // not be served as an ordinary success. + warn!( + repo = %repo_name, + "refusing a quarantined read: no attempt identity to reconcile against" + ); + return refuse(); + }; + match tigris.attempt_landed(owner_slug, repo_name, &attempt).await { + Ok(true) => { + info!( + repo = %repo_name, + attempt = %attempt, + "quarantine lifted: object storage holds this attempt's archive" + ); + clear_quarantine(local_path, repo_name); + Ok(()) + } + Ok(false) => { + // The store does not hold this attempt. That is NOT proof it + // never will — an abandoned PUT can still be in flight — so the + // tree stays put and the read is refused rather than served or + // deleted. + warn!( + repo = %repo_name, + attempt = %attempt, + "refusing a quarantined read: object storage holds a different generation \ + than the unresolved write on local disk" + ); + refuse() + } + Err(e) => { + warn!( + repo = %repo_name, + err = %e, + "refusing a quarantined read: could not reach object storage to reconcile" + ); + refuse() + } + } + } + /// Non-mutating snapshot of a repo's **latest** Tigris state, for reads that /// must see fresh data but must NOT write into the live repo path. /// @@ -475,6 +561,13 @@ impl RepoStore { // configured, in which case `release` publishes nothing at all. publish_fence: UploadPrecondition::Unconditional, refresh_swap_authority: Some(refresh_swap_authority.clone()), + // Seeded from whether a backend exists at all, so a cancelled + // release can tell "there was nothing to publish" from "a publish + // was possible and never started". + publish_stage: Arc::new(PublishStageCell::seeded(match self.tigris { + Some(_) => PublishStage::Idle, + None => PublishStage::NoBackend, + })), #[cfg(test)] test_pre_unlock_gate: self.pre_unlock_gate.clone(), #[cfg(test)] @@ -537,7 +630,14 @@ impl RepoStore { .await; match refreshed { - Some(Ok(fence)) => guard.publish_fence = fence, + Some(Ok(fence)) => { + // The tree at the live path is now the generation this HEAD + // observed (downloaded, or confirmed absent), so any + // quarantine an earlier unresolved write left on this path is + // answered by the refresh itself. + clear_quarantine(&local_path, repo_name); + guard.publish_fence = fence; + } Some(Err(RefreshFailure::Download { err, .. })) => { // HEAD established a stored generation but the GET failed, so we // do not know whether the local tree matches it. Proceeding on a @@ -623,7 +723,7 @@ impl RepoStore { .upload(&owner_slug, &repo_name, &path, UploadPrecondition::IfAbsent) .await { - Ok(()) => {} + Ok(_) => {} // Distinct from the warn arm: the fence refusing is the // design working, not a storage failure. Err(UploadError::PreconditionLost { status }) => { @@ -648,39 +748,42 @@ impl RepoStore { /// former to refuse the fork rather than create a DB record shadowed by an /// orphan other nodes would fetch. Plain upload failures are propagated so fork /// creation does not insert a DB row when the archive never landed. + /// + /// `attempt` is the identity the bytes are stamped with, and fork creation + /// passes the `record.id` it is about to insert. That is what makes the + /// object, the disk clone and the database row all name ONE attempt, so a + /// later cleanup can be conditional on still owning the thing it is about to + /// destroy instead of trusting the logical owner/name. pub async fn release_after_write( &self, owner_did: &str, repo_name: &str, + attempt: &PublishAttemptId, ) -> Result<(), UploadError> { if let Some(ref tigris) = self.tigris { let (owner_slug, local_path) = match self.local_path(owner_did, repo_name) { Ok(p) => p, Err(e) => { warn!(repo = %repo_name, err = %e, "rejected unsafe path in release_after_write"); - return Err(UploadError::Other(e)); + // A path this node refused to build is a path nothing was + // ever sent to. + return Err(UploadError::NotPublished(e)); } }; // Create-only. The sole caller is fork creation, which rejects a // name conflict in the database before it clones anything, so the - // key is expected absent here (and archive keys are never deleted: - // `delete` has no callers). A refusal therefore means someone else - // already published this key. - match tigris - .upload( + // key is expected absent here. A refusal therefore means someone + // else already published this key. + tigris + .upload_tracked( &owner_slug, repo_name, &local_path, UploadPrecondition::IfAbsent, + attempt.clone(), + None, ) - .await - { - Ok(()) => {} - // Propagated, not logged as success: an orphan archive under - // this key shadows the fork for every other node. - Err(e @ UploadError::PreconditionLost { .. }) => return Err(e), - Err(e) => return Err(e), - } + .await?; } Ok(()) } @@ -705,21 +808,59 @@ impl RepoStore { Ok((owner_slug, local_path)) } + /// Did `attempt`'s fork archive land? Used to recover a create-only publish + /// whose response was lost, instead of compensating it as a definite failure + /// and fencing the fork name behind its own orphan. + /// + /// `Ok(false)` with no backend configured, where there is nothing to have + /// landed in. + pub async fn fork_attempt_landed( + &self, + owner_did: &str, + repo_name: &str, + attempt: &PublishAttemptId, + ) -> Result { + let Some(ref tigris) = self.tigris else { + return Ok(false); + }; + let (owner_slug, _) = self.local_path(owner_did, repo_name)?; + tigris.attempt_landed(&owner_slug, repo_name, attempt).await + } + /// Best-effort cleanup when fork creation published an archive but failed to persist - /// the database row. Removes the object-store key this attempt owns and the local - /// mirror clone so a retry is not blocked by its own orphan. Object-store deletion - /// is retried asynchronously when the first attempt fails so a transient DELETE does - /// not tombstone the fork name. + /// the database row. Removes the object-store key and the local mirror clone THIS + /// ATTEMPT OWNS, so a retry is not blocked by its own orphan. + /// + /// Every destructive step is conditional on the resource still belonging to + /// `attempt`. The unconditional version this replaces authorized itself with the + /// logical owner/name, which identifies a namespace and not the attempt that owns a + /// row, an object generation or a filesystem tree: recovery for a failed fork could + /// observe `get_repo == None`, be overtaken by a successor that committed and + /// returned 201, and then delete that successor's archive and directory. A second + /// name lookup immediately before the delete only moves that window; the attempt + /// guard removes it. pub async fn compensate_fork_archive( &self, owner_did: &str, repo_name: &str, disk_path: &Path, + attempt: &PublishAttemptId, ) { if let Some(ref tigris) = self.tigris { if let Ok((owner_slug, _)) = self.local_path(owner_did, repo_name) { - match tigris.delete(&owner_slug, repo_name).await { - Ok(()) => {} + match tigris + .delete_if_attempt_matches(&owner_slug, repo_name, attempt) + .await + { + Ok(AttemptDelete::Deleted) | Ok(AttemptDelete::Absent) => {} + Ok(AttemptDelete::NotOurs) => { + info!( + repo = %repo_name, + attempt = %attempt, + "fork compensation left the stored archive alone: it belongs to \ + another attempt" + ); + } Err(e) => { warn!( repo = %repo_name, @@ -729,43 +870,141 @@ impl RepoStore { let tigris = tigris.clone(); let slug = owner_slug.clone(); let name = repo_name.to_string(); + let attempt = attempt.clone(); tokio::spawn(async move { - retry_fork_archive_delete(&tigris, &slug, &name).await; + retry_fork_archive_delete(&tigris, &slug, &name, &attempt).await; }); } } } } - if let Err(e) = std::fs::remove_dir_all(disk_path) { - warn!( - path = %disk_path.display(), - err = %e, - "failed to remove fork clone during create_repo compensation" - ); - } + remove_fork_clone_if_ours(disk_path, attempt, "create_repo compensation"); + } +} + +/// Sidecar naming the attempt that owns a fork's on-disk clone, beside the +/// directory for the same reason the quarantine marker is: anything inside the +/// bare repo would be tarred into the archive and shipped to every node. +fn fork_attempt_path(disk_path: &Path) -> Option { + let parent = disk_path.parent()?; + let file_name = disk_path.file_name()?.to_string_lossy().to_string(); + Some(parent.join(format!(".{file_name}.fork-attempt"))) +} + +/// Stamp a freshly cloned fork mirror with the attempt that created it. Written +/// before the archive upload, so every later cleanup can ask "is this still +/// mine?" of the directory as well as of the object. +pub(crate) fn claim_fork_disk_path(disk_path: &Path, attempt: &PublishAttemptId) { + let Some(path) = fork_attempt_path(disk_path) else { + return; + }; + if let Err(e) = std::fs::write(&path, attempt.as_str()) { + warn!( + path = %disk_path.display(), + err = %e, + "failed to stamp the fork clone with its attempt id — cleanup will refuse to \ + remove it rather than risk removing a successor's clone" + ); + } +} + +/// Drop the attempt stamp once the fork's row has committed and no cleanup may +/// ever remove this directory again. +/// +/// Leaving the stamp would be harmless but misleading; removing it also means a +/// LATER attempt's cleanup finds no owner and therefore refuses to delete, which +/// is the safe direction. +pub(crate) fn release_fork_disk_claim(disk_path: &Path) { + if let Some(path) = fork_attempt_path(disk_path) { + let _ = std::fs::remove_file(path); + } +} + +/// Does the clone at `disk_path` still belong to `attempt`? +/// +/// A missing stamp answers NO. That is the fail-safe direction: an unstamped +/// directory is one this attempt cannot prove it owns, and refusing to delete +/// leaves an orphan for an operator, while deleting wrongly destroys a +/// successor's repository after that successor has already returned success. +fn fork_clone_is_ours(disk_path: &Path, attempt: &PublishAttemptId) -> bool { + fork_attempt_path(disk_path) + .and_then(|p| std::fs::read_to_string(p).ok()) + .is_some_and(|owner| owner.trim() == attempt.as_str()) +} + +/// Remove a fork's clone only while it is still this attempt's. +pub(crate) fn remove_fork_clone_if_ours( + disk_path: &Path, + attempt: &PublishAttemptId, + reason: &str, +) { + if !disk_path.exists() { + let _ = fork_attempt_path(disk_path).map(std::fs::remove_file); + return; + } + if !fork_clone_is_ours(disk_path, attempt) { + info!( + path = %disk_path.display(), + attempt = %attempt, + reason, + "left the fork clone alone: it no longer belongs to this attempt" + ); + return; + } + if let Err(e) = std::fs::remove_dir_all(disk_path) { + warn!( + path = %disk_path.display(), + err = %e, + reason, + "failed to remove fork clone" + ); + return; } + let _ = fork_attempt_path(disk_path).map(std::fs::remove_file); } -async fn retry_fork_archive_delete(tigris: &TigrisClient, owner_slug: &str, repo_name: &str) { +async fn retry_fork_archive_delete( + tigris: &TigrisClient, + owner_slug: &str, + repo_name: &str, + attempt: &PublishAttemptId, +) { const MAX_ATTEMPTS: u32 = 6; - for attempt in 0..MAX_ATTEMPTS { - match tigris.delete(owner_slug, repo_name).await { - Ok(()) => { + for n in 0..MAX_ATTEMPTS { + // Re-evaluated on EVERY retry, not resolved once before the loop. The + // whole point of retrying is that time passes, and the ownership gap the + // unconditional version left open widened with each attempt: a successor + // that commits between retry 2 and retry 3 would still have had its + // archive deleted by retry 3. + match tigris + .delete_if_attempt_matches(owner_slug, repo_name, attempt) + .await + { + Ok(AttemptDelete::Deleted) => { info!( repo = %repo_name, - attempt, + attempt = n, "fork archive compensation delete succeeded on retry" ); return; } - Err(e) if attempt + 1 < MAX_ATTEMPTS => { + Ok(AttemptDelete::Absent) => return, + Ok(AttemptDelete::NotOurs) => { + info!( + repo = %repo_name, + "fork archive compensation stopped retrying: the stored archive now \ + belongs to another attempt" + ); + return; + } + Err(e) if n + 1 < MAX_ATTEMPTS => { warn!( repo = %repo_name, - attempt, + attempt = n, err = %e, "fork archive compensation delete failed — retrying" ); - tokio::time::sleep(Duration::from_secs(1u64 << attempt.min(4))).await; + tokio::time::sleep(Duration::from_secs(1u64 << n.min(4))).await; } Err(e) => { warn!( @@ -890,12 +1129,110 @@ pub(crate) fn swap_extracted_into_validated_repo( Ok(()) } +/// How long the under-lock reconciliation HEAD gets. Separate from (and much +/// smaller than) the publish bound it follows: it runs after a transfer that has +/// already used its whole budget, with the advisory lock and a lock-pool slot +/// still pinned, so it must be an addendum rather than a second budget. +const RECONCILE_BOUND: Duration = Duration::from_secs(5); + +/// The sidecar that marks a live repo tree as QUARANTINED: present on disk, but +/// with no confirmed object-store generation behind it. +/// +/// A sibling dotfile rather than something inside the repo directory, for two +/// reasons. Anything under the bare repo would be tarred into the next archive +/// and shipped to every node that downloads it, and the marker has to survive +/// exactly as long as the directory it describes — a swap that replaces the +/// directory wholesale must not carry the old marker along inside it. +#[derive(Debug, serde::Serialize, serde::Deserialize)] +struct QuarantineMarker { + /// The attempt whose PUT was left unresolved. `None` when the bound expired + /// with no dispatched attempt to name (a state that should compensate rather + /// than quarantine, kept representable so a marker is never unparseable). + attempt: Option, + at: chrono::DateTime, +} + +/// Sidecar path for a live repo directory: `.{name}.git.quarantine` beside it. +fn quarantine_path(local_path: &Path) -> Option { + let parent = local_path.parent()?; + let file_name = local_path.file_name()?.to_string_lossy().to_string(); + Some(parent.join(format!(".{file_name}.quarantine"))) +} + +/// Mark the live tree as carrying an unresolved generation. Reads must reconcile +/// it before serving; nothing may delete it, because the PUT may have landed and +/// this can be the only local copy. +fn quarantine_local_tree(local_path: &Path, repo_name: &str, attempt: Option<&PublishAttemptId>) { + let Some(path) = quarantine_path(local_path) else { + return; + }; + let marker = QuarantineMarker { + attempt: attempt.map(|a| a.as_str().to_string()), + at: chrono::Utc::now(), + }; + let body = match serde_json::to_vec(&marker) { + Ok(body) => body, + Err(e) => { + warn!(repo = %repo_name, err = %e, "could not serialize the quarantine marker"); + return; + } + }; + match std::fs::write(&path, body) { + Ok(()) => warn!( + repo = %repo_name, + attempt = ?marker.attempt, + "quarantined the local tree: its publish outcome is unresolved, so reads must \ + reconcile it against the store before serving it" + ), + Err(e) => warn!( + repo = %repo_name, + err = %e, + "failed to write the quarantine marker — reads may serve an unconfirmed tree" + ), + } +} + +/// Lift a quarantine. Called wherever the live tree becomes a CONFIRMED +/// generation again: a publish the store acknowledged, an under-lock refresh +/// that overwrote the tree from the stored archive, a reconciliation that found +/// the attempt did land, or an invalidation that removed the tree entirely. +fn clear_quarantine(local_path: &Path, repo_name: &str) { + let Some(path) = quarantine_path(local_path) else { + return; + }; + match std::fs::remove_file(&path) { + Ok(()) => debug!(repo = %repo_name, "cleared the local tree's quarantine"), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => warn!( + repo = %repo_name, + err = %e, + "failed to clear the quarantine marker — reads will keep refusing this repo" + ), + } +} + +/// The quarantine on a live tree, when there is one. +fn read_quarantine(local_path: &Path) -> Option { + let path = quarantine_path(local_path)?; + let body = std::fs::read(&path).ok()?; + // An unparseable marker still means quarantined. Failing open on a corrupt + // sidecar would serve exactly the tree the sidecar exists to withhold. + Some(serde_json::from_slice(&body).unwrap_or(QuarantineMarker { + attempt: None, + at: chrono::Utc::now(), + })) +} + /// Remove a refused write from the unlocked read cache so `acquire` cannot serve /// a tree that never landed in object storage. fn invalidate_local_write_cache(local_path: &Path, repo_name: &str, reason: &str) { if !local_path.exists() { + clear_quarantine(local_path, repo_name); return; } + // Order matters: drop the marker only after the tree it describes is gone, + // so a failed removal leaves the quarantine standing rather than clearing + // the way for the tree it could not delete. if let Err(e) = std::fs::remove_dir_all(local_path) { warn!( repo = %repo_name, @@ -905,6 +1242,7 @@ fn invalidate_local_write_cache(local_path: &Path, repo_name: &str, reason: &str "failed to invalidate local write cache after a refused publish" ); } else { + clear_quarantine(local_path, repo_name); debug!( repo = %repo_name, reason, @@ -1235,6 +1573,11 @@ pub struct RepoWriteGuard { /// guard drops so a detached `spawn_blocking` extraction cannot swap into the live /// tree after its advisory-lock ownership ends. refresh_swap_authority: Option>, + /// How far THIS guard's publish attempt got, readable from outside the + /// `release` future. A cancelled handler never returns a `ReleaseOutcome`, so + /// this is the only thing that can tell "the PUT was never constructed" from + /// "the PUT is on the wire and may commit" after the future is gone. + publish_stage: Arc, /// Test-only seam: when set, `release` parks on this gate at the exact point /// it is about to await `pg_advisory_unlock` (connection still owned, not yet /// released). Dropping the `release` future while it is parked reproduces a @@ -1275,15 +1618,25 @@ impl RepoWriteGuard { /// third attempt would have no more reason to terminate than the second. async fn publish(&self, tigris: &TigrisClient) -> std::result::Result<(), PublishRefusal> { match tigris - .upload( + .upload_tracked( &self.owner_slug, &self.repo_name, &self.local_path, self.publish_fence.clone(), + PublishAttemptId::new(), + Some(&self.publish_stage), ) .await { - Ok(()) => return Ok(()), + Ok(receipt) => { + debug!( + repo = %self.repo_name, + attempt = %receipt.attempt, + etag = ?receipt.etag, + "release published the writer's tree" + ); + return Ok(()); + } Err(UploadError::PreconditionLost { status }) => { // EPISTEMIC ASYMMETRY, and it is why one retry is sound here // while the timeout arm in `release` deliberately does nothing. @@ -1320,13 +1673,31 @@ impl RepoWriteGuard { // Nothing is stored now, so create-only is the fence that matches // what was just observed. Ok(None) => UploadPrecondition::IfAbsent, - Err(e) => return Err(PublishRefusal::Failed(UploadError::Other(e))), + // A HEAD is a read. It cannot have published anything, so failing it + // is a definite non-publication for THIS attempt — the first PUT was + // already refused outright above. + Err(e) => return Err(PublishRefusal::Failed(UploadError::NotPublished(e))), }; match tigris - .upload(&self.owner_slug, &self.repo_name, &self.local_path, fresh) + .upload_tracked( + &self.owner_slug, + &self.repo_name, + &self.local_path, + fresh, + PublishAttemptId::new(), + Some(&self.publish_stage), + ) .await { - Ok(()) => Ok(()), + Ok(receipt) => { + debug!( + repo = %self.repo_name, + attempt = %receipt.attempt, + etag = ?receipt.etag, + "the supersede-retry published the writer's tree" + ); + Ok(()) + } Err(UploadError::PreconditionLost { status }) => { // Two definite losses in a row: something is publishing this key // without the lock faster than we can fence on it. Refuse rather @@ -1345,6 +1716,76 @@ impl RepoWriteGuard { } } + /// The publish stage of this guard's attempt, shared with whoever needs to + /// classify a CANCELLED release. See [`PublishStageCell`]. + pub fn publish_stage(&self) -> Arc { + Arc::clone(&self.publish_stage) + } + + /// Decide what a publish that ran out its bound actually left behind. + /// + /// "The bounded transfer returned `None`" is not one state. `publish` awaits + /// a blocking compression before it constructs the request, so the bound can + /// expire with nothing on the wire at all. Only the stage distinguishes them, + /// and only a dispatched attempt is genuinely unknowable. + /// + /// For a dispatched attempt this spends one short, separately bounded HEAD + /// asking the store whether THIS attempt's bytes are what it holds. A `true` + /// there is a real confirmation — the attempt id travelled with the bytes — + /// and upgrades the outcome to `Released`. A `false` is NOT a refutation: the + /// request may still be in flight, so it stays unknowable and the tree is + /// quarantined rather than deleted. + async fn resolve_unfinished_publish(&self, tigris: &TigrisClient) -> ReleaseOutcome { + let stage = self.publish_stage.get(); + if !stage.may_have_published() { + warn!( + repo = %self.repo_name, + ?stage, + "release upload exceeded its bound before any PUT was dispatched — the write \ + definitively did not publish" + ); + return ReleaseOutcome::UploadFailed; + } + warn!( + repo = %self.repo_name, + "release upload exceeded its bound; the PUT may still land, so the outcome is \ + unknowable and the conditional upload is what keeps a late publish from \ + overwriting a successor's archive" + ); + let Some(attempt) = stage.unresolved_attempt().cloned() else { + // `Published` reaches here only if the bound expired between the + // acknowledged PUT and the outer future resuming. The store already + // answered, so this is durable. + return ReleaseOutcome::Released; + }; + // A separate, deliberately small slice rather than a share of the + // already-exhausted publish budget: this runs with the lock still held, + // and a store that just stalled for the whole bound must not be able to + // hold it for a second one. + match bounded_transfer( + "release-reconcile", + &self.repo_name, + RECONCILE_BOUND, + tigris.attempt_landed(&self.owner_slug, &self.repo_name, &attempt), + ) + .await + { + Some(Ok(true)) => { + info!( + repo = %self.repo_name, + attempt = %attempt, + "reconciled an abandoned publish: the store holds this attempt's archive" + ); + self.publish_stage.set(PublishStage::Published { + attempt, + etag: None, + }); + ReleaseOutcome::Released + } + _ => ReleaseOutcome::UploadUnknowable, + } + } + /// Upload to Tigris (only when the write succeeded) and release the advisory /// lock. Pass `success = false` when the write operation failed — uploading a /// half-applied or otherwise inconsistent repo would propagate corruption to @@ -1401,8 +1842,25 @@ impl RepoWriteGuard { // the refusal out to the caller. Some(Err(PublishRefusal::Fenced)) => outcome = ReleaseOutcome::Fenced, Some(Err(PublishRefusal::Failed(e))) => { - warn!(repo = %self.repo_name, err = %e, "failed to upload repo to tigris after write"); - outcome = ReleaseOutcome::UploadFailed; + // SPLIT BY WHAT THE FAILURE PROVES, not by "it was an + // error". `UploadFailed` runs destructive compensation + // below, and a response-loss error does not license it: + // the store can accept a complete conditional PUT and + // lose the response before the client reads it, so + // invalidating the cache and undoing the write would + // discard state that IS published. + if e.proves_not_published() { + warn!(repo = %self.repo_name, err = %e, "failed to upload repo to tigris after write"); + outcome = ReleaseOutcome::UploadFailed; + } else { + warn!( + repo = %self.repo_name, + err = %e, + "release upload failed WITHOUT proving it did not commit; \ + quarantining the local tree rather than compensating" + ); + outcome = ReleaseOutcome::UploadUnknowable; + } } None => { // Timed out is UNKNOWABLE, not failed: the PUT may well @@ -1417,13 +1875,15 @@ impl RepoWriteGuard { // the lifetime of this lock. The caller still must not // report success: `into_result` maps this to a retryable // refusal. - warn!( - repo = %self.repo_name, - "release upload exceeded its bound; the PUT may still land, so the \ - outcome is unknowable and the conditional upload is what keeps a \ - late publish from overwriting a successor's archive" - ); - outcome = ReleaseOutcome::UploadUnknowable; + // + // But "timed out" is not one state, it is two, and only + // the STAGE can tell them apart. The publish awaits a + // blocking compression before it constructs the request + // at all, so a bound that expires there means no PUT was + // ever dispatched — a definite non-publication, safe to + // compensate. A bound that expires after dispatch is the + // genuinely unknowable case. + outcome = self.resolve_unfinished_publish(&tigris).await; } } } @@ -1449,7 +1909,30 @@ impl RepoWriteGuard { } } } - ReleaseOutcome::Released | ReleaseOutcome::UploadUnknowable => {} + ReleaseOutcome::Released => { + // Publication confirmed: the live tree IS the stored + // generation, so any quarantine an earlier attempt left on + // this path is answered and must be lifted, or reads would + // stay refused forever after a single unresolved write. + clear_quarantine(&self.local_path, &self.repo_name); + } + ReleaseOutcome::UploadUnknowable => { + // THE CACHE CONTRACT. The local tree carries this writer's + // refs but nothing ties it to a confirmed object-store + // generation, and deleting it is not safe either — the PUT + // may have landed and this could be the only local copy. + // + // So mark it, and make `acquire` reconcile before it serves. + // Without the marker the existing-path fast path hands the + // tree to every subsequent read on filesystem existence + // alone, and a refused write is served indefinitely while + // durable storage still holds the previous generation. + quarantine_local_tree( + &self.local_path, + &self.repo_name, + self.publish_stage.get().unresolved_attempt(), + ); + } } } @@ -1798,6 +2281,8 @@ pub(crate) fn advisory_lock_key(owner_slug: &str, repo_name: &str) -> i64 { #[cfg(test)] mod tests { use super::*; + use crate::git::tigris::ATTEMPT_METADATA_KEY; + use std::collections::HashMap; #[test] fn non_durable_release_outcomes_do_not_report_success() { @@ -2849,6 +3334,7 @@ mod tests { lock_held_transfer_timeout: Duration::from_secs(300), publish_fence: UploadPrecondition::Unconditional, refresh_swap_authority: None, + publish_stage: Arc::new(PublishStageCell::new()), #[cfg(test)] test_pre_unlock_gate: None, #[cfg(test)] @@ -3216,6 +3702,7 @@ mod tests { // No backend, so nothing is ever published and the fence is unread. publish_fence: UploadPrecondition::Unconditional, refresh_swap_authority: None, + publish_stage: Arc::new(PublishStageCell::new()), #[cfg(test)] test_pre_unlock_gate: None, #[cfg(test)] @@ -4054,9 +4541,22 @@ mod tests { /// decided later) with no timing in it. #[derive(Clone, Debug)] struct CapturedPut { + key: String, body: Vec, if_match: Option, if_none_match: Option, + attempt: Option, + } + + /// One stored object. + #[derive(Clone, Debug)] + struct MockObject { + body: Vec, + etag: String, + /// The `x-amz-meta-gitlawb-attempt` it was written with. This is what + /// makes "are the published bytes MINE" answerable, and the mock has to + /// model it or no reconciliation test proves anything. + attempt: Option, } /// One PUT as the mock judged it, for tests that assert on attempt counts. @@ -4071,15 +4571,29 @@ mod tests { #[derive(Default)] struct MockState { - object: Option>, - etag: Option, + /// Keyed by request path. A single-slot store was enough while every test + /// drove one repo; fork creation touches the SOURCE key and the FORK key + /// in one request, and collapsing them would make an assertion about one + /// silently read the other. + objects: HashMap, + /// The key the last successful PUT wrote, so the single-key accessors + /// below keep meaning what they meant when this mock served one key. + last_key: Option, next_etag: u64, puts: Vec, /// Set by `park_next_put`, consumed by the next arriving PUT. park_next_put: bool, + /// The response-loss arm, set by `commit_then_lose_next_put_response` / + /// `fail_next_put_after_delivery`. The request is fully delivered and the + /// client is told it failed; `Some(true)` also COMMITS it first, which is + /// the state that makes the failure a lie rather than a truth. + lose_next_response: Option, /// Set by `roll_generation_after_next_heads`, decremented per HEAD. roll_after_heads: u32, captured: Option, + /// Conditional DELETEs the mock accepted, so a compensation test can + /// assert that a guarded delete did NOT run. + deletes: u32, } /// An in-process S3-compatible server with REAL conditional semantics. @@ -4106,9 +4620,11 @@ mod tests { /// always the state as of the CALL, never as of capture. fn evaluate_put( st: &mut MockState, + key: &str, body: Vec, if_match: Option<&str>, if_none_match: Option<&str>, + attempt: Option<&str>, ) -> (u16, Option) { let refuse = |st: &mut MockState| { st.puts.push(PutAttempt { @@ -4121,12 +4637,12 @@ mod tests { if let Some(want) = if_match { // An absent object matches nothing, so If-Match cannot pass. - match st.etag.as_deref() { + match st.objects.get(key).map(|o| o.etag.as_str()) { Some(have) if unquote_etag(have) == unquote_etag(want) => {} _ => return refuse(st), } } - if if_none_match.map(str::trim) == Some("*") && st.object.is_some() { + if if_none_match.map(str::trim) == Some("*") && st.objects.contains_key(key) { return refuse(st); } @@ -4136,8 +4652,15 @@ mod tests { // never observed. st.next_etag += 1; let etag = format!("\"mock-etag-{}\"", st.next_etag); - st.object = Some(body); - st.etag = Some(etag.clone()); + st.objects.insert( + key.to_string(), + MockObject { + body, + etag: etag.clone(), + attempt: attempt.map(str::to_string), + }, + ); + st.last_key = Some(key.to_string()); st.puts.push(PutAttempt { if_match: if_match.map(str::to_string), if_none_match: if_none_match.map(str::to_string), @@ -4159,11 +4682,13 @@ mod tests { let state = state.clone(); let gate = gate.clone(); move |method: axum::http::Method, + uri: axum::http::Uri, headers: axum::http::HeaderMap, body: axum::body::Bytes| { let state = state.clone(); let gate = gate.clone(); async move { + let key = uri.path().trim_start_matches('/').to_string(); let header = |name: &str| { headers .get(name) @@ -4174,6 +4699,8 @@ mod tests { axum::http::Method::PUT => { let if_match = header("if-match"); let if_none_match = header("if-none-match"); + let attempt = + header(&format!("x-amz-meta-{ATTEMPT_METADATA_KEY}")); // A parked PUT records what arrived and then // waits. The client will usually be gone by @@ -4189,9 +4716,11 @@ mod tests { status: None, }); st.captured = Some(CapturedPut { + key: key.clone(), body: body.to_vec(), if_match: if_match.clone(), if_none_match: if_none_match.clone(), + attempt: attempt.clone(), }); true } else { @@ -4203,15 +4732,38 @@ mod tests { return axum::http::StatusCode::OK.into_response(); } - let (status, etag) = { + let (status, etag, lost_response) = { + #[allow(clippy::needless_late_init)] let mut st = state.lock().unwrap(); - evaluate_put( + let lost_response = st.lose_next_response.take(); + if lost_response == Some(false) { + st.puts.push(PutAttempt { + if_match: if_match.clone(), + if_none_match: if_none_match.clone(), + status: Some(500), + }); + return axum::http::StatusCode::INTERNAL_SERVER_ERROR + .into_response(); + } + let (status, etag) = evaluate_put( &mut st, + &key, body.to_vec(), if_match.as_deref(), if_none_match.as_deref(), - ) + attempt.as_deref(), + ); + (status, etag, lost_response) }; + // The response-loss arm: the write is COMMITTED + // above and the client is told it failed. This + // is the state an S3-compatible store reaches + // when it durably records a conditional PUT and + // then loses or corrupts the response. + if lost_response == Some(true) && status == 200 { + return axum::http::StatusCode::INTERNAL_SERVER_ERROR + .into_response(); + } match etag { Some(etag) => ( axum::http::StatusCode::OK, @@ -4223,9 +4775,26 @@ mod tests { .into_response(), } } + axum::http::Method::DELETE => { + let if_match = header("if-match"); + let mut st = state.lock().unwrap(); + let Some(existing) = st.objects.get(&key).cloned() else { + return axum::http::StatusCode::NO_CONTENT.into_response(); + }; + let stale = if_match.as_deref().is_some_and(|want| { + unquote_etag(&existing.etag) != unquote_etag(want) + }); + if stale { + return axum::http::StatusCode::PRECONDITION_FAILED + .into_response(); + } + st.objects.remove(&key); + st.deletes += 1; + axum::http::StatusCode::NO_CONTENT.into_response() + } axum::http::Method::HEAD | axum::http::Method::GET => { let mut st = state.lock().unwrap(); - let answered = (st.object.clone(), st.etag.clone()); + let answered = st.objects.get(&key).cloned(); // Fault injection for the two-consecutive- // losses arm, and the only deterministic way // to sit BETWEEN a caller's HEAD and the @@ -4244,20 +4813,35 @@ mod tests { // count only the caller's own PUTs. if method == axum::http::Method::HEAD && st.roll_after_heads > 0 - && st.object.is_some() + && st.objects.contains_key(&key) { st.roll_after_heads -= 1; st.next_etag += 1; - st.etag = Some(format!("\"mock-etag-{}\"", st.next_etag)); + let rolled = format!("\"mock-etag-{}\"", st.next_etag); + if let Some(obj) = st.objects.get_mut(&key) { + obj.etag = rolled; + } } match answered { - (Some(bytes), Some(etag)) => ( - axum::http::StatusCode::OK, - [(axum::http::header::ETAG, etag)], - axum::body::Body::from(bytes), - ) - .into_response(), - _ => axum::http::StatusCode::NOT_FOUND.into_response(), + Some(obj) => { + let mut resp = ( + axum::http::StatusCode::OK, + [(axum::http::header::ETAG, obj.etag)], + axum::body::Body::from(obj.body), + ) + .into_response(); + if let Some(attempt) = obj.attempt { + resp.headers_mut().insert( + axum::http::HeaderName::from_static( + "x-amz-meta-gitlawb-attempt", + ), + axum::http::HeaderValue::from_str(&attempt) + .expect("attempt id is ascii"), + ); + } + resp + } + None => axum::http::StatusCode::NOT_FOUND.into_response(), } } _ => axum::http::StatusCode::METHOD_NOT_ALLOWED.into_response(), @@ -4285,12 +4869,26 @@ mod tests { &self.endpoint } + /// The object the last successful PUT wrote. Every test that uses this + /// drives a single key; a test touching two keys reads them by key. + fn last(&self) -> Option { + let st = self.state.lock().unwrap(); + let key = st.last_key.clone()?; + st.objects.get(&key).cloned() + } + fn current_etag(&self) -> Option { - self.state.lock().unwrap().etag.clone() + self.last().map(|o| o.etag) } fn object(&self) -> Option> { - self.state.lock().unwrap().object.clone() + self.last().map(|o| o.body) + } + + /// The stored object under a specific repo key. + fn object_for(&self, owner_slug: &str, repo_name: &str) -> Option { + let key = format!("test-bucket/repos/v1/{owner_slug}/{repo_name}.tar.zst"); + self.state.lock().unwrap().objects.get(&key).cloned() } fn put_attempts(&self) -> Vec { @@ -4312,6 +4910,29 @@ mod tests { self.state.lock().unwrap().park_next_put = true; } + /// Accept and COMMIT the next PUT, then answer 500. The response-loss + /// arm: the write is durable and the client is told it failed. + fn commit_then_lose_next_put_response(&self) { + self.state.lock().unwrap().lose_next_response = Some(true); + } + + /// Deliver the next PUT in full and then fail it WITHOUT committing. The + /// client's knowledge is identical to the arm above — that is the whole + /// point — so only asking the store can tell the two apart. + fn fail_next_put_after_delivery(&self) { + self.state.lock().unwrap().lose_next_response = Some(false); + } + + /// The attempt id stamped on whatever the last successful PUT stored. + fn stored_attempt(&self) -> Option { + self.last().and_then(|o| o.attempt) + } + + /// How many DELETEs the mock actually carried out. + fn deletes(&self) -> u32 { + self.state.lock().unwrap().deletes + } + /// Let a parked handler go. Only the socket-level arm needs this; the /// deterministic assertion is `replay_captured`. fn open_gate(&self) { @@ -4329,9 +4950,11 @@ mod tests { let captured = st.captured.clone().expect("a PUT was captured"); evaluate_put( &mut st, + &captured.key, captured.body, captured.if_match.as_deref(), captured.if_none_match.as_deref(), + captured.attempt.as_deref(), ) .0 } @@ -4364,12 +4987,27 @@ mod tests { body: &[u8], if_match: Option<&str>, if_none_match: Option<&str>, + ) -> Result { + mock_put_as(client, body, if_match, if_none_match, None).await + } + + /// The same, stamped with an attempt id, so a test can seed an object that + /// belongs to a NAMED attempt (its own, or a foreign one). + async fn mock_put_as( + client: &aws_sdk_s3::Client, + body: &[u8], + if_match: Option<&str>, + if_none_match: Option<&str>, + attempt: Option<&str>, ) -> Result { let mut req = client .put_object() .bucket("test-bucket") .key("repos/v1/owner/repo.tar.zst") .body(aws_sdk_s3::primitives::ByteStream::from(body.to_vec())); + if let Some(attempt) = attempt { + req = req.metadata(ATTEMPT_METADATA_KEY, attempt); + } if let Some(v) = if_match { req = req.if_match(v); } @@ -4917,7 +5555,7 @@ mod tests { // colons so its slug is exactly "owner" and the IfAbsent upload is // refused against the seeded key. let err = store - .release_after_write("owner", "repo") + .release_after_write("owner", "repo", &PublishAttemptId::new()) .await .expect_err("a create-only upload over an existing key must be refused"); assert!( @@ -4935,10 +5573,14 @@ mod tests { /// MUST-NOT. A 404 is permanent (no such bucket, a misrouted endpoint), so /// reporting it as a lost precondition would tell a client to retry - /// something that can never succeed. `delete` has no callers, so a racing - /// delete cannot produce this. + /// something that can never succeed. Archive keys are never deleted by the + /// write path, so a racing delete cannot produce this. + /// + /// It must land as `NotPublished` rather than `Ambiguous`: a 4xx is an answer + /// the server gave BEFORE storing anything, so it does prove the write did + /// not commit, which is what licenses a caller to compensate. #[tokio::test] - async fn upload_classifies_404_as_other_under_either_precondition() { + async fn upload_classifies_404_as_a_definite_non_publication() { let (endpoint, server) = start_fixed_status_stub(404).await; let client = TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint); @@ -4952,16 +5594,25 @@ mod tests { .await .expect_err("404 must be an error"); assert!( - matches!(err, UploadError::Other(_)), + matches!(err, UploadError::NotPublished(_)), "404 under {precondition:?} must NOT be a lost precondition, got {err:?}" ); + assert!( + err.proves_not_published(), + "a 404 is a definite refusal, so compensation is licensed" + ); } server.abort(); } + /// THE P2 SPLIT. A 5xx says the server FAILED, not that it did not commit: + /// an S3-compatible store can accept and durably record a conditional PUT and + /// then fail while producing the response. Classifying that as a definite + /// failure is what let guarded writes invalidate their cache and fork + /// creation drop its only local clone for a write that had landed. #[tokio::test] - async fn upload_classifies_500_as_other() { + async fn upload_classifies_500_as_ambiguous_not_definite_failure() { let (endpoint, server) = start_fixed_status_stub(500).await; let client = TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint); @@ -4976,8 +5627,12 @@ mod tests { .await .expect_err("500 must be an error"); assert!( - matches!(err, UploadError::Other(_)), - "500 under {precondition:?} must be Other, got {err:?}" + matches!(err, UploadError::Ambiguous { .. }), + "500 under {precondition:?} must be ambiguous, got {err:?}" + ); + assert!( + !err.proves_not_published(), + "a 5xx must never license destructive compensation, got {err:?}" ); } @@ -5748,4 +6403,955 @@ mod tests { mock.open_gate(); mock.shutdown(); } + + // ── attempt identity and publication stage (#285) ────────────────────── + + /// The mock has to MODEL attempt metadata or every reconciliation test below + /// is vacuous: a HEAD that never echoes what the PUT stamped would make + /// `attempt_landed` answer `false` unconditionally and the fix would look + /// like it worked by doing nothing. + #[tokio::test] + async fn mock_round_trips_the_attempt_metadata_a_put_stamped() { + let mock = S3Mock::start().await; + let client = TigrisClient::for_testing_with_endpoint("test-bucket", mock.endpoint()); + + let dir = payload_dir("stamped"); + let attempt = PublishAttemptId::new(); + let receipt = client + .upload_tracked( + "owner", + "repo", + dir.path(), + UploadPrecondition::IfAbsent, + attempt.clone(), + None, + ) + .await + .expect("the create must publish"); + assert_eq!(receipt.attempt, attempt); + assert!( + receipt.etag.is_some(), + "an acknowledged PUT carries the generation it minted" + ); + assert_eq!(mock.stored_attempt().as_deref(), Some(attempt.as_str())); + + let stored = client + .head_generation("owner", "repo") + .await + .expect("HEAD") + .expect("an object is stored"); + assert!( + stored.belongs_to(&attempt), + "HEAD must report the attempt the PUT stamped, got {stored:?}" + ); + assert!( + !stored.belongs_to(&PublishAttemptId::new()), + "a different attempt must not match" + ); + assert!(client + .attempt_landed("owner", "repo", &attempt) + .await + .expect("reconcile")); + + mock.shutdown(); + } + + /// P2 FINDING 4, THE HEADLINE. A server that accepts the COMPLETE conditional + /// PUT, commits it, and then loses or corrupts the response before the SDK + /// can return success. + /// + /// Classifying that as a definite failure is what made fork creation drop its + /// only local clone and skip the DB insert for a write that HAD landed, after + /// which every retry saw the orphan object under `If-None-Match: *` and + /// returned `RepoExists` — the fork name unusable until an operator cleaned + /// up. The outcome must be ambiguous, and it must be RECONCILABLE: the + /// attempt id travelled with the bytes, so the client can ask the store what + /// it holds. + #[tokio::test] + async fn a_put_that_commits_and_loses_its_response_is_ambiguous_and_reconcilable() { + let mock = S3Mock::start().await; + let client = TigrisClient::for_testing_with_endpoint("test-bucket", mock.endpoint()); + + let dir = payload_dir("committed-but-unreported"); + let attempt = PublishAttemptId::new(); + mock.commit_then_lose_next_put_response(); + let stage = PublishStageCell::new(); + let err = client + .upload_tracked( + "owner", + "repo", + dir.path(), + UploadPrecondition::IfAbsent, + attempt.clone(), + Some(&stage), + ) + .await + .expect_err("the client never saw a success response"); + + assert!( + matches!(err, UploadError::Ambiguous { .. }), + "a lost response is not proof of failure, got {err:?}" + ); + assert!( + !err.proves_not_published(), + "compensating this would delete a write that landed" + ); + assert_eq!( + stage.get(), + PublishStage::Ambiguous { + attempt: attempt.clone() + }, + "the stage must record the attempt whose fate is unresolved" + ); + + // The write IS durable, and the attempt id is what proves it. + assert!( + client + .attempt_landed("owner", "repo", &attempt) + .await + .expect("reconcile"), + "the store committed this attempt's bytes, so reconciliation must recover it" + ); + assert_eq!(mock.stored_attempt().as_deref(), Some(attempt.as_str())); + + mock.shutdown(); + } + + /// The other half of "closes or corrupts": a peer that reads the whole + /// request and then drops the socket without answering at all. There is no + /// HTTP status to classify on, so the SdkError variant is all there is — and + /// only a construction failure proves the request never left. + #[tokio::test] + async fn a_closed_response_with_no_http_status_is_ambiguous() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port()); + let server = tokio::spawn(async move { + while let Ok((mut sock, _)) = listener.accept().await { + tokio::spawn(async move { + use tokio::io::AsyncReadExt; + // Consume what the client sends — the request IS delivered — + // then hang up without a response. + let mut buf = vec![0u8; 64 * 1024]; + let _ = tokio::time::timeout( + std::time::Duration::from_millis(250), + sock.read(&mut buf), + ) + .await; + drop(sock); + }); + } + }); + + let client = TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint); + let dir = payload_dir("no-answer"); + let err = client + .upload("owner", "repo", dir.path(), UploadPrecondition::IfAbsent) + .await + .expect_err("a dropped connection must not read as success"); + assert!( + matches!(err, UploadError::Ambiguous { .. }), + "a request delivered with no response read must stay ambiguous, got {err:?}" + ); + assert!(!err.proves_not_published()); + + server.abort(); + } + + /// The conditional delete must be atomic, not merely narrowed. A successor + /// publishing between the ownership HEAD and the DELETE is the exact race a + /// "look it up again first" guard leaves open, and `roll_generation_after_ + /// next_heads` is the seam that sits in that window deterministically. + #[tokio::test] + async fn a_generation_that_moves_between_the_head_and_the_delete_is_not_deleted() { + let mock = S3Mock::start().await; + let client = TigrisClient::for_testing_with_endpoint("test-bucket", mock.endpoint()); + let attempt = PublishAttemptId::new(); + + mock_put_as( + &mock_s3_client(mock.endpoint()), + b"ours", + None, + None, + Some(attempt.as_str()), + ) + .await + .expect("seed this attempt's object"); + + // The ownership HEAD sees our attempt and our ETag; the store then moves + // on before the DELETE fenced on that ETag arrives. + mock.roll_generation_after_next_heads(1); + let outcome = client + .delete_if_attempt_matches("owner", "repo", &attempt) + .await + .expect("a refused conditional delete is an outcome, not an error"); + assert_eq!( + outcome, + AttemptDelete::NotOurs, + "a delete whose generation moved under it must be refused, not retried blind" + ); + assert_eq!(mock.deletes(), 0, "nothing may have been deleted"); + assert!(mock.object().is_some(), "the object must survive"); + + mock.shutdown(); + } + + /// The must-do direction: an attempt's OWN orphan is still cleanable, or a + /// failed fork would tombstone its name forever. + #[tokio::test] + async fn an_attempts_own_object_is_deleted_and_a_foreign_one_is_not() { + let mock = S3Mock::start().await; + let client = TigrisClient::for_testing_with_endpoint("test-bucket", mock.endpoint()); + let mine = PublishAttemptId::new(); + let theirs = PublishAttemptId::new(); + + assert_eq!( + client + .delete_if_attempt_matches("owner", "repo", &mine) + .await + .expect("delete"), + AttemptDelete::Absent, + "an empty key is nothing to clean up" + ); + + mock_put_as( + &mock_s3_client(mock.endpoint()), + b"theirs", + None, + None, + Some(theirs.as_str()), + ) + .await + .expect("seed a foreign object"); + assert_eq!( + client + .delete_if_attempt_matches("owner", "repo", &mine) + .await + .expect("delete"), + AttemptDelete::NotOurs + ); + assert!(mock.object().is_some(), "a foreign object must survive"); + + mock_put_as( + &mock_s3_client(mock.endpoint()), + b"mine", + None, + None, + Some(mine.as_str()), + ) + .await + .expect("replace it with ours"); + assert_eq!( + client + .delete_if_attempt_matches("owner", "repo", &mine) + .await + .expect("delete"), + AttemptDelete::Deleted + ); + assert!(mock.object().is_none(), "our own orphan must be removable"); + + mock.shutdown(); + } + + /// P2 FINDING 4, the fork path. `release_after_write` is fork creation's + /// publish, and a lost response there must reach the handler as ambiguous so + /// it can recover the committed attempt instead of dropping its clone. + #[sqlx::test] + async fn fork_publish_that_loses_its_response_stays_recoverable(pool: PgPool) { + let mock = S3Mock::start().await; + let tmp = TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + let local = repos_dir.join("owner").join("repo.git"); + std::fs::create_dir_all(local.parent().unwrap()).unwrap(); + store::init_bare(&local).expect("a bare repo to upload"); + + let store = RepoStore::new( + repos_dir, + Some(mock_tigris(&mock)), + pool.clone(), + Duration::from_secs(30), + ); + let attempt = PublishAttemptId::new(); + mock.commit_then_lose_next_put_response(); + let err = store + .release_after_write("owner", "repo", &attempt) + .await + .expect_err("the client saw no success"); + assert!( + !err.proves_not_published(), + "the fork must not treat a lost response as licence to delete its clone, got {err:?}" + ); + assert!( + store + .fork_attempt_landed("owner", "repo", &attempt) + .await + .expect("reconcile"), + "the archive IS published under this attempt, so the fork must be recoverable \ + rather than fenced behind its own orphan" + ); + + mock.shutdown(); + } + + /// P1 FINDING 3, the object half, driven through the REAL recovery path. + /// + /// The interleaving: the background recovery observed no row for this fork + /// name (the DB below has none), a successor then committed and published + /// under that name, and only afterwards did the recovery resume its cleanup. + /// Before the attempt guard, that cleanup deleted the successor's archive and + /// directory after the successor had already answered 201. + #[sqlx::test] + async fn fork_recovery_resuming_after_a_successor_deletes_neither_object_nor_path( + pool: PgPool, + ) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let tmp = TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + let store = RepoStore::new( + repos_dir.clone(), + Some(mock_tigris(&mock)), + pool.clone(), + Duration::from_secs(30), + ); + + let failed = PublishAttemptId::new(); + let successor = PublishAttemptId::new(); + + // The successor owns both resources by the time cleanup resumes. + mock_put_as( + &mock_s3_client(mock.endpoint()), + b"successor-archive", + None, + None, + Some(successor.as_str()), + ) + .await + .expect("the successor published"); + let disk_path = repos_dir.join("owner").join("repo.git"); + std::fs::create_dir_all(&disk_path).unwrap(); + std::fs::write(disk_path.join("HEAD"), b"successor").unwrap(); + claim_fork_disk_path(&disk_path, &successor); + + // The failed attempt resumes its compensation. + store + .compensate_fork_archive("owner", "repo", &disk_path, &failed) + .await; + + assert_eq!( + mock.deletes(), + 0, + "a failed attempt must not delete the successor's archive" + ); + assert_eq!( + mock.stored_attempt().as_deref(), + Some(successor.as_str()), + "the successor's object must still be what is stored" + ); + assert!( + disk_path.exists(), + "a failed attempt must not delete the successor's repository directory" + ); + assert_eq!( + std::fs::read_to_string(disk_path.join("HEAD")).unwrap(), + "successor" + ); + + mock.shutdown(); + } + + /// The must-do direction of the same guard: an attempt that still owns both + /// resources must be able to clean up after itself, or a failed fork leaves + /// an orphan that fences its own name. + #[sqlx::test] + async fn fork_compensation_removes_what_this_attempt_still_owns(pool: PgPool) { + let mock = S3Mock::start().await; + let tmp = TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + let store = RepoStore::new( + repos_dir.clone(), + Some(mock_tigris(&mock)), + pool.clone(), + Duration::from_secs(30), + ); + + let attempt = PublishAttemptId::new(); + mock_put_as( + &mock_s3_client(mock.endpoint()), + b"my-orphan", + None, + None, + Some(attempt.as_str()), + ) + .await + .expect("this attempt published"); + let disk_path = repos_dir.join("owner").join("repo.git"); + std::fs::create_dir_all(&disk_path).unwrap(); + claim_fork_disk_path(&disk_path, &attempt); + + store + .compensate_fork_archive("owner", "repo", &disk_path, &attempt) + .await; + + assert!( + mock.object().is_none(), + "this attempt's orphan must be gone" + ); + assert!(!disk_path.exists(), "this attempt's clone must be gone"); + + mock.shutdown(); + } + + // ── the quarantined generation (#285 finding 2) ──────────────────────── + + /// A store whose Tigris compression parks on `gate`, so a publish can be + /// stalled at `PublishStage::PreparingArchive` — the window in which no + /// request has been constructed, let alone sent. + async fn compression_gated_store( + mock: &S3Mock, + opts: &sqlx::postgres::PgConnectOptions, + repos_dir: &Path, + bound: std::time::Duration, + gate: Arc, + ) -> RepoStore { + RepoStore::new( + repos_dir.to_path_buf(), + Some(mock_tigris(mock).with_compress_gate(gate)), + no_reap_pool(opts, 2).await, + bound, + ) + } + + /// P1 FINDING 1, at the release boundary. A publish whose bound expires while + /// the archive is still being COMPRESSED never constructed a request, so it + /// is a definite non-publication — not the unknowable in-flight PUT the + /// timeout arm used to report for every stall alike. + /// + /// The observable is the mock's PUT log: zero attempts. That is what + /// distinguishes this from the parked-PUT tests above, where exactly one + /// request arrived. + #[sqlx::test] + async fn a_bound_that_expires_before_dispatch_is_a_definite_failure(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let gate = Arc::new(crate::git::tigris::BlockingGate::shut()); + let owner = "did:key:z6MkStageCompress"; + let repo = "stage-compress-repo"; + + // The acquire-side refresh must run BEFORE the gate closes: it does no + // compression of its own, but the store is shared and holding the gate + // early would prove nothing about the release. + let store = compression_gated_store( + &mock, + &opts, + repos.path(), + std::time::Duration::from_millis(600), + Arc::clone(&gate), + ) + .await; + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + marked_repo(&guard.local_path, "writer"); + let local_path = guard.local_path.clone(); + + let outcome = guard.release(true).await; + + assert!( + matches!(outcome, ReleaseOutcome::UploadFailed), + "a bound that expired before any PUT was dispatched is a DEFINITE failure, \ + got {outcome:?}" + ); + assert!( + mock.put_attempts().is_empty(), + "no request may have reached the store, got {:?}", + mock.put_attempts() + ); + assert!( + !local_path.exists(), + "a definite non-publication must invalidate the local write cache; only an \ + unresolved dispatch may leave the tree in place" + ); + assert!( + warn_lines_for(repo).contains("before any PUT was dispatched"), + "the distinct verdict must be visible in the log, got {:?}", + warn_lines_for(repo) + ); + + // Let the parked blocking thread finish so it is not stranded. + gate.open(); + mock.shutdown(); + } + + /// P1 FINDING 2. A bounded release whose PUT is in flight leaves the writer's + /// tree at the ORDINARY live path with no confirmed generation behind it. The + /// writer gets its 503 — and then a same-node read must not hand that tree + /// out as an ordinary successful read. + /// + /// Deleting the tree is deliberately not the fix and is asserted against: the + /// PUT may have landed, and this can be the only local copy of it. + #[sqlx::test] + async fn an_unresolved_publish_quarantines_the_tree_and_a_later_read_refuses(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let bound = std::time::Duration::from_millis(750); + let store = fenced_store_with_bound(&mock, &opts, repos.path(), bound).await; + let owner = "did:key:z6MkQuarantineRead"; + let repo = "quarantine-read-repo"; + let slug = owner_slug_of(owner); + + let seed = TempDir::new().unwrap(); + marked_repo(seed.path(), "seed"); + mock_tigris(&mock) + .upload(&slug, repo, seed.path(), UploadPrecondition::Unconditional) + .await + .expect("seeding the archive"); + + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + std::fs::write(guard.local_path.join("MARKER"), "unresolved").unwrap(); + let local_path = guard.local_path.clone(); + mock.park_next_put(); + let outcome = guard.release(true).await; + + assert!( + matches!(outcome, ReleaseOutcome::UploadUnknowable), + "the PUT was consumed and left unresolved, got {outcome:?}" + ); + assert!( + outcome.into_result().is_err(), + "the writer must get a retryable refusal, not a 2xx" + ); + assert!( + local_path.exists(), + "the tree must NOT be deleted: the PUT may have landed and this could be the \ + only local copy" + ); + + // THE READ. Pre-fix this returned the live path on filesystem existence + // alone and served the unresolved refs indefinitely. + let read = store.acquire(owner, repo).await; + let err = read.expect_err("an unconfirmed tree must not be served as an ordinary read"); + assert!( + err.downcast_ref::().is_some(), + "the refusal must be the retryable one, got {err:#}" + ); + assert!( + local_path.exists(), + "refusing must not delete the tree either" + ); + + mock.open_gate(); + mock.shutdown(); + } + + /// The release valve, and what keeps the quarantine from being a permanent + /// outage: once the abandoned PUT actually commits, the stored object carries + /// THIS attempt's id, reconciliation says so, and the read is served. + #[sqlx::test] + async fn a_quarantined_tree_is_served_once_the_store_confirms_the_attempt(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let bound = std::time::Duration::from_millis(750); + let store = fenced_store_with_bound(&mock, &opts, repos.path(), bound).await; + let owner = "did:key:z6MkQuarantineLift"; + let repo = "quarantine-lift-repo"; + + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + marked_repo(&guard.local_path, "unresolved"); + let local_path = guard.local_path.clone(); + mock.park_next_put(); + assert!(matches!( + guard.release(true).await, + ReleaseOutcome::UploadUnknowable + )); + assert!( + store.acquire(owner, repo).await.is_err(), + "refused while unresolved" + ); + + // The abandoned PUT reaches the store's commit point and lands. + assert_eq!(mock.replay_captured(), 200); + + let served = store + .acquire(owner, repo) + .await + .expect("a confirmed attempt must lift the quarantine"); + assert_eq!(served, local_path.as_path()); + assert!( + store.acquire(owner, repo).await.is_ok(), + "the marker must have been cleared, not re-evaluated on every read" + ); + + mock.open_gate(); + mock.shutdown(); + } + + /// THE CONTROL. An ordinary confirmed write must leave no quarantine behind, + /// or every read after every push would refuse. It asserts an outcome the + /// quarantine does not change, which is what lets the two tests above + /// attribute their red. + #[sqlx::test] + async fn a_confirmed_publish_leaves_the_tree_readable(pool: PgPool) { + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let store = fenced_store(&mock, &opts, repos.path()).await; + let owner = "did:key:z6MkQuarantineControl"; + let repo = "quarantine-control-repo"; + + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + marked_repo(&guard.local_path, "clean"); + guard + .release(true) + .await + .into_result() + .expect("an uncontended publish lands"); + + store + .acquire(owner, repo) + .await + .expect("a confirmed write must be readable with no reconciliation at all"); + + mock.shutdown(); + } + + /// P2 FINDING 4, at a GUARDED WRITE. The review names this arm explicitly: + /// "guarded issue/push writes also take definite-failure cache and + /// compensation paths despite not knowing whether their generation landed." + /// + /// The store here accepts the complete PUT, commits it, and then loses the + /// response. Pre-fix that arrived as `UploadError::Other`, which `release` + /// read as `UploadFailed` and answered by deleting the local tree and running + /// the caller's compensator (`create_issue` deletes the issue ref it just + /// wrote) — undoing a write that IS durable in object storage, so the next + /// reader downloads an archive containing the "undone" work. + /// + /// The correct outcome is a retryable refusal with the tree kept and + /// quarantined, and — because the attempt id travelled with the bytes — a + /// later read that RECONCILES and is served. + #[sqlx::test] + async fn a_guarded_write_whose_response_is_lost_is_not_compensated(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let store = fenced_store(&mock, &opts, repos.path()).await; + let owner = "did:key:z6MkLostResponseWrite"; + let repo = "lost-response-repo"; + + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + marked_repo(&guard.local_path, "writer"); + let local_path = guard.local_path.clone(); + + let compensated = Arc::new(std::sync::atomic::AtomicBool::new(false)); + mock.commit_then_lose_next_put_response(); + let outcome = { + let compensated = Arc::clone(&compensated); + guard + .release_compensating(true, move |_path| { + compensated.store(true, std::sync::atomic::Ordering::SeqCst); + Ok(()) + }) + .await + }; + + assert!( + matches!(outcome, ReleaseOutcome::UploadUnknowable), + "a lost response is not a definite failure, got {outcome:?}" + ); + assert!( + outcome.into_result().is_err(), + "the writer still must not be told it succeeded" + ); + assert!( + !compensated.load(std::sync::atomic::Ordering::SeqCst), + "RED: the caller's undo ran for a write that IS published — create_issue would \ + have deleted the issue ref that other nodes will fetch" + ); + assert!( + local_path.exists(), + "RED: the local tree was invalidated for a write that landed" + ); + + // ...and because the attempt travelled with the bytes, the quarantine is + // answerable rather than a standing outage. + store + .acquire(owner, repo) + .await + .expect("the committed attempt must reconcile and be served"); + + mock.shutdown(); + } + + /// A write that follows an unresolved one heals the path: the under-lock + /// refresh replaces the tree with the stored generation, so the quarantine it + /// inherited is answered rather than carried forever. + #[sqlx::test] + async fn the_next_write_clears_an_inherited_quarantine(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let bound = std::time::Duration::from_millis(750); + let store = fenced_store_with_bound(&mock, &opts, repos.path(), bound).await; + let owner = "did:key:z6MkQuarantineHeal"; + let repo = "quarantine-heal-repo"; + let slug = owner_slug_of(owner); + + let seed = TempDir::new().unwrap(); + marked_repo(seed.path(), "seed"); + mock_tigris(&mock) + .upload(&slug, repo, seed.path(), UploadPrecondition::Unconditional) + .await + .expect("seeding the archive"); + + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + std::fs::write(guard.local_path.join("MARKER"), "unresolved").unwrap(); + mock.park_next_put(); + assert!(matches!( + guard.release(true).await, + ReleaseOutcome::UploadUnknowable + )); + assert!(store.acquire(owner, repo).await.is_err()); + + // A successor takes the freed lock; its refresh downloads the confirmed + // archive over the quarantined tree. + let successor = store.acquire_write(owner, repo).await.expect("successor"); + std::fs::write(successor.local_path.join("MARKER"), "successor").unwrap(); + successor + .release(true) + .await + .into_result() + .expect("the successor publishes"); + + store + .acquire(owner, repo) + .await + .expect("the healed path must read normally again"); + + mock.open_gate(); + mock.shutdown(); + } + + // ── fork creation through the handler (#285 findings 3 and 4) ────────── + + /// A state whose repo store publishes to `mock`, plus a PUBLIC source repo + /// that is already on disk and already in object storage — so the fork's own + /// create-only PUT is the first and only PUT the handler makes, and a + /// one-shot response-loss flag can be aimed at it. + async fn fork_state( + mock: &S3Mock, + pool: &PgPool, + repos_dir: &Path, + source_owner: &str, + source_name: &str, + ) -> crate::state::AppState { + let opts = (*pool.connect_options()).clone(); + let mut state = crate::test_support::test_state(pool.clone()).await; + state.repo_store = fenced_store(mock, &opts, repos_dir).await; + // `fork_repo` derives the clone's destination from `config.repos_dir`, + // not from the store, so the two have to agree or the validated join + // rejects the default (relative) config path. + let mut config = (*state.config).clone(); + config.repos_dir = repos_dir.to_path_buf(); + state.config = Arc::new(config); + + let now = chrono::Utc::now(); + state + .db + .create_repo(&crate::db::RepoRecord { + id: uuid::Uuid::new_v4().to_string(), + name: source_name.to_string(), + owner_did: source_owner.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: format!("/unused/{source_name}"), + forked_from: None, + machine_id: None, + }) + .await + .expect("seed the source repo row"); + + let slug = owner_slug_of(source_owner); + let source_path = repos_dir.join(&slug).join(format!("{source_name}.git")); + store::init_bare(&source_path).expect("a real bare source repo"); + // Publish the SOURCE key so `acquire` finds it already migrated and does + // not lazily upload it; the fork's PUT must be the only one. + mock_tigris(mock) + .upload( + &slug, + source_name, + &source_path, + UploadPrecondition::Unconditional, + ) + .await + .expect("seed the source archive"); + state + } + + async fn do_fork( + state: &crate::state::AppState, + source_owner: &str, + source_name: &str, + forker: &str, + fork_name: &str, + ) -> std::result::Result<(axum::http::StatusCode, String), crate::error::AppError> { + crate::api::repos::fork_repo( + axum::extract::State(state.clone()), + axum::Extension(crate::auth::AuthenticatedDid(forker.to_string())), + axum::extract::Path(( + crate::db::normalize_owner_key(source_owner).to_string(), + source_name.to_string(), + )), + axum::http::HeaderMap::new(), + axum::Json(crate::api::repos::ForkRepoRequest { + name: Some(fork_name.to_string()), + }), + ) + .await + .map(|(status, body)| (status, body.0.id)) + } + + /// P2 FINDING 4, THE USER-VISIBLE FAILURE MODE. The fork's create-only PUT + /// commits, and the response is lost before the SDK can report success. + /// + /// Pre-fix that arrived as a definite failure: `ForkCloneGuard` removed the + /// only local clone and no DB row was inserted, leaving an orphan object + /// under the fork's key. Every retry then sent `If-None-Match: *`, saw the + /// orphan and answered `RepoExists` — the fork name unusable until an + /// operator cleaned up. The attempt id stamped into the object is what turns + /// "did my request succeed" (undecidable) into "are the published bytes mine" + /// (decidable), so the committed attempt is RECOVERED. + #[sqlx::test] + async fn a_fork_whose_publish_lost_its_response_is_recovered_not_fenced(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let repos = TempDir::new().unwrap(); + let source_owner = "did:key:z6MkForkSourceAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let forker = "did:key:z6MkForkerLostRespAAAAAAAAAAAAAAAAAAAAA"; + let state = fork_state(&mock, &pool, repos.path(), source_owner, "src").await; + + mock.commit_then_lose_next_put_response(); + let (status, id) = do_fork(&state, source_owner, "src", forker, "recovered") + .await + .expect("a committed publish must not be reported as a failure"); + + assert_eq!(status, axum::http::StatusCode::CREATED); + let row = state + .db + .get_repo(crate::db::normalize_owner_key(forker), "recovered") + .await + .expect("lookup") + .expect("the fork row must have been inserted"); + assert_eq!(row.id, id); + + // The object is the fork's, stamped with the row that owns it. + let stored = mock + .object_for(&owner_slug_of(forker), "recovered") + .expect("the fork archive is published"); + assert_eq!( + stored.attempt.as_deref(), + Some(row.id.as_str()), + "the published archive must name the row that owns it" + ); + assert!( + repos + .path() + .join(owner_slug_of(forker)) + .join("recovered.git") + .exists(), + "the fork's clone must still be on disk" + ); + + // ...and the name is NOT fenced: it resolves to a real repository. + assert!(state + .db + .get_repo_by_id(&row.id) + .await + .expect("lookup") + .is_some()); + + mock.shutdown(); + } + + /// The other half of the ambiguity, where the client's knowledge is + /// IDENTICAL: the PUT was delivered in full and failed, and this time it did + /// not commit. + /// + /// Nothing may be destroyed on this path either — the request could still be + /// in flight — so the answer is a retryable refusal with the only local clone + /// left in place, not a deletion. + #[sqlx::test] + async fn a_fork_whose_publish_is_unresolved_keeps_its_clone_and_refuses_retryably( + pool: PgPool, + ) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let repos = TempDir::new().unwrap(); + let source_owner = "did:key:z6MkForkSourceBBBBBBBBBBBBBBBBBBBBBBBBBB"; + let forker = "did:key:z6MkForkerUnresolvedAAAAAAAAAAAAAAAAAAA"; + let state = fork_state(&mock, &pool, repos.path(), source_owner, "src").await; + + mock.fail_next_put_after_delivery(); + let err = do_fork(&state, source_owner, "src", forker, "unresolved") + .await + .expect_err("an unresolved publish must not report success"); + assert!( + matches!(err, crate::error::AppError::RepoUnavailable), + "the refusal must be the retryable one, got {err:?}" + ); + + assert!( + repos + .path() + .join(owner_slug_of(forker)) + .join("unresolved.git") + .exists(), + "RED: the only local copy of a write that MAY have landed was deleted" + ); + assert_eq!( + mock.deletes(), + 0, + "nothing may be deleted while the outcome is unresolved" + ); + assert!( + state + .db + .get_repo(crate::db::normalize_owner_key(forker), "unresolved") + .await + .expect("lookup") + .is_none(), + "no row may be inserted for a publish that is not confirmed" + ); + + mock.shutdown(); + } + + /// THE CONTROL. An ordinary fork must still work end to end, or the two + /// assertions above would pass on a handler that simply never forks. + #[sqlx::test] + async fn an_ordinary_fork_publishes_and_commits(pool: PgPool) { + let mock = S3Mock::start().await; + let repos = TempDir::new().unwrap(); + let source_owner = "did:key:z6MkForkSourceCCCCCCCCCCCCCCCCCCCCCCCCCC"; + let forker = "did:key:z6MkForkerPlainAAAAAAAAAAAAAAAAAAAAAAAA"; + let state = fork_state(&mock, &pool, repos.path(), source_owner, "src").await; + + let (status, id) = do_fork(&state, source_owner, "src", forker, "plain") + .await + .expect("an uncontended fork must succeed"); + assert_eq!(status, axum::http::StatusCode::CREATED); + let stored = mock + .object_for(&owner_slug_of(forker), "plain") + .expect("the fork archive is published"); + assert_eq!(stored.attempt.as_deref(), Some(id.as_str())); + + mock.shutdown(); + } } From 535c5dca1866a5f830f18b84c303bdbc9e87745d Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sun, 6 Sep 2026 22:32:45 -0500 Subject: [PATCH 52/54] fix(node): settle the cache contract on the abandoned path and the attempt The publication boundary this branch introduced was applied at call sites that each had to be remembered, and five were not. Close the eight gaps that survived, grouped by what actually failed rather than by symptom. The marker's lifecycle was answered by the wrong event in three places. The swap that replaces a live tree now owns the clear, so a cache-miss download no longer leaves a stale marker that fences the archive it just installed, and the under-lock refresh no longer lifts a quarantine on a HEAD that downloaded nothing. init clears a predecessor's marker. A cancelled handler never reached release, so it applied the tail contract and not the cache contract. RepoWriteGuard::drop now settles the tree it abandoned, keyed on how far the publish actually got, and a guard that handed out its path for writing marks the tree whatever stage it reached. A marker naming no attempt is a definite non-publication rather than a permanent refusal: it invalidates and the next read serves the stored generation, so an abandoned first push cannot wedge a repo at 503. Only an unreadable sidecar still fails shut. Both of the store's live-path hand-outs now go through one confirming helper, so read_snapshot can no longer return an unconfirmed tree as an ordinary snapshot. The gate binds ahead of the lazy-migration spawn, because a quarantined tree backfilled to object storage by an IfAbsent PUT would publish exactly the refs the marker exists to withhold. The marker no longer rests on a single fs::write; an unwritable sidecar falls back to an in-memory record that reads consult. Fork cleanup claims ownership by renaming the stamp before it touches the tree, so a successor that claims the same path cannot lose its directory to a predecessor's check-then-unlink, and the directory-absent path no longer unlinks a stamp it does not own. A create-only precondition loss whose reconciliation HEAD fails is refused retryably instead of reported as a permanent name conflict, and its clone is released so the retry can re-clone rather than finding the destination occupied. The pin sweep skips a tree whose publish is unresolved, so provider CIDs are not rewritten from refs that may never become durable. That is the sidecar marker, distinct from the operator-level quarantined row flag. 26 tests, each written and observed failing before its fix. Suite 1211 passed, 0 failed, 1 ignored; fmt, clippy -D warnings and --locked clean. --- crates/gitlawb-node/src/api/issues.rs | 57 + crates/gitlawb-node/src/api/repos.rs | 278 +++- crates/gitlawb-node/src/git/repo_store.rs | 1646 +++++++++++++++++++-- crates/gitlawb-node/src/ipfs_pin.rs | 25 +- crates/gitlawb-node/src/test_support.rs | 171 +++ 5 files changed, 2035 insertions(+), 142 deletions(-) diff --git a/crates/gitlawb-node/src/api/issues.rs b/crates/gitlawb-node/src/api/issues.rs index 5150c2e0..f214d96a 100644 --- a/crates/gitlawb-node/src/api/issues.rs +++ b/crates/gitlawb-node/src/api/issues.rs @@ -938,6 +938,63 @@ mod lock_pool_shed_tests { ); } + /// #285 U2, gap-driving. `close_issue` decides authorship out of a + /// `read_snapshot`, and on a node whose only copy is the live tree that + /// snapshot IS the live tree. When a write left that tree carrying refs the + /// store never confirmed, the pre-check reads authorship out of state no + /// other node can see, so it must be refused as retryable rather than + /// answered. + /// + /// No backend is configured here, which is the sharpest form: nothing can + /// ever confirm the marked tree, so serving it is not a race, it is a + /// standing wrong answer. + #[sqlx::test] + async fn close_issue_refuses_a_quarantined_snapshot_before_authorization(pool: PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let owner = "did:key:zISSUECLOSEQUARANTINEBBBBBBBBBBBBBBBBBBB"; + let mut state = crate::test_support::test_state(pool.clone()).await; + state.repo_store = + crate::git::repo_store::RepoStore::for_testing(tmp.path().to_path_buf(), pool.clone()); + state + .db + .create_repo(&seed_repo(owner, "quar-close")) + .await + .expect("seed repo"); + + // A live tree with an unresolved publish beside it. + let live = state + .repo_store + .acquire(owner, "quar-close") + .await + .expect("resolve the live path"); + let _ = std::fs::remove_dir_all(&live); + crate::git::store::init_bare(&live).expect("init bare repo"); + crate::git::repo_store::quarantine_local_tree( + &live, + "quar-close", + Some(&crate::git::publish::PublishAttemptId::new()), + ); + + let refused = close_issue( + State(state.clone()), + Extension(AuthenticatedDid( + "did:key:zISSUECLOSEQUARSTRANGER".to_string(), + )), + Path(( + owner.to_string(), + "quar-close".to_string(), + "deadbeef".to_string(), + )), + axum::http::HeaderMap::new(), + crate::rate_limit::PeerAddr(None), + ) + .await; + assert!( + matches!(refused, Err(AppError::RepoUnavailable)), + "close_issue must not read authorship out of a quarantined tree the store cannot confirm" + ); + } + #[sqlx::test] async fn close_issue_lock_pool_exhaustion_sheds_503_not_500(pool: PgPool) { let owner = "did:key:zISSUECLOSELOCKPOOLBBBBBBBBBBBBBBBBBBBBB"; diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index ed028ab2..b646e4bf 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -3562,26 +3562,51 @@ pub async fn fork_repo( // the create-only fence refuse the second copy. The stored object // carrying our own attempt id says the publish succeeded, and // refusing it would fence the fork name behind our own work. - if state + match state .repo_store .fork_attempt_landed(&forker_did, &fork_name, &attempt) .await - .unwrap_or(false) { - tracing::info!( - fork = %fork_name, - status, - "fork create-only PUT was refused but the stored archive is this \ - attempt's own — continuing" - ); - } else { - tracing::warn!( - forker = %forker_did, - fork = %fork_name, - status, - "fork refused: an archive already exists under the fork's key" - ); - return Err(AppError::RepoExists(fork_name.clone())); + Ok(true) => { + tracing::info!( + fork = %fork_name, + status, + "fork create-only PUT was refused but the stored archive is this \ + attempt's own — continuing" + ); + } + Ok(false) => { + tracing::warn!( + forker = %forker_did, + fork = %fork_name, + status, + "fork refused: an archive already exists under the fork's key" + ); + return Err(AppError::RepoExists(fork_name.clone())); + } + // The store could not be asked whether the stored archive is ours, + // so "not ours" is not an answer we have. Reporting a permanent name + // conflict from a transient store failure sends the client away from + // a name that may well be free; refuse retryably instead. + // + // The clone guard is deliberately NOT disarmed here. Unlike the + // ambiguous arm below, a lost precondition proves this attempt's PUT + // was refused, so the local clone protects nothing the store does not + // already hold. Keeping it would wedge the fork name: the retry this + // 503 invites re-clones into the same destination and fails because + // it exists, and the leftover carries a dead attempt's stamp that no + // cleanup will ever claim. Let Drop remove it. + Err(e) => { + tracing::warn!( + forker = %forker_did, + fork = %fork_name, + status, + err = %e, + "fork refused retryably: the store could not be asked whether the \ + archive under the fork's key is this attempt's own" + ); + return Err(AppError::RepoUnavailable); + } } } crate::git::tigris::UploadError::NotPublished(other) => { @@ -11347,6 +11372,227 @@ mod tests { server.abort(); } + // ── #285 U3: the abandoned write path settles the live tree ──────────── + // + // The sibling above proves the replication TAIL is withheld when a push is + // cancelled before its PUT is dispatched. Nothing yet stops the next READ + // from serving the very refs that tail refused to announce: the write guard + // drops without classifying the tree, so `acquire` finds a live directory, + // no marker, and hands it out. + + /// The sidecar marker path for a live tree, derived the way production + /// derives it (`.{name}.git.quarantine` beside the validated path) rather + /// than returned by the state builder. + #[cfg(unix)] + fn p3_quarantine_marker( + repos_dir: &std::path::Path, + owner_did: &str, + name: &str, + ) -> std::path::PathBuf { + let live = crate::git::repo_store::validated_repo_disk_path(repos_dir, owner_did, name) + .expect("test repo path"); + let file = live + .file_name() + .expect("a live tree always has a file name") + .to_string_lossy() + .to_string(); + live.parent() + .expect("a live tree always has a parent") + .join(format!(".{file}.quarantine")) + } + + /// #285 U3, gap-driving. A push cancelled inside the release-side + /// compression definitely never attempted publication, so the refs it wrote + /// exist only on this node's disk. The next read must not serve them. + /// + /// The p3 store answers every HEAD 404, so there is nothing stored to serve + /// in their place: the correct outcome is that the abandoned tree is gone, + /// not that a stale copy is handed out. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_cancelled_during_compression_leaves_no_servable_refs(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let gate = Arc::new(crate::git::tigris::BlockingGate::shut()); + let (state, log, puts, server) = + p3_compression_gated_state(pool, tmp.path(), "z6p3quar", "c1", Arc::clone(&gate)).await; + let rec = state.db.get_repo("z6p3quar", "c1").await.unwrap().unwrap(); + let repos_dir = tmp.path().join("repos"); + let live = + crate::git::repo_store::validated_repo_disk_path(&repos_dir, &rec.owner_did, "c1") + .expect("test repo path") + .into_path_buf(); + let marker = p3_quarantine_marker(&repos_dir, &rec.owner_did, "c1"); + + let mut fut = Box::pin(p2_push(&state, "z6p3quar", "c1")); + let mut ran = false; + for _ in 0..1000 { + let step = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + assert!( + step.is_err(), + "the handler must park inside the release-side compression, not return" + ); + if p2_logged(&log, "receive-pack") { + ran = true; + break; + } + } + assert!(ran, "the push must reach receive-pack"); + for _ in 0..10 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + } + assert_eq!( + puts.load(std::sync::atomic::Ordering::SeqCst), + 0, + "compression has not finished, so no PUT can have been built yet" + ); + + // THE DISCONNECT, inside the compression window. + drop(fut); + tokio::time::sleep(std::time::Duration::from_millis(1500)).await; + // Release the orphaned compression BEFORE the assertions. The guard has + // already dropped, so nothing below depends on the gate, and a failing + // assertion would otherwise leave a blocking task parked forever and + // hang the runtime teardown instead of reporting. + gate.open(); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + + assert!( + marker.exists(), + "a push cancelled before its PUT was dispatched must leave a marker beside the live tree" + ); + // The read resolves the marker: a definite non-publication, so the tree + // is dropped and the stored generation (there is none here) is what + // would serve. + let _ = state.repo_store.acquire(&rec.owner_did, "c1").await; + assert!( + !live.exists(), + "a push cancelled before its PUT was dispatched left its refs servable on the live path" + ); + + server.abort(); + } + + /// #285 U3, gap-driving. The disconnect one step earlier: receive-pack has + /// already APPLIED a ref and the client goes away before `release` is ever + /// entered, so the publish stage is still `Idle`. The guard rides the + /// admission guard into the detached reaper, so it drops once the git group + /// is torn down, and at that point the tree carries refs no PUT ever + /// described. + /// + /// The fake git applies the ref and then hangs, which is what puts the + /// disconnect in that window; argv is + /// ` receive-pack --stateless-rpc `, so `$3` is the tree. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_disconnected_after_refs_are_applied_leaves_no_servable_refs( + pool: sqlx::PgPool, + ) { + let tmp = tempfile::TempDir::new().unwrap(); + let gate = Arc::new(crate::git::tigris::BlockingGate::shut()); + let (mut state, _log, _puts, server) = + p3_compression_gated_state(pool, tmp.path(), "z6p3disc", "c1", Arc::clone(&gate)).await; + // Its own directory: `write_fake_git` always writes `fakegit`, and the + // state builder has already put the logging shim at that name. + let gitdir = tmp.path().join("hanging-git"); + std::fs::create_dir_all(&gitdir).unwrap(); + state.git_bin = write_fake_git( + &gitdir, + r#"#!/bin/sh +case "$1" in + receive-pack) + mkdir -p "$3/refs/heads" + printf '%s\n' 1111111111111111111111111111111111111111 > "$3/refs/heads/main" + sleep 30 + ;; + *) : ;; +esac +exit 0 +"#, + ); + + let rec = state.db.get_repo("z6p3disc", "c1").await.unwrap().unwrap(); + let repos_dir = tmp.path().join("repos"); + let live = + crate::git::repo_store::validated_repo_disk_path(&repos_dir, &rec.owner_did, "c1") + .expect("test repo path") + .into_path_buf(); + let marker = p3_quarantine_marker(&repos_dir, &rec.owner_did, "c1"); + let ref_path = live.join("refs").join("heads").join("main"); + + let mut fut = Box::pin(p2_push(&state, "z6p3disc", "c1")); + let mut applied = false; + for _ in 0..2000 { + let step = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + assert!( + step.is_err(), + "the handler must park inside the hanging receive-pack, not return" + ); + if ref_path.exists() { + applied = true; + break; + } + } + assert!(applied, "the push must apply its ref before the disconnect"); + + // THE DISCONNECT, after the refs landed and before release is entered. + drop(fut); + // Nothing below depends on the gate: this disconnect never reaches + // release, so no compression is parked on it. Opened here anyway so a + // failing assertion reports rather than hanging the teardown. + gate.open(); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + while !marker.exists() { + assert!( + std::time::Instant::now() < deadline, + "a push disconnected after receive-pack applied its refs must mark the tree, or acquire serves refs object storage does not hold" + ); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + + let _ = state.repo_store.acquire(&rec.owner_did, "c1").await; + assert!( + !ref_path.exists(), + "refs applied by a disconnected push must not be served" + ); + + server.abort(); + } + + /// THE CONTROL for both of the above. Same builder, no disconnect: the + /// publish completes, the store acknowledges the PUT, and the tree stays + /// readable with nothing withholding it. Without this a green refusal above + /// would prove only that this harness never serves. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_that_completes_its_publish_leaves_the_tree_servable(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let gate = Arc::new(crate::git::tigris::BlockingGate::shut()); + gate.open(); + let (state, _log, puts, server) = + p3_compression_gated_state(pool, tmp.path(), "z6p3serv", "c1", gate).await; + let rec = state.db.get_repo("z6p3serv", "c1").await.unwrap().unwrap(); + let repos_dir = tmp.path().join("repos"); + let marker = p3_quarantine_marker(&repos_dir, &rec.owner_did, "c1"); + + p2_push(&state, "z6p3serv", "c1") + .await + .expect("an uncontended push must succeed"); + assert_eq!( + puts.load(std::sync::atomic::Ordering::SeqCst), + 1, + "the release must have published exactly once" + ); + + let served = state.repo_store.acquire(&rec.owner_did, "c1").await; + assert!( + served.is_ok() && !marker.exists(), + "a confirmed publish must leave the tree readable with no marker" + ); + + server.abort(); + } + // ── #285 P1 finding 3: fork confirmation is bound to the attempt ─────── /// A repo row under `owner/name` owned by some OTHER attempt. diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 9878ab28..b1c6aa79 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -26,6 +26,30 @@ use super::tigris::{ UploadPrecondition, }; +/// A live tree this store has decided it may hand out: its quarantine sidecar +/// has been read and answered. +/// +/// The point of the newtype is that inside this module a hand-out of the live +/// path that skipped the gate does not type-check, so the next +/// `read_snapshot`-shaped reader cannot forget it the way `read_snapshot` did. +/// +/// Module-private, constructor included, and deliberately NOT `pub(crate)`: a +/// `pub(crate)` signature mentioning a private type trips clippy's +/// `private-interfaces` under `-D warnings`, and both callers live in this file. +struct ConfirmedLiveTree(ValidatedRepoDiskPath); + +impl ConfirmedLiveTree { + /// Is the tree still there? A marker naming no attempt resolves by deleting + /// the tree, so a confirmed path can legitimately be absent. + fn exists(&self) -> bool { + self.0.exists() + } + + fn into_path_buf(self) -> PathBuf { + self.0.into_path_buf() + } +} + /// Centralized repo storage: local disk cache + optional Tigris backend. #[derive(Clone)] pub struct RepoStore { @@ -146,68 +170,78 @@ impl RepoStore { // possibly-refused write to every reader for as long as the directory // survives. Reconcile the quarantine FIRST, before the migration // bookkeeping and before the path is handed out. - if let Some(marker) = read_quarantine(&local_path) { - self.reconcile_quarantine(&owner_slug, repo_name, &local_path, &marker) - .await?; - } - // Lazy migration: if Tigris is enabled and we haven't confirmed this - // repo is in Tigris yet, check and upload in the background. - if let Some(ref tigris) = self.tigris { - let key = format!("{owner_slug}/{repo_name}"); - let already_migrated = self.migrated.lock().await.contains(&key); - if !already_migrated { - let tigris = tigris.clone(); - let slug = owner_slug.clone(); - let name = repo_name.to_string(); - let path = local_path.clone(); - let migrated = Arc::clone(&self.migrated); - tokio::spawn(async move { - // Check if already in Tigris before uploading - match tigris.exists(&slug, &name).await { - Ok(true) => { - debug!(repo = %name, "repo already in tigris — skipping migration"); - } - Ok(false) => { - info!(repo = %name, "migrating local repo to tigris"); - // Create-only. This backfill was decided on a - // negative existence check that is already - // stale, so a refusal means someone else - // published this key in between and dropping - // our bytes is the correct outcome. An - // unconditional PUT here would overwrite their - // archive, which is the exact bug this fence - // exists to close. - match tigris - .upload(&slug, &name, &path, UploadPrecondition::IfAbsent) - .await - { - Ok(_) => { - info!(repo = %name, "lazy migration to tigris complete"); - } - // Logged apart from the warn arm below so a - // refusal, which is the fence working, does - // not read as a storage failure. The key is - // populated either way, so this still - // counts as migrated. - Err(UploadError::PreconditionLost { status }) => { - info!(repo = %name, status, "lazy migration dropped: another writer already published this repo"); - } - Err(e) => { - warn!(repo = %name, err = %e, "lazy migration to tigris failed"); - return; + // + // FIRST is load-bearing, not tidiness. The lazy migration below + // backfills this very tree to object storage on an IfAbsent PUT. A + // gate that only sat at the returns would refuse the read and, on the + // same call, publish the unconfirmed refs it refused to serve, which + // is a worse outcome than the ungated read. + let confirmed = self + .confirm_live_tree(&owner_slug, repo_name, local_path.clone()) + .await?; + // A marker naming no attempt is resolved by deleting the tree, so the + // path can be gone by the time it is confirmed. Fall through to the + // download branch and serve the stored generation instead. + if confirmed.exists() { + // Lazy migration: if Tigris is enabled and we haven't confirmed this + // repo is in Tigris yet, check and upload in the background. + if let Some(ref tigris) = self.tigris { + let key = format!("{owner_slug}/{repo_name}"); + let already_migrated = self.migrated.lock().await.contains(&key); + if !already_migrated { + let tigris = tigris.clone(); + let slug = owner_slug.clone(); + let name = repo_name.to_string(); + let path = local_path.clone(); + let migrated = Arc::clone(&self.migrated); + tokio::spawn(async move { + // Check if already in Tigris before uploading + match tigris.exists(&slug, &name).await { + Ok(true) => { + debug!(repo = %name, "repo already in tigris — skipping migration"); + } + Ok(false) => { + info!(repo = %name, "migrating local repo to tigris"); + // Create-only. This backfill was decided on a + // negative existence check that is already + // stale, so a refusal means someone else + // published this key in between and dropping + // our bytes is the correct outcome. An + // unconditional PUT here would overwrite their + // archive, which is the exact bug this fence + // exists to close. + match tigris + .upload(&slug, &name, &path, UploadPrecondition::IfAbsent) + .await + { + Ok(_) => { + info!(repo = %name, "lazy migration to tigris complete"); + } + // Logged apart from the warn arm below so a + // refusal, which is the fence working, does + // not read as a storage failure. The key is + // populated either way, so this still + // counts as migrated. + Err(UploadError::PreconditionLost { status }) => { + info!(repo = %name, status, "lazy migration dropped: another writer already published this repo"); + } + Err(e) => { + warn!(repo = %name, err = %e, "lazy migration to tigris failed"); + return; + } } } + Err(e) => { + warn!(repo = %name, err = %e, "tigris existence check failed"); + return; + } } - Err(e) => { - warn!(repo = %name, err = %e, "tigris existence check failed"); - return; - } - } - migrated.lock().await.insert(format!("{slug}/{name}")); - }); + migrated.lock().await.insert(format!("{slug}/{name}")); + }); + } } + return Ok(confirmed.into_path_buf()); } - return Ok(local_path.into_path_buf()); } // Try downloading from Tigris @@ -223,13 +257,81 @@ impl RepoStore { .lock() .await .insert(format!("{owner_slug}/{repo_name}")); - return Ok(local_path.into_path_buf()); + // Through the gate like every other hand-out of the live path. + // Cheap here: the swap that installed the archive cleared any + // marker, so this is one failed stat. + return Ok(self + .confirm_live_tree(&owner_slug, repo_name, local_path) + .await? + .into_path_buf()); } } // Not found anywhere — return path anyway; caller will get a meaningful - // error from git when the path doesn't exist. - Ok(local_path.into_path_buf()) + // error from git when the path doesn't exist. Still through the gate: an + // attempt-bearing marker with no tree beside it means a PUT that may have + // landed is unresolved, and answering that with a path is what the gate + // exists to prevent. + Ok(self + .confirm_live_tree(&owner_slug, repo_name, local_path) + .await? + .into_path_buf()) + } + + /// The one confirming hand-out of a live tree inside this store. + /// + /// Three answers, because the sidecar carries three different facts: + /// + /// - no marker: the tree is a confirmed generation, hand it out; + /// - a marker naming an ATTEMPT: unresolved, reconcile it against the store + /// and refuse unless the store holds that attempt; + /// - a marker naming NO attempt: a DEFINITE non-publication left by a write + /// that was abandoned before anything reached the wire. Resolvable, and it + /// must be resolved: it is handled exactly the way `release` handles the + /// same state, by invalidating the local write cache, after which the next + /// read serves the stored generation. A definite non-publication that + /// refused forever would wedge a repo at 503 with nothing able to lift it; + /// - an UNREADABLE sidecar: fail shut. That is the corrupt-marker case the + /// shape exists for, and it must not be the routine outcome of a + /// disconnect, which is why it is a distinct value from "no attempt". + async fn confirm_live_tree( + &self, + owner_slug: &str, + repo_name: &str, + local_path: ValidatedRepoDiskPath, + ) -> Result { + match read_quarantine(&local_path) { + None => Ok(ConfirmedLiveTree(local_path)), + Some(Quarantine::Marker(marker)) => match marker.attempt.as_deref() { + Some(attempt) => { + let attempt = PublishAttemptId::from_owned(attempt); + self.reconcile_quarantine(owner_slug, repo_name, &local_path, &attempt) + .await?; + Ok(ConfirmedLiveTree(local_path)) + } + None => { + invalidate_local_write_cache( + &local_path, + repo_name, + "definite non-publication left by an abandoned write", + ); + // An absent tree is trivially confirmed: nothing at that path + // can carry unconfirmed refs. + Ok(ConfirmedLiveTree(local_path)) + } + }, + Some(Quarantine::Unreadable) => { + warn!( + repo = %repo_name, + "refusing a quarantined read: the marker beside this tree could not be \ + parsed, so nothing can say which attempt it is waiting on" + ); + Err(anyhow::Error::new(RepoUnavailable).context(format!( + "local tree for {owner_slug}/{repo_name} is quarantined by a marker that \ + could not be read" + ))) + } + } } /// Decide whether a quarantined live tree may be served. @@ -250,7 +352,7 @@ impl RepoStore { owner_slug: &str, repo_name: &str, local_path: &ValidatedRepoDiskPath, - marker: &QuarantineMarker, + attempt: &PublishAttemptId, ) -> Result<()> { let refuse = || { Err(anyhow::Error::new(RepoUnavailable).context(format!( @@ -258,20 +360,16 @@ impl RepoStore { is unresolved and could not be reconciled against object storage" ))) }; - let (Some(tigris), Some(attempt)) = ( - self.tigris.as_ref(), - marker.attempt.as_deref().map(PublishAttemptId::from_owned), - ) else { - // No backend to ask, or no attempt to ask about. Either way nothing - // can confirm this tree, and a read that cannot be confirmed must - // not be served as an ordinary success. + let Some(tigris) = self.tigris.as_ref() else { + // No backend to ask, so nothing can confirm this tree, and a read + // that cannot be confirmed must not be served as an ordinary success. warn!( repo = %repo_name, - "refusing a quarantined read: no attempt identity to reconcile against" + "refusing a quarantined read: no object-storage backend to reconcile against" ); return refuse(); }; - match tigris.attempt_landed(owner_slug, repo_name, &attempt).await { + match tigris.attempt_landed(owner_slug, repo_name, attempt).await { Ok(true) => { info!( repo = %repo_name, @@ -361,11 +459,13 @@ impl RepoStore { } } - // Tigris disabled or repo not in Tigris — fall back to local. - Ok(RepoSnapshot { - path: local_path.into_path_buf(), - owned: false, - }) + // Tigris disabled or repo not in Tigris — fall back to local, through + // the SAME gate `acquire` uses. This reader had none, so the tree one + // reader refused as unconfirmed the other handed out as a snapshot. + Ok(RepoSnapshot::live( + self.confirm_live_tree(&owner_slug, repo_name, local_path) + .await?, + )) } /// Take a write lock (Postgres advisory lock), ensure repo is local, return guard. @@ -568,6 +668,8 @@ impl RepoStore { Some(_) => PublishStage::Idle, None => PublishStage::NoBackend, })), + tree_settled: false, + path_handed_out: AtomicBool::new(false), #[cfg(test)] test_pre_unlock_gate: self.pre_unlock_gate.clone(), #[cfg(test)] @@ -631,11 +733,17 @@ impl RepoStore { match refreshed { Some(Ok(fence)) => { - // The tree at the live path is now the generation this HEAD - // observed (downloaded, or confirmed absent), so any - // quarantine an earlier unresolved write left on this path is - // answered by the refresh itself. - clear_quarantine(&local_path, repo_name); + // The refresh does NOT answer a quarantine. Only a swap that + // replaced the live tree with a stored generation does, and + // that swap now clears the marker itself. A HEAD that answered + // "nothing stored" downloads nothing and swaps nothing, so the + // tree it leaves behind is still the unresolved one. + // + // Consequence, stated rather than hidden: a writer on that arm + // proceeds on a tree that may still be quarantined and, if it + // publishes, publishes those refs. That is today's behavior; + // what changes is that the marker survives until the publish is + // acknowledged instead of being dropped before the write. guard.publish_fence = fence; } Some(Err(RefreshFailure::Download { err, .. })) => { @@ -706,6 +814,11 @@ impl RepoStore { store::init_bare(&local_path).context("initializing bare repo")?; + // A fresh repo is a confirmed tree by the same rule as the swap: nothing + // has ever been published from it, so a marker a deleted predecessor left + // at this path describes nothing and must not fence the new repo. + clear_quarantine(&local_path, repo_name); + // Upload to Tigris in background if let Some(ref tigris) = self.tigris { let tigris = tigris.clone(); @@ -932,16 +1045,41 @@ fn fork_clone_is_ours(disk_path: &Path, attempt: &PublishAttemptId) -> bool { .is_some_and(|owner| owner.trim() == attempt.as_str()) } -/// Remove a fork's clone only while it is still this attempt's. -pub(crate) fn remove_fork_clone_if_ours( +/// The name a claimed stamp is renamed to while the clone it names is being +/// removed: `.{name}.git.fork-attempt.removing.{attempt}`. +fn fork_removal_stamp_path(disk_path: &Path, attempt: &PublishAttemptId) -> Option { + let live = fork_attempt_path(disk_path)?; + let name = live.file_name()?.to_string_lossy().to_string(); + Some(live.with_file_name(format!("{name}.removing.{attempt}"))) +} + +/// A cleanup that has established the clone at `disk_path` is still this +/// attempt's, kept apart from the removal itself so a test can interleave a +/// successor's clone and stamp between the steps. +struct ForkRemovalClaim { + disk_path: PathBuf, + /// The stamp this claim renamed aside. Only this path is ever removed, so a + /// successor's stamp written at the live path afterwards survives. + claimed_stamp: PathBuf, + reason: String, +} + +/// Claim the clone at `disk_path` for `attempt`, by RENAMING its stamp. +/// +/// Read first, then claim, and never the other way round. The refusal path is a +/// pure read that touches nothing, so the rightful owner's own concurrent +/// cleanup always finds its own stamp where it left it; a protocol that renamed +/// first and renamed back on a foreign stamp would open exactly the window it +/// was meant to close. +/// +/// `None` means the stamp is missing, names another attempt, or was claimed by a +/// concurrent remover for the same attempt. In every one of those cases nothing +/// on disk has been changed by this call. +fn begin_fork_removal( disk_path: &Path, attempt: &PublishAttemptId, reason: &str, -) { - if !disk_path.exists() { - let _ = fork_attempt_path(disk_path).map(std::fs::remove_file); - return; - } +) -> Option { if !fork_clone_is_ours(disk_path, attempt) { info!( path = %disk_path.display(), @@ -949,18 +1087,93 @@ pub(crate) fn remove_fork_clone_if_ours( reason, "left the fork clone alone: it no longer belongs to this attempt" ); - return; + return None; } - if let Err(e) = std::fs::remove_dir_all(disk_path) { - warn!( - path = %disk_path.display(), - err = %e, - reason, - "failed to remove fork clone" - ); + let live = fork_attempt_path(disk_path)?; + let claimed_stamp = fork_removal_stamp_path(disk_path, attempt)?; + // The rename is the claim, and it is taken BEFORE the tree is touched. The + // ownership read above is already stale by the time it returns: a successor + // can clone and stamp between it and any later unlink of the live path, and + // an unconditional unlink would then destroy the successor's ownership. + match std::fs::rename(&live, &claimed_stamp) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + // A concurrent remover for this same attempt claimed it first. It + // owns the removal; doing it twice is what would race. + info!( + path = %disk_path.display(), + attempt = %attempt, + reason, + "left the fork clone alone: another cleanup for this attempt already claimed it" + ); + return None; + } + Err(e) => { + warn!( + path = %disk_path.display(), + attempt = %attempt, + err = %e, + reason, + "could not claim the fork clone's stamp — leaving the clone alone rather than \ + removing a tree this attempt cannot prove it still owns" + ); + return None; + } + } + Some(ForkRemovalClaim { + disk_path: disk_path.to_path_buf(), + claimed_stamp, + reason: reason.to_string(), + }) +} + +impl ForkRemovalClaim { + /// Remove the clone. `false` means the tree is still there, so the caller + /// must not go on to drop the stamp that describes it. + fn remove_tree(&self) -> bool { + match std::fs::remove_dir_all(&self.disk_path) { + Ok(()) => true, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => true, + Err(e) => { + warn!( + path = %self.disk_path.display(), + err = %e, + reason = %self.reason, + "failed to remove fork clone" + ); + false + } + } + } + + /// Drop the RENAMED stamp now that the clone it named is gone. + /// + /// The live stamp path is never touched after the claim, so a successor that + /// cloned into the freed name and stamped it keeps its ownership. + fn finish(self) { + let _ = std::fs::remove_file(&self.claimed_stamp); + } +} + +/// Remove a fork's clone only while it is still this attempt's. +pub(crate) fn remove_fork_clone_if_ours( + disk_path: &Path, + attempt: &PublishAttemptId, + reason: &str, +) { + // No directory-absent early return. The old one unlinked the live stamp + // unconditionally on an `exists()` that is stale the moment it answers, which + // is the most reachable form of the race this protocol closes: a successor + // that clones and stamps in that window lost its stamp. An absent directory + // routes through the claim like every other case, and `remove_tree` tolerates + // the `NotFound` it gets. + let Some(claim) = begin_fork_removal(disk_path, attempt, reason) else { + return; + }; + if !claim.remove_tree() { return; } - let _ = fork_attempt_path(disk_path).map(std::fs::remove_file); + claim.finish(); } async fn retry_fork_archive_delete( @@ -1126,6 +1339,22 @@ pub(crate) fn swap_extracted_into_validated_repo( std::fs::remove_dir_all(live).context("removing stale repo dir")?; } std::fs::rename(tmp_dir, live).context("swapping extracted repo into place")?; + // THE CLEAR BELONGS HERE, to the event that installs a confirmed generation, + // not to the callers that happen to observe one. This is the only function + // that replaces a live tree with a stored archive, so every present and + // future download site inherits the clear with no rule to remember, and a + // marker left beside a tree that no longer exists cannot fence the archive + // that replaces it. + // + // The log field is derived from the directory name (so it reads `name.git` + // at this one site) rather than threaded down: carrying `repo_name` through + // `decompress_repo` and `download_to` costs two signatures and a clone into + // the spawn_blocking closure, for a log field. + let logged_name = live + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_default(); + clear_quarantine(live, &logged_name); Ok(()) } @@ -1143,7 +1372,7 @@ const RECONCILE_BOUND: Duration = Duration::from_secs(5); /// and shipped to every node that downloads it, and the marker has to survive /// exactly as long as the directory it describes — a swap that replaces the /// directory wholesale must not carry the old marker along inside it. -#[derive(Debug, serde::Serialize, serde::Deserialize)] +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] struct QuarantineMarker { /// The attempt whose PUT was left unresolved. `None` when the bound expired /// with no dispatched attempt to name (a state that should compensate rather @@ -1152,6 +1381,37 @@ struct QuarantineMarker { at: chrono::DateTime, } +/// What the sidecar says about a live tree, when it says anything. +/// +/// `Unreadable` is a value of its own rather than a marker with no attempt, +/// because the two license opposite actions: a corrupt sidecar must fail shut, +/// and a marker that genuinely names no attempt is a definite non-publication +/// that has to be resolvable or a disconnected first push wedges the repo. +enum Quarantine { + Marker(QuarantineMarker), + Unreadable, +} + +/// Quarantines whose sidecar could not be written, held here so the withholding +/// contract does not rest on a single `fs::write`. A full disk or a read-only +/// mount fails that write, and without this the tree it was meant to withhold is +/// served as an ordinary read. +/// +/// The two media are complementary and both are consulted: disk survives a +/// restart, the map survives a filesystem that will not take the file. +/// +/// Never evicted, the same shape and bound as `tigris::publish_lock`: one entry +/// per repo path that ever failed to write a marker, removed on every clear and +/// on every later successful write. +static UNWRITTEN_QUARANTINES: std::sync::OnceLock< + std::sync::Mutex>, +> = std::sync::OnceLock::new(); + +fn unwritten_quarantines( +) -> &'static std::sync::Mutex> { + UNWRITTEN_QUARANTINES.get_or_init(Default::default) +} + /// Sidecar path for a live repo directory: `.{name}.git.quarantine` beside it. fn quarantine_path(local_path: &Path) -> Option { let parent = local_path.parent()?; @@ -1162,7 +1422,11 @@ fn quarantine_path(local_path: &Path) -> Option { /// Mark the live tree as carrying an unresolved generation. Reads must reconcile /// it before serving; nothing may delete it, because the PUT may have landed and /// this can be the only local copy. -fn quarantine_local_tree(local_path: &Path, repo_name: &str, attempt: Option<&PublishAttemptId>) { +pub(crate) fn quarantine_local_tree( + local_path: &Path, + repo_name: &str, + attempt: Option<&PublishAttemptId>, +) { let Some(path) = quarantine_path(local_path) else { return; }; @@ -1170,25 +1434,38 @@ fn quarantine_local_tree(local_path: &Path, repo_name: &str, attempt: Option<&Pu attempt: attempt.map(|a| a.as_str().to_string()), at: chrono::Utc::now(), }; - let body = match serde_json::to_vec(&marker) { - Ok(body) => body, + let written = serde_json::to_vec(&marker) + .map_err(|e| e.to_string()) + .and_then(|body| std::fs::write(&path, body).map_err(|e| e.to_string())); + match written { + Ok(()) => { + // The disk marker is now the authority for this path, so the shadow + // an earlier failed write left has to go. Leaving it would keep + // refusing after a confirmation cleared the file. + unwritten_quarantines() + .lock() + .expect("quarantine map poisoned") + .remove(&path); + warn!( + repo = %repo_name, + attempt = ?marker.attempt, + "quarantined the local tree: its publish outcome is unresolved, so reads must \ + reconcile it against the store before serving it" + ); + } Err(e) => { - warn!(repo = %repo_name, err = %e, "could not serialize the quarantine marker"); - return; + unwritten_quarantines() + .lock() + .expect("quarantine map poisoned") + .insert(path.clone(), marker.clone()); + warn!( + repo = %repo_name, + err = %e, + attempt = ?marker.attempt, + "could not write the quarantine marker — holding the quarantine in memory \ + for this process instead, so reads still refuse this tree" + ); } - }; - match std::fs::write(&path, body) { - Ok(()) => warn!( - repo = %repo_name, - attempt = ?marker.attempt, - "quarantined the local tree: its publish outcome is unresolved, so reads must \ - reconcile it against the store before serving it" - ), - Err(e) => warn!( - repo = %repo_name, - err = %e, - "failed to write the quarantine marker — reads may serve an unconfirmed tree" - ), } } @@ -1196,10 +1473,16 @@ fn quarantine_local_tree(local_path: &Path, repo_name: &str, attempt: Option<&Pu /// generation again: a publish the store acknowledged, an under-lock refresh /// that overwrote the tree from the stored archive, a reconciliation that found /// the attempt did land, or an invalidation that removed the tree entirely. -fn clear_quarantine(local_path: &Path, repo_name: &str) { +pub(crate) fn clear_quarantine(local_path: &Path, repo_name: &str) { let Some(path) = quarantine_path(local_path) else { return; }; + // The map first: a clear that dropped the file and left the shadow would + // keep refusing a tree the store has already confirmed. + unwritten_quarantines() + .lock() + .expect("quarantine map poisoned") + .remove(&path); match std::fs::remove_file(&path) { Ok(()) => debug!(repo = %repo_name, "cleared the local tree's quarantine"), Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} @@ -1211,16 +1494,38 @@ fn clear_quarantine(local_path: &Path, repo_name: &str) { } } -/// The quarantine on a live tree, when there is one. -fn read_quarantine(local_path: &Path) -> Option { +/// The quarantine on a live tree, when there is one. Disk first, then the +/// in-memory shadow for the markers that could not be written. +fn read_quarantine(local_path: &Path) -> Option { let path = quarantine_path(local_path)?; - let body = std::fs::read(&path).ok()?; - // An unparseable marker still means quarantined. Failing open on a corrupt - // sidecar would serve exactly the tree the sidecar exists to withhold. - Some(serde_json::from_slice(&body).unwrap_or(QuarantineMarker { - attempt: None, - at: chrono::Utc::now(), - })) + match std::fs::read(&path) { + // An unparseable marker still means quarantined, and it means something + // NARROWER than "no attempt": nothing can say what it is waiting on, so + // it fails shut rather than resolving as a definite non-publication. + Ok(body) => Some(match serde_json::from_slice::(&body) { + Ok(marker) => Quarantine::Marker(marker), + Err(_) => Quarantine::Unreadable, + }), + Err(_) => unwritten_quarantines() + .lock() + .expect("quarantine map poisoned") + .get(&path) + .cloned() + .map(Quarantine::Marker), + } +} + +/// Is the live tree at `local_path` carrying a publish nobody has resolved? +/// +/// For readers that resolve the disk path themselves and never ask the store, so +/// they cannot go through [`RepoStore::confirm_live_tree`]. Both media and both +/// marker shapes count: an unresolved publish is unresolved whether the sidecar +/// names an attempt, names none, or cannot be parsed at all. +/// +/// Distinct from the operator-level `quarantined` flag on a repo row, which is a +/// different concept with the same word attached to it. +pub(crate) fn live_tree_publish_unresolved(local_path: &Path) -> bool { + read_quarantine(local_path).is_some() } /// Remove a refused write from the unlocked read cache so `acquire` cannot serve @@ -1539,6 +1844,16 @@ impl RepoSnapshot { pub(crate) fn from_owned_path(path: PathBuf) -> Self { Self { path, owned: true } } + + /// A snapshot borrowing the live tree. Constructible only from a + /// [`ConfirmedLiveTree`], so the fallback cannot skip the quarantine gate. + /// Private, not `pub(crate)`, for the reason the newtype states. + fn live(tree: ConfirmedLiveTree) -> Self { + Self { + path: tree.into_path_buf(), + owned: false, + } + } } impl Drop for RepoSnapshot { @@ -1578,6 +1893,23 @@ pub struct RepoWriteGuard { /// this is the only thing that can tell "the PUT was never constructed" from /// "the PUT is on the wire and may commit" after the future is gone. publish_stage: Arc, + /// Has the live tree already been settled by `release`? + /// + /// `release` classifies the tree on every outcome it reaches, so `Drop` must + /// only settle the ABANDONED path. Without this a definite refusal, which + /// deletes the tree and clears its marker, would be followed by a `Drop` that + /// writes a marker back beside a directory that no longer exists. + tree_settled: bool, + /// Was the writable tree ever handed out through `path()`? + /// + /// This is what answers "may a write have landed" at stage `Idle`, where no + /// publish was ever started: the receive-pack disconnect applies its refs and + /// then loses the guard to the reaper before `release` is entered. Every + /// production writer obtains the tree through `path()`, so obtaining it IS + /// the declaration and no call site has a rule to remember. + /// + /// An atomic rather than a `Cell` because `path()` takes `&self`. + path_handed_out: AtomicBool, /// Test-only seam: when set, `release` parks on this gate at the exact point /// it is about to await `pg_advisory_unlock` (connection still owned, not yet /// released). Dropping the `release` future while it is parked reproduces a @@ -1607,7 +1939,13 @@ impl RepoWriteGuard { } /// Path to the bare repo on local disk. + /// + /// Handing the tree out is also the DECLARATION that a write may land on it. + /// Every production writer goes through here, so an abandoned guard can tell + /// "the refs were applied and nothing published them" from "nothing ever + /// touched this tree" without a per-caller flag anyone could forget to set. pub fn path(&self) -> &Path { + self.path_handed_out.store(true, Ordering::Release); self.local_path.as_path() } @@ -1936,6 +2274,11 @@ impl RepoWriteGuard { } } + // Whatever `release` reached, it has now classified the tree: published, + // refused and invalidated, quarantined, or (on `success == false`) left + // deliberately alone. `Drop` settles only the path that never got here. + self.tree_settled = true; + // Release the advisory lock on the SAME session that took it. Unlocking // through the pool would land on an arbitrary backend, where the call is a // silent no-op. @@ -2007,6 +2350,55 @@ impl Drop for RepoWriteGuard { revoke_swap_authority(&authority); } + // SETTLE THE TREE. A guard that never reached `release` is a write whose + // publication nobody classified, and the live tree it leaves behind is + // served by every later read on filesystem existence alone. The slot's + // own Drop already classified the abandoned attempt; this is the same + // question asked of the tree. + // + // Exhaustive on purpose, no wildcard arm: a new stage must be classified + // here rather than inheriting whatever the catch-all happened to do. + if !self.tree_settled { + let stage = self.publish_stage.get(); + match &stage { + // The store acknowledged this write, so the live tree IS the + // stored generation. Mirrors `release`'s `Released` arm. + PublishStage::Published { .. } => { + clear_quarantine(&self.local_path, &self.repo_name) + } + // Nothing to publish to: the local write is the durable copy. + PublishStage::NoBackend => {} + PublishStage::Idle => { + // A publish was possible and never started. If the tree was + // handed out, refs may already be on it (the receive-pack + // disconnect: git applied them, the reaper killed the group, + // and the guard dropped before `release` was entered), and + // nothing dispatched, so this is a definite non-publication. + // If it was never handed out, nothing touched the tree. + if self.path_handed_out.load(Ordering::Acquire) { + quarantine_local_tree(&self.local_path, &self.repo_name, None); + } + } + // A definite non-publication after a successful write. The marker + // names no attempt, and the next read resolves it by invalidating + // the cache and serving the stored generation. Deliberately NOT a + // `remove_dir_all` here: that would run a blocking delete inside + // `Drop` on a runtime worker, when the read path already runs one. + PublishStage::PreparingArchive | PublishStage::Refused => { + quarantine_local_tree(&self.local_path, &self.repo_name, None) + } + // The PUT may have committed. The marker carries the attempt, so + // a landed PUT can still lift it through reconciliation. + PublishStage::PutDispatched { .. } | PublishStage::Ambiguous { .. } => { + quarantine_local_tree( + &self.local_path, + &self.repo_name, + stage.unresolved_attempt(), + ) + } + } + } + let Some(mut conn) = self.conn.take() else { // release() already unlocked and handed the connection back. return; @@ -3335,6 +3727,8 @@ mod tests { publish_fence: UploadPrecondition::Unconditional, refresh_swap_authority: None, publish_stage: Arc::new(PublishStageCell::new()), + tree_settled: false, + path_handed_out: AtomicBool::new(false), #[cfg(test)] test_pre_unlock_gate: None, #[cfg(test)] @@ -3703,6 +4097,8 @@ mod tests { publish_fence: UploadPrecondition::Unconditional, refresh_swap_authority: None, publish_stage: Arc::new(PublishStageCell::new()), + tree_settled: false, + path_handed_out: AtomicBool::new(false), #[cfg(test)] test_pre_unlock_gate: None, #[cfg(test)] @@ -4594,6 +4990,11 @@ mod tests { /// Conditional DELETEs the mock accepted, so a compensation test can /// assert that a guarded delete did NOT run. deletes: u32, + /// Set by `fail_next_head_for`, decremented per HEAD and answering 500 + /// while positive. Keyed by request path rather than globally, so a + /// test can fail the reconciliation HEAD for ONE repo without also + /// failing the lazy-migration HEAD another key answers. + fail_heads_for: HashMap, } /// An in-process S3-compatible server with REAL conditional semantics. @@ -4794,6 +5195,20 @@ mod tests { } axum::http::Method::HEAD | axum::http::Method::GET => { let mut st = state.lock().unwrap(); + // Fault injection for the reconciliation + // HEAD, which is the one request that turns + // "is the stored archive ours" from a + // decision into a non-answer. GET is left + // alone so a download still works. + if method == axum::http::Method::HEAD { + if let Some(n) = + st.fail_heads_for.get_mut(&key).filter(|n| **n > 0) + { + *n -= 1; + return axum::http::StatusCode::INTERNAL_SERVER_ERROR + .into_response(); + } + } let answered = st.objects.get(&key).cloned(); // Fault injection for the two-consecutive- // losses arm, and the only deterministic way @@ -4928,6 +5343,19 @@ mod tests { self.last().and_then(|o| o.attempt) } + /// Fail the next HEAD for ONE repo key with a 500, so a caller's + /// reconciliation cannot be answered either way. + fn fail_next_head_for(&self, owner_slug: &str, repo_name: &str) { + let key = format!("test-bucket/repos/v1/{owner_slug}/{repo_name}.tar.zst"); + *self + .state + .lock() + .unwrap() + .fail_heads_for + .entry(key) + .or_insert(0) += 1; + } + /// How many DELETEs the mock actually carried out. fn deletes(&self) -> u32 { self.state.lock().unwrap().deletes @@ -7354,4 +7782,972 @@ mod tests { mock.shutdown(); } + + // ── Cycle 1 RED: the residual cache-contract and attempt-identity gaps ── + + /// U1 / gap C. The under-lock refresh clears the quarantine whenever the + /// HEAD answered, including when it answered "nothing stored" and therefore + /// swapped no tree at all. Only an event that REPLACES the live tree with a + /// confirmed generation may lift the marker. + #[sqlx::test] + async fn a_create_only_refresh_does_not_lift_a_quarantine_it_did_not_answer(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let bound = std::time::Duration::from_millis(750); + let store = fenced_store_with_bound(&mock, &opts, repos.path(), bound).await; + let owner = "did:key:z6MkRefreshNoSwapAAAAAAAAAAAAAAAAAAAAAA"; + let repo = "refresh-no-swap-repo"; + + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + marked_repo(&guard.local_path, "unresolved"); + mock.park_next_put(); + assert!(matches!( + guard.release(true).await, + ReleaseOutcome::UploadUnknowable + )); + assert!( + store.acquire(owner, repo).await.is_err(), + "refused while unresolved" + ); + + // A successor whose refresh downloads NOTHING: the parked PUT never + // committed, so the HEAD answers absent and no swap replaces the tree. + let successor = store.acquire_write(owner, repo).await.expect("successor"); + let _ = successor.release(false).await; + + let err = store.acquire(owner, repo).await.expect_err( + "a refresh that downloaded nothing must not lift a quarantine: the live tree \ + still carries the unresolved write", + ); + assert!( + err.downcast_ref::().is_some(), + "the refusal must be the retryable one, got {err:#}" + ); + + // The control: the abandoned PUT reaches the commit point and the + // marker is answered by the store, not by a writer that swapped nothing. + assert_eq!(mock.replay_captured(), 200); + store.acquire(owner, repo).await.expect( + "once the store holds the attempt the quarantine lifts through reconciliation, \ + not through the write", + ); + + mock.open_gate(); + mock.shutdown(); + } + + /// U1 / gap D. A marker can outlive the tree it describes (a crash between + /// an invalidation's `remove_dir_all` and its clear, or a swap that removed + /// the live tree and then failed to rename). The cache-miss download + /// installs a CONFIRMED archive at that path, so it must clear the marker. + #[sqlx::test] + async fn a_quarantine_marker_with_no_tree_does_not_fence_the_downloaded_archive(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let bound = std::time::Duration::from_millis(750); + let store = fenced_store_with_bound(&mock, &opts, repos.path(), bound).await; + let owner = "did:key:z6MkMarkerNoTreeAAAAAAAAAAAAAAAAAAAAAAA"; + let repo = "marker-no-tree-repo"; + let slug = owner_slug_of(owner); + + let seed = TempDir::new().unwrap(); + marked_repo(seed.path(), "seed"); + mock_tigris(&mock) + .upload(&slug, repo, seed.path(), UploadPrecondition::Unconditional) + .await + .expect("seeding the archive"); + + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + std::fs::write(guard.local_path.join("MARKER"), "unresolved").unwrap(); + let local_path = guard.local_path.clone(); + mock.park_next_put(); + assert!(matches!( + guard.release(true).await, + ReleaseOutcome::UploadUnknowable + )); + + std::fs::remove_dir_all(&local_path).unwrap(); + + store.acquire(owner, repo).await.expect( + "a cache-miss download of the confirmed archive must not be refused by a marker \ + left by a tree that no longer exists", + ); + assert!( + !quarantine_path(&local_path).unwrap().exists(), + "the swap that installed the confirmed archive must have cleared the stale marker" + ); + store.acquire(owner, repo).await.expect( + "a tree downloaded from the confirmed archive must not 503 on a later read either", + ); + + mock.open_gate(); + mock.shutdown(); + } + + /// U1 / review open question 2. `init` installs a tree nothing has ever + /// published from, so a marker left by a deleted predecessor at the same + /// path describes nothing and must not fence the new repo. + #[sqlx::test] + async fn init_clears_a_predecessors_quarantine(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let store = fenced_store(&mock, &opts, repos.path()).await; + let owner = "did:key:z6MkInitClearsAAAAAAAAAAAAAAAAAAAAAAAAA"; + let repo = "init-clears-repo"; + + let path = validated_repo_disk_path(repos.path(), owner, repo).expect("path validates"); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + quarantine_local_tree(&path, repo, Some(&PublishAttemptId::new())); + + store.init(owner, repo).await.expect("init"); + + store.acquire(owner, repo).await.expect( + "a freshly initialised repo must not inherit a deleted predecessor's quarantine", + ); + + mock.shutdown(); + } + + /// U2 / gap B. `acquire` reconciles the quarantine; `read_snapshot`'s + /// live-path fallback does not, so the same unconfirmed tree one reader + /// refuses the other hands out. + #[sqlx::test] + async fn read_snapshot_refuses_a_quarantined_tree_when_nothing_is_stored(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let bound = std::time::Duration::from_millis(750); + let store = fenced_store_with_bound(&mock, &opts, repos.path(), bound).await; + let owner = "did:key:z6MkSnapshotRefuseAAAAAAAAAAAAAAAAAAAAA"; + let repo = "snapshot-refuse-repo"; + + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + marked_repo(&guard.local_path, "unresolved"); + let local_path = guard.local_path.clone(); + mock.park_next_put(); + assert!(matches!( + guard.release(true).await, + ReleaseOutcome::UploadUnknowable + )); + + // `let ... else` rather than `expect_err`: `RepoSnapshot` is not `Debug`, + // and deriving it just to phrase an assertion is not this cycle's work. + let Err(err) = store.read_snapshot(owner, repo).await else { + panic!( + "read_snapshot must not hand out a quarantined live tree as a snapshot when \ + the store cannot confirm it" + ); + }; + assert!( + err.downcast_ref::().is_some(), + "the refusal must be the retryable one, got {err:#}" + ); + + // The control. Once the attempt is in the store the read is served, and + // it is served from the store rather than from the live path. + assert_eq!(mock.replay_captured(), 200); + let snapshot = store.read_snapshot(owner, repo).await.expect( + "a confirmed attempt must let read_snapshot serve, from the store, not the live path", + ); + assert_ne!( + snapshot.path(), + local_path.as_path(), + "a confirmed attempt must let read_snapshot serve, from the store, not the live path" + ); + + mock.open_gate(); + mock.shutdown(); + } + + /// U2 / gap B, the degenerate half. With no backend configured nothing can + /// ever confirm the tree, so the fallback is the ONLY path and it is the + /// one that serves. + #[sqlx::test] + async fn read_snapshot_refuses_a_quarantined_tree_with_no_backend(pool: PgPool) { + let _sink = log_sink(); + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let store = RepoStore::new( + repos.path().to_path_buf(), + None, + no_reap_pool(&opts, 2).await, + std::time::Duration::from_secs(30), + ); + let owner = "did:key:z6MkSnapshotNoBackendAAAAAAAAAAAAAAAAAA"; + let repo = "snapshot-no-backend-repo"; + + let path = validated_repo_disk_path(repos.path(), owner, repo).expect("path validates"); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + seed_bare_repo(&path); + quarantine_local_tree(&path, repo, Some(&PublishAttemptId::new())); + + let Err(err) = store.read_snapshot(owner, repo).await else { + panic!( + "with no backend nothing can confirm a quarantined tree, so read_snapshot \ + must refuse rather than serve it" + ); + }; + assert!( + err.downcast_ref::().is_some(), + "the refusal must be the retryable one, got {err:#}" + ); + } + + /// A state whose store publishes to `mock`, plus a PUBLIC repo row whose + /// bare tree lives on local disk and is NOT in object storage, so a parked + /// write can leave it quarantined with nothing stored to confirm it. + async fn quarantined_advert_state( + mock: &S3Mock, + pool: &PgPool, + repos_dir: &Path, + owner: &str, + name: &str, + bound: std::time::Duration, + ) -> crate::state::AppState { + let opts = (*pool.connect_options()).clone(); + let mut state = crate::test_support::test_state(pool.clone()).await; + state.repo_store = fenced_store_with_bound(mock, &opts, repos_dir, bound).await; + let mut config = (*state.config).clone(); + config.repos_dir = repos_dir.to_path_buf(); + state.config = Arc::new(config); + + let now = chrono::Utc::now(); + state + .db + .create_repo(&crate::db::RepoRecord { + id: uuid::Uuid::new_v4().to_string(), + name: name.to_string(), + owner_did: owner.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: format!("/unused/{name}"), + forked_from: None, + machine_id: None, + }) + .await + .expect("seed the repo row"); + state + } + + async fn advertise_receive_pack( + state: &crate::state::AppState, + owner: &str, + name: &str, + ) -> std::result::Result { + crate::api::repos::git_info_refs( + axum::extract::State(state.clone()), + axum::extract::Path(( + crate::db::normalize_owner_key(owner).to_string(), + name.to_string(), + )), + axum::extract::Query(crate::api::repos::InfoRefsQuery { + service: Some("git-receive-pack".to_string()), + }), + crate::rate_limit::PeerAddr(Some("203.0.113.201:5000".parse().unwrap())), + axum::http::HeaderMap::new(), + None, + ) + .await + } + + /// U2 / gap B at the handler. The receive-pack advertisement reads through + /// `read_snapshot`, so an unconfirmed tree is advertised to a pushing client + /// as the base its push will be built on. + #[sqlx::test] + async fn info_refs_for_receive_pack_refuses_a_quarantined_tree_with_503(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let repos = TempDir::new().unwrap(); + let owner = "did:key:z6MkAdvertQuarantineAAAAAAAAAAAAAAAAAAA"; + let repo = "advert-quarantine-repo"; + let bound = std::time::Duration::from_millis(750); + let state = quarantined_advert_state(&mock, &pool, repos.path(), owner, repo, bound).await; + + let guard = state + .repo_store + .acquire_write(owner, repo) + .await + .expect("acquire"); + // A REAL bare repo, not the minimal marked shape: this tree is what git + // advertises from, both on the live path and out of the archive. + store::init_bare(&guard.local_path).expect("a real bare repo"); + mock.park_next_put(); + assert!(matches!( + guard.release(true).await, + ReleaseOutcome::UploadUnknowable + )); + + let err = advertise_receive_pack(&state, owner, repo) + .await + .expect_err( + "info/refs for receive-pack must refuse a quarantined tree the store cannot \ + confirm, not advertise its refs", + ); + assert!( + matches!(err, crate::error::AppError::RepoUnavailable), + "the refusal must be the retryable one, got {err:?}" + ); + + assert_eq!(mock.replay_captured(), 200); + advertise_receive_pack(&state, owner, repo) + .await + .expect("once confirmed, the advertisement serves"); + + mock.open_gate(); + mock.shutdown(); + } + + /// U3 / gap A. `release`'s unknowable arm quarantines; the guard's own + /// `Drop` does not, so a handler cancelled with its PUT on the wire leaves + /// exactly the same state unmarked and servable. + #[sqlx::test] + async fn a_write_guard_dropped_with_its_put_in_flight_quarantines_the_tree(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let store = fenced_store(&mock, &opts, repos.path()).await; + let owner = "did:key:z6MkDropInFlightAAAAAAAAAAAAAAAAAAAAAAA"; + let repo = "drop-in-flight-repo"; + + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + marked_repo(&guard.local_path, "in-flight"); + let local_path = guard.local_path.clone(); + mock.park_next_put(); + + let mut fut = Box::pin(guard.release(true)); + let mut dispatched = false; + for _ in 0..1000 { + let step = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + assert!( + step.is_err(), + "the release must park on the in-flight PUT, not return" + ); + if mock.captured_put().is_some() { + dispatched = true; + break; + } + } + assert!(dispatched, "the PUT must reach the store before the drop"); + + // THE DISCONNECT, with the request on the wire. + drop(fut); + + let err = store.acquire(owner, repo).await.expect_err( + "a write guard dropped with its PUT in flight must leave the tree quarantined, \ + not servable", + ); + assert!( + err.downcast_ref::().is_some(), + "the refusal must be the retryable one, got {err:#}" + ); + assert!( + local_path.exists(), + "the abandoned path must not delete a tree whose PUT may have landed" + ); + + // The control. The marker has to name the attempt, or a PUT that does + // land can never lift it. + assert_eq!(mock.replay_captured(), 200); + store.acquire(owner, repo).await.expect( + "the quarantine a Drop writes must carry the attempt so a landed PUT can lift it", + ); + + mock.open_gate(); + mock.shutdown(); + } + + /// U3 / gap A, the definite half. A cancellation inside the compression + /// window dispatched nothing at all, so the abandoned refs must not be what + /// the next read serves. + #[sqlx::test] + async fn a_write_guard_dropped_during_compression_leaves_reads_served_from_the_store( + pool: PgPool, + ) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let owner = "did:key:z6MkDropCompressAAAAAAAAAAAAAAAAAAAAAAA"; + let repo = "drop-compress-repo"; + let slug = owner_slug_of(owner); + + // Seeded through an ungated client, so the seeding upload's own + // compression does not park. + let seed = TempDir::new().unwrap(); + marked_repo(seed.path(), "seed"); + mock_tigris(&mock) + .upload(&slug, repo, seed.path(), UploadPrecondition::Unconditional) + .await + .expect("seeding the archive"); + + let gate = Arc::new(crate::git::tigris::BlockingGate::shut()); + let store = compression_gated_store( + &mock, + &opts, + repos.path(), + std::time::Duration::from_secs(30), + Arc::clone(&gate), + ) + .await; + + // Snapshot AFTER seeding: the fixture published the seed archive through this + // same mock, so the recorded PUTs are never empty. The property under test is + // that THIS guard's abandoned release dispatched nothing, not that the mock + // never saw a PUT at all. + let puts_before = mock.put_attempts().len(); + + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + std::fs::write(guard.local_path.join("MARKER"), "abandoned").unwrap(); + let local_path = guard.local_path.clone(); + let stage = guard.publish_stage(); + + let mut fut = Box::pin(guard.release(true)); + let mut parked = false; + for _ in 0..1000 { + let step = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + assert!( + step.is_err(), + "the release must park inside the compression, not return" + ); + if stage.get() == PublishStage::PreparingArchive { + parked = true; + break; + } + } + assert!(parked, "the release must reach the compression stage"); + + drop(fut); + gate.open(); + + let served = store.acquire(owner, repo).await.expect( + "a write abandoned before any PUT was dispatched is a definite non-publication: \ + the next read must serve the stored generation, not the abandoned refs", + ); + assert_eq!( + std::fs::read_to_string(served.join("MARKER")).unwrap_or_default(), + "seed", + "a write abandoned before any PUT was dispatched is a definite non-publication: \ + the next read must serve the stored generation, not the abandoned refs" + ); + assert!( + !quarantine_path(&local_path).unwrap().exists(), + "a no-attempt marker must resolve on the next read, never stand as a permanent 503" + ); + assert_eq!( + mock.put_attempts().len(), + puts_before, + "nothing was dispatched, so nothing may have been published" + ); + + mock.shutdown(); + } + + /// U3 / gap A, the same cancellation on a repo with nothing stored. There + /// is no confirmed generation to fall back to, so the only safe answer is + /// that the abandoned refs are not left on disk. + #[sqlx::test] + async fn a_write_guard_dropped_during_compression_with_nothing_stored_leaves_no_servable_refs( + pool: PgPool, + ) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let owner = "did:key:z6MkDropCompressBareAAAAAAAAAAAAAAAAAAA"; + let repo = "drop-compress-bare-repo"; + + let gate = Arc::new(crate::git::tigris::BlockingGate::shut()); + let store = compression_gated_store( + &mock, + &opts, + repos.path(), + std::time::Duration::from_secs(30), + Arc::clone(&gate), + ) + .await; + + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + marked_repo(&guard.local_path, "abandoned"); + let local_path = guard.local_path.clone(); + let stage = guard.publish_stage(); + + let mut fut = Box::pin(guard.release(true)); + let mut parked = false; + for _ in 0..1000 { + let step = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + assert!( + step.is_err(), + "the release must park inside the compression, not return" + ); + if stage.get() == PublishStage::PreparingArchive { + parked = true; + break; + } + } + assert!(parked, "the release must reach the compression stage"); + + drop(fut); + gate.open(); + + let _ = store.acquire(owner, repo).await; + assert!( + !local_path.exists(), + "a definite non-publication on a repo with nothing stored must not leave its \ + refs on disk" + ); + + mock.shutdown(); + } + + /// U3 / gap A at stage `Idle`: the receive-pack disconnect. The refs are + /// already applied to the live tree and the guard is dropped by the reaper + /// before `release` is ever entered, so no stage past `Idle` is reached and + /// nothing records that the tree was written. + #[sqlx::test] + async fn a_write_guard_dropped_at_idle_after_handing_out_its_path_quarantines_the_tree( + pool: PgPool, + ) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let store = fenced_store(&mock, &opts, repos.path()).await; + let owner = "did:key:z6MkDropIdleHandedAAAAAAAAAAAAAAAAAAAAA"; + let repo = "drop-idle-handed-repo"; + let slug = owner_slug_of(owner); + + let seed = TempDir::new().unwrap(); + marked_repo(seed.path(), "seed"); + mock_tigris(&mock) + .upload(&slug, repo, seed.path(), UploadPrecondition::Unconditional) + .await + .expect("seeding the archive"); + + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + let p = guard.path().to_path_buf(); + std::fs::write(p.join("MARKER"), "applied").unwrap(); + let local_path = guard.local_path.clone(); + drop(guard); + + let marker_path = quarantine_path(&local_path).expect("the marker path"); + assert!( + marker_path.exists(), + "a guard dropped after its tree was handed out for writing must mark the tree, \ + whatever stage the publish reached" + ); + let body: serde_json::Value = serde_json::from_slice(&std::fs::read(&marker_path).unwrap()) + .expect("the marker is JSON"); + assert_eq!( + body["attempt"], + serde_json::Value::Null, + "a guard dropped after its tree was handed out for writing must mark the tree, \ + whatever stage the publish reached" + ); + + let served = store + .acquire(owner, repo) + .await + .expect("refs applied by a write that never reached release must not be served"); + assert_eq!( + std::fs::read_to_string(served.join("MARKER")).unwrap_or_default(), + "seed", + "refs applied by a write that never reached release must not be served" + ); + + mock.shutdown(); + } + + /// THE CONTROL for the arm above. Every production writer obtains the tree + /// through `path()`, so a guard that never handed it out cannot have been + /// written through and has nothing to settle. + #[sqlx::test] + async fn a_write_guard_dropped_at_idle_without_handing_out_its_path_leaves_no_marker( + pool: PgPool, + ) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let store = fenced_store(&mock, &opts, repos.path()).await; + let owner = "did:key:z6MkDropIdleUntouchedAAAAAAAAAAAAAAAAAA"; + let repo = "drop-idle-untouched-repo"; + + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + let local_path = guard.local_path.clone(); + drop(guard); + + assert!( + !quarantine_path(&local_path).unwrap().exists(), + "a guard that never handed out its tree has nothing to settle" + ); + + mock.shutdown(); + } + + /// THE CONTROL that pins the settlement to the ABANDONED path only. A + /// release that ended in a definite refusal already deleted the tree and + /// cleared the marker; the guard's Drop must not then write one back. + #[sqlx::test] + async fn a_definite_refusal_leaves_no_marker_beside_the_deleted_tree(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let store = fenced_store(&mock, &opts, repos.path()).await; + let owner = "did:key:z6MkDefiniteRefusalAAAAAAAAAAAAAAAAAAAA"; + let repo = "definite-refusal-repo"; + let slug = owner_slug_of(owner); + + let seed = TempDir::new().unwrap(); + marked_repo(seed.path(), "seed"); + mock_tigris(&mock) + .upload(&slug, repo, seed.path(), UploadPrecondition::Unconditional) + .await + .expect("seeding the archive"); + + // Both of the release's HEADs lose their generation, so the fence and + // its one supersede-retry are both refused. + mock.roll_generation_after_next_heads(2); + + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + std::fs::write(guard.local_path.join("MARKER"), "writer").unwrap(); + let local_path = guard.local_path.clone(); + let outcome = guard.release(true).await; + + assert!( + matches!(outcome, ReleaseOutcome::Fenced), + "the write must be definitively refused, got {outcome:?}" + ); + assert!( + !local_path.exists(), + "a definite refusal invalidates the local write cache" + ); + assert!( + !quarantine_path(&local_path).unwrap().exists(), + "a release that already settled the tree must not be re-settled by Drop: no \ + marker may exist beside a tree the definite refusal deleted" + ); + + mock.shutdown(); + } + + /// U4 / gap E. The whole withholding contract rests on one `fs::write`. A + /// full disk or a read-only mount fails it, and the tree is then served as + /// an ordinary read. + #[sqlx::test] + async fn an_unwritable_quarantine_marker_still_refuses_reads(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let bound = std::time::Duration::from_millis(750); + let store = fenced_store_with_bound(&mock, &opts, repos.path(), bound).await; + let owner = "did:key:z6MkUnwritableMarkerAAAAAAAAAAAAAAAAAAA"; + let repo = "unwritable-marker-repo"; + + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + marked_repo(&guard.local_path, "unresolved"); + let local_path = guard.local_path.clone(); + // A directory at the marker path makes `fs::write` fail with EISDIR + // regardless of privilege; a chmod is silently ineffective as root. + std::fs::create_dir(quarantine_path(&local_path).unwrap()).unwrap(); + mock.park_next_put(); + assert!(matches!( + guard.release(true).await, + ReleaseOutcome::UploadUnknowable + )); + + let err = store.acquire(owner, repo).await.expect_err( + "an unwritable quarantine marker must still refuse reads: the contract cannot \ + rest on one fs::write", + ); + assert!( + err.downcast_ref::().is_some(), + "the refusal must be the retryable one, got {err:#}" + ); + + // The control: the in-memory half must lift the same way the on-disk + // half does, or it is a permanent outage instead of a withholding. + assert_eq!(mock.replay_captured(), 200); + store.acquire(owner, repo).await.expect( + "the in-memory quarantine must lift through the same reconciliation as the \ + on-disk one", + ); + + mock.open_gate(); + mock.shutdown(); + } + + /// THE CONTROL for the two media. A marker that reaches disk has to replace + /// the in-memory shadow an earlier failed write left, or clearing the file + /// leaves the shadow refusing forever. + #[sqlx::test] + async fn a_marker_that_reaches_disk_replaces_its_in_memory_shadow(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let bound = std::time::Duration::from_millis(750); + let store = fenced_store_with_bound(&mock, &opts, repos.path(), bound).await; + let owner = "did:key:z6MkMarkerShadowAAAAAAAAAAAAAAAAAAAAAAA"; + let repo = "marker-shadow-repo"; + + // First unresolved write: the marker path is a directory, so the write + // fails and only the in-memory half can hold the quarantine. + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + marked_repo(&guard.local_path, "first"); + let local_path = guard.local_path.clone(); + let marker_path = quarantine_path(&local_path).unwrap(); + std::fs::create_dir(&marker_path).unwrap(); + mock.park_next_put(); + assert!(matches!( + guard.release(true).await, + ReleaseOutcome::UploadUnknowable + )); + + // The path becomes writable, and a second unresolved write reaches disk. + std::fs::remove_dir(&marker_path).unwrap(); + let second = store.acquire_write(owner, repo).await.expect("acquire"); + std::fs::write(second.local_path.join("MARKER"), "second").unwrap(); + mock.park_next_put(); + assert!(matches!( + second.release(true).await, + ReleaseOutcome::UploadUnknowable + )); + assert!( + marker_path.exists(), + "the second quarantine must reach disk once the path is writable" + ); + + // Lift the second attempt and clear the file the way a confirmation + // does. Nothing may still be refusing after that. + assert_eq!(mock.replay_captured(), 200); + clear_quarantine(&local_path, repo); + assert!( + read_quarantine(&local_path).is_none(), + "a marker that reached disk must have replaced the in-memory shadow, or the \ + shadow keeps refusing after the disk marker was cleared" + ); + + mock.open_gate(); + mock.shutdown(); + } + + // ── U5: fork clone removal claims the stamp (#285 gap F) ─────────────── + + /// U5 / gap F. The removal reads ownership and then unlinks the LIVE stamp + /// path, so a successor that clones and stamps between the two loses its + /// stamp and its clone becomes unowned. + #[test] + fn fork_cleanup_after_the_directory_is_freed_leaves_a_successors_stamp_alone() { + let repos = TempDir::new().unwrap(); + let disk = repos.path().join("owner").join("forked.git"); + std::fs::create_dir_all(&disk).unwrap(); + let failed = PublishAttemptId::new(); + let successor = PublishAttemptId::new(); + claim_fork_disk_path(&disk, &failed); + + let claim = begin_fork_removal(&disk, &failed, "test").expect("A owns the clone"); + assert!(claim.remove_tree(), "the tree removal must succeed"); + + // The successor clones into the freed name and stamps it before the + // failed attempt finishes its cleanup. + std::fs::create_dir_all(&disk).unwrap(); + claim_fork_disk_path(&disk, &successor); + + claim.finish(); + + let stamp = fork_attempt_path(&disk).unwrap(); + assert_eq!( + std::fs::read_to_string(&stamp).unwrap_or_default().trim(), + successor.as_str(), + "a failed attempt's cleanup must not remove the stamp a successor wrote after \ + the directory was freed" + ); + assert!( + disk.exists(), + "the successor's clone must survive the failed attempt's cleanup" + ); + } + + /// U5 / gap F, the protocol itself. Ownership has to be claimed by RENAME + /// before the tree is touched; a read leaves a window a successor can land + /// in. + #[test] + fn fork_cleanup_claims_the_stamp_by_rename_before_touching_the_tree() { + let repos = TempDir::new().unwrap(); + let disk = repos.path().join("owner").join("claimed.git"); + std::fs::create_dir_all(&disk).unwrap(); + let attempt = PublishAttemptId::new(); + claim_fork_disk_path(&disk, &attempt); + + let _claim = begin_fork_removal(&disk, &attempt, "test").expect("this attempt owns it"); + + let live = fork_attempt_path(&disk).unwrap(); + let renamed = fork_removal_stamp_path(&disk, &attempt).unwrap(); + assert!( + !live.exists() && renamed.exists(), + "ownership must be claimed by rename before the tree is touched, or a successor \ + can re-stamp between the read and the unlink" + ); + } + + /// U5 / gap F, the F1 arm. The directory-absent early return unlinks the + /// live stamp unconditionally, on an `exists()` that is already stale. + #[test] + fn fork_cleanup_with_the_directory_absent_leaves_a_successors_stamp_alone() { + let repos = TempDir::new().unwrap(); + let disk = repos.path().join("owner").join("absent.git"); + std::fs::create_dir_all(disk.parent().unwrap()).unwrap(); + let failed = PublishAttemptId::new(); + let successor = PublishAttemptId::new(); + claim_fork_disk_path(&disk, &successor); + + remove_fork_clone_if_ours(&disk, &failed, "test"); + + let stamp = fork_attempt_path(&disk).unwrap(); + assert_eq!( + std::fs::read_to_string(&stamp).unwrap_or_default().trim(), + successor.as_str(), + "a cleanup that finds no directory must not unlink a stamp that is not its own" + ); + } + + /// THE CONTROL. A refusal must be a pure read: renaming a foreign stamp and + /// renaming it back is not equivalent, because the rightful owner's own + /// cleanup can look during the window and find nothing. + #[cfg(unix)] + #[test] + fn fork_cleanup_refuses_a_foreign_stamp_without_touching_it() { + use std::os::unix::fs::MetadataExt; + + let repos = TempDir::new().unwrap(); + let disk = repos.path().join("owner").join("foreign.git"); + std::fs::create_dir_all(&disk).unwrap(); + let successor = PublishAttemptId::new(); + let failed = PublishAttemptId::new(); + claim_fork_disk_path(&disk, &successor); + + let stamp = fork_attempt_path(&disk).unwrap(); + let ctime_before = std::fs::metadata(&stamp).unwrap().ctime(); + let ctime_ns_before = std::fs::metadata(&stamp).unwrap().ctime_nsec(); + + assert!( + begin_fork_removal(&disk, &failed, "test").is_none(), + "a stamp naming another attempt must refuse the claim" + ); + + let meta = std::fs::metadata(&stamp).unwrap(); + assert!( + std::fs::read_to_string(&stamp).unwrap_or_default().trim() == successor.as_str() + && meta.ctime() == ctime_before + && meta.ctime_nsec() == ctime_ns_before, + "a refused cleanup must not move the successor's stamp even briefly, or the \ + owner's own cleanup can miss it" + ); + } + + // ── U6: a failed reconciliation HEAD on the fork path (#285 gap G) ───── + + /// Seed a FOREIGN archive under the fork's key, so the fork's create-only + /// PUT is refused and the handler has to reconcile. + async fn seed_foreign_fork_archive(mock: &S3Mock, forker: &str, fork_name: &str) { + let foreign = TempDir::new().unwrap(); + marked_repo(foreign.path(), "someone-else"); + mock_tigris(mock) + .upload_tracked( + &owner_slug_of(forker), + fork_name, + foreign.path(), + UploadPrecondition::Unconditional, + PublishAttemptId::from_owned("someone-else"), + None, + ) + .await + .expect("seed the foreign archive under the fork key"); + } + + /// U6 / gap G. A reconciliation HEAD that FAILS is collapsed into "the + /// archive is not ours" and answered with a permanent name conflict, so a + /// storage blip burns the fork name. + #[sqlx::test] + async fn a_fork_whose_reconciliation_head_fails_after_a_lost_precondition_is_refused_retryably( + pool: PgPool, + ) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let repos = TempDir::new().unwrap(); + let source_owner = "did:key:z6MkForkSourceHeadFailAAAAAAAAAAAAAAAA"; + let forker = "did:key:z6MkForkerHeadFailAAAAAAAAAAAAAAAAAAAAA"; + let state = fork_state(&mock, &pool, repos.path(), source_owner, "src").await; + + seed_foreign_fork_archive(&mock, forker, "headfail").await; + // Keyed by object key, so the source repo's lazy-migration HEAD is not + // the one that fails. + mock.fail_next_head_for(&owner_slug_of(forker), "headfail"); + + let err = do_fork(&state, source_owner, "src", forker, "headfail") + .await + .expect_err("an unanswered reconciliation must not report a definite conflict"); + assert!( + matches!(err, crate::error::AppError::RepoUnavailable), + "a fork whose reconciliation HEAD failed must be refused retryably, not \ + reported as a permanent name conflict, got {err:?}" + ); + assert!( + !repos + .path() + .join(owner_slug_of(forker)) + .join("headfail.git") + .exists(), + "a refused precondition proves this attempt's PUT never landed, so its clone \ + protects nothing and must not wedge the fork name against the retry the 503 \ + invites" + ); + assert_eq!( + mock.deletes(), + 0, + "nothing may be compensated on an unanswered reconciliation" + ); + + mock.shutdown(); + } + + /// THE CONTROL. A reconciliation that ANSWERS, and answers "not ours", is a + /// real name conflict and stays a permanent refusal. + #[sqlx::test] + async fn a_fork_whose_precondition_is_lost_to_a_foreign_archive_is_refused_as_exists( + pool: PgPool, + ) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let repos = TempDir::new().unwrap(); + let source_owner = "did:key:z6MkForkSourceForeignAAAAAAAAAAAAAAAAA"; + let forker = "did:key:z6MkForkerForeignAAAAAAAAAAAAAAAAAAAAAA"; + let state = fork_state(&mock, &pool, repos.path(), source_owner, "src").await; + + seed_foreign_fork_archive(&mock, forker, "foreign").await; + + let err = do_fork(&state, source_owner, "src", forker, "foreign") + .await + .expect_err("an archive that is not ours under the fork name refuses the fork"); + assert!( + matches!(err, crate::error::AppError::RepoExists(_)), + "a stored archive that is not ours under the fork name is a real conflict, \ + got {err:?}" + ); + + mock.shutdown(); + } } diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 5ee09aa5..c459a742 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -676,7 +676,20 @@ async fn warm_candidates( &repo.owner_did, &repo.name, ) { - Ok(p) if p.is_dir() => out.push((repo, created_at_key, p.into_path_buf())), + // A tree whose publish is unresolved is filtered HERE, before the + // candidate is ever probed: discovery records the repo as a source + // for a CID, and refs that may never become durable must not become + // durable provenance. This is a different filter from the `quarantined` + // status flag dropped at candidate-load time; that one is an operator + // decision about a mirror, this one is "the store cannot confirm what + // is on disk right now". An unresolved candidate falls into the arm + // below and is accounted exactly as a cold one: absent, not evidence. + Ok(p) + if p.is_dir() + && !crate::git::repo_store::live_tree_publish_unresolved(p.as_path()) => + { + out.push((repo, created_at_key, p.into_path_buf())) + } Ok(_) => {} Err(e) => { tracing::warn!(repo_id = %repo.id, err = %e, "sweep discovery: rejected unsafe repo path"); @@ -1015,6 +1028,16 @@ async fn sweep_pass( row_retryable = true; continue; } + // The repo is on disk but its last publish is unresolved, so the refs it + // holds may never become durable. The sweep does not merely read here: it + // rewrites the legacy provider CID from whatever these objects say, which + // is state the resolver then serves. Skip, and count it retryable rather + // than settled: an unresolved publish resolves one way or the other, so + // the row is worth walking again, unlike a repo that is simply gone. + if crate::git::repo_store::live_tree_publish_unresolved(repo_path.as_path()) { + row_retryable = true; + continue; + } // The sweep holds no pin permit and has no batch to overrun, so the plain // `git_timeout` is the right budget here. row_read_attempted = true; diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index e5c9b0c5..3e8040a2 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -8238,6 +8238,177 @@ mod tests { ); } + // ── #285 U7: the sweep skips trees whose publish is unresolved ───────── + // + // Two different things are called "quarantine" on this node. The operator + // status flag on the repo row, which the test above pins, and the sidecar + // marker beside a live tree whose publish never resolved. The sweep has + // never heard of the second. It does not merely read: it rewrites provider + // CIDs and records the repo as a source, durable state fed to the resolver, + // out of refs that may never become durable. + + /// #285 U7, gap-driving. A KNOWN source (the row carries `repo_id`) whose + /// live tree is marked unresolved must be skipped before any object bytes + /// are read, and accounted the way a cold candidate is: retryable, so the + /// row is re-walked once the publish resolves. + #[sqlx::test] + async fn sweep_skips_a_source_whose_publish_is_unresolved_and_leaves_the_row_retryable( + pool: PgPool, + ) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + let fx = seed_cid_repos(&slug, &short, &["swsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("swsrc.git"); + let repo = seed_repo(&owner_did, "swsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + let (_raw_cid, provider_cid) = + seed_legacy_pin(&pool, &bare, &fx.public_oid, Some(&repo.id)).await; + + // The sidecar an unresolved publish leaves beside the live tree. + crate::git::repo_store::quarantine_local_tree( + &bare, + "swsrc", + Some(&crate::git::publish::PublishAttemptId::new()), + ); + + crate::ipfs_pin::reset_legacy_repair_reads(); + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the sweep terminates"); + + assert_eq!( + stats.repaired, 0, + "the sweep must not read objects out of a tree whose publish is unresolved" + ); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "an unresolved tree is skipped before any object bytes are read" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + provider_cid, + "the legacy key must not be rewritten from an unconfirmed tree" + ); + assert_eq!( + stats.retryable_skips, 1, + "an unresolved source must be a retryable skip, like a cold one, so the row is re-walked once the publish resolves" + ); + + // THE CONTROL: the same row, the same bytes, the marker gone. + crate::git::repo_store::clear_quarantine(&bare, "swsrc"); + state.db.set_pin_repair_cursor("").await.unwrap(); + crate::ipfs_pin::reset_legacy_repair_reads(); + let second = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the second run terminates"); + assert_eq!( + second.repaired, 1, + "once the publish resolves the same row repairs" + ); + } + + /// #285 U7, gap-driving. The discovery half: the row names no repo, so the + /// sweep goes looking for a warm holder. A holder whose publish is + /// unresolved must be filtered at warm-check time, before any probe, and + /// must never be recorded as a source. + #[sqlx::test] + async fn sweep_discovery_skips_a_warm_candidate_whose_publish_is_unresolved(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["unressrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("unressrc.git"); + let repo = seed_repo(&owner_did, "unressrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + // The DB status is deliberately left alone: this candidate is healthy as + // far as the operator flag is concerned, and only the sidecar withholds it. + let (_raw_cid, provider_cid) = seed_legacy_pin(&pool, &bare, &fx.public_oid, None).await; + crate::git::repo_store::quarantine_local_tree( + &bare, + "unressrc", + Some(&crate::git::publish::PublishAttemptId::new()), + ); + + crate::ipfs_pin::reset_legacy_repair_reads(); + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the sweep terminates"); + + assert_eq!( + stats.repaired, 0, + "discovery must not probe the objects of a candidate whose publish is unresolved" + ); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "the marker filters the candidate at warm-check time, before any probe" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + provider_cid, + "the row keeps its provider key" + ); + assert_eq!( + state + .db + .pin_sources_for_oid(&fx.public_oid) + .await + .unwrap() + .len(), + 0, + "no source may be recorded from an unconfirmed tree" + ); + } + /// F1 scenario 4 (#173, MUST-NOT): a candidate that is not on local disk is COLD. /// Discovery must not pull it back from remote storage (the sweep is opportunistic /// background maintenance, not a bulk restore), and it must not mark the row From f048a1e50171438a147fbf539da3240e0e07ae1d Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 7 Sep 2026 01:29:45 -0500 Subject: [PATCH 53/54] fix(node): settle the abandoned tree at the disconnect, not at reaper exit The settlement that marks an abandoned write ran from RepoWriteGuard's Drop, and on a client disconnect that guard is held by the Arc clone riding the admission guard into the detached reaper, so it ran only once the process group had been reaped. Reads take no advisory lock and consult only the sidecar, so for the width of that window a concurrent fetch was served the abandoned refs off the live tree with no marker beside it. Confirmed by execution before the fix: no marker on disk, acquire returning the live path, and the abandoned ref still on the tree it served. Only the lock release has to wait for the reaper; the settlement does not. TreeSettlement is a token carrying the path, the publish stage and the hand-out flag, holding the exhaustive stage match that used to live in Drop. The handler takes it out of the guard before the guard is shared with the reaper, so on a disconnect the handler future drops first and the marker lands at the disconnect instant while the reaper still holds the lock. A successful release disarms it. Drop still settles a guard nobody took the token from, so the arms have one home. The regression test reads inside the window and asserts the abandoned refs are not served. It samples pg_locks from an independent session first, so a run where the reaper had already finished reports itself inconclusive rather than passing for the wrong reason. Suite 1212 passed, 0 failed, 1 ignored. The five settlement mutations are load-bearing, including one that reverts this split and reddens the window test. --- crates/gitlawb-node/src/api/repos.rs | 130 ++++++++++++++- crates/gitlawb-node/src/git/repo_store.rs | 185 +++++++++++++++++----- 2 files changed, 271 insertions(+), 44 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index b646e4bf..72cbf32d 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -2278,7 +2278,7 @@ pub async fn git_receive_pack( // permit is a handler local here (moved into the AdmissionGuard only after this), // so the early return on timeout drops it and frees the slot; shed a bounded 503. let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); - let guard = tokio::time::timeout( + let mut guard = tokio::time::timeout( acquire_deadline, state .repo_store @@ -2314,6 +2314,17 @@ pub async fn git_receive_pack( // this copy dies with the future and the reaper's copy is last, so the lock frees // after the reap with no upload, which is the release(success = false) semantics an // interrupted push must have. + // THE SPLIT (#285 U3). Settling the tree and freeing the advisory lock do not + // have to happen at the same moment, and only the unlock has to wait for the + // reaper. The guard below is shared, so on a disconnect its last copy is the + // one riding `KillGroupOnDrop`, and a settlement done from `RepoWriteGuard::drop` + // would therefore land only after the process group is torn down. `acquire` + // takes no advisory lock on its way to the live tree, so a read inside that + // window would be served refs no publish ever confirmed. Holding the + // settlement in THIS future puts the marker on disk at the disconnect instant, + // while the reaper still holds the lock. The success path disarms it below, + // after `release` has classified the tree itself. + let mut settlement = guard.take_settlement(); let guard = std::sync::Arc::new(std::sync::Mutex::new(Some(guard))); // Clone (a) of the write lease rides this AdmissionGuard: on a client disconnect the // guard moves into KillGroupOnDrop's detached reaper, so the lease frees only after @@ -2405,6 +2416,12 @@ pub async fn git_receive_pack( // certificates or answering 200 would all be reporting a write no other // node can read. let outcome = reclaimed.release(push_succeeded).await; + // `release` classified the tree on whatever outcome it reached, so this + // future's copy of the settlement must not classify it a second time: a + // definite refusal has already deleted the tree and cleared its marker. + if let Some(settlement) = settlement.as_mut() { + settlement.disarm(); + } if push_succeeded { publish_durability.record(outcome).await; } @@ -11559,6 +11576,117 @@ exit 0 server.abort(); } + /// #285 U3, the REAP WINDOW. The sibling above waits for the marker before + /// it reads, so it proves the settled state and cannot see the interval + /// between the disconnect and the settlement. This one reads inside that + /// interval: `RepoWriteGuard::drop` runs on the guard clone that rode the + /// admission guard into `KillGroupOnDrop`'s detached reaper, so the marker + /// cannot exist until the receive-pack group is torn down, and `acquire` + /// takes no advisory lock on the way to the live tree. + /// + /// The read happens immediately after the handler future is dropped, while + /// the fake receive-pack is still sleeping, so the guard is still alive in + /// the reaper. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_read_inside_the_reap_window_serves_no_abandoned_refs(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let gate = Arc::new(crate::git::tigris::BlockingGate::shut()); + let (mut state, _log, _puts, server) = p3_compression_gated_state( + pool.clone(), + tmp.path(), + "z6p3rwin", + "c1", + Arc::clone(&gate), + ) + .await; + let gitdir = tmp.path().join("hanging-git"); + std::fs::create_dir_all(&gitdir).unwrap(); + state.git_bin = write_fake_git( + &gitdir, + r#"#!/bin/sh +case "$1" in + receive-pack) + mkdir -p "$3/refs/heads" + printf '%s\n' 1111111111111111111111111111111111111111 > "$3/refs/heads/main" + sleep 30 + ;; + *) : ;; +esac +exit 0 +"#, + ); + + let rec = state.db.get_repo("z6p3rwin", "c1").await.unwrap().unwrap(); + let repos_dir = tmp.path().join("repos"); + let marker = p3_quarantine_marker(&repos_dir, &rec.owner_did, "c1"); + let live = + crate::git::repo_store::validated_repo_disk_path(&repos_dir, &rec.owner_did, "c1") + .expect("test repo path") + .into_path_buf(); + let ref_path = live.join("refs").join("heads").join("main"); + + let mut fut = Box::pin(p2_push(&state, "z6p3rwin", "c1")); + let mut applied = false; + for _ in 0..2000 { + let step = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + assert!( + step.is_err(), + "the handler must park inside the hanging receive-pack, not return" + ); + if ref_path.exists() { + applied = true; + break; + } + } + assert!(applied, "the push must apply its ref before the disconnect"); + + // THE DISCONNECT. Nothing parks on the gate here, opened only so a + // failing assertion reports instead of hanging teardown. + drop(fut); + gate.open(); + tokio::task::yield_now().await; + + // THE PROBE, inside the window. What makes the read INSIDE it is that the + // receive-pack group is still being reaped, and the write lock rides that + // reaper: the guard's last copy is held by `KillGroupOnDrop`, so the + // advisory lock is still taken at this instant. Sampled before the read, + // from an independent session, and filtered to this test's own database, + // which `#[sqlx::test]` gives it exclusively. + // + // Keying conclusiveness on the LOCK rather than on the marker's absence: + // the marker is what the settlement writes, so requiring it to be missing + // would demand the very state the fix removes, and no correct + // implementation could satisfy it. + let locks_held: (i64,) = sqlx::query_as( + "SELECT count(*) FROM pg_locks WHERE locktype = 'advisory' \ + AND database = (SELECT oid FROM pg_database WHERE datname = current_database())", + ) + .fetch_one(&pool) + .await + .expect("count the advisory locks this test's database holds"); + let marker_before = marker.exists(); + let served = state.repo_store.acquire(&rec.owner_did, "c1").await; + let refs_after = ref_path.exists(); + let served_ok = served.is_ok(); + + assert!( + locks_held.0 >= 1, + "INCONCLUSIVE: the write lock was already free when the probe ran, so the \ + reaper had finished and this read never observed the reap window" + ); + assert!( + !(served_ok && refs_after), + "a read inside the reap window served the abandoned refs: marker_before={}, \ + acquire={:?}, ref still on the served tree={}", + marker_before, + served, + refs_after + ); + + server.abort(); + } + /// THE CONTROL for both of the above. Same builder, no disconnect: the /// publish completes, the store acknowledges the PUT, and the tree stays /// readable with nothing withholding it. Without this a green refusal above diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index b1c6aa79..0dbc92d0 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -669,7 +669,7 @@ impl RepoStore { None => PublishStage::NoBackend, })), tree_settled: false, - path_handed_out: AtomicBool::new(false), + path_handed_out: Arc::new(AtomicBool::new(false)), #[cfg(test)] test_pre_unlock_gate: self.pre_unlock_gate.clone(), #[cfg(test)] @@ -1908,8 +1908,9 @@ pub struct RepoWriteGuard { /// production writer obtains the tree through `path()`, so obtaining it IS /// the declaration and no call site has a rule to remember. /// - /// An atomic rather than a `Cell` because `path()` takes `&self`. - path_handed_out: AtomicBool, + /// An atomic rather than a `Cell` because `path()` takes `&self`, and + /// shared with the settlement token so a hand-out is seen from either side. + path_handed_out: Arc, /// Test-only seam: when set, `release` parks on this gate at the exact point /// it is about to await `pg_advisory_unlock` (connection still owned, not yet /// released). Dropping the `release` future while it is parked reproduces a @@ -2060,6 +2061,36 @@ impl RepoWriteGuard { Arc::clone(&self.publish_stage) } + /// Take the tree settlement out of this guard. + /// + /// The caller becomes responsible for classifying the live tree an + /// abandoned write leaves behind, and this guard stops doing it. That is + /// the point: a shared guard frees its advisory lock only when the last + /// copy drops, which on a disconnect is the detached reaper, whereas the + /// marker has to be on disk before the next read reaches the live tree. + /// + /// `None` once the tree is settled: `release` classifies it on every + /// outcome it reaches, and there is nothing left to hand out. + pub fn take_settlement(&mut self) -> Option { + if self.tree_settled { + return None; + } + self.tree_settled = true; + Some(self.settlement()) + } + + /// An armed settlement over this guard's tree. Shares the publish stage and + /// the hand-out flag, so it reads the same state the guard would. + fn settlement(&self) -> TreeSettlement { + TreeSettlement { + local_path: self.local_path.clone(), + repo_name: self.repo_name.clone(), + publish_stage: Arc::clone(&self.publish_stage), + path_handed_out: Arc::clone(&self.path_handed_out), + armed: true, + } + } + /// Decide what a publish that ran out its bound actually left behind. /// /// "The bounded transfer returned `None`" is not one state. `publish` awaits @@ -2344,6 +2375,92 @@ impl RepoWriteGuard { } } +/// The half of a write guard's settlement that does not need the advisory lock. +/// +/// A `RepoWriteGuard` can be shared, and the lock frees only when the LAST copy +/// drops. On a receive-pack disconnect that copy rides the admission guard into +/// the detached process reaper, so a settlement done from `Drop` lands only +/// after the process group is torn down. `acquire` takes no advisory lock on +/// the way to the live tree, so a read inside that window is served the +/// abandoned refs. +/// +/// Only the UNLOCK has to wait for the reaper. This token carries everything +/// the settlement itself needs, so the request future can hold it and settle +/// the tree at the disconnect instant while the reaper still holds the lock. +pub(crate) struct TreeSettlement { + local_path: ValidatedRepoDiskPath, + repo_name: String, + /// Shared with the guard, so the stage read here is the stage the publish + /// actually reached, not a copy taken when the token was handed out. + publish_stage: Arc, + path_handed_out: Arc, + armed: bool, +} + +impl TreeSettlement { + /// `release` classified the tree itself, so this token has nothing to do. + /// + /// Disarming rather than forgetting the token: a settlement that ran after + /// a definite refusal would write a marker beside a tree the refusal has + /// already deleted. + pub(crate) fn disarm(&mut self) { + self.armed = false; + } + + /// Classify the live tree an abandoned write left behind. + /// + /// Exhaustive on purpose, no wildcard arm: a new stage must be classified + /// here rather than inheriting whatever the catch-all happened to do. + fn settle(&mut self) { + if !self.armed { + return; + } + self.armed = false; + let stage = self.publish_stage.get(); + match &stage { + // The store acknowledged this write, so the live tree IS the + // stored generation. Mirrors `release`'s `Released` arm. + PublishStage::Published { .. } => clear_quarantine(&self.local_path, &self.repo_name), + // Nothing to publish to: the local write is the durable copy. + PublishStage::NoBackend => {} + PublishStage::Idle => { + // A publish was possible and never started. If the tree was + // handed out, refs may already be on it (the receive-pack + // disconnect: git applied them, the reaper killed the group, + // and the guard dropped before `release` was entered), and + // nothing dispatched, so this is a definite non-publication. + // If it was never handed out, nothing touched the tree. + if self.path_handed_out.load(Ordering::Acquire) { + quarantine_local_tree(&self.local_path, &self.repo_name, None); + } + } + // A definite non-publication after a successful write. The marker + // names no attempt, and the next read resolves it by invalidating + // the cache and serving the stored generation. Deliberately NOT a + // `remove_dir_all` here: that would run a blocking delete inside + // `Drop` on a runtime worker, when the read path already runs one. + PublishStage::PreparingArchive | PublishStage::Refused => { + quarantine_local_tree(&self.local_path, &self.repo_name, None) + } + // The PUT may have committed. The marker carries the attempt, so + // a landed PUT can still lift it through reconciliation. + PublishStage::PutDispatched { .. } | PublishStage::Ambiguous { .. } => { + quarantine_local_tree( + &self.local_path, + &self.repo_name, + stage.unresolved_attempt(), + ) + } + } + } +} + +impl Drop for TreeSettlement { + fn drop(&mut self) { + self.settle(); + } +} + impl Drop for RepoWriteGuard { fn drop(&mut self) { if let Some(authority) = self.refresh_swap_authority.take() { @@ -2358,45 +2475,15 @@ impl Drop for RepoWriteGuard { // // Exhaustive on purpose, no wildcard arm: a new stage must be classified // here rather than inheriting whatever the catch-all happened to do. + // SETTLE THE TREE, if the handler did not take the settlement out. + // + // `take_settlement` exists because the lock and the tree do not have to + // be freed at the same moment; when it was called this guard's copy has + // nothing left to classify. Otherwise this is the only classifier a + // guard that never reached `release` gets. if !self.tree_settled { - let stage = self.publish_stage.get(); - match &stage { - // The store acknowledged this write, so the live tree IS the - // stored generation. Mirrors `release`'s `Released` arm. - PublishStage::Published { .. } => { - clear_quarantine(&self.local_path, &self.repo_name) - } - // Nothing to publish to: the local write is the durable copy. - PublishStage::NoBackend => {} - PublishStage::Idle => { - // A publish was possible and never started. If the tree was - // handed out, refs may already be on it (the receive-pack - // disconnect: git applied them, the reaper killed the group, - // and the guard dropped before `release` was entered), and - // nothing dispatched, so this is a definite non-publication. - // If it was never handed out, nothing touched the tree. - if self.path_handed_out.load(Ordering::Acquire) { - quarantine_local_tree(&self.local_path, &self.repo_name, None); - } - } - // A definite non-publication after a successful write. The marker - // names no attempt, and the next read resolves it by invalidating - // the cache and serving the stored generation. Deliberately NOT a - // `remove_dir_all` here: that would run a blocking delete inside - // `Drop` on a runtime worker, when the read path already runs one. - PublishStage::PreparingArchive | PublishStage::Refused => { - quarantine_local_tree(&self.local_path, &self.repo_name, None) - } - // The PUT may have committed. The marker carries the attempt, so - // a landed PUT can still lift it through reconciliation. - PublishStage::PutDispatched { .. } | PublishStage::Ambiguous { .. } => { - quarantine_local_tree( - &self.local_path, - &self.repo_name, - stage.unresolved_attempt(), - ) - } - } + let mut settlement = self.settlement(); + settlement.settle(); } let Some(mut conn) = self.conn.take() else { @@ -3728,7 +3815,7 @@ mod tests { refresh_swap_authority: None, publish_stage: Arc::new(PublishStageCell::new()), tree_settled: false, - path_handed_out: AtomicBool::new(false), + path_handed_out: Arc::new(AtomicBool::new(false)), #[cfg(test)] test_pre_unlock_gate: None, #[cfg(test)] @@ -4098,7 +4185,7 @@ mod tests { refresh_swap_authority: None, publish_stage: Arc::new(PublishStageCell::new()), tree_settled: false, - path_handed_out: AtomicBool::new(false), + path_handed_out: Arc::new(AtomicBool::new(false)), #[cfg(test)] test_pre_unlock_gate: None, #[cfg(test)] @@ -8529,6 +8616,18 @@ mod tests { marker_path.exists(), "the second quarantine must reach disk once the path is writable" ); + // THE PROPERTY, observed BEFORE any clear. `clear_quarantine` drops the + // map entry ahead of the file, so an assertion taken after one passes + // whatever the successful-write arm did with the shadow. This is the + // only point where the shadow is observable. + assert!( + !unwritten_quarantines() + .lock() + .expect("quarantine map poisoned") + .contains_key(&marker_path), + "a marker that reached disk left the in-memory shadow of the earlier failed \ + write behind it, so the disk marker is no longer the authority for this path" + ); // Lift the second attempt and clear the file the way a confirmation // does. Nothing may still be refusing after that. From 957d50bd7482df04f13e85906e4a1e840cb253e2 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 7 Sep 2026 02:36:30 -0500 Subject: [PATCH 54/54] fix(node): let exactly one classifier settle an abandoned tree Moving settlement into a token the handler holds left two classifiers able to run for one write. release() classifies the tree and only then awaits the unlock, while the handler disarms the token after that await returns, so a cancel in the unlock gap dropped a still-armed token and settled a second time. After a definite refusal that is a quarantine marker written beside the tree release had just deleted, which sends every later read down the refuse-and-reconcile path for a repo whose state was already resolved. Observed before the fix: the tree gone, the marker there. The arm is now an AtomicBool shared between the guard and its token. release stores false at the classification point, above the unlock await, and settle takes the arm with compare_exchange, so whichever classifier gets there first is the only one that acts. tree_settled stays, and now has a test that says why: after the handler takes the settlement the arm is still true, so tree_settled is the only thing stopping the guard's own Drop, which on a disconnect runs in the detached reaper, from settling again at reaper exit. The reap-window test was passing for weaker reasons than it claimed. Its fake git took SIGTERM and let the group die, so the reaper could free the advisory lock before the probe read; it now traps TERM and re-parks so only SIGKILL ends it. It asserts the marker exists at the disconnect instant rather than only mentioning it, and the served-refs half no longer passes when acquire fails for an unrelated reason. Suite 1214 passed, 0 failed, 1 ignored. Nineteen mutations load-bearing, including one that reverts the release-side disarm and one that reverts the settlement split. --- crates/gitlawb-node/src/api/repos.rs | 35 ++++- crates/gitlawb-node/src/git/repo_store.rs | 158 +++++++++++++++++++++- 2 files changed, 182 insertions(+), 11 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 72cbf32d..1947c04e 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -11602,14 +11602,26 @@ exit 0 .await; let gitdir = tmp.path().join("hanging-git"); std::fs::create_dir_all(&gitdir).unwrap(); + // Ignores SIGTERM and re-parks after each interrupted `sleep`, so the + // process group outlives the reaper's grace period and only SIGKILL ends + // it. A plain `sleep 30` does NOT hold this window open: the group kill + // TERMs the sleep, the shell falls through the case and exits, and the + // reaper can free the advisory lock before the probe below runs, which + // turns this test into its own INCONCLUSIVE branch instead of an + // observation of the window it names. state.git_bin = write_fake_git( &gitdir, r#"#!/bin/sh +trap '' TERM case "$1" in receive-pack) mkdir -p "$3/refs/heads" printf '%s\n' 1111111111111111111111111111111111111111 > "$3/refs/heads/main" - sleep 30 + i=0 + while [ "$i" -lt 300 ]; do + sleep 1 + i=$((i+1)) + done ;; *) : ;; esac @@ -11675,13 +11687,24 @@ exit 0 "INCONCLUSIVE: the write lock was already free when the probe ran, so the \ reaper had finished and this read never observed the reap window" ); + // THE FIX'S OWN PROPERTY, asserted rather than only reported. The marker + // has to be on disk at the DISCONNECT instant, which is what moving the + // settlement out of the reaper-held guard buys; without this the test + // reddens only through the refs below, which a settlement arriving late + // can still satisfy by accident once the reaper catches up. assert!( - !(served_ok && refs_after), - "a read inside the reap window served the abandoned refs: marker_before={}, \ - acquire={:?}, ref still on the served tree={}", marker_before, - served, - refs_after + "the settlement must land at the disconnect, not at reaper exit: no quarantine \ + marker existed when the probe read inside the reap window (acquire={served:?})" + ); + // Unconditional on the refs, not `!(served_ok && refs_after)`. That form + // also passed when `acquire` failed for a reason unrelated to the + // withholding while the abandoned refs were still sitting on the live + // tree, which is a pass for the wrong reason. + assert!( + !refs_after, + "a read inside the reap window left the abandoned refs on the live tree: \ + marker_before={marker_before}, acquire={served:?}, served_ok={served_ok}" ); server.abort(); diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 0dbc92d0..80f51cfa 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -669,6 +669,7 @@ impl RepoStore { None => PublishStage::NoBackend, })), tree_settled: false, + settlement_armed: Arc::new(AtomicBool::new(true)), path_handed_out: Arc::new(AtomicBool::new(false)), #[cfg(test)] test_pre_unlock_gate: self.pre_unlock_gate.clone(), @@ -1900,6 +1901,16 @@ pub struct RepoWriteGuard { /// deletes the tree and clears its marker, would be followed by a `Drop` that /// writes a marker back beside a directory that no longer exists. tree_settled: bool, + /// Is the tree still waiting to be classified by SOMEONE? + /// + /// Shared with every [`TreeSettlement`] this guard hands out, because + /// `tree_settled` above is private to the guard and a token that has already + /// left it cannot see it. `release` classifies the tree and only THEN awaits + /// the connection-affine unlock, so a cancel in that gap drops a token the + /// handler has not reached its `disarm` for yet. Clearing this at the moment + /// `release` classifies, rather than when it returns, is what makes the + /// second settle impossible instead of merely unlikely. + settlement_armed: Arc, /// Was the writable tree ever handed out through `path()`? /// /// This is what answers "may a write have landed" at stage `Idle`, where no @@ -2087,7 +2098,7 @@ impl RepoWriteGuard { repo_name: self.repo_name.clone(), publish_stage: Arc::clone(&self.publish_stage), path_handed_out: Arc::clone(&self.path_handed_out), - armed: true, + armed: Arc::clone(&self.settlement_armed), } } @@ -2308,7 +2319,14 @@ impl RepoWriteGuard { // Whatever `release` reached, it has now classified the tree: published, // refused and invalidated, quarantined, or (on `success == false`) left // deliberately alone. `Drop` settles only the path that never got here. + // + // Disarmed HERE, above the unlock await, not after `release` returns. + // The unlock is an await, so the handler's copy of the settlement can be + // dropped between this classification and the handler's own `disarm`; a + // token that still believed the tree unclassified would then write a + // marker beside the directory the refusal arm above just deleted. self.tree_settled = true; + self.settlement_armed.store(false, Ordering::Release); // Release the advisory lock on the SAME session that took it. Unlocking // through the pool would land on an arbitrary backend, where the call is a @@ -2394,7 +2412,12 @@ pub(crate) struct TreeSettlement { /// actually reached, not a copy taken when the token was handed out. publish_stage: Arc, path_handed_out: Arc, - armed: bool, + /// Shared with the guard and with every other token over the same tree, so + /// the first settler wins and every later one is a no-op. A plain `bool` + /// here left the window this closes: `release` deletes the tree on a + /// definite refusal and then awaits the unlock, and a cancel in between + /// dropped a token that still believed nothing had classified the tree. + armed: Arc, } impl TreeSettlement { @@ -2404,7 +2427,7 @@ impl TreeSettlement { /// a definite refusal would write a marker beside a tree the refusal has /// already deleted. pub(crate) fn disarm(&mut self) { - self.armed = false; + self.armed.store(false, Ordering::Release); } /// Classify the live tree an abandoned write left behind. @@ -2412,10 +2435,15 @@ impl TreeSettlement { /// Exhaustive on purpose, no wildcard arm: a new stage must be classified /// here rather than inheriting whatever the catch-all happened to do. fn settle(&mut self) { - if !self.armed { + // Take the arm, don't just read it: the guard, this token, and any other + // copy all race to be the one classifier, and only the winner may act. + if self + .armed + .compare_exchange(true, false, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { return; } - self.armed = false; let stage = self.publish_stage.get(); match &stage { // The store acknowledged this write, so the live tree IS the @@ -3815,6 +3843,7 @@ mod tests { refresh_swap_authority: None, publish_stage: Arc::new(PublishStageCell::new()), tree_settled: false, + settlement_armed: Arc::new(AtomicBool::new(true)), path_handed_out: Arc::new(AtomicBool::new(false)), #[cfg(test)] test_pre_unlock_gate: None, @@ -4185,6 +4214,7 @@ mod tests { refresh_swap_authority: None, publish_stage: Arc::new(PublishStageCell::new()), tree_settled: false, + settlement_armed: Arc::new(AtomicBool::new(true)), path_handed_out: Arc::new(AtomicBool::new(false)), #[cfg(test)] test_pre_unlock_gate: None, @@ -8529,6 +8559,124 @@ mod tests { mock.shutdown(); } + /// The POST-CLASSIFY CANCEL WINDOW. `release` classifies the tree and only + /// then awaits the connection-affine unlock. The receive-pack handler holds + /// the settlement token across that await and disarms it after `release` + /// returns, so a disconnect inside the unlock gap drops a still-armed token + /// on a tree the definite refusal has already deleted. The second settle + /// would write a marker beside nothing, and every later `acquire` on that + /// path would take the refuse/reconcile branch for a write that definitively + /// did not publish. + #[sqlx::test] + async fn a_cancel_in_the_unlock_gap_after_a_definite_refusal_leaves_no_marker(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let store = fenced_store(&mock, &opts, repos.path()).await; + let owner = "did:key:z6MkCancelAfterRefusalAAAAAAAAAAAAAAAAA"; + let repo = "cancel-after-refusal-repo"; + let slug = owner_slug_of(owner); + + let seed = TempDir::new().unwrap(); + marked_repo(seed.path(), "seed"); + mock_tigris(&mock) + .upload(&slug, repo, seed.path(), UploadPrecondition::Unconditional) + .await + .expect("seeding the archive"); + + // Both of the release's HEADs lose their generation, so the fence and + // its one supersede-retry are both refused: a DEFINITE refusal. + mock.roll_generation_after_next_heads(2); + + let mut guard = store.acquire_write(owner, repo).await.expect("acquire"); + std::fs::write(guard.local_path.join("MARKER"), "writer").unwrap(); + let local_path = guard.local_path.clone(); + // Exactly what `git_receive_pack` does: the handler takes the token so + // the marker can land at the disconnect instant rather than after the + // detached reaper frees the lock. + let settlement = guard.take_settlement(); + assert!( + settlement.is_some(), + "the handler must be able to take the settlement out of a fresh guard" + ); + // Park `release` at its pre-unlock point: classification has happened, + // the unlock has not. + let gate = Arc::new(tokio::sync::Notify::new()); + guard.test_pre_unlock_gate = Some(gate); + + let mut fut = Box::pin(guard.release(true)); + let parked = tokio::time::timeout(std::time::Duration::from_secs(20), fut.as_mut()).await; + assert!( + parked.is_err(), + "release must park on the pre-unlock gate, not complete" + ); + assert!( + !local_path.exists(), + "release must have reached its definite-refusal arm and deleted the tree before \ + parking on the unlock" + ); + + // The disconnect. Handler locals drop in reverse declaration order, so + // the release future goes first and the settlement token after it. + drop(fut); + drop(settlement); + + assert!( + !quarantine_path(&local_path).unwrap().exists(), + "a cancel in the unlock gap must not settle a tree release already classified: no \ + marker may exist beside a tree the definite refusal deleted" + ); + + mock.shutdown(); + } + + /// The OTHER half of the hand-off, and the one property `tree_settled` still + /// carries on its own. Once the handler has taken the settlement out, the + /// guard has given up responsibility for the tree: on a receive-pack + /// disconnect the guard's last copy rides `KillGroupOnDrop`'s detached + /// reaper, so a guard that still settled from its own `Drop` would put the + /// marker on disk at reaper exit and reopen the window U3 closed. The shared + /// arm alone does not answer this: it is still armed at this point, so + /// whichever of the two dropped first would win the race. + #[sqlx::test] + async fn a_guard_that_handed_off_its_settlement_does_not_settle_when_it_drops(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let store = fenced_store(&mock, &opts, repos.path()).await; + let owner = "did:key:z6MkHandedOffSettlementAAAAAAAAAAAAAAAA"; + let repo = "handed-off-settlement-repo"; + + let mut guard = store.acquire_write(owner, repo).await.expect("acquire"); + // The hand-out is what makes an Idle abandonment worth settling at all. + let handed_out = guard.path().to_path_buf(); + marked_repo(&handed_out, "writer"); + let local_path = guard.local_path.clone(); + let marker = quarantine_path(&local_path).unwrap(); + let mut settlement = guard + .take_settlement() + .expect("a fresh guard hands out its token"); + + drop(guard); + assert!( + !marker.exists(), + "a guard that handed its settlement to the handler must not be re-settled by Drop: \ + on a disconnect that Drop is the detached reaper's, which is the whole window the \ + hand-off exists to close" + ); + + // THE CONTROL: the token that took the responsibility still discharges it. + settlement.settle(); + assert!( + marker.exists(), + "the token that took the settlement must classify the tree it took" + ); + + mock.shutdown(); + } + /// U4 / gap E. The whole withholding contract rests on one `fs::write`. A /// full disk or a read-only mount fails it, and the tree is then served as /// an ordinary read.