diff --git a/crates/trusted-server-adapter-fastly/src/ec_kv.rs b/crates/trusted-server-adapter-fastly/src/ec_kv.rs index 229c6c2d3..c64fb5131 100644 --- a/crates/trusted-server-adapter-fastly/src/ec_kv.rs +++ b/crates/trusted-server-adapter-fastly/src/ec_kv.rs @@ -5,12 +5,56 @@ //! writes (`if_generation_match`). use error_stack::{Report, ResultExt}; -use fastly::kv_store::{InsertMode, KVStore}; +use fastly::kv_store::{InsertMode, KVStore, KVStoreError, ListPage}; use trusted_server_core::ec::kv_backend::{ EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteMode, EcKvWriteOutcome, }; +use trusted_server_core::ec::log_id; use trusted_server_core::error::TrustedServerError; +/// Bounds the work a withdrawal can trigger when keys share its prefix. +const EXISTENCE_PAGE_SIZE: u32 = 100; +const EXISTENCE_MAX_PAGES: usize = 4; + +/// Checks exact equality while driving one strong list page at a time. +/// +/// Explicit pagination avoids the SDK iterator reissuing a failed page. +fn contains_exact_key( + store_name: &str, + key: &str, + mut fetch_page: impl FnMut(Option<&str>) -> Result, +) -> Result> { + let mut cursor = None; + for _ in 0..EXISTENCE_MAX_PAGES { + let page = match fetch_page(cursor.as_deref()) { + Ok(page) => page, + Err(KVStoreError::ItemNotFound) => return Ok(false), + Err(error) => { + return Err( + Report::new(error).change_context(TrustedServerError::KvStore { + store_name: store_name.to_owned(), + message: format!("Failed to confirm existence of key '{}'", log_id(key)), + }), + ); + } + }; + if page.keys().iter().any(|listed| listed == key) { + return Ok(true); + } + cursor = page.next_cursor(); + if cursor.is_none() { + return Ok(false); + } + } + Err(Report::new(TrustedServerError::KvStore { + store_name: store_name.to_owned(), + message: format!( + "Existence check exceeds page budget for key '{}'", + log_id(key) + ), + })) +} + /// Fastly KV Store backend for the EC identity graph. #[derive(Debug, Clone)] pub struct FastlyEcKvStore { @@ -56,7 +100,7 @@ impl EcKvStore for FastlyEcKvStore { return Err( Report::new(err).change_context(TrustedServerError::KvStore { store_name: self.store_name.clone(), - message: format!("Failed to read key '{key}'"), + message: format!("Failed to read key '{}'", log_id(key)), }), ); } @@ -73,6 +117,18 @@ impl EcKvStore for FastlyEcKvStore { })) } + fn key_exists(&self, key: &str) -> Result> { + let store = self.open_store()?; + contains_exact_key(&self.store_name, key, |cursor| { + // build_list defaults to ListMode::Strong, reading primary state. + let mut request = store.build_list().prefix(key).limit(EXISTENCE_PAGE_SIZE); + if let Some(cursor) = cursor { + request = request.cursor(cursor); + } + request.execute() + }) + } + fn insert( &self, key: &str, @@ -98,7 +154,7 @@ impl EcKvStore for FastlyEcKvStore { Err(err) => Err( Report::new(err).change_context(TrustedServerError::KvStore { store_name: self.store_name.clone(), - message: format!("Failed to write entry for key '{key}'"), + message: format!("Failed to write entry for key '{}'", log_id(key)), }), ), } @@ -117,10 +173,7 @@ impl EcKvStore for FastlyEcKvStore { .execute() .change_context(TrustedServerError::KvStore { store_name: self.store_name.clone(), - message: format!( - "Failed to list keys with prefix '{}'", - prefix.get(..8).unwrap_or(prefix), - ), + message: format!("Failed to list keys with prefix '{}'", log_id(prefix)), })?; #[allow(clippy::cast_possible_truncation)] @@ -134,7 +187,316 @@ impl EcKvStore for FastlyEcKvStore { .delete(key) .change_context(TrustedServerError::KvStore { store_name: self.store_name.clone(), - message: format!("Failed to delete key '{key}'"), + message: format!("Failed to delete key '{}'", log_id(key)), + }) + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + /// KV store declared for the local simulator in `fastly.toml`. + const TEST_STORE: &str = "ec_identity_store"; + + /// Entry metadata. Opaque to the backend, which only round-trips bytes. + const METADATA: &str = "entry-metadata"; + + fn store() -> FastlyEcKvStore { + FastlyEcKvStore::new(TEST_STORE) + } + + fn write(key: &str, body: &str, mode: EcKvWriteMode) -> EcKvWriteOutcome { + store() + .insert( + key, + EcKvWrite { + body, + metadata: METADATA, + ttl: Duration::from_secs(60), + mode, + }, + ) + .expect("should reach the store") + } + + fn page(keys: &[&str], next_cursor: Option<&str>) -> ListPage { + serde_json::from_value(serde_json::json!({ + "data": keys, + "meta": { "limit": EXISTENCE_PAGE_SIZE, "next_cursor": next_cursor, "prefix": "key" } + })) + .expect("should build a list page") + } + + #[test] + fn exact_existence_follows_cursors_and_rejects_prefix_collisions() { + let mut cursors = Vec::new(); + let found = contains_exact_key(TEST_STORE, "key", |cursor| { + cursors.push(cursor.map(str::to_owned)); + Ok(if cursor.is_none() { + page(&["key-longer"], Some("next")) + } else { + page(&["key"], None) }) + }) + .expect("should find an exact match on a later page"); + assert!(found, "should find the exact key"); + assert_eq!( + cursors, + [None, Some("next".to_string())], + "should pass the returned cursor to the next request" + ); + assert!( + !contains_exact_key(TEST_STORE, "key", |_| Ok(page(&["key-longer"], None))) + .expect("should finish listing"), + "should not accept a longer prefix match" + ); + } + + #[test] + fn exact_existence_stops_at_a_first_page_match() { + let mut calls = 0; + assert!( + contains_exact_key(TEST_STORE, "key", |_| { + calls += 1; + Ok(page(&["key"], Some("unused"))) + }) + .expect("should find the key"), + "should recognize exact equality" + ); + assert_eq!(calls, 1, "should not fetch another page after a match"); + } + + #[test] + fn exact_existence_terminates_on_item_not_found() { + let mut calls = 0; + assert!( + !contains_exact_key(TEST_STORE, "key", |_| { + calls += 1; + Err(KVStoreError::ItemNotFound) + }) + .expect("should report absence"), + "should return absence immediately" + ); + assert_eq!(calls, 1, "should not reissue a failed page"); + } + + #[test] + fn exact_existence_reports_errors_without_retrying() { + let mut calls = 0; + assert!( + contains_exact_key(TEST_STORE, "key", |_| { + calls += 1; + Err(KVStoreError::TooManyRequests) + }) + .is_err(), + "should propagate list failures" + ); + assert_eq!(calls, 1, "should not retry a store error"); + } + + #[test] + fn exact_existence_reports_exhaustion_instead_of_absence() { + let mut calls = 0; + let key = format!("{}.ABC123", "a".repeat(64)); + let error = contains_exact_key(TEST_STORE, &key, |_| { + calls += 1; + Ok(page(&[], Some("more"))) + }) + .expect_err("should report an inconclusive bounded check"); + assert_eq!( + calls, EXISTENCE_MAX_PAGES, + "should enforce the page budget even for empty pages" + ); + assert!( + !format!("{error:?}").contains(&key), + "should redact the identifier in budget errors" + ); + } + + #[test] + fn exact_existence_accepts_a_match_on_the_last_allowed_page() { + let mut calls = 0; + let found = contains_exact_key(TEST_STORE, "key", |_| { + calls += 1; + Ok(if calls == EXISTENCE_MAX_PAGES { + page(&["key"], Some("more")) + } else { + page(&[], Some("next")) + }) + }) + .expect("should inspect the last allowed page"); + assert!(found, "should not discard a match at the budget boundary"); + assert_eq!( + calls, EXISTENCE_MAX_PAGES, + "should not request a page beyond the budget" + ); + } + + #[test] + fn strong_existence_checks_real_store_keys_exactly() { + let backend = store(); + let key = "existence-test-key"; + let longer = "existence-test-key-longer"; + write(longer, "body", EcKvWriteMode::Overwrite); + assert!( + !backend.key_exists(key).expect("should list the prefix"), + "should not confuse a prefix with an exact key" + ); + write(key, "body", EcKvWriteMode::Overwrite); + assert!( + backend.key_exists(key).expect("should list the issued key"), + "should see a completed issuance" + ); + backend.delete(key).expect("should delete exact key"); + backend.delete(longer).expect("should delete longer key"); + assert!( + !backend.key_exists(key).expect("should list after deletion"), + "should report absence after deletion" + ); + } + + #[test] + fn opening_a_store_this_service_does_not_have_is_an_error() { + let error = FastlyEcKvStore::new("no_such_store") + .lookup("any-key") + .expect_err("should not resolve against a store that is not linked"); + + assert!( + matches!( + error.current_context(), + TrustedServerError::KvStore { store_name, .. } if store_name == "no_such_store" + ), + "should name the store it could not open: {error:?}" + ); + } + + #[test] + fn a_missing_key_is_absent_rather_than_an_error() { + let absent = format!("{}.ABC123", "1".repeat(64)); + + assert!( + store() + .lookup(&absent) + .expect("should reach the store") + .is_none(), + "a key the store does not hold is absent, not a failure" + ); + } + + #[test] + fn an_entry_round_trips_through_insert_lookup_and_delete() { + let key = format!("{}.ABC123", "2".repeat(64)); + let backend = store(); + + assert_eq!( + write(&key, "entry-body-1", EcKvWriteMode::Overwrite), + EcKvWriteOutcome::Written, + "should write the entry" + ); + + let found = backend + .lookup(&key) + .expect("should reach the store") + .expect("should hold the entry just written"); + assert_eq!(found.body, b"entry-body-1", "should read back the body"); + assert_eq!( + found.metadata.as_deref(), + Some(METADATA.as_bytes()), + "should read back the metadata" + ); + + backend.delete(&key).expect("should delete the entry"); + assert!( + backend + .lookup(&key) + .expect("should reach the store") + .is_none(), + "a deleted key is absent" + ); + } + + #[test] + fn add_mode_refuses_a_key_that_already_exists() { + let key = format!("{}.ABC123", "3".repeat(64)); + let backend = store(); + + assert_eq!( + write(&key, "entry-body-1", EcKvWriteMode::Add), + EcKvWriteOutcome::Written, + "should create a key nothing holds" + ); + assert_eq!( + write(&key, "entry-body-2", EcKvWriteMode::Add), + EcKvWriteOutcome::PreconditionFailed, + "a precondition failure is control flow, not an error" + ); + + backend.delete(&key).expect("should delete the entry"); + } + + #[test] + fn a_generation_mismatch_is_reported_as_a_precondition_failure() { + let key = format!("{}.ABC123", "4".repeat(64)); + let backend = store(); + + write(&key, "entry-body-1", EcKvWriteMode::Overwrite); + let generation = backend + .lookup(&key) + .expect("should reach the store") + .expect("should hold the entry just written") + .generation; + + assert_eq!( + write( + &key, + "entry-body-2", + EcKvWriteMode::IfGenerationMatch(generation) + ), + EcKvWriteOutcome::Written, + "should write when the generation still matches" + ); + assert_eq!( + write( + &key, + "entry-body-3", + EcKvWriteMode::IfGenerationMatch(generation) + ), + EcKvWriteOutcome::PreconditionFailed, + "the generation moved on with the previous write" + ); + + backend.delete(&key).expect("should delete the entry"); + } + + #[test] + fn counting_a_prefix_counts_only_the_keys_under_it() { + let hash = "5".repeat(64); + let backend = store(); + let keys = [format!("{hash}.AAA111"), format!("{hash}.BBB222")]; + for key in &keys { + write(key, "entry-body-1", EcKvWriteMode::Overwrite); + } + + assert_eq!( + backend + .count_keys_with_prefix(&hash, 100) + .expect("should list the prefix"), + 2, + "should count both keys issued under this hash" + ); + assert_eq!( + backend + .count_keys_with_prefix(&"6".repeat(64), 100) + .expect("should list the prefix"), + 0, + "should count nothing under a hash nothing was issued for" + ); + + for key in &keys { + backend.delete(key).expect("should delete the entry"); + } } } diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index e44708742..fb8ef5192 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -677,6 +677,8 @@ mod tests { use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; + use crate::ec::kv::TombstoneOutcome; + use super::*; use crate::ec::kv_backend::test_support::InMemoryEcKv; use crate::ec::kv_backend::{EcKvStore as _, EcKvWrite, EcKvWriteMode}; @@ -1114,9 +1116,18 @@ mod tests { #[test] fn reports_tombstone_entries() { let ec_id = test_ec_id(); - let kv = KvIdentityGraph::in_memory("test-store"); - kv.write_withdrawal_tombstone(&ec_id) - .expect("should write tombstone"); + // Only an identity the store already holds can be tombstoned, so seed + // the live entry the withdrawal replaces. + let kv = kv_with_entry( + &ec_id, + &KvEntry::minimal("bidstream.example", "uid-live", 1_741_824_000), + ); + assert_eq!( + kv.write_withdrawal_tombstone(&ec_id, drop) + .expect("should write tombstone"), + TombstoneOutcome::Written, + "should tombstone the seeded identity" + ); let req = get_request(&format!("/_ts/admin/ec/{ec_id}")); let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index fe2ac7203..1fe61fa1f 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -6,15 +6,19 @@ use std::collections::HashSet; use edgezero_core::body::Body as EdgeBody; +use error_stack::Report; use http::Response; use super::consent::{ec_consent_granted, ec_consent_withdrawn}; +use crate::error::TrustedServerError; use crate::settings::Settings; use super::EcContext; use super::cookies::{expire_ec_cookie, set_ec_cookie}; use super::generation::{generate_ec_id, is_valid_ec_id}; -use super::kv::{CreateIfAbsentOutcome, KvIdentityGraph, apply_partner_id_updates}; +use super::kv::{ + CreateIfAbsentOutcome, KvIdentityGraph, TombstoneOutcome, apply_partner_id_updates, +}; use super::kv_types::KvEntry; use super::prebid_eids::collect_eid_cookie_updates; use super::registry::PartnerRegistry; @@ -52,49 +56,14 @@ pub fn ec_finalize_response( let consent_withdrawn = ec_consent_withdrawn(ec_context.consent()); if !consent_allows_ec { - // Always strip EC-specific response headers when consent is not - // currently usable for this request. This covers both explicit - // revocation and fail-closed cases such as missing geo or undecodable - // consent input. - clear_ec_headers_on_response(response, Some(registry)); - - // Only expire the browser cookie and tombstone the identity-graph row - // when the request carries an explicit withdrawal signal. - if consent_withdrawn && ec_context.cookie_was_present() { - expire_ec_cookie(settings, response); - - // Compute once for the authoritative identity-graph tombstones. - let ids_to_withdraw = withdrawal_ec_ids(ec_context); - - // The identity-graph tombstone is the authoritative withdrawal marker - // for subsequent EC behavior. - if let Some(graph) = kv { - apply_withdrawal_tombstones(&ids_to_withdraw, |ec_id| { - let initial = if ec_context.kv_snapshot().belongs_to(ec_id) { - ec_context.kv_snapshot().clone() - } else { - EcKvSnapshot::NotRead - }; - let outcome = graph.tombstone_existing_from_snapshot(ec_id, initial); - // The browser cookie is already cleared, so a failed - // tombstone leaves a live row that server-side consumers - // still read as consented. Report every failure, including - // the non-active cookie ID whose outcome is not retained on - // the request context. - if matches!(outcome, EcKvSnapshot::Failed { .. }) { - log::warn!( - "EC withdrawal tombstone failed for '{}': the identity-graph row may \ - still be live with consent granted", - log_id(ec_id) - ); - } - if ec_context.ec_value() == Some(ec_id) { - ec_context.set_kv_snapshot(outcome); - } - }); - } - } - + finalize_unusable_consent( + settings, + ec_context, + kv, + registry, + consent_withdrawn, + response, + ); return; } @@ -314,6 +283,84 @@ pub fn clear_ec_on_response(settings: &Settings, response: &mut Response, + registry: &PartnerRegistry, + consent_withdrawn: bool, + response: &mut Response, +) { + clear_ec_headers_on_response(response, Some(registry)); + + if !(consent_withdrawn && ec_context.cookie_was_present()) { + return; + } + + expire_ec_cookie(settings, response); + + // Compute once for the authoritative identity-graph tombstones. + let ids_to_withdraw = withdrawal_ec_ids(ec_context); + + // The identity-graph tombstone is the authoritative withdrawal marker + // for subsequent EC behavior. + if let Some(graph) = kv { + apply_withdrawal_tombstones(&ids_to_withdraw, |ec_id| { + // The graph hands back the post-withdrawal snapshot rather than + // leaving the caller to rebuild it, so post-send work that reads + // the context — pull sync discloses the raw EC ID to partners — + // sees the tombstone that was just written. Only the active ID has + // a snapshot in the context to correct. + let outcome = graph.write_withdrawal_tombstone(ec_id, |snapshot| { + if ec_context.ec_value() == Some(ec_id) { + ec_context.set_kv_snapshot(snapshot); + } + }); + log_tombstone_outcome(ec_id, outcome); + }); + } +} + +/// Records what happened to one withdrawal tombstone. +/// +/// An unknown identity is expected traffic rather than a fault: the identifier +/// comes from a client-supplied cookie, so it may name something this +/// deployment never issued. An error is different: nothing was recorded, so a +/// real row may have gone unmarked, and that is logged as a fault. The browser +/// cookie is expired in every case, and that is the primary enforcement. +fn log_tombstone_outcome( + ec_id: &str, + outcome: Result>, +) { + match outcome { + Ok(TombstoneOutcome::Written) => {} + Ok(TombstoneOutcome::UnknownIdentity) => { + log::debug!( + "Skipping withdrawal tombstone for unknown EC ID '{}'", + log_id(ec_id), + ); + } + Err(err) => { + // Covers both a failed write and a check that could not determine + // whether the identity exists. Either way no marker was recorded, + // so a withdrawal may go unrecorded for the batch-sync window; the + // browser cookie is expired regardless. + log::error!( + "Could not record the withdrawal of EC ID '{}', so it may go unrecorded \ + for the batch-sync window; the browser cookie is still expired: {err:?}", + log_id(ec_id), + ); + } + } +} + fn withdrawal_ec_ids(ec_context: &EcContext) -> HashSet { let mut hashes = HashSet::new(); @@ -555,6 +602,152 @@ mod tests { ); } + #[test] + fn finalize_withdrawal_does_not_create_a_row_for_an_unheld_identity() { + let settings = create_test_settings(); + // The cookie value is chosen by the client, so a withdrawal naming an + // identity this deployment never issued must not put a row in the + // identity graph. + let ec_id = sample_ec_id("zz9999"); + let consent = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + source: ConsentSource::Cookie, + ..Default::default() + }; + let mut ec_context = + make_context_with_consent(Some(&ec_id), Some(&ec_id), true, false, consent); + let kv = KvIdentityGraph::in_memory("test-store"); + let mut response = empty_response(); + let registry = PartnerRegistry::from_config(&[]).expect("should build registry"); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&kv), + ®istry, + None, + None, + &mut response, + ); + + assert!( + kv.get(&ec_id).expect("should read back").is_none(), + "should not write a tombstone for an identity that was never issued" + ); + assert!( + matches!(ec_context.kv_snapshot(), EcKvSnapshot::Missing { .. }), + "should record the confirmed missing identity" + ); + let set_cookie = get_header_str(&response, "set-cookie").unwrap_or_default(); + assert!( + set_cookie.contains("Max-Age=0"), + "should still expire the browser cookie, which is the primary enforcement" + ); + } + + #[test] + fn finalize_withdrawal_tombstones_a_held_identity() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("held01"); + let consent = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + source: ConsentSource::Cookie, + ..Default::default() + }; + let mut ec_context = + make_context_with_consent(Some(&ec_id), Some(&ec_id), true, false, consent); + let kv = KvIdentityGraph::stale_lookup("test-store", 1); + kv.create( + &ec_id, + &crate::ec::kv_types::KvEntry::minimal("p.example", "uid", 1), + ) + .expect("should seed the identity"); + ec_context.set_kv_snapshot(kv.load_snapshot(&ec_id)); + assert!(matches!( + ec_context.kv_snapshot(), + EcKvSnapshot::Missing { .. } + )); + let mut response = empty_response(); + let registry = PartnerRegistry::from_config(&[]).expect("should build registry"); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&kv), + ®istry, + None, + None, + &mut response, + ); + + let (entry, _) = kv + .get(&ec_id) + .expect("should read back") + .expect("should still hold the identity"); + assert!( + !entry.consent.ok, + "a genuine withdrawal must still tombstone the identity" + ); + let snapshot_entry = ec_context + .kv_snapshot() + .entry_for(&ec_id) + .expect("should replace the stale miss with a tombstone snapshot"); + assert!(!snapshot_entry.consent.ok && snapshot_entry.ids.is_empty()); + assert_eq!(ec_context.kv_snapshot().generation_for(&ec_id), None); + } + + #[test] + fn withdrawal_still_expires_the_cookie_when_the_store_is_unavailable() { + // Cookie expiry is the primary enforcement, so it has to survive a + // store that cannot answer at all — the case where the identity-graph + // marker is exactly what goes missing. + let settings = create_test_settings(); + let ec_id = sample_ec_id("dead01"); + let consent = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + source: ConsentSource::Cookie, + ..Default::default() + }; + let mut ec_context = + make_context_with_consent(Some(&ec_id), Some(&ec_id), true, false, consent); + ec_context.set_kv_snapshot(EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(live_entry()), + generation: Some(1), + }); + let kv = KvIdentityGraph::failing("test-store"); + let mut response = empty_response(); + set_header(&mut response, "x-ts-ec", "stale"); + let registry = PartnerRegistry::from_config(&[]).expect("should build registry"); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&kv), + ®istry, + None, + None, + &mut response, + ); + + let set_cookie = get_header_str(&response, "set-cookie").unwrap_or_default(); + assert!( + set_cookie.contains("Max-Age=0"), + "should expire the EC cookie even when the store is unavailable: {set_cookie}" + ); + assert!( + get_header(&response, "x-ts-ec").is_none(), + "should still strip EC response headers" + ); + assert!( + matches!(ec_context.kv_snapshot(), EcKvSnapshot::Failed { .. }), + "should invalidate the live snapshot when withdrawal cannot be confirmed" + ); + } + #[test] fn finalize_withdrawal_clears_cookie_and_headers() { let settings = create_test_settings(); @@ -942,6 +1135,73 @@ mod tests { ); } + #[test] + fn finalize_rotates_when_only_a_longer_key_shares_the_orphan_prefix() { + // `key_exists_confirmed` gates orphan recovery as well as withdrawal. + // It matches whole keys, so a neighbouring key that merely starts with + // the orphaned ID cannot answer for it. Under the prefix count this + // path used before, that neighbour reported the orphan as still held + // and suppressed a rotation the visitor needed — the same collision + // this PR closes on the withdrawal path. + let settings = create_test_settings(); + let orphan = sample_ec_id("prefix"); + let neighbour = format!("{orphan}-longer"); + let graph = KvIdentityGraph::in_memory("test_store"); + let live = KvEntry::new( + &granting_consent(), + None, + current_timestamp(), + &settings.publisher.domain, + ); + graph + .create(&neighbour, &live) + .expect("should seed the neighbouring row"); + assert!( + !graph + .key_exists_confirmed(&orphan) + .expect("should confirm against the store"), + "a longer key sharing the prefix must not prove the orphan exists" + ); + let mut ec_context = returning_user_context( + &orphan, + EcKvSnapshot::Missing { + ec_id: orphan.clone(), + }, + true, + ); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + let replacement = ec_context.ec_value().expect("should rotate the orphan"); + assert_ne!( + replacement, orphan, + "a proven-absent orphan must rotate even with a prefix neighbour present" + ); + assert!( + graph + .get(replacement) + .expect("should read the replacement") + .is_some(), + "the replacement cookie should have a backing row" + ); + assert!( + graph + .get(&neighbour) + .expect("should read the neighbour") + .is_some(), + "the neighbouring identity must be left untouched" + ); + } + #[test] fn finalize_does_not_rotate_when_the_existence_check_fails() { // Absence is unprovable when the list itself errors. Rotation abandons a @@ -1255,7 +1515,7 @@ mod tests { .expect("active row should remain as a tombstone"); assert!( !active_stored.consent.ok, - "the present active ID should be tombstoned via its carried snapshot" + "the present active ID should be tombstoned after strong confirmation" ); assert!( graph.get(&cookie_ec).expect("should read store").is_none(), diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index 40c10ebe2..fd9aa5773 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -132,6 +132,16 @@ impl fmt::Debug for KvIdentityGraph { } } +/// Result of [`KvIdentityGraph::write_withdrawal_tombstone`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[must_use] +pub enum TombstoneOutcome { + /// The identity was found and is now tombstoned. + Written, + /// No such identity is held, so there was nothing to mark withdrawn. + UnknownIdentity, +} + impl KvIdentityGraph { /// Creates a new identity graph backed by the given store primitives. #[must_use] @@ -246,13 +256,16 @@ impl KvIdentityGraph { let entry: KvEntry = serde_json::from_slice(body_bytes).change_context(TrustedServerError::KvStore { store_name: store_name.to_owned(), - message: format!("Failed to deserialize entry for key '{ec_id}'"), + message: format!("Failed to deserialize entry for key '{}'", log_id(ec_id)), })?; entry.validate().map_err(|message| { Report::new(TrustedServerError::KvStore { store_name: store_name.to_owned(), - message: format!("Loaded invalid entry for key '{ec_id}': {message}"), + message: format!( + "Loaded invalid entry for key '{}': {message}", + log_id(ec_id) + ), }) })?; @@ -281,7 +294,7 @@ impl KvIdentityGraph { let meta: KvMetadata = serde_json::from_slice(&meta_bytes).change_context(TrustedServerError::KvStore { store_name: self.store_name().to_owned(), - message: format!("Failed to deserialize metadata for key '{ec_id}'"), + message: format!("Failed to deserialize metadata for key '{}'", log_id(ec_id)), })?; Ok(Some(meta)) @@ -301,7 +314,7 @@ impl KvIdentityGraph { match self.write_entry(ec_id, &body, &meta_str, ENTRY_TTL, EcKvWriteMode::Add)? { EcKvWriteOutcome::Written => Ok(()), EcKvWriteOutcome::PreconditionFailed => { - Err(self.kv_error(format!("Key '{ec_id}' already exists"))) + Err(self.kv_error(format!("Key '{}' already exists", log_id(ec_id)))) } } } @@ -429,7 +442,8 @@ impl KvIdentityGraph { } Err(self.kv_error(format!( - "CAS conflict after {MAX_CAS_RETRIES} retries reviving tombstone for '{ec_id}'" + "CAS conflict after {MAX_CAS_RETRIES} retries reviving tombstone for '{}'", + log_id(ec_id), ))) } @@ -463,8 +477,9 @@ impl KvIdentityGraph { updates.len(), ); return Err(self.kv_error(format!( - "Cannot upsert {} partner IDs for missing key '{ec_id}'", + "Cannot upsert {} partner IDs for missing key '{}'", updates.len(), + log_id(ec_id), ))); } }; @@ -478,8 +493,9 @@ impl KvIdentityGraph { updates.len(), ); return Err(self.kv_error(format!( - "Cannot upsert {} partner IDs for withdrawn key '{ec_id}'", + "Cannot upsert {} partner IDs for withdrawn key '{}'", updates.len(), + log_id(ec_id), ))); } @@ -509,8 +525,9 @@ impl KvIdentityGraph { } Err(self.kv_error(format!( - "CAS conflict after {MAX_CAS_RETRIES} retries upserting {} partner IDs for '{ec_id}'", + "CAS conflict after {MAX_CAS_RETRIES} retries upserting {} partner IDs for '{}'", updates.len(), + log_id(ec_id), ))) } @@ -702,7 +719,8 @@ impl KvIdentityGraph { log_id(ec_id) ); return Err(self.kv_error(format!( - "Cannot upsert partner '{partner_id}' for missing key '{ec_id}'" + "Cannot upsert partner '{partner_id}' for missing key '{}'", + log_id(ec_id), ))); } }; @@ -715,7 +733,8 @@ impl KvIdentityGraph { log_id(ec_id), ); return Err(self.kv_error(format!( - "Cannot upsert partner '{partner_id}' for withdrawn key '{ec_id}'" + "Cannot upsert partner '{partner_id}' for withdrawn key '{}'", + log_id(ec_id), ))); } @@ -758,7 +777,8 @@ impl KvIdentityGraph { } Err(self.kv_error(format!( - "CAS conflict after {MAX_CAS_RETRIES} retries upserting partner '{partner_id}' for '{ec_id}'" + "CAS conflict after {MAX_CAS_RETRIES} retries upserting partner '{partner_id}' for '{}'", + log_id(ec_id), ))) } @@ -828,10 +848,20 @@ impl KvIdentityGraph { } Err(self.kv_error(format!( - "CAS conflict after {MAX_CAS_RETRIES} retries upserting partner '{partner_id}' for '{ec_id}'" + "CAS conflict after {MAX_CAS_RETRIES} retries upserting partner '{partner_id}' for '{}'", + log_id(ec_id), ))) } + /// Checks exact existence against strongly consistent store state. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::KvStore`] if existence cannot be confirmed. + pub fn key_exists_confirmed(&self, ec_id: &str) -> Result> { + self.store.key_exists(ec_id) + } + /// Writes a withdrawal tombstone for consent enforcement. /// /// Overwrites the entry with `consent.ok = false`, empty partner IDs, @@ -841,203 +871,108 @@ impl KvIdentityGraph { /// The tombstone preserves consent enforcement for batch sync clients /// (`POST /_ts/api/v1/batch-sync`) during the 24-hour revocation window. /// - /// # Errors + /// Only an identity this store already holds is tombstoned. The marker + /// exists to stop later reads of a real row, so writing one for an ID that + /// was never issued enforces nothing while still consuming a write and a + /// row; the identifier in a request is chosen by the client, so that write + /// would be the client's to trigger at will. Existence is confirmed with + /// [`Self::key_exists_confirmed`], which checks exact equality against + /// strongly consistent state, so no neighbouring key can answer for it. /// - /// Returns [`TrustedServerError::KvStore`] on store error. Callers on - /// the browser path should log at `error` level and continue — cookie - /// deletion is the primary enforcement mechanism. - pub fn write_withdrawal_tombstone( - &self, - ec_id: &str, - ) -> Result<(), Report> { - let entry = KvEntry::tombstone(current_timestamp()); - let (body, meta_str) = Self::serialize_entry(&entry, self.store_name())?; - - match self.write_entry( - ec_id, - &body, - &meta_str, - TOMBSTONE_TTL, - EcKvWriteMode::Overwrite, - ) { - Ok(_) => Ok(()), - Err(report) => Err(report.change_context(TrustedServerError::KvStore { - store_name: self.store_name().to_owned(), - message: format!("Failed to write tombstone for key '{ec_id}'"), - })), - } - } - - /// Reports whether a row exists for `ec_id`, reading the primary data source. + /// The check and the write are not one atomic operation: an entry that + /// expires between them is still tombstoned, briefly restoring a row that + /// had gone. That is deliberate — the write stays unconditional so a + /// withdrawal is not lost to a concurrent update — and it cannot be used to + /// create an identity, because the entry must have existed to pass the + /// check at all. /// - /// Point lookups on edge data stores are eventually consistent: a recently - /// created key can read absent at a POP that has not converged yet, so - /// `Ok(None)` from [`get`](Self::get) is *not* an absence proof. The list - /// API is the consistency-safe alternative — Fastly's KV list reads the - /// primary data source unless `eventual_consistency()` is requested — so an - /// empty prefix page proves absence where a stale point read cannot. + /// An eventually consistent lookup miss cannot establish absence: a + /// recently issued identity may already have been shared with a partner. + /// The strong check prevents that replication gap from losing withdrawal. + /// An inconclusive check is reported as an error without a lookup fallback. /// - /// EC IDs are fixed-width (`{64hex}.{6alnum}`), so listing with the full ID - /// as the prefix matches at most the key itself. A limit of 1 is enough: - /// only existence is in question, not the count. + /// # Propagating the result + /// + /// A withdrawal is only half-enforced by the store write. Post-send work in + /// the same request — pull sync in particular — decides what to disclose to + /// partners from the in-request snapshot, not from a fresh read, so a + /// tombstone that never reaches that snapshot still leaks the identity it + /// just withdrew. `record_snapshot` is therefore a parameter rather than + /// something the caller may remember to do afterwards: every path out of + /// this method, including the error path, hands back the state the caller + /// must now hold, and a caller that drops it cannot compile. /// /// # Errors /// - /// Returns [`TrustedServerError::KvStore`] on store open or list failure. - /// Callers must treat an error as "existence unknown" and fail closed — - /// never as absence. - pub fn key_exists_confirmed(&self, ec_id: &str) -> Result> { - Ok(self.store.count_keys_with_prefix(ec_id, 1)? > 0) - } + /// Returns [`TrustedServerError::KvStore`] when the tombstone write fails, + /// and when it cannot be determined whether the identity exists — in that + /// case nothing is written and the recorded snapshot is + /// [`EcKvSnapshot::Failed`]. Callers on the browser path should log at + /// `error` level and continue: cookie deletion is the primary enforcement + /// mechanism. + pub fn write_withdrawal_tombstone( + &self, + ec_id: &str, + record_snapshot: impl FnOnce(EcKvSnapshot), + ) -> Result> { + let written = self.tombstone_held_identity(ec_id); - /// Resolves a tombstone attempt whose point read reported the row absent. - /// - /// A proven-absent key is a no-op: there is nothing to withdraw, and a - /// forged cookie must not mint a row. A key that provably exists is - /// tombstoned unconditionally — no CAS generation is available after a - /// missed read, and a withdrawal must win over any concurrent write. An - /// existence check that itself fails leaves the withdrawal unresolved - /// rather than silently dropped. - fn tombstone_unproven_missing(&self, ec_id: &str, missing: EcKvSnapshot) -> EcKvSnapshot { - match self.key_exists_confirmed(ec_id) { - Ok(false) => missing, - Ok(true) => { - log::warn!( - "withdrawal tombstone for '{}': point read missed a row the store still \ - lists; writing an unconditional tombstone", - log_id(ec_id) - ); - let tombstone = KvEntry::tombstone(current_timestamp()); - match self.write_withdrawal_tombstone(ec_id) { - Ok(()) => EcKvSnapshot::Present { - ec_id: ec_id.to_owned(), - entry: Box::new(tombstone), - generation: None, - }, - Err(err) => { - log::warn!( - "unconditional withdrawal tombstone failed for '{}': {err:?}", - log_id(ec_id) - ); - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - } - } - } - } - Err(err) => { - log::warn!( - "withdrawal tombstone for '{}': existence check failed, cannot confirm \ - absence: {err:?}", - log_id(ec_id) - ); - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - } + record_snapshot(match &written { + Ok(Some(entry)) => EcKvSnapshot::Present { + ec_id: ec_id.to_owned(), + entry: Box::new(entry.clone()), + generation: None, + }, + Ok(None) => EcKvSnapshot::Missing { + ec_id: ec_id.to_owned(), + }, + Err(_) => EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }, + }); + + written.map(|entry| { + if entry.is_some() { + TombstoneOutcome::Written + } else { + TombstoneOutcome::UnknownIdentity } - } + }) } - /// Writes a tombstone only when an existing row can be confirmed. - /// - /// Existing-key-only behavior is deliberate: a forged or expired `ts-ec` - /// cookie must not mint a row. But a *point read* cannot prove absence on - /// an eventually-consistent store, and dropping a withdrawal is worse than - /// a redundant read, so absence is established in two stages: + /// Tombstones a held identity, returning the entry written. /// - /// 1. Any snapshot that is not a usable `Present` for this EC ID — a - /// publisher preload that read `Missing`, a read that `Failed`, or one - /// lacking a CAS generation — is re-read. On the publisher path that - /// re-read is separated from the preload by the full origin round trip, - /// which gives replication time to converge. - /// 2. A re-read that still reports the row absent is checked against - /// [`key_exists_confirmed`](Self::key_exists_confirmed), which reads the - /// primary data source. - /// - /// Resolving the initial snapshot happens outside the retry counter, so all - /// [`MAX_CAS_RETRIES`] iterations stay available for the tombstone write. - pub(crate) fn tombstone_existing_from_snapshot( + /// `Ok(None)` means the store does not hold the identity, so nothing was + /// written. + fn tombstone_held_identity( &self, ec_id: &str, - snapshot: EcKvSnapshot, - ) -> EcKvSnapshot { - let mut current = match snapshot { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - generation: Some(_), - .. - } if snapshot_id == ec_id => snapshot, - _ => self.load_snapshot(ec_id), - }; - - for _attempt in 0..MAX_CAS_RETRIES { - let generation = match current { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - generation: Some(generation), - .. - } if snapshot_id == ec_id => generation, - // A missing row (including one that disappeared mid-retry) is - // only a no-op once absence is proven against the primary data - // source. - EcKvSnapshot::Missing { - ec_id: ref snapshot_id, - } if snapshot_id == ec_id => { - return self.tombstone_unproven_missing(ec_id, current); - } - // A refreshed read that failed (or any other unusable state) - // fails closed rather than silently dropping the withdrawal. - _ => { - return EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }; - } - }; - let tombstone = KvEntry::tombstone(current_timestamp()); - let Ok((body, meta_str)) = Self::serialize_entry(&tombstone, self.store_name()) else { - return EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }; - }; - match self.write_entry( - ec_id, - &body, - &meta_str, - TOMBSTONE_TTL, - EcKvWriteMode::IfGenerationMatch(generation), - ) { - Ok(EcKvWriteOutcome::Written) => { - return EcKvSnapshot::Present { - ec_id: ec_id.to_owned(), - entry: Box::new(tombstone), - generation: None, - }; - } - Ok(EcKvWriteOutcome::PreconditionFailed) => { - current = self.load_snapshot(ec_id); - } - Err(err) => { - log::warn!( - "conditional withdrawal tombstone failed for '{}': {err:?}", - log_id(ec_id) - ); - return EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }; - } - } - } - // Withdrawal enforcement lost every CAS race, so the row can still be - // live with consent granted while the browser cookie is cleared. That - // divergence is only visible to operators if it is logged here. - log::warn!( - "withdrawal tombstone for '{}': CAS conflict after {MAX_CAS_RETRIES} retries; the \ - identity-graph row may still be live with consent granted", - log_id(ec_id) - ); - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), + ) -> Result, Report> { + // A store failure is an error, not a third outcome: writing blind + // would restore the unconditional write whenever the store can be made + // to fail, and an extra `Ok` variant would be discarded in silence by a + // caller that only inspects the error case. + if !self.key_exists_confirmed(ec_id)? { + return Ok(None); } + + let entry = KvEntry::tombstone(current_timestamp()); + let (body, meta_str) = Self::serialize_entry(&entry, self.store_name())?; + + self.write_entry( + ec_id, + &body, + &meta_str, + TOMBSTONE_TTL, + EcKvWriteMode::Overwrite, + ) + .map(|_| Some(entry)) + .map_err(|report| { + report.change_context(TrustedServerError::KvStore { + store_name: self.store_name().to_owned(), + message: format!("Failed to write tombstone for key '{}'", log_id(ec_id)), + }) + }) } /// Counts the number of keys sharing the same EC hash prefix. @@ -1210,6 +1145,11 @@ impl KvIdentityGraph { #[cfg(test)] mod tests { use super::*; + use crate::ec::kv_backend::test_support::InMemoryEcKv; + + fn snapshot_ec_id() -> String { + format!("{}.ABC123", "a".repeat(64)) + } #[test] fn constants_have_expected_values() { @@ -1300,13 +1240,6 @@ mod tests { entry } - // ----------------------------------------------------------------------- - // CAS-conflict injection tests - // ----------------------------------------------------------------------- - - use crate::ec::kv_backend::EcKvLookup; - use crate::ec::kv_backend::test_support::InMemoryEcKv; - /// [`EcKvStore`] wrapper that injects generation conflicts: the first /// `conflicts_remaining` `IfGenerationMatch` inserts return /// [`EcKvWriteOutcome::PreconditionFailed`] without writing, optionally @@ -1344,6 +1277,23 @@ mod tests { ) .expect("should seed tombstone"); } + + fn seed_live(&self, ec_id: &str) { + let (body, meta) = + KvIdentityGraph::serialize_entry(&live_entry(), self.inner.store_name()) + .expect("should serialize live entry"); + self.inner + .insert( + ec_id, + EcKvWrite { + body: &body, + metadata: &meta, + ttl: TOMBSTONE_TTL, + mode: EcKvWriteMode::Add, + }, + ) + .expect("should seed live entry"); + } } impl EcKvStore for ConflictInjectingEcKv { @@ -1355,6 +1305,10 @@ mod tests { self.inner.lookup(key) } + fn key_exists(&self, key: &str) -> Result> { + self.inner.key_exists(key) + } + fn insert( &self, key: &str, @@ -1749,8 +1703,12 @@ mod tests { let ec_id = format!("{}.ABC123", "a".repeat(64)); kv.create(&ec_id, &live_entry()).expect("should create"); - kv.write_withdrawal_tombstone(&ec_id) - .expect("should write tombstone"); + assert_eq!( + kv.write_withdrawal_tombstone(&ec_id, drop) + .expect("should write tombstone"), + TombstoneOutcome::Written, + "should tombstone an identity the store holds" + ); let (loaded, _) = kv .get(&ec_id) @@ -1759,49 +1717,6 @@ mod tests { assert!(!loaded.consent.ok, "should be withdrawn after tombstone"); } - #[test] - fn tombstone_existing_from_snapshot_never_creates_missing_key() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let snapshot = EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }; - - let outcome = kv.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!(matches!(outcome, EcKvSnapshot::Missing { .. })); - assert!( - kv.get(&ec_id).expect("should read store").is_none(), - "withdrawal must not create a tombstone for an absent key" - ); - } - - #[test] - fn tombstone_existing_from_snapshot_uses_existing_generation() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - kv.create(&ec_id, &live_entry()).expect("should create"); - let snapshot = kv.load_snapshot(&ec_id); - - let outcome = kv.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "should return the persisted tombstone" - ); - let (stored, _) = kv - .get(&ec_id) - .expect("should read store") - .expect("should preserve existing key"); - assert!(!stored.consent.ok, "should persist withdrawal state"); - } - - // ----------------------------------------------------------------------- - // Snapshot-aware mutation stores and tests - // ----------------------------------------------------------------------- - /// [`EcKvStore`] whose reads succeed but every write fails, simulating a /// store that becomes unwritable mid-request. struct WriteFailingEcKv { @@ -1823,6 +1738,10 @@ mod tests { fn lookup(&self, key: &str) -> Result, Report> { self.inner.lookup(key) } + fn key_exists(&self, key: &str) -> Result> { + self.inner.key_exists(key) + } + fn insert( &self, _key: &str, @@ -1845,108 +1764,34 @@ mod tests { } } - /// [`EcKvStore`] wrapper whose first CAS write both fails the precondition - /// and deletes the key, simulating a concurrent withdrawal that removes the - /// row between this writer's read and its write. - struct DisappearOnConflictEcKv { - inner: InMemoryEcKv, - conflicts_remaining: std::sync::Mutex, - } + #[test] + fn snapshot_upsert_with_generation_writes_without_reading() { + let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let graph = KvIdentityGraph::counting("counting-store", lookups.clone()); + let ec_id = snapshot_ec_id(); + graph.create(&ec_id, &live_entry()).expect("should seed"); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(live_entry()), + generation: Some(1), + }; + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - impl DisappearOnConflictEcKv { - fn new(conflicts: u32) -> Self { - Self { - inner: InMemoryEcKv::new("disappear-store"), - conflicts_remaining: std::sync::Mutex::new(conflicts), - } - } - fn seed_live(&self, ec_id: &str) { - let (body, meta) = - KvIdentityGraph::serialize_entry(&live_entry(), self.inner.store_name()) - .expect("should serialize seeded entry"); - self.inner - .insert( - ec_id, - EcKvWrite { - body: &body, - metadata: &meta, - ttl: ENTRY_TTL, - mode: EcKvWriteMode::Add, - }, - ) - .expect("should seed live entry"); - } - } - - impl EcKvStore for DisappearOnConflictEcKv { - fn store_name(&self) -> &str { - self.inner.store_name() - } - fn lookup(&self, key: &str) -> Result, Report> { - self.inner.lookup(key) - } - fn insert( - &self, - key: &str, - write: EcKvWrite<'_>, - ) -> Result> { - if matches!(write.mode, EcKvWriteMode::IfGenerationMatch(_)) { - let mut remaining = self - .conflicts_remaining - .lock() - .expect("should lock conflict counter"); - if *remaining > 0 { - *remaining -= 1; - self.inner.delete(key).expect("should delete on conflict"); - return Ok(EcKvWriteOutcome::PreconditionFailed); - } - } - self.inner.insert(key, write) - } - fn count_keys_with_prefix( - &self, - prefix: &str, - limit: u32, - ) -> Result> { - self.inner.count_keys_with_prefix(prefix, limit) - } - fn delete(&self, key: &str) -> Result<(), Report> { - self.inner.delete(key) - } - } - - fn snapshot_ec_id() -> String { - format!("{}.ABC123", "a".repeat(64)) - } - - #[test] - fn snapshot_upsert_with_generation_writes_without_reading() { - let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let graph = KvIdentityGraph::counting("counting-store", lookups.clone()); - let ec_id = snapshot_ec_id(); - graph.create(&ec_id, &live_entry()).expect("should seed"); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: Some(1), - }; - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - - assert_eq!( - lookups.load(std::sync::atomic::Ordering::Relaxed), - 0, - "a usable generation must avoid the initial read" - ); - assert_eq!( - outcome - .entry_for(&ec_id) - .and_then(|entry| entry.ids.get("ssp_x")) - .map(|id| id.uid.as_str()), - Some("uid-1") - ); - assert_eq!(outcome.generation_for(&ec_id), None); + let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); + + assert_eq!( + lookups.load(std::sync::atomic::Ordering::Relaxed), + 0, + "a usable generation must avoid the initial read" + ); + assert_eq!( + outcome + .entry_for(&ec_id) + .and_then(|entry| entry.ids.get("ssp_x")) + .map(|id| id.uid.as_str()), + Some("uid-1") + ); + assert_eq!(outcome.generation_for(&ec_id), None); } #[test] @@ -2178,140 +2023,6 @@ mod tests { ); } - #[test] - fn tombstone_existing_from_snapshot_retries_cas_conflict() { - let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(1, false)); - let ec_id = snapshot_ec_id(); - graph.create(&ec_id, &live_entry()).expect("should seed"); - let snapshot = graph.load_snapshot(&ec_id); - - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "should retry the conflict and persist the tombstone" - ); - } - - #[test] - fn tombstone_gen_unavailable_survives_four_conflicts_then_writes() { - // A generation-unavailable snapshot refreshes once before its CAS. That - // refresh must not spend a CAS attempt, so a withdrawal tombstone still - // persists after four conflicts and a successful fifth write. - let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(4, false)); - let ec_id = snapshot_ec_id(); - graph.create(&ec_id, &live_entry()).expect("should seed"); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: None, - }; - - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "the fifth CAS attempt must persist the tombstone after a refresh and four conflicts" - ); - } - - #[test] - fn tombstone_existing_from_snapshot_returns_failed_after_cas_exhaustion() { - // Every CAS attempt loses its race, so the row stays live with consent - // granted while the browser cookie is already cleared. The caller must - // see a failure it can report rather than a silent no-op. - let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(MAX_CAS_RETRIES, false)); - let ec_id = snapshot_ec_id(); - graph.create(&ec_id, &live_entry()).expect("should seed"); - let snapshot = graph.load_snapshot(&ec_id); - - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!( - matches!(outcome, EcKvSnapshot::Failed { .. }), - "CAS exhaustion must report a failed withdrawal" - ); - let (stored, _) = graph - .get(&ec_id) - .expect("should read store") - .expect("row should remain"); - assert!( - stored.consent.ok, - "the row is still live, which is exactly why the failure must be reported" - ); - } - - #[test] - fn tombstone_existing_from_snapshot_store_failure_returns_failed() { - let graph = KvIdentityGraph::new(WriteFailingEcKv::new()); - let ec_id = snapshot_ec_id(); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: Some(1), - }; - - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!(matches!(outcome, EcKvSnapshot::Failed { .. })); - } - - #[test] - fn tombstone_existing_from_snapshot_noop_when_row_disappears_on_retry() { - let store = DisappearOnConflictEcKv::new(1); - store.seed_live(&snapshot_ec_id()); - let graph = KvIdentityGraph::new(store); - let ec_id = snapshot_ec_id(); - let snapshot = graph.load_snapshot(&ec_id); - - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!( - matches!(outcome, EcKvSnapshot::Missing { .. }), - "a row that disappears during retry becomes a no-op" - ); - assert!( - graph.get(&ec_id).expect("should read store").is_none(), - "must not recreate the disappeared key" - ); - } - - #[test] - fn tombstone_existing_from_snapshot_reretries_failed_snapshot_read() { - // A prior request-scoped read failed, so the snapshot is `Failed`. A - // withdrawal must not silently drop consent removal: re-read the store - // and tombstone the row if it is authoritatively present. - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - kv.create(&ec_id, &live_entry()).expect("should seed live"); - - let outcome = kv.tombstone_existing_from_snapshot( - &ec_id, - EcKvSnapshot::Failed { - ec_id: ec_id.clone(), - }, - ); - - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "a failed snapshot must re-read and persist the tombstone" - ); - let (stored, _) = kv - .get(&ec_id) - .expect("should read store") - .expect("should preserve existing key"); - assert!(!stored.consent.ok, "withdrawal must reach the store"); - } - // ----------------------------------------------------------------------- - // Eventual-consistency guards - // ----------------------------------------------------------------------- - #[test] fn key_exists_confirmed_distinguishes_absence_from_a_stale_point_read() { let graph = KvIdentityGraph::stale_lookup("stale-store", 1); @@ -2405,83 +2116,442 @@ mod tests { } #[test] - fn tombstone_revalidates_preloaded_missing_and_writes_when_row_is_present() { - // The publisher preload read `Missing` at a POP that had not converged. - // Finalization runs after the origin round trip, so the re-read sees the - // row and the withdrawal must reach the store. - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - kv.create(&ec_id, &live_entry()).expect("should seed live"); + fn a_store_error_never_carries_the_whole_identifier() { + // Every message in this module goes through `log_id`, so a report that + // reaches a log cannot disclose the identifier it is about. + let kv = KvIdentityGraph::failing("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); - let outcome = kv.tombstone_existing_from_snapshot( - &ec_id, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, + let report = kv + .create(&ec_id, &live_entry()) + .expect_err("the failing store should error"); + + let rendered = format!("{report:?}"); + assert!( + !rendered.contains(&ec_id), + "a store error must not disclose the identifier: {rendered}" ); + } + + #[test] + fn a_locally_built_error_never_carries_the_whole_identifier() { + // The injected-failure case above covers errors the backend produces. + // These are built in this module from the identifier itself, on every + // path a request can reach: a duplicate create, single and batched + // upserts naming a key the store does not hold or has withdrawn, and + // the CAS-exhaustion terminal errors. + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + kv.create(&ec_id, &live_entry()).expect("should create"); + + let duplicate = kv + .create(&ec_id, &live_entry()) + .expect_err("a second create should be refused"); + let missing = kv + .upsert_partner_id(&format!("{}.ZZZ999", "b".repeat(64)), "partner", "uid") + .expect_err("an upsert on a missing key should be refused"); + let batched_missing = kv + .upsert_partner_ids( + &format!("{}.ZZZ999", "b".repeat(64)), + &[PartnerIdUpdate::new("partner", "uid")], + ) + .expect_err("a batched upsert on a missing key should be refused"); + let withdrawn = { + assert_eq!( + kv.write_withdrawal_tombstone(&ec_id, drop) + .expect("should tombstone"), + TombstoneOutcome::Written, + "should tombstone the seeded identity" + ); + kv.upsert_partner_id(&ec_id, "partner", "uid") + .expect_err("an upsert on a withdrawn key should be refused") + }; + let batched_withdrawn = kv + .upsert_partner_ids(&ec_id, &[PartnerIdUpdate::new("partner", "uid")]) + .expect_err("a batched upsert on a withdrawn key should be refused"); + + // The CAS-exhaustion paths build their message the same way, and a + // store that never lets a write land is the only way to reach them. + let cas_revive = { + let store = ConflictInjectingEcKv::new(MAX_CAS_RETRIES + 1, false); + store.seed_tombstone(&ec_id); + KvIdentityGraph::new(store) + .create_or_revive(&ec_id, &live_entry()) + .expect_err("should exhaust CAS retries") + }; + let cas_upsert = { + let store = ConflictInjectingEcKv::new(MAX_CAS_RETRIES + 1, false); + store.seed_live(&ec_id); + KvIdentityGraph::new(store) + .upsert_partner_id(&ec_id, "partner", "uid") + .expect_err("should exhaust CAS retries") + }; + let cas_batched = { + let store = ConflictInjectingEcKv::new(MAX_CAS_RETRIES + 1, false); + store.seed_live(&ec_id); + KvIdentityGraph::new(store) + .upsert_partner_ids(&ec_id, &[PartnerIdUpdate::new("partner", "uid")]) + .expect_err("should exhaust CAS retries") + }; + let cas_if_exists = { + let store = ConflictInjectingEcKv::new(MAX_CAS_RETRIES + 1, false); + store.seed_live(&ec_id); + KvIdentityGraph::new(store) + .upsert_partner_id_if_exists(&ec_id, "partner", "uid") + .expect_err("should exhaust CAS retries") + }; + for (label, report) in [ + ("duplicate create", duplicate), + ("missing key", missing), + ("batched missing key", batched_missing), + ("withdrawn key", withdrawn), + ("batched withdrawn key", batched_withdrawn), + ("CAS exhaustion reviving", cas_revive), + ("CAS exhaustion upserting", cas_upsert), + ("CAS exhaustion batch upserting", cas_batched), + ("CAS exhaustion upserting if present", cas_if_exists), + ] { + let rendered = format!("{report:?}"); + assert!( + !rendered.contains(&ec_id) && !rendered.contains(&"b".repeat(64)), + "the {label} error must not disclose the identifier: {rendered}" + ); + } + } + + #[test] + fn write_withdrawal_tombstone_ignores_an_identity_the_store_does_not_hold() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "b".repeat(64)); + + assert_eq!( + kv.write_withdrawal_tombstone(&ec_id, drop) + .expect("should resolve the withdrawal"), + TombstoneOutcome::UnknownIdentity, + "an identity that was never issued has nothing to withdraw" + ); assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "a stale preloaded miss must not drop the withdrawal" + kv.get(&ec_id).expect("should read back").is_none(), + "should not create a row for an identity the store never held" ); - let (stored, _) = kv - .get(&ec_id) - .expect("should read store") - .expect("should preserve existing key"); - assert!(!stored.consent.ok, "withdrawal must reach the store"); } #[test] - fn tombstone_writes_unconditionally_when_both_point_reads_miss_a_listed_row() { - // Both the preload and the confirming re-read are stale. A point read - // cannot prove absence, so the list API decides: the row exists, and a - // withdrawal must win even without a CAS generation. - let kv = KvIdentityGraph::stale_lookup("stale-store", 1); - let ec_id = snapshot_ec_id(); - kv.create(&ec_id, &live_entry()).expect("should seed live"); + fn withdrawing_many_unheld_identities_creates_no_rows() { + let kv = KvIdentityGraph::in_memory("test_store"); + let hash = "c".repeat(64); + + // The suffix is caller-supplied, so a shared hash prefix must not be + // enough to have a row written under it. + for suffix in ["aaaaaa", "bbbbbb", "cccccc", "dddddd"] { + assert_eq!( + kv.write_withdrawal_tombstone(&format!("{hash}.{suffix}"), drop) + .expect("should resolve the withdrawal"), + TombstoneOutcome::UnknownIdentity, + "suffix `{suffix}` was never issued" + ); + } - let outcome = kv.tombstone_existing_from_snapshot( - &ec_id, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, + assert_eq!( + kv.count_hash_prefix_keys(&hash) + .expect("should count the prefix"), + 0, + "should hold no rows under a hash nothing was issued for" ); + } + #[test] + fn withdrawal_does_not_lose_a_new_identity_to_lookup_lag() { + let (mut store, _) = CountingEcKv::new(); + store.lag_live_entries = true; + let kv = KvIdentityGraph::new(store); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + kv.create(&ec_id, &live_entry()) + .expect("should issue an identity"); assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "a listed row must be tombstoned even when point reads miss it" + kv.lookup_raw(&ec_id) + .expect("should read lagging replica") + .is_none(), + "should model the issuance replication gap" ); - let (stored, _) = kv - .get(&ec_id) - .expect("should read store") - .expect("should preserve existing key"); + assert_eq!( + kv.write_withdrawal_tombstone(&ec_id, drop) + .expect("should process withdrawal"), + TombstoneOutcome::Written, + "should tombstone an issued identity despite a lagging lookup" + ); + assert_eq!( + kv.upsert_partner_id_if_exists(&ec_id, "partner", "uid") + .expect("should check batch-sync eligibility"), + UpsertResult::ConsentWithdrawn, + "should not revive the withdrawn identity through batch sync" + ); + } + + #[test] + fn a_withdrawal_checks_strong_existence_once_without_eventual_lookup() { + // One existence operation may page at the backend, but must not be + // followed by an eventually consistent lookup. + let (store, reads) = CountingEcKv::new(); + let lookups = std::sync::Arc::clone(&store.lookups); + let kv = KvIdentityGraph::new(store); + let count = || *reads.lock().expect("should lock the read counter"); + + let absent = format!("{}.ABC123", "e".repeat(64)); + assert_eq!( + kv.write_withdrawal_tombstone(&absent, drop) + .expect("should resolve the withdrawal"), + TombstoneOutcome::UnknownIdentity, + "an absent identity is not held" + ); + assert_eq!(count(), 1, "an unknown identity costs a single read"); + + let held = format!("{}.ABC123", "a".repeat(64)); + kv.create(&held, &live_entry()).expect("should create"); + let before = count(); + assert_eq!( + kv.write_withdrawal_tombstone(&held, drop) + .expect("should resolve the withdrawal"), + TombstoneOutcome::Written, + "a held identity is tombstoned" + ); + assert_eq!( + count() - before, + 1, + "a held identity is checked once, then written" + ); + assert_eq!( + *lookups.lock().expect("should lock lookup counter"), + 0, + "should not use an eventual lookup for withdrawal" + ); + } + + #[test] + fn write_withdrawal_tombstone_refuses_an_empty_identifier() { + let kv = KvIdentityGraph::in_memory("test_store"); + kv.create(&format!("{}.ABC123", "f".repeat(64)), &live_entry()) + .expect("should create"); + + assert_eq!( + kv.write_withdrawal_tombstone("", drop) + .expect("should resolve the withdrawal"), + TombstoneOutcome::UnknownIdentity, + "an empty identifier names no key and must not withdraw anything" + ); + let (held, _) = kv + .get(&format!("{}.ABC123", "f".repeat(64))) + .expect("should read back") + .expect("should still hold the identity"); assert!( - !stored.consent.ok, - "the live row must not keep consent.ok after an explicit withdrawal" + held.consent.ok, + "should not have withdrawn an unrelated row" + ); + } + + /// Store double that records how many reads reach it. + struct CountingEcKv { + inner: super::super::kv_backend::test_support::InMemoryEcKv, + reads: std::sync::Arc>, + lag_live_entries: bool, + lookups: std::sync::Arc>, + } + + impl CountingEcKv { + /// Returns the store and a handle to its counter, which stays readable + /// after the store moves into the graph. + fn new() -> (Self, std::sync::Arc>) { + let counter = std::sync::Arc::new(std::sync::Mutex::new(0)); + ( + Self { + inner: super::super::kv_backend::test_support::InMemoryEcKv::new("test_store"), + reads: std::sync::Arc::clone(&counter), + lag_live_entries: false, + lookups: std::sync::Arc::new(std::sync::Mutex::new(0)), + }, + counter, + ) + } + } + + impl EcKvStore for CountingEcKv { + fn store_name(&self) -> &str { + self.inner.store_name() + } + + fn lookup(&self, key: &str) -> Result, Report> { + *self.lookups.lock().expect("should lock the lookup counter") += 1; + let found = self.inner.lookup(key)?; + if self.lag_live_entries + && found.as_ref().is_some_and(|entry| { + serde_json::from_slice::(&entry.body) + .expect("should decode test entry") + .consent + .ok + }) + { + return Ok(None); + } + Ok(found) + } + + fn key_exists(&self, key: &str) -> Result> { + *self.reads.lock().expect("should lock the read counter") += 1; + self.inner.key_exists(key) + } + + fn insert( + &self, + key: &str, + write: EcKvWrite<'_>, + ) -> Result> { + self.inner.insert(key, write) + } + + fn count_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result> { + self.inner.count_keys_with_prefix(prefix, limit) + } + + fn delete(&self, key: &str) -> Result<(), Report> { + self.inner.delete(key) + } + } + + /// Store double whose reads always fail while writes still work. + struct ReadFailingEcKv { + inner: super::super::kv_backend::test_support::InMemoryEcKv, + } + + impl ReadFailingEcKv { + fn new() -> Self { + Self { + inner: super::super::kv_backend::test_support::InMemoryEcKv::new("test_store"), + } + } + } + + impl EcKvStore for ReadFailingEcKv { + fn store_name(&self) -> &str { + self.inner.store_name() + } + + fn lookup(&self, _key: &str) -> Result, Report> { + Err(Report::new(TrustedServerError::KvStore { + store_name: "test_store".to_owned(), + message: "reads unavailable".to_owned(), + })) + } + + fn key_exists(&self, key: &str) -> Result> { + self.lookup(key).map(|entry| entry.is_some()) + } + + fn insert( + &self, + key: &str, + write: EcKvWrite<'_>, + ) -> Result> { + self.inner.insert(key, write) + } + + // Left working so a test can prove no row was created without going + // through the read path it just made fail. + fn count_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result> { + self.inner.count_keys_with_prefix(prefix, limit) + } + + fn delete(&self, key: &str) -> Result<(), Report> { + self.inner.delete(key) + } + } + + #[test] + fn a_failing_check_is_not_a_way_to_write_for_an_identity_that_was_never_issued() { + // The caller controls the identifier and can drive load, so a store + // failure must not become a route to the write this gate exists to + // prevent. + let kv = KvIdentityGraph::new(ReadFailingEcKv::new()); + let hash = "8".repeat(64); + let ec_id = format!("{hash}.ABC123"); + + assert!( + kv.write_withdrawal_tombstone(&ec_id, drop).is_err(), + "a check that cannot answer is a fault, so a caller inspecting only \ + the error case still reports it" + ); + assert_eq!( + kv.count_hash_prefix_keys(&hash) + .expect("should count the prefix"), + 0, + "should not create a row while the store is degraded" ); } #[test] - fn tombstone_fails_closed_when_the_existence_check_fails() { - // A forged-cookie no-op requires proof of absence. When the list itself - // fails, the withdrawal is left unresolved rather than silently dropped. - let kv = KvIdentityGraph::failing("failing-store"); - let ec_id = snapshot_ec_id(); + fn a_caller_that_only_inspects_the_error_case_still_sees_a_failed_check() { + // The withdrawal call site is edited by more than one branch. Reporting + // a failed check through `Err` means the common + // `if let Err(..) = ...` shape cannot discard it, where a third `Ok` + // variant would be dropped without a compiler complaint. + let kv = KvIdentityGraph::new(ReadFailingEcKv::new()); + let ec_id = format!("{}.ABC123", "6".repeat(64)); + + let mut reported = false; + if let Err(_err) = kv.write_withdrawal_tombstone(&ec_id, drop) { + reported = true; + } - let outcome = kv.tombstone_existing_from_snapshot( - &ec_id, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, + assert!(reported, "a failed check must reach an error-only caller"); + } + + #[test] + fn a_longer_key_does_not_answer_for_the_identity_it_starts_with() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "7".repeat(64)); + // Only a longer key exists. The identity itself was never issued, so a + // check that matched by prefix would report it as held and tombstone it. + kv.create(&format!("{ec_id}trailing"), &live_entry()) + .expect("should create"); + + assert!( + !kv.key_exists_confirmed(&ec_id).expect("should check"), + "a longer key is a different identity" ); + assert_eq!( + kv.write_withdrawal_tombstone(&ec_id, drop) + .expect("should resolve the withdrawal"), + TombstoneOutcome::UnknownIdentity, + "should not tombstone an identity the store never held" + ); + assert!( + kv.get(&ec_id).expect("should read back").is_none(), + "should not create a row via a prefix match" + ); + } + + #[test] + fn key_exists_confirmed_distinguishes_held_identities() { + let kv = KvIdentityGraph::in_memory("test_store"); + let held = format!("{}.ABC123", "d".repeat(64)); + let sibling = format!("{}.ZZZ999", "d".repeat(64)); + kv.create(&held, &live_entry()).expect("should create"); assert!( - matches!(outcome, EcKvSnapshot::Failed { .. }), - "an unprovable absence must not report a completed withdrawal" + kv.key_exists_confirmed(&held).expect("should check"), + "should confirm a held identity" + ); + assert!( + !kv.key_exists_confirmed(&sibling).expect("should check"), + "a different suffix under the same hash is a different identity" ); } } diff --git a/crates/trusted-server-core/src/ec/kv_backend.rs b/crates/trusted-server-core/src/ec/kv_backend.rs index 5a5c3766d..43b835972 100644 --- a/crates/trusted-server-core/src/ec/kv_backend.rs +++ b/crates/trusted-server-core/src/ec/kv_backend.rs @@ -83,6 +83,33 @@ pub trait EcKvStore { /// Returns [`TrustedServerError::KvStore`] on store open or read failure. fn lookup(&self, key: &str) -> Result, Report>; + /// Checks exact-key existence against strongly consistent store state. + /// + /// A completed issuance must be visible even when [`Self::lookup`] lags. + /// Prefix matches are insufficient and an inconclusive check is an error. + /// + /// # Implementing this method + /// + /// Strong consistency is a contract this signature cannot express, so a + /// new backend has to establish it deliberately. An implementation whose + /// listing or point read is eventually consistent must not answer from it: + /// doing so reintroduces the bug this check exists to prevent, where a + /// replication lag reports a recently issued identity as absent and its + /// withdrawal is silently discarded while the identity stays live for + /// batch sync. If the platform offers no strongly consistent read, return + /// an error rather than a `false` the caller will trust. + /// + /// The in-memory double used across the core tests is trivially strong, so + /// those tests cannot catch a backend that breaks this. Cover a new backend + /// against its own platform. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::KvStore`] on store failure or when a + /// bounded check cannot determine existence. Never falls back to an + /// eventually consistent read. + fn key_exists(&self, key: &str) -> Result>; + /// Writes an entry according to the requested precondition mode. /// /// # Errors @@ -152,6 +179,10 @@ pub(crate) mod test_support { self.inner.lookup(key) } + fn key_exists(&self, key: &str) -> Result> { + self.inner.key_exists(key) + } + fn insert( &self, key: &str, @@ -215,6 +246,16 @@ pub(crate) mod test_support { self.inner.lookup(key) } + fn key_exists(&self, key: &str) -> Result> { + if self.list_fails { + return Err(Report::new(TrustedServerError::KvStore { + store_name: self.inner.store_name().to_owned(), + message: "existence check unavailable".to_owned(), + })); + } + self.inner.key_exists(key) + } + fn insert( &self, key: &str, @@ -277,6 +318,11 @@ pub(crate) mod test_support { })) } + fn key_exists(&self, key: &str) -> Result> { + let entries = self.entries.lock().expect("should lock in-memory store"); + Ok(entries.contains_key(key)) + } + fn insert( &self, key: &str, @@ -360,6 +406,11 @@ pub(crate) mod test_support { Err(self.error("lookup")) } + fn key_exists(&self, key: &str) -> Result> { + let _ = key; + Err(self.error("key_exists")) + } + fn insert( &self, _key: &str, diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index 6bb1b54d8..3dc6e299e 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -49,14 +49,20 @@ pub mod pull_sync; pub mod rate_limiter; pub mod registry; +/// Characters of an identifier kept when redacting it for a log. +const LOG_ID_PREFIX_CHARS: usize = 8; + /// Truncates an EC ID for safe inclusion in log messages. /// -/// Returns the first 8 characters followed by `…` to aid debugging without -/// writing the full user identifier to logs (satisfies the `CodeQL` -/// "cleartext logging of sensitive information" rule). +/// Returns the first [`LOG_ID_PREFIX_CHARS`] characters followed by `…` to aid +/// debugging without writing the full user identifier to logs (satisfies the +/// `CodeQL` "cleartext logging of sensitive information" rule). #[must_use] pub fn log_id(ec_id: &str) -> String { - let prefix = ec_id.get(..8).unwrap_or(ec_id); + // Truncated by character, not by byte. A byte index that lands inside a + // multi-byte character makes `get` return `None`, and falling back to the + // whole value would print in full the identifier this exists to redact. + let prefix: String = ec_id.chars().take(LOG_ID_PREFIX_CHARS).collect(); format!("{prefix}\u{2026}") } @@ -652,6 +658,10 @@ mod tests { fn lookup(&self, key: &str) -> Result, Report> { self.inner.lookup(key) } + fn key_exists(&self, key: &str) -> Result> { + self.inner.key_exists(key) + } + fn insert( &self, key: &str, @@ -947,6 +957,30 @@ mod tests { assert!(!ec.ec_generated(), "should not mark as generated"); } + #[test] + fn log_id_never_emits_more_than_the_redacted_prefix() { + // A byte index inside a multi-byte character used to make the + // truncation fall back to the whole value, printing in full the + // identifier this redacts. + let boundary_splitting = "abcdefg\u{e9}-tail-that-must-not-be-logged"; + let redacted = log_id(boundary_splitting); + + assert!( + !redacted.contains("must-not-be-logged"), + "should not disclose the rest of the identifier: {redacted}" + ); + assert_eq!( + redacted.chars().count(), + 9, + "should be eight characters plus the ellipsis: {redacted}" + ); + + // The ordinary case is unchanged. + assert_eq!(log_id("0123456789abcdef.ABC123"), "01234567\u{2026}"); + // A value shorter than the prefix is emitted whole, which is all there is. + assert_eq!(log_id("abc"), "abc\u{2026}"); + } + #[test] fn existing_cookie_ec_id_returns_cookie_value() { let settings = create_test_settings(); diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs index be4110d43..66b468d3c 100644 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ b/crates/trusted-server-core/src/ec/pull_sync.rs @@ -767,6 +767,7 @@ mod tests { // Snapshot-driven eligibility and request-wide aggregation // ----------------------------------------------------------------------- + use crate::ec::kv::TombstoneOutcome; use crate::error::TrustedServerError; use crate::platform::test_support::{StubHttpClient, build_services_with_http_client}; use crate::settings::EcPartner; @@ -967,9 +968,13 @@ mod tests { let snapshot = seed_present_snapshot(&graph, &ec_id); // Concurrent withdrawal lands after the snapshot was captured. - graph - .write_withdrawal_tombstone(&ec_id) - .expect("should tombstone the row"); + assert_eq!( + graph + .write_withdrawal_tombstone(&ec_id, drop) + .expect("should tombstone the row"), + TombstoneOutcome::Written, + "should tombstone the row the snapshot still reports as present" + ); let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, br#"{"uid":"leaked-uid"}"#.to_vec()); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 031e6b638..05daf0b4e 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -6967,6 +6967,10 @@ mod tests { // Report a miss: the scheduling assertions only care about ordering. Ok(None) } + fn key_exists(&self, key: &str) -> Result> { + self.inner.key_exists(key) + } + fn insert( &self, key: &str,