From b07ae02a80d1c59dc82a6687a66491cd56830756 Mon Sep 17 00:00:00 2001 From: Jussi Saurio Date: Mon, 19 May 2025 09:24:49 +0300 Subject: [PATCH 1/5] remote encryption poc --- libsql/src/database.rs | 8 +++-- libsql/src/database/builder.rs | 19 +++++++++++- libsql/src/hrana/hyper.rs | 22 ++++++++++++-- libsql/src/lib.rs | 1 + libsql/src/local/database.rs | 3 +- libsql/src/sync.rs | 54 ++++++++++++++++++++++++++++++++++ 6 files changed, 101 insertions(+), 6 deletions(-) diff --git a/libsql/src/database.rs b/libsql/src/database.rs index 7069799caa..ba1cfbb7ce 100644 --- a/libsql/src/database.rs +++ b/libsql/src/database.rs @@ -100,6 +100,7 @@ enum DbType { auth_token: String, connector: crate::util::ConnectorService, _bg_abort: Option>, + remote_encryption: Option, }, #[cfg(feature = "remote")] Remote { @@ -214,7 +215,7 @@ cfg_replication! { endpoint, auth_token, https, - encryption_config + encryption_config, ).await } @@ -524,7 +525,7 @@ cfg_remote! { url: impl Into, auth_token: impl Into, connector: C, - version: Option + version: Option, ) -> Result where C: tower::Service + Send + Clone + Sync + 'static, @@ -677,6 +678,7 @@ impl Database { url, auth_token, connector, + remote_encryption, .. } => { use crate::{ @@ -708,6 +710,7 @@ impl Database { connector.clone(), None, None, + remote_encryption.clone() ), read_your_writes: *read_your_writes, context: db.sync_ctx.clone().unwrap(), @@ -738,6 +741,7 @@ impl Database { connector.clone(), version.as_ref().map(|s| s.as_str()), namespace.as_ref().map(|s| s.as_str()), + None, ), ); diff --git a/libsql/src/database/builder.rs b/libsql/src/database/builder.rs index ef70479430..30e68a1c66 100644 --- a/libsql/src/database/builder.rs +++ b/libsql/src/database/builder.rs @@ -5,6 +5,8 @@ cfg_core! { use super::DbType; use crate::{Database, Result}; +pub use crate::sync::EncryptionContext; + /// A builder for [`Database`]. This struct can be used to build /// all variants of [`Database`]. These variants include: /// @@ -50,6 +52,8 @@ impl Builder<()> { path: impl AsRef, url: String, auth_token: String, + #[cfg(feature = "sync")] + remote_encryption: Option, ) -> Builder { Builder { inner: RemoteReplica { @@ -68,6 +72,8 @@ impl Builder<()> { skip_safety_assert: false, #[cfg(feature = "sync")] sync_protocol: Default::default(), + #[cfg(feature = "sync")] + remote_encryption, }, } } @@ -92,6 +98,7 @@ impl Builder<()> { path: impl AsRef, url: String, auth_token: String, + remote_encryption: Option, ) -> Builder { Builder { inner: SyncedDatabase { @@ -109,6 +116,7 @@ impl Builder<()> { remote_writes: false, push_batch_size: 0, sync_interval: None, + remote_encryption, }, } } @@ -229,6 +237,8 @@ cfg_replication! { skip_safety_assert: bool, #[cfg(feature = "sync")] sync_protocol: super::SyncProtocol, + #[cfg(feature = "sync")] + remote_encryption: Option, } /// Local replica configuration type in [`Builder`]. @@ -345,6 +355,8 @@ cfg_replication! { skip_safety_assert, #[cfg(feature = "sync")] sync_protocol, + #[cfg(feature = "sync")] + remote_encryption, } = self.inner; let connector = if let Some(connector) = connector { @@ -403,7 +415,7 @@ cfg_replication! { if res.status().is_success() { tracing::trace!("Using sync protocol v2 for {}", url); - let builder = Builder::new_synced_database(path, url, auth_token) + let builder = Builder::new_synced_database(path, url, auth_token, remote_encryption) .connector(connector) .remote_writes(true) .read_your_writes(read_your_writes); @@ -553,6 +565,7 @@ cfg_sync! { read_your_writes: bool, push_batch_size: u32, sync_interval: Option, + remote_encryption: Option, } impl Builder { @@ -617,6 +630,7 @@ cfg_sync! { read_your_writes, push_batch_size, sync_interval, + remote_encryption, } = self.inner; let path = path.to_str().ok_or(crate::Error::InvalidUTF8Path)?.to_owned(); @@ -640,6 +654,7 @@ cfg_sync! { flags, url.clone(), auth_token.clone(), + remote_encryption.clone(), ) .await?; @@ -708,6 +723,8 @@ cfg_sync! { auth_token, connector, _bg_abort: bg_abort, + #[cfg(feature = "sync")] + remote_encryption, }, max_write_replication_index: Default::default(), }) diff --git a/libsql/src/hrana/hyper.rs b/libsql/src/hrana/hyper.rs index b78838d11a..129784ffb7 100644 --- a/libsql/src/hrana/hyper.rs +++ b/libsql/src/hrana/hyper.rs @@ -27,6 +27,8 @@ pub struct HttpSender { inner: hyper::Client, version: HeaderValue, namespace: Option, + #[cfg(feature = "sync")] + remote_encryption: Option, } impl HttpSender { @@ -34,6 +36,7 @@ impl HttpSender { connector: ConnectorService, version: Option<&str>, namespace: Option<&str>, + #[cfg(feature = "sync")] remote_encryption: Option, ) -> Self { let ver = version.unwrap_or(env!("CARGO_PKG_VERSION")); @@ -41,11 +44,12 @@ impl HttpSender { let namespace = namespace.map(|v| HeaderValue::try_from(v).unwrap()); let inner = hyper::Client::builder().build(connector); - Self { inner, version, namespace, + #[cfg(feature = "sync")] + remote_encryption, } } @@ -63,6 +67,13 @@ impl HttpSender { req_builder = req_builder.header("x-namespace", namespace); } + if let Some(remote_encryption) = &self.remote_encryption { + req_builder = req_builder.header( + "x-turso-encryption-key", + remote_encryption.key_16_bytes_base64_encoded.as_str(), + ); + } + let req = req_builder .body(hyper::Body::from(body)) .map_err(|err| HranaError::Http(format!("{:?}", err)))?; @@ -126,8 +137,15 @@ impl HttpConnection { connector: ConnectorService, version: Option<&str>, namespace: Option<&str>, + #[cfg(feature = "sync")] remote_encryption: Option, ) -> Self { - let inner = HttpSender::new(connector, version, namespace); + let inner = HttpSender::new( + connector, + version, + namespace, + #[cfg(feature = "sync")] + remote_encryption, + ); Self::new(url.into(), token.into(), inner) } } diff --git a/libsql/src/lib.rs b/libsql/src/lib.rs index 15f98d8869..fe47e56d16 100644 --- a/libsql/src/lib.rs +++ b/libsql/src/lib.rs @@ -132,6 +132,7 @@ pub mod params; cfg_sync! { mod sync; pub use database::SyncProtocol; + pub use sync::EncryptionContext; } cfg_replication! { diff --git a/libsql/src/local/database.rs b/libsql/src/local/database.rs index 7cfcf330fe..4723133a31 100644 --- a/libsql/src/local/database.rs +++ b/libsql/src/local/database.rs @@ -212,6 +212,7 @@ impl Database { flags: OpenFlags, endpoint: String, auth_token: String, + remote_encryption: Option, ) -> Result { let db_path = db_path.into(); let endpoint = if endpoint.starts_with("libsql:") { @@ -222,7 +223,7 @@ impl Database { let mut db = Database::open(&db_path, flags)?; let sync_ctx = - SyncContext::new(connector, db_path.into(), endpoint, Some(auth_token)).await?; + SyncContext::new(connector, db_path.into(), endpoint, Some(auth_token), remote_encryption).await?; db.sync_ctx = Some(Arc::new(Mutex::new(sync_ctx))); Ok(db) diff --git a/libsql/src/sync.rs b/libsql/src/sync.rs index 9a74b118b0..63df912dde 100644 --- a/libsql/src/sync.rs +++ b/libsql/src/sync.rs @@ -118,6 +118,16 @@ struct PushFramesResult { baton: Option, } +#[derive(Debug, Clone)] +pub struct EncryptionContext { + /// The base64-encoded key for the encryption, sent on every request. + pub key_16_bytes_base64_encoded: String, + /// Whether the pushed frames are already encrypted. + pub push_is_encrypted: bool, + /// Whether to request the server to decrypt the pulled frames. + pub decrypt_pull: bool, +} + pub struct SyncContext { db_path: String, client: hyper::Client, @@ -133,6 +143,8 @@ pub struct SyncContext { /// whenever sync is called very first time, we will call the remote server /// to get the generation information and sync the db file if needed initial_server_sync: bool, + /// The encryption context for the sync. + remote_encryption: Option, } impl SyncContext { @@ -141,6 +153,7 @@ impl SyncContext { db_path: String, sync_url: String, auth_token: Option, + remote_encryption: Option, ) -> Result { let client = hyper::client::Client::builder().build::<_, hyper::Body>(connector); @@ -163,6 +176,7 @@ impl SyncContext { durable_generation: 0, durable_frame_num: 0, initial_server_sync: false, + remote_encryption, }; if let Err(e) = me.read_metadata().await { @@ -303,6 +317,16 @@ impl SyncContext { None => {} } + if let Some(remote_encryption) = &self.remote_encryption { + if remote_encryption.decrypt_pull { + req = req.header("x-turso-decrypt-response", "true"); + } + if remote_encryption.push_is_encrypted { + req = req.header("x-turso-encrypted-request", "true"); + } + req = req.header("x-turso-encryption-key", remote_encryption.key_16_bytes_base64_encoded.as_str()); + } + let req = req.body(body.clone().into()).expect("valid body"); let res = self @@ -414,6 +438,16 @@ impl SyncContext { None => {} } + if let Some(remote_encryption) = &self.remote_encryption { + if remote_encryption.decrypt_pull { + req = req.header("x-turso-decrypt-response", "true"); + } + if remote_encryption.push_is_encrypted { + req = req.header("x-turso-encrypted-request", "true"); + } + req = req.header("x-turso-encryption-key", remote_encryption.key_16_bytes_base64_encoded.as_str()); + } + let req = req.body(Body::empty()).expect("valid request"); let res = self @@ -577,6 +611,16 @@ impl SyncContext { req = req.header("Authorization", auth_token); } + if let Some(remote_encryption) = &self.remote_encryption { + if remote_encryption.decrypt_pull { + req = req.header("x-turso-decrypt-response", "true"); + } + if remote_encryption.push_is_encrypted { + req = req.header("x-turso-encrypted-request", "true"); + } + req = req.header("x-turso-encryption-key", remote_encryption.key_16_bytes_base64_encoded.as_str()); + } + let req = req.body(Body::empty()).expect("valid request"); let res = self @@ -673,6 +717,16 @@ impl SyncContext { req = req.header("Authorization", auth_token); } + if let Some(remote_encryption) = &self.remote_encryption { + if remote_encryption.decrypt_pull { + req = req.header("x-turso-decrypt-response", "true"); + } + if remote_encryption.push_is_encrypted { + req = req.header("x-turso-encrypted-request", "true"); + } + req = req.header("x-turso-encryption-key", remote_encryption.key_16_bytes_base64_encoded.as_str()); + } + let req = req.body(Body::empty()).expect("valid request"); let (res, http_duration) = From ce853e671d2a67b7b9273ff47956fb491b2f5f6e Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Sat, 28 Jun 2025 23:40:16 +0530 Subject: [PATCH 2/5] simplify encryption parameters --- libsql/src/database.rs | 2 +- libsql/src/hrana/hyper.rs | 2 +- libsql/src/sync.rs | 50 +++++++++++++-------------------------- 3 files changed, 19 insertions(+), 35 deletions(-) diff --git a/libsql/src/database.rs b/libsql/src/database.rs index ba1cfbb7ce..eaf34f8075 100644 --- a/libsql/src/database.rs +++ b/libsql/src/database.rs @@ -710,7 +710,7 @@ impl Database { connector.clone(), None, None, - remote_encryption.clone() + remote_encryption.clone(), ), read_your_writes: *read_your_writes, context: db.sync_ctx.clone().unwrap(), diff --git a/libsql/src/hrana/hyper.rs b/libsql/src/hrana/hyper.rs index 129784ffb7..12bbdfc60b 100644 --- a/libsql/src/hrana/hyper.rs +++ b/libsql/src/hrana/hyper.rs @@ -70,7 +70,7 @@ impl HttpSender { if let Some(remote_encryption) = &self.remote_encryption { req_builder = req_builder.header( "x-turso-encryption-key", - remote_encryption.key_16_bytes_base64_encoded.as_str(), + remote_encryption.key_32_bytes_base64_encoded.as_str(), ); } diff --git a/libsql/src/sync.rs b/libsql/src/sync.rs index 63df912dde..5136b95df2 100644 --- a/libsql/src/sync.rs +++ b/libsql/src/sync.rs @@ -121,11 +121,7 @@ struct PushFramesResult { #[derive(Debug, Clone)] pub struct EncryptionContext { /// The base64-encoded key for the encryption, sent on every request. - pub key_16_bytes_base64_encoded: String, - /// Whether the pushed frames are already encrypted. - pub push_is_encrypted: bool, - /// Whether to request the server to decrypt the pulled frames. - pub decrypt_pull: bool, + pub key_32_bytes_base64_encoded: String, } pub struct SyncContext { @@ -318,13 +314,10 @@ impl SyncContext { } if let Some(remote_encryption) = &self.remote_encryption { - if remote_encryption.decrypt_pull { - req = req.header("x-turso-decrypt-response", "true"); - } - if remote_encryption.push_is_encrypted { - req = req.header("x-turso-encrypted-request", "true"); - } - req = req.header("x-turso-encryption-key", remote_encryption.key_16_bytes_base64_encoded.as_str()); + req = req.header( + "x-turso-encryption-key", + remote_encryption.key_32_bytes_base64_encoded.as_str(), + ); } let req = req.body(body.clone().into()).expect("valid body"); @@ -439,13 +432,10 @@ impl SyncContext { } if let Some(remote_encryption) = &self.remote_encryption { - if remote_encryption.decrypt_pull { - req = req.header("x-turso-decrypt-response", "true"); - } - if remote_encryption.push_is_encrypted { - req = req.header("x-turso-encrypted-request", "true"); - } - req = req.header("x-turso-encryption-key", remote_encryption.key_16_bytes_base64_encoded.as_str()); + req = req.header( + "x-turso-encryption-key", + remote_encryption.key_32_bytes_base64_encoded.as_str(), + ); } let req = req.body(Body::empty()).expect("valid request"); @@ -612,13 +602,10 @@ impl SyncContext { } if let Some(remote_encryption) = &self.remote_encryption { - if remote_encryption.decrypt_pull { - req = req.header("x-turso-decrypt-response", "true"); - } - if remote_encryption.push_is_encrypted { - req = req.header("x-turso-encrypted-request", "true"); - } - req = req.header("x-turso-encryption-key", remote_encryption.key_16_bytes_base64_encoded.as_str()); + req = req.header( + "x-turso-encryption-key", + remote_encryption.key_32_bytes_base64_encoded.as_str(), + ); } let req = req.body(Body::empty()).expect("valid request"); @@ -718,13 +705,10 @@ impl SyncContext { } if let Some(remote_encryption) = &self.remote_encryption { - if remote_encryption.decrypt_pull { - req = req.header("x-turso-decrypt-response", "true"); - } - if remote_encryption.push_is_encrypted { - req = req.header("x-turso-encrypted-request", "true"); - } - req = req.header("x-turso-encryption-key", remote_encryption.key_16_bytes_base64_encoded.as_str()); + req = req.header( + "x-turso-encryption-key", + remote_encryption.key_32_bytes_base64_encoded.as_str(), + ); } let req = req.body(Body::empty()).expect("valid request"); From 330a3be64be2fe15ff83d4768abefe6afbf5ab7c Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Mon, 30 Jun 2025 15:07:19 +0530 Subject: [PATCH 3/5] Add offline writes with encryption example --- libsql/examples/encryption_sync.rs | 81 ++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 libsql/examples/encryption_sync.rs diff --git a/libsql/examples/encryption_sync.rs b/libsql/examples/encryption_sync.rs new file mode 100644 index 0000000000..5a381a127a --- /dev/null +++ b/libsql/examples/encryption_sync.rs @@ -0,0 +1,81 @@ +// Example of using offline writes with encryption + +use libsql::{params, Builder, EncryptionContext}; + +#[tokio::main] +async fn main() { + tracing_subscriber::fmt::init(); + + // The local database path where the data will be stored. + let db_path = std::env::var("LIBSQL_DB_PATH").unwrap(); + + // The remote sync URL to use. + let sync_url = std::env::var("LIBSQL_SYNC_URL").unwrap(); + + // The authentication token for the remote sync server. + let auth_token = std::env::var("LIBSQL_AUTH_TOKEN").unwrap_or("".to_string()); + + // Optional encryption key for the database, if provided. + let encryption = if let Ok(key) = std::env::var("LIBSQL_ENCRYPTION_KEY") { + Some(EncryptionContext { + key_32_bytes_base64_encoded: key.to_string(), + }) + } else { + None + }; + + let db_builder = Builder::new_synced_database(db_path, sync_url, auth_token, encryption); + + let db = match db_builder.build().await { + Ok(db) => db, + Err(error) => { + eprintln!("Error connecting to remote sync server: {}", error); + return; + } + }; + + let conn = db.connect().unwrap(); + + print!("Syncing with remote database..."); + db.sync().await.unwrap(); + println!(" done"); + + let mut results = conn.query("SELECT count(*) FROM dummy", ()).await.unwrap(); + let count: u32 = results.next().await.unwrap().unwrap().get(0).unwrap(); + println!("dummy table has {} entries", count); + + conn.execute( + r#" + CREATE TABLE IF NOT EXISTS guest_book_entries ( + text TEXT + )"#, + (), + ) + .await + .unwrap(); + + let mut input = String::new(); + println!("Please write your entry to the guestbook:"); + match std::io::stdin().read_line(&mut input) { + Ok(_) => { + println!("You entered: {}", input); + let params = params![input.as_str()]; + conn.execute("INSERT INTO guest_book_entries (text) VALUES (?)", params) + .await + .unwrap(); + } + Err(error) => { + eprintln!("Error reading input: {}", error); + } + } + db.sync().await.unwrap(); + let mut results = conn + .query("SELECT * FROM guest_book_entries", ()) + .await + .unwrap(); + println!("Guest book entries:"); + while let Some(row) = results.next().await.unwrap() { + let text: String = row.get(0).unwrap(); + println!(" {}", text); + } +} From 5da95cc1b44fbfff881df2526cf94bef07faa983 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Mon, 30 Jun 2025 17:25:55 +0530 Subject: [PATCH 4/5] Add `EncryptionKey` enum for the key --- libsql/examples/encryption_sync.rs | 4 +-- libsql/src/hrana/hyper.rs | 6 ++-- libsql/src/lib.rs | 1 + libsql/src/sync.rs | 44 +++++++++++++++++------------- 4 files changed, 30 insertions(+), 25 deletions(-) diff --git a/libsql/examples/encryption_sync.rs b/libsql/examples/encryption_sync.rs index 5a381a127a..2d28f07898 100644 --- a/libsql/examples/encryption_sync.rs +++ b/libsql/examples/encryption_sync.rs @@ -1,6 +1,6 @@ // Example of using offline writes with encryption -use libsql::{params, Builder, EncryptionContext}; +use libsql::{params, Builder, EncryptionContext, EncryptionKey}; #[tokio::main] async fn main() { @@ -18,7 +18,7 @@ async fn main() { // Optional encryption key for the database, if provided. let encryption = if let Ok(key) = std::env::var("LIBSQL_ENCRYPTION_KEY") { Some(EncryptionContext { - key_32_bytes_base64_encoded: key.to_string(), + key: EncryptionKey::Base64Encoded(key), }) } else { None diff --git a/libsql/src/hrana/hyper.rs b/libsql/src/hrana/hyper.rs index 12bbdfc60b..c546179a8a 100644 --- a/libsql/src/hrana/hyper.rs +++ b/libsql/src/hrana/hyper.rs @@ -68,10 +68,8 @@ impl HttpSender { } if let Some(remote_encryption) = &self.remote_encryption { - req_builder = req_builder.header( - "x-turso-encryption-key", - remote_encryption.key_32_bytes_base64_encoded.as_str(), - ); + req_builder = + req_builder.header("x-turso-encryption-key", remote_encryption.key.as_string()); } let req = req_builder diff --git a/libsql/src/lib.rs b/libsql/src/lib.rs index fe47e56d16..64761c19fa 100644 --- a/libsql/src/lib.rs +++ b/libsql/src/lib.rs @@ -133,6 +133,7 @@ cfg_sync! { mod sync; pub use database::SyncProtocol; pub use sync::EncryptionContext; + pub use sync::EncryptionKey; } cfg_replication! { diff --git a/libsql/src/sync.rs b/libsql/src/sync.rs index 5136b95df2..65b9ad4f4c 100644 --- a/libsql/src/sync.rs +++ b/libsql/src/sync.rs @@ -1,11 +1,12 @@ use crate::{local::Connection, util::ConnectorService, Error, Result}; -use std::path::Path; - +use base64::engine::general_purpose; +use base64::Engine; use bytes::Bytes; use chrono::Utc; use http::{HeaderValue, StatusCode}; use hyper::Body; +use std::path::Path; use tokio::io::AsyncWriteExt as _; use uuid::Uuid; @@ -118,10 +119,27 @@ struct PushFramesResult { baton: Option, } +#[derive(Debug, Clone)] +pub enum EncryptionKey { + /// The key is a base64-encoded string. + Base64Encoded(String), + /// The key is a byte array. + Bytes(Vec), +} + +impl EncryptionKey { + pub fn as_string(&self) -> String { + match self { + EncryptionKey::Base64Encoded(s) => s.clone(), + EncryptionKey::Bytes(b) => general_purpose::STANDARD.encode(b), + } + } +} + #[derive(Debug, Clone)] pub struct EncryptionContext { /// The base64-encoded key for the encryption, sent on every request. - pub key_32_bytes_base64_encoded: String, + pub key: EncryptionKey, } pub struct SyncContext { @@ -314,10 +332,7 @@ impl SyncContext { } if let Some(remote_encryption) = &self.remote_encryption { - req = req.header( - "x-turso-encryption-key", - remote_encryption.key_32_bytes_base64_encoded.as_str(), - ); + req = req.header("x-turso-encryption-key", remote_encryption.key.as_string()); } let req = req.body(body.clone().into()).expect("valid body"); @@ -432,10 +447,7 @@ impl SyncContext { } if let Some(remote_encryption) = &self.remote_encryption { - req = req.header( - "x-turso-encryption-key", - remote_encryption.key_32_bytes_base64_encoded.as_str(), - ); + req = req.header("x-turso-encryption-key", remote_encryption.key.as_string()); } let req = req.body(Body::empty()).expect("valid request"); @@ -602,10 +614,7 @@ impl SyncContext { } if let Some(remote_encryption) = &self.remote_encryption { - req = req.header( - "x-turso-encryption-key", - remote_encryption.key_32_bytes_base64_encoded.as_str(), - ); + req = req.header("x-turso-encryption-key", remote_encryption.key.as_string()); } let req = req.body(Body::empty()).expect("valid request"); @@ -705,10 +714,7 @@ impl SyncContext { } if let Some(remote_encryption) = &self.remote_encryption { - req = req.header( - "x-turso-encryption-key", - remote_encryption.key_32_bytes_base64_encoded.as_str(), - ); + req = req.header("x-turso-encryption-key", remote_encryption.key.as_string()); } let req = req.body(Body::empty()).expect("valid request"); From 3c42428aad56336bcd69c8021155e9f1aac353f7 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Fri, 4 Jul 2025 18:40:24 +0530 Subject: [PATCH 5/5] enable remote encryption support for remote and synced databases --- libsql/Cargo.toml | 2 ++ libsql/examples/encryption_sync.rs | 6 ++-- libsql/src/database.rs | 35 +++++++++++++++++-- libsql/src/database/builder.rs | 54 ++++++++++++++++++++++++------ libsql/src/hrana/hyper.rs | 17 ++++++---- libsql/src/lib.rs | 4 +-- libsql/src/local/database.rs | 12 +++++-- libsql/src/sync.rs | 26 +------------- libsql/src/sync/test.rs | 8 +++++ 9 files changed, 113 insertions(+), 51 deletions(-) diff --git a/libsql/Cargo.toml b/libsql/Cargo.toml index c02bbe229f..9df2c3e537 100644 --- a/libsql/Cargo.toml +++ b/libsql/Cargo.toml @@ -102,6 +102,7 @@ sync = [ "stream", "remote", "replication", + "dep:base64", "dep:tower", "dep:hyper", "dep:http", @@ -131,6 +132,7 @@ hrana = [ serde = ["dep:serde"] remote = [ "hrana", + "dep:base64", "dep:tower", "dep:hyper", "dep:hyper", diff --git a/libsql/examples/encryption_sync.rs b/libsql/examples/encryption_sync.rs index 2d28f07898..15a3b22eaf 100644 --- a/libsql/examples/encryption_sync.rs +++ b/libsql/examples/encryption_sync.rs @@ -1,6 +1,7 @@ // Example of using offline writes with encryption -use libsql::{params, Builder, EncryptionContext, EncryptionKey}; +use libsql::{params, Builder}; +use libsql::{EncryptionContext, EncryptionKey}; #[tokio::main] async fn main() { @@ -24,7 +25,8 @@ async fn main() { None }; - let db_builder = Builder::new_synced_database(db_path, sync_url, auth_token, encryption); + let db_builder = + Builder::new_synced_database(db_path, sync_url, auth_token).remote_encryption(encryption); let db = match db_builder.build().await { Ok(db) => db, diff --git a/libsql/src/database.rs b/libsql/src/database.rs index eaf34f8075..322913eefc 100644 --- a/libsql/src/database.rs +++ b/libsql/src/database.rs @@ -8,6 +8,8 @@ pub use builder::Builder; pub use libsql_sys::{Cipher, EncryptionConfig}; use crate::{Connection, Result}; +#[cfg(any(feature = "remote", feature = "sync"))] +use base64::{engine::general_purpose, Engine}; use std::fmt; use std::sync::atomic::AtomicU64; @@ -100,7 +102,7 @@ enum DbType { auth_token: String, connector: crate::util::ConnectorService, _bg_abort: Option>, - remote_encryption: Option, + remote_encryption: Option, }, #[cfg(feature = "remote")] Remote { @@ -109,6 +111,7 @@ enum DbType { connector: crate::util::ConnectorService, version: Option, namespace: Option, + remote_encryption: Option, }, } @@ -545,6 +548,7 @@ cfg_remote! { connector: crate::util::ConnectorService::new(svc), version, namespace: None, + remote_encryption: None }, max_write_replication_index: Default::default(), }) @@ -733,6 +737,7 @@ impl Database { connector, version, namespace, + remote_encryption, } => { let conn = std::sync::Arc::new( crate::hrana::connection::HttpConnection::new_with_connector( @@ -741,7 +746,7 @@ impl Database { connector.clone(), version.as_ref().map(|s| s.as_str()), namespace.as_ref().map(|s| s.as_str()), - None, + remote_encryption.clone(), ), ); @@ -785,3 +790,29 @@ impl std::fmt::Debug for Database { f.debug_struct("Database").finish() } } + +#[cfg(any(feature = "remote", feature = "sync"))] +#[derive(Debug, Clone)] +pub enum EncryptionKey { + /// The key is a base64-encoded string. + Base64Encoded(String), + /// The key is a byte array. + Bytes(Vec), +} + +#[cfg(any(feature = "remote", feature = "sync"))] +impl EncryptionKey { + pub fn as_string(&self) -> String { + match self { + EncryptionKey::Base64Encoded(s) => s.clone(), + EncryptionKey::Bytes(b) => general_purpose::STANDARD.encode(b), + } + } +} + +#[cfg(any(feature = "remote", feature = "sync"))] +#[derive(Debug, Clone)] +pub struct EncryptionContext { + /// The base64-encoded key for the encryption, sent on every request. + pub key: EncryptionKey, +} diff --git a/libsql/src/database/builder.rs b/libsql/src/database/builder.rs index 30e68a1c66..623ec24a3e 100644 --- a/libsql/src/database/builder.rs +++ b/libsql/src/database/builder.rs @@ -5,7 +5,8 @@ cfg_core! { use super::DbType; use crate::{Database, Result}; -pub use crate::sync::EncryptionContext; +#[cfg(any(feature = "remote", feature = "sync"))] +pub use crate::database::EncryptionContext; /// A builder for [`Database`]. This struct can be used to build /// all variants of [`Database`]. These variants include: @@ -52,8 +53,6 @@ impl Builder<()> { path: impl AsRef, url: String, auth_token: String, - #[cfg(feature = "sync")] - remote_encryption: Option, ) -> Builder { Builder { inner: RemoteReplica { @@ -64,6 +63,8 @@ impl Builder<()> { connector: None, version: None, namespace: None, + #[cfg(any(feature = "remote", feature = "sync"))] + remote_encryption: None, }, encryption_config: None, read_your_writes: true, @@ -73,7 +74,7 @@ impl Builder<()> { #[cfg(feature = "sync")] sync_protocol: Default::default(), #[cfg(feature = "sync")] - remote_encryption, + remote_encryption: None }, } } @@ -98,7 +99,6 @@ impl Builder<()> { path: impl AsRef, url: String, auth_token: String, - remote_encryption: Option, ) -> Builder { Builder { inner: SyncedDatabase { @@ -110,13 +110,14 @@ impl Builder<()> { connector: None, version: None, namespace: None, + remote_encryption: None, }, connector: None, read_your_writes: true, remote_writes: false, push_batch_size: 0, sync_interval: None, - remote_encryption, + remote_encryption: None, }, } } @@ -132,6 +133,7 @@ impl Builder<()> { connector: None, version: None, namespace: None, + remote_encryption: None, }, } } @@ -146,6 +148,8 @@ cfg_replication_or_remote_or_sync! { connector: Option, version: Option, namespace: Option, + #[cfg(any(feature = "remote", feature = "sync"))] + remote_encryption: Option, } } @@ -238,7 +242,7 @@ cfg_replication! { #[cfg(feature = "sync")] sync_protocol: super::SyncProtocol, #[cfg(feature = "sync")] - remote_encryption: Option, + remote_encryption: Option, } /// Local replica configuration type in [`Builder`]. @@ -300,6 +304,13 @@ cfg_replication! { self } + /// Set the encryption context if the database is encrypted in remote server. + #[cfg(feature = "sync")] + pub fn remote_encryption(mut self, encryption_context: Option) -> Builder { + self.inner.remote_encryption = encryption_context; + self + } + pub fn http_request_callback(mut self, f: F) -> Builder where F: Fn(&mut http::Request<()>) + Send + Sync + 'static @@ -347,6 +358,7 @@ cfg_replication! { connector, version, namespace, + .. }, encryption_config, read_your_writes, @@ -415,10 +427,11 @@ cfg_replication! { if res.status().is_success() { tracing::trace!("Using sync protocol v2 for {}", url); - let builder = Builder::new_synced_database(path, url, auth_token, remote_encryption) + let builder = Builder::new_synced_database(path, url, auth_token) .connector(connector) .remote_writes(true) - .read_your_writes(read_your_writes); + .read_your_writes(read_your_writes) + .remote_encryption(remote_encryption); let builder = if let Some(sync_interval) = sync_interval { builder.sync_interval(sync_interval) @@ -475,7 +488,10 @@ cfg_replication! { Ok(Database { - db_type: DbType::Sync { db, encryption_config }, + db_type: DbType::Sync { + db, + encryption_config, + }, max_write_replication_index: Default::default(), }) } @@ -515,6 +531,7 @@ cfg_replication! { connector, version, namespace, + .. }) = remote { let connector = if let Some(connector) = connector { @@ -565,7 +582,7 @@ cfg_sync! { read_your_writes: bool, push_batch_size: u32, sync_interval: Option, - remote_encryption: Option, + remote_encryption: Option, } impl Builder { @@ -598,6 +615,12 @@ cfg_sync! { self } + /// Set the encryption context if the database is encrypted in remote server. + pub fn remote_encryption(mut self, encryption_context: Option) -> Builder { + self.inner.remote_encryption = encryption_context; + self + } + /// Provide a custom http connector that will be used to create http connections. pub fn connector(mut self, connector: C) -> Builder where @@ -624,6 +647,7 @@ cfg_sync! { connector: _, version: _, namespace: _, + .. }, connector, remote_writes, @@ -759,6 +783,12 @@ cfg_remote! { self } + /// Set the encryption context if the database is encrypted in remote server. + pub fn remote_encryption(mut self, encryption_context: Option) -> Builder { + self.inner.remote_encryption = encryption_context; + self + } + /// Build the remote database client. pub async fn build(self) -> Result { let Remote { @@ -767,6 +797,7 @@ cfg_remote! { connector, version, namespace, + remote_encryption, } = self.inner; let connector = if let Some(connector) = connector { @@ -789,6 +820,7 @@ cfg_remote! { connector, version, namespace, + remote_encryption }, max_write_replication_index: Default::default(), }) diff --git a/libsql/src/hrana/hyper.rs b/libsql/src/hrana/hyper.rs index c546179a8a..d32341796b 100644 --- a/libsql/src/hrana/hyper.rs +++ b/libsql/src/hrana/hyper.rs @@ -27,8 +27,8 @@ pub struct HttpSender { inner: hyper::Client, version: HeaderValue, namespace: Option, - #[cfg(feature = "sync")] - remote_encryption: Option, + #[cfg(any(feature = "remote", feature = "sync"))] + remote_encryption: Option, } impl HttpSender { @@ -36,7 +36,9 @@ impl HttpSender { connector: ConnectorService, version: Option<&str>, namespace: Option<&str>, - #[cfg(feature = "sync")] remote_encryption: Option, + #[cfg(any(feature = "remote", feature = "sync"))] remote_encryption: Option< + crate::database::EncryptionContext, + >, ) -> Self { let ver = version.unwrap_or(env!("CARGO_PKG_VERSION")); @@ -48,7 +50,7 @@ impl HttpSender { inner, version, namespace, - #[cfg(feature = "sync")] + #[cfg(any(feature = "remote", feature = "sync"))] remote_encryption, } } @@ -67,6 +69,7 @@ impl HttpSender { req_builder = req_builder.header("x-namespace", namespace); } + #[cfg(any(feature = "remote", feature = "sync"))] if let Some(remote_encryption) = &self.remote_encryption { req_builder = req_builder.header("x-turso-encryption-key", remote_encryption.key.as_string()); @@ -135,13 +138,15 @@ impl HttpConnection { connector: ConnectorService, version: Option<&str>, namespace: Option<&str>, - #[cfg(feature = "sync")] remote_encryption: Option, + #[cfg(any(feature = "remote", feature = "sync"))] remote_encryption: Option< + crate::database::EncryptionContext, + >, ) -> Self { let inner = HttpSender::new( connector, version, namespace, - #[cfg(feature = "sync")] + #[cfg(any(feature = "remote", feature = "sync"))] remote_encryption, ); Self::new(url.into(), token.into(), inner) diff --git a/libsql/src/lib.rs b/libsql/src/lib.rs index 64761c19fa..a42b0a4940 100644 --- a/libsql/src/lib.rs +++ b/libsql/src/lib.rs @@ -132,8 +132,8 @@ pub mod params; cfg_sync! { mod sync; pub use database::SyncProtocol; - pub use sync::EncryptionContext; - pub use sync::EncryptionKey; + pub use database::EncryptionContext; + pub use database::EncryptionKey; } cfg_replication! { diff --git a/libsql/src/local/database.rs b/libsql/src/local/database.rs index 4723133a31..5d5eda0b06 100644 --- a/libsql/src/local/database.rs +++ b/libsql/src/local/database.rs @@ -212,7 +212,7 @@ impl Database { flags: OpenFlags, endpoint: String, auth_token: String, - remote_encryption: Option, + remote_encryption: Option, ) -> Result { let db_path = db_path.into(); let endpoint = if endpoint.starts_with("libsql:") { @@ -222,8 +222,14 @@ impl Database { }; let mut db = Database::open(&db_path, flags)?; - let sync_ctx = - SyncContext::new(connector, db_path.into(), endpoint, Some(auth_token), remote_encryption).await?; + let sync_ctx = SyncContext::new( + connector, + db_path.into(), + endpoint, + Some(auth_token), + remote_encryption, + ) + .await?; db.sync_ctx = Some(Arc::new(Mutex::new(sync_ctx))); Ok(db) diff --git a/libsql/src/sync.rs b/libsql/src/sync.rs index 65b9ad4f4c..1071646ea2 100644 --- a/libsql/src/sync.rs +++ b/libsql/src/sync.rs @@ -1,7 +1,6 @@ use crate::{local::Connection, util::ConnectorService, Error, Result}; -use base64::engine::general_purpose; -use base64::Engine; +use crate::database::EncryptionContext; use bytes::Bytes; use chrono::Utc; use http::{HeaderValue, StatusCode}; @@ -119,29 +118,6 @@ struct PushFramesResult { baton: Option, } -#[derive(Debug, Clone)] -pub enum EncryptionKey { - /// The key is a base64-encoded string. - Base64Encoded(String), - /// The key is a byte array. - Bytes(Vec), -} - -impl EncryptionKey { - pub fn as_string(&self) -> String { - match self { - EncryptionKey::Base64Encoded(s) => s.clone(), - EncryptionKey::Bytes(b) => general_purpose::STANDARD.encode(b), - } - } -} - -#[derive(Debug, Clone)] -pub struct EncryptionContext { - /// The base64-encoded key for the encryption, sent on every request. - pub key: EncryptionKey, -} - pub struct SyncContext { db_path: String, client: hyper::Client, diff --git a/libsql/src/sync/test.rs b/libsql/src/sync/test.rs index edd6d33eee..2232aecad5 100644 --- a/libsql/src/sync/test.rs +++ b/libsql/src/sync/test.rs @@ -20,6 +20,7 @@ async fn test_sync_context_push_frame() { db_path.to_str().unwrap().to_string(), server.url(), None, + None, ) .await .unwrap(); @@ -49,6 +50,7 @@ async fn test_sync_context_with_auth() { db_path.to_str().unwrap().to_string(), server.url(), Some("test_token".to_string()), + None, ) .await .unwrap(); @@ -73,6 +75,7 @@ async fn test_sync_context_multiple_frames() { db_path.to_str().unwrap().to_string(), server.url(), None, + None, ) .await .unwrap(); @@ -102,6 +105,7 @@ async fn test_sync_context_corrupted_metadata() { db_path.to_str().unwrap().to_string(), server.url(), None, + None, ) .await .unwrap(); @@ -123,6 +127,7 @@ async fn test_sync_context_corrupted_metadata() { db_path.to_str().unwrap().to_string(), server.url(), None, + None, ) .await .unwrap(); @@ -146,6 +151,7 @@ async fn test_sync_restarts_with_lower_max_frame_no() { db_path.to_str().unwrap().to_string(), server.url(), None, + None, ) .await .unwrap(); @@ -171,6 +177,7 @@ async fn test_sync_restarts_with_lower_max_frame_no() { db_path.to_str().unwrap().to_string(), server.url(), None, + None, ) .await .unwrap(); @@ -210,6 +217,7 @@ async fn test_sync_context_retry_on_error() { db_path.to_str().unwrap().to_string(), server.url(), None, + None, ) .await .unwrap();