diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 1d97b754..2bee5417 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -4268,7 +4268,7 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "mdk-core" version = "0.5.3" -source = "git+https://github.com/parres-hq/mdk?rev=7c3157c#7c3157c803746f79a9399b9f4a3e2ff7f0064beb" +source = "git+https://github.com/parres-hq/mdk?rev=1ad73229ab712018c078d5295e11ec38e10d6a24#1ad73229ab712018c078d5295e11ec38e10d6a24" dependencies = [ "blurhash", "chacha20poly1305", @@ -4292,7 +4292,7 @@ dependencies = [ [[package]] name = "mdk-sqlite-storage" version = "0.5.1" -source = "git+https://github.com/parres-hq/mdk?rev=7c3157c#7c3157c803746f79a9399b9f4a3e2ff7f0064beb" +source = "git+https://github.com/parres-hq/mdk?rev=1ad73229ab712018c078d5295e11ec38e10d6a24#1ad73229ab712018c078d5295e11ec38e10d6a24" dependencies = [ "getrandom 0.3.2", "hex", @@ -4313,7 +4313,7 @@ dependencies = [ [[package]] name = "mdk-storage-traits" version = "0.5.1" -source = "git+https://github.com/parres-hq/mdk?rev=7c3157c#7c3157c803746f79a9399b9f4a3e2ff7f0064beb" +source = "git+https://github.com/parres-hq/mdk?rev=1ad73229ab712018c078d5295e11ec38e10d6a24#1ad73229ab712018c078d5295e11ec38e10d6a24" dependencies = [ "nostr", "openmls", @@ -9399,7 +9399,6 @@ dependencies = [ "cpal", "data-encoding", "futures-util", - "hex", "hound", "http 1.3.1", "image", @@ -9421,6 +9420,7 @@ dependencies = [ "openssl", "parking_lot", "rand 0.8.5", + "rayon", "reqwest", "ripemd", "rubato", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 69be3df2..b444a192 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -24,9 +24,9 @@ tauri-build = { version = "2.5.3", features = [] } [dependencies] nostr-sdk = { version = "0.44.1", features = ["nip06", "nip44", "nip59"] } nostr-blossom = "0.44.0" -mdk-core = { git = "https://github.com/parres-hq/mdk", rev = "7c3157c", features = ["mip04"] } -mdk-sqlite-storage = { git = "https://github.com/parres-hq/mdk", rev = "7c3157c" } -mdk-storage-traits = { git = "https://github.com/parres-hq/mdk", rev = "7c3157c" } +mdk-core = { git = "https://github.com/parres-hq/mdk", rev = "1ad73229ab712018c078d5295e11ec38e10d6a24", features = ["mip04"] } +mdk-sqlite-storage = { git = "https://github.com/parres-hq/mdk", rev = "1ad73229ab712018c078d5295e11ec38e10d6a24" } +mdk-storage-traits = { git = "https://github.com/parres-hq/mdk", rev = "1ad73229ab712018c078d5295e11ec38e10d6a24" } bip39 = { version = "2.2.2", features = ["rand"] } tokio = { version = "1.49.0", features = ["sync", "time"] } futures-util = "0.3.31" @@ -37,7 +37,6 @@ reqwest = { version = "0.12", features = ["rustls-tls", "json", "stream", "block scraper = "0.24.0" aes = "0.8.4" aes-gcm = "0.10.3" -hex = "0.4.3" sha2 = "0.10.9" once_cell = "1.21.3" lazy_static = "1.5.0" @@ -58,6 +57,7 @@ hound = "3.5.1" rubato = "0.16.2" symphonia = { version = "0.5.5", features = ["mp3", "wav", "flac", "pcm"] } rusqlite = { version = "0.32", features = ["bundled"] } +rayon = "1.11.0" # Mini Apps (WebXDC) support zip = "2.4" diff --git a/src-tauri/build.rs b/src-tauri/build.rs index f9808fec..20234d72 100644 --- a/src-tauri/build.rs +++ b/src-tauri/build.rs @@ -69,7 +69,6 @@ fn main() { "get_messages_around_id", "get_system_events", "get_chat_message_count", - "get_file_hash_index", "evict_chat_messages", "generate_blurhash_preview", "decode_blurhash", diff --git a/src-tauri/capabilities/default.json b/src-tauri/capabilities/default.json index 8153b50f..ba7105c1 100644 --- a/src-tauri/capabilities/default.json +++ b/src-tauri/capabilities/default.json @@ -67,7 +67,6 @@ "allow-get-messages-around-id", "allow-get-system-events", "allow-get-chat-message-count", - "allow-get-file-hash-index", "allow-evict-chat-messages", "allow-generate-blurhash-preview", "allow-decode-blurhash", diff --git a/src-tauri/permissions/autogenerated/get_file_hash_index.toml b/src-tauri/permissions/autogenerated/get_file_hash_index.toml deleted file mode 100644 index f1cc7640..00000000 --- a/src-tauri/permissions/autogenerated/get_file_hash_index.toml +++ /dev/null @@ -1,11 +0,0 @@ -# Automatically generated - DO NOT EDIT! - -[[permission]] -identifier = "allow-get-file-hash-index" -description = "Enables the get_file_hash_index command without any pre-configured scope." -commands.allow = ["get_file_hash_index"] - -[[permission]] -identifier = "deny-get-file-hash-index" -description = "Denies the get_file_hash_index command without any pre-configured scope." -commands.deny = ["get_file_hash_index"] diff --git a/src-tauri/src/account_manager.rs b/src-tauri/src/account_manager.rs index 10194079..9ef2ec6f 100644 --- a/src-tauri/src/account_manager.rs +++ b/src-tauri/src/account_manager.rs @@ -203,6 +203,7 @@ CREATE TABLE IF NOT EXISTS events ( failed INTEGER NOT NULL DEFAULT 0, wrapper_event_id TEXT, npub TEXT, + preview_metadata TEXT, FOREIGN KEY (chat_id) REFERENCES chats(id) ON DELETE CASCADE, FOREIGN KEY (user_id) REFERENCES profiles(id) ON DELETE SET NULL ); @@ -909,10 +910,26 @@ fn run_migrations(conn: &mut rusqlite::Connection) -> Result<(), String> { Ok(()) })?; + // Migration 13: Add preview_metadata column to events table for link preview caching + run_atomic_migration(conn, 13, "Add preview_metadata to events table", |tx| { + // Check if column already exists (may have been added by a prior dev build) + let col_exists: bool = tx.query_row( + "SELECT COUNT(*) FROM pragma_table_info('events') WHERE name='preview_metadata'", + [], |row| row.get::<_, i32>(0) + ).map(|c| c > 0).unwrap_or(false); + if !col_exists { + tx.execute( + "ALTER TABLE events ADD COLUMN preview_metadata TEXT", + [] + ).map_err(|e| format!("Failed to add preview_metadata column: {}", e))?; + } + Ok(()) + })?; + // ========================================================================= - // Future migrations (13+) follow the same pattern: + // Future migrations (14+) follow the same pattern: // - // run_atomic_migration(conn, 13, "Description here", |tx| { + // run_atomic_migration(conn, 14, "Description here", |tx| { // tx.execute("...", [])?; // Ok(()) // })?; diff --git a/src-tauri/src/android/filesystem.rs b/src-tauri/src/android/filesystem.rs index 1b238789..ca95aeee 100644 --- a/src-tauri/src/android/filesystem.rs +++ b/src-tauri/src/android/filesystem.rs @@ -1,3 +1,4 @@ +use std::sync::Arc; use jni::objects::{JObject, JValue, JString}; use crate::message::{AttachmentFile, FileInfo}; @@ -418,7 +419,7 @@ fn read_from_android_uri_internal( let _ = env.call_method(&input_stream, "close", "()V", &[]); Ok(AttachmentFile { - bytes, + bytes: Arc::new(bytes), img_meta: None, extension, }) diff --git a/src-tauri/src/android/miniapp_jni.rs b/src-tauri/src/android/miniapp_jni.rs index ca7b22a7..f217d96a 100644 --- a/src-tauri/src/android/miniapp_jni.rs +++ b/src-tauri/src/android/miniapp_jni.rs @@ -10,6 +10,8 @@ use jni::JNIEnv; use log::{debug, error, info, warn}; use std::io::Read; use std::path::Path; +use crate::util::bytes_to_hex_string; +use crate::TAURI_APP; // ============================================================================ // Constants @@ -482,22 +484,17 @@ fn get_user_display_name() -> String { } fn get_granted_permissions_for_package(package_path: &str) -> Result { - // Compute file hash for permission lookup - let path = Path::new(package_path); - if !path.exists() { - return Err("Package file not found".to_string()); - } - - let bytes = std::fs::read(path).map_err(|e| format!("Failed to read package: {}", e))?; + // Compute file hash for permission lookup - fs::read fails with NotFound if missing + let bytes = std::fs::read(package_path).map_err(|e| format!("Failed to read package: {}", e))?; use sha2::{Sha256, Digest}; let mut hasher = Sha256::new(); hasher.update(&bytes); - let _file_hash = hex::encode(hasher.finalize()); + let file_hash = bytes_to_hex_string(hasher.finalize().as_slice()); - // TODO: Look up permissions from database using _file_hash - // For now, return empty (no permissions granted) - Ok(String::new()) + // Look up permissions from database using file_hash + let handle = TAURI_APP.get().ok_or("Tauri app not initialized")?; + crate::db::get_miniapp_granted_permissions(handle, &file_hash) } fn serve_file_from_package( @@ -505,12 +502,7 @@ fn serve_file_from_package( package_path: &str, path: &str, ) -> Result { - let package_file = Path::new(package_path); - if !package_file.exists() { - return Err("Package file not found".to_string()); - } - - let file = std::fs::File::open(package_file).map_err(|e| format!("Failed to open package: {}", e))?; + let file = std::fs::File::open(package_path).map_err(|e| format!("Failed to open package: {}", e))?; let mut archive = zip::ZipArchive::new(file).map_err(|e| format!("Failed to read ZIP: {}", e))?; diff --git a/src-tauri/src/blossom.rs b/src-tauri/src/blossom.rs index 88f60ede..532bf241 100644 --- a/src-tauri/src/blossom.rs +++ b/src-tauri/src/blossom.rs @@ -21,27 +21,27 @@ struct ProgressTrackingStream { } impl ProgressTrackingStream { - fn new(data: Vec, bytes_sent: Arc>) -> Self { + fn new(data: Arc>, bytes_sent: Arc>) -> Self { let (tx, rx) = mpsc::channel(8); // Buffer size of 8 chunks - + // Spawn a background task to feed the stream tokio::spawn(async move { - let chunk_size = 64 * 1024; // 64 KB chunks + let chunk_size = 64 * 1024; // 64 KB chunks - only unavoidable copy let mut position = 0; - + while position < data.len() { let end = std::cmp::min(position + chunk_size, data.len()); let chunk = data[position..end].to_vec(); - + // Send chunk through channel if tx.send(Ok(chunk)).await.is_err() { break; // Receiver was dropped } - + position = end; } }); - + Self { bytes_sent, inner: rx, @@ -113,7 +113,7 @@ where pub async fn upload_blob_with_progress( signer: T, server_url: &Url, - file_data: Vec, + file_data: Arc>, mime_type: Option<&str>, progress_callback: ProgressCallback, retry_count: Option, @@ -124,16 +124,16 @@ where { let retry_count = retry_count.unwrap_or(0); let retry_spacing = retry_spacing.unwrap_or(std::time::Duration::from_secs(1)); - + let mut last_error = None; - + for attempt in 0..=retry_count { // Log retry attempt if not the first attempt if attempt > 0 { // Sleep before retry tokio::time::sleep(retry_spacing).await; } - + match upload_attempt( signer.clone(), server_url, @@ -148,7 +148,7 @@ where } } } - + // All attempts failed, return the last error Err(last_error.unwrap_or_else(|| "No upload attempts were made".to_string())) } @@ -157,7 +157,7 @@ where async fn upload_attempt( signer: T, server_url: &Url, - file_data: Vec, + file_data: Arc>, mime_type: Option<&str>, progress_callback: &ProgressCallback, ) -> Result @@ -166,7 +166,7 @@ where { let upload_url = server_url.join("upload") .map_err(|e| format!("Invalid server URL: {}", e))?; - + let total_size = file_data.len() as u64; let hash = Sha256Hash::hash(&file_data); @@ -262,7 +262,7 @@ where pub async fn upload_blob( signer: T, server_url: &Url, - file_data: Vec, + file_data: Arc>, mime_type: Option<&str>, ) -> Result where @@ -270,7 +270,7 @@ where { let upload_url = server_url.join("upload") .map_err(|e| format!("Invalid server URL: {}", e))?; - + let hash = Sha256Hash::hash(&file_data); // Build authorization header @@ -292,11 +292,12 @@ where .build() .map_err(|e| format!("Failed to create HTTP client: {}", e))?; - // Perform the upload + // Perform the upload - convert Arc to owned Vec for body + let body_data = Arc::try_unwrap(file_data).unwrap_or_else(|arc| (*arc).clone()); let response = client .put(upload_url) .headers(headers) - .body(file_data) + .body(body_data) .send() .await .map_err(|e| format!("Upload request failed: {}", e))?; @@ -320,14 +321,14 @@ where pub async fn upload_blob_with_failover( signer: T, server_urls: Vec, - file_data: Vec, + file_data: Arc>, mime_type: Option<&str>, ) -> Result where T: NostrSigner + Clone, { let mut last_error = String::from("No servers available"); - + for (index, server_url_str) in server_urls.iter().enumerate() { let server_url = match Url::parse(server_url_str) { Ok(url) => url, @@ -337,10 +338,10 @@ where continue; } }; - + eprintln!("[Blossom] Attempting upload to server {} of {}: {}", index + 1, server_urls.len(), server_url_str); - + match upload_blob(signer.clone(), &server_url, file_data.clone(), mime_type).await { Ok(url) => { eprintln!("[Blossom] Upload successful to: {}", server_url_str); @@ -353,7 +354,7 @@ where } } } - + // All servers failed Err(format!("All Blossom servers failed. Last error: {}", last_error)) } @@ -363,7 +364,7 @@ where pub async fn upload_blob_with_progress_and_failover( signer: T, server_urls: Vec, - file_data: Vec, + file_data: Arc>, mime_type: Option<&str>, progress_callback: ProgressCallback, retry_count: Option, @@ -373,7 +374,7 @@ where T: NostrSigner + Clone, { let mut last_error = String::from("No servers available"); - + for (index, server_url_str) in server_urls.iter().enumerate() { let server_url = match Url::parse(server_url_str) { Ok(url) => url, @@ -383,10 +384,10 @@ where continue; } }; - + eprintln!("[Blossom] Attempting upload to server {} of {}: {}", index + 1, server_urls.len(), server_url_str); - + // Try uploading to this server with progress tracking and retries match upload_blob_with_progress( signer.clone(), @@ -410,7 +411,7 @@ where } } } - + // All servers failed Err(format!("All Blossom servers failed. Last error: {}", last_error)) } \ No newline at end of file diff --git a/src-tauri/src/chat.rs b/src-tauri/src/chat.rs index 491f2082..c28cc3ad 100644 --- a/src-tauri/src/chat.rs +++ b/src-tauri/src/chat.rs @@ -1,20 +1,36 @@ +//! Chat types and management. +//! +//! This module provides: +//! - `Chat`: Core chat struct with compact message storage +//! - `SerializableChat`: Frontend-friendly format for Tauri communication +//! - `ChatType`, `ChatMetadata`: Supporting types + use serde::{Deserialize, Serialize}; -use crate::Message; use std::collections::HashMap; -#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] +use crate::message::compact::{CompactMessage, CompactMessageVec, NpubInterner}; +use crate::Message; + +// ============================================================================ +// Chat (Internal Storage) +// ============================================================================ + +/// Chat with compact message storage for memory efficiency. +/// +/// Messages are stored as `CompactMessage` with binary IDs and interned npubs. +/// Use `to_serializable()` to convert to frontend-friendly format. +#[derive(Clone, Debug)] pub struct Chat { pub id: String, pub chat_type: ChatType, - pub participants: Vec, // List of npubs - pub messages: Vec, + pub participants: Vec, + /// Compact message storage - O(log n) lookup by ID + pub messages: CompactMessageVec, pub last_read: String, pub created_at: u64, pub metadata: ChatMetadata, pub muted: bool, - /// Typing participants for group chats (npub -> expires_at timestamp) - /// Memory-only, never persisted to disk - #[serde(skip)] + /// Typing participants (npub -> expires_at), memory-only pub typing_participants: HashMap, } @@ -24,7 +40,7 @@ impl Chat { id, chat_type, participants, - messages: Vec::new(), + messages: CompactMessageVec::new(), last_read: String::new(), created_at: std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -36,7 +52,7 @@ impl Chat { } } - /// Create a new DM chat with another user + /// Create a new DM chat pub fn new_dm(their_npub: String) -> Self { Self::new(their_npub.clone(), ChatType::DirectMessage, vec![their_npub]) } @@ -46,79 +62,154 @@ impl Chat { Self::new(group_id, ChatType::MlsGroup, participants) } - /// Get the last message timestamp + // ======================================================================== + // Message Access (Compact) + // ======================================================================== + + /// Number of messages + #[inline] + pub fn message_count(&self) -> usize { + self.messages.len() + } + + /// Check if chat has no messages + #[inline] + pub fn is_empty(&self) -> bool { + self.messages.is_empty() + } + + /// Get last message timestamp + #[inline] pub fn last_message_time(&self) -> Option { - self.messages.last().map(|msg| msg.at) + self.messages.last_timestamp() + } + + /// Check if a message exists by ID - O(log n) + #[inline] + pub fn has_message(&self, id: &str) -> bool { + self.messages.contains_hex_id(id) + } + + /// Get a compact message by ID - O(log n) + #[inline] + pub fn get_compact_message(&self, id: &str) -> Option<&CompactMessage> { + self.messages.find_by_hex_id(id) } - /// Get a mutable message by ID - pub fn get_message_mut(&mut self, id: &str) -> Option<&mut Message> { - self.messages.iter_mut().find(|msg| msg.id == id) + /// Get a mutable compact message by ID - O(log n) + #[inline] + pub fn get_compact_message_mut(&mut self, id: &str) -> Option<&mut CompactMessage> { + self.messages.find_by_hex_id_mut(id) } - /// Set the Last Received message as the "Last Read" message + /// Get a message by ID, converting to full Message format - O(log n) + pub fn get_message(&self, id: &str, interner: &NpubInterner) -> Option { + self.messages.find_by_hex_id(id) + .map(|cm| cm.to_message(interner)) + } + + /// Iterate over compact messages (no conversion, supports .rev()) + #[inline] + pub fn iter_compact(&self) -> std::slice::Iter<'_, CompactMessage> { + self.messages.iter() + } + + /// Get all messages as full Message format (for serialization) + pub fn get_all_messages(&self, interner: &NpubInterner) -> Vec { + self.messages.iter() + .map(|cm| cm.to_message(interner)) + .collect() + } + + /// Get last N messages as full Message format + pub fn get_last_messages(&self, n: usize, interner: &NpubInterner) -> Vec { + let len = self.messages.len(); + let start = len.saturating_sub(n); + self.messages.messages()[start..] + .iter() + .map(|cm| cm.to_message(interner)) + .collect() + } + + // ======================================================================== + // Message Mutation + // ======================================================================== + + /// Add a Message, converting to compact format - O(log n) duplicate check + pub fn add_message(&mut self, message: Message, interner: &mut NpubInterner) -> bool { + let compact = CompactMessage::from_message(&message, interner); + self.messages.insert(compact) + } + + /// Add a pre-converted CompactMessage directly + #[inline] + pub fn add_compact_message(&mut self, message: CompactMessage) -> bool { + self.messages.insert(message) + } + + /// Set the last non-mine message as read pub fn set_as_read(&mut self) -> bool { - // Ensure we have at least one message received from others for msg in self.messages.iter().rev() { - if !msg.mine { - // Found the most recent message from others - self.last_read = msg.id.clone(); + if !msg.flags.is_mine() { + self.last_read = msg.id_hex(); return true; } } - - // No messages from others, can't mark anything as read false } - /// Add a Message to this Chat - /// - /// This method internally checks for and avoids duplicate messages. - pub fn internal_add_message(&mut self, message: Message) -> bool { - // Make sure we don't add the same message twice - if self.messages.iter().any(|m| m.id == message.id) { - // Message is already known by the state - return false; - } + // ======================================================================== + // Compatibility Methods (for gradual migration) + // ======================================================================== - // Fast path for common cases: newest or oldest messages - if self.messages.is_empty() { - // First message - self.messages.push(message); - } else if message.at >= self.messages.last().unwrap().at { - // Common case 1: Latest message (append to end) - self.messages.push(message); - } else if message.at <= self.messages.first().unwrap().at { - // Common case 2: Oldest message (insert at beginning) - self.messages.insert(0, message); - } else { - // Less common case: Message belongs somewhere in the middle - self.messages.insert( - self.messages.binary_search_by(|m| m.at.cmp(&message.at)).unwrap_or_else(|idx| idx), - message - ); + /// Legacy: Add message (calls add_message internally) + /// Used during migration - prefer add_message() with explicit interner + pub fn internal_add_message(&mut self, message: Message, interner: &mut NpubInterner) -> bool { + self.add_message(message, interner) + } + + /// Get mutable message by ID (returns compact, caller must handle) + #[inline] + pub fn get_message_mut(&mut self, id: &str) -> Option<&mut CompactMessage> { + self.get_compact_message_mut(id) + } + + // ======================================================================== + // Serialization + // ======================================================================== + + /// Convert to SerializableChat for frontend communication + pub fn to_serializable(&self, interner: &NpubInterner) -> SerializableChat { + SerializableChat { + id: self.id.clone(), + chat_type: self.chat_type.clone(), + participants: self.participants.clone(), + messages: self.get_all_messages(interner), + last_read: self.last_read.clone(), + created_at: self.created_at, + metadata: self.metadata.clone(), + muted: self.muted, } - true } - /// Add a Reaction - if it was not already added - pub fn add_reaction(&mut self, reaction: crate::Reaction, message_id: &str) -> bool { - // Find the message - if let Some(msg) = self.get_message_mut(message_id) { - // Make sure we don't add the same reaction twice - if !msg.reactions.iter().any(|r| r.id == reaction.id) { - msg.reactions.push(reaction); - true - } else { - // Reaction was already added previously - false - } - } else { - false + /// Convert to SerializableChat with only the last N messages (for efficiency) + pub fn to_serializable_with_last_n(&self, n: usize, interner: &NpubInterner) -> SerializableChat { + SerializableChat { + id: self.id.clone(), + chat_type: self.chat_type.clone(), + participants: self.participants.clone(), + messages: self.get_last_messages(n, interner), + last_read: self.last_read.clone(), + created_at: self.created_at, + metadata: self.metadata.clone(), + muted: self.muted, } } - /// Get other participant for DM chats + // ======================================================================== + // Chat Metadata & Participants + // ======================================================================== + pub fn get_other_participant(&self, my_npub: &str) -> Option { match self.chat_type { ChatType::DirectMessage => { @@ -126,33 +217,29 @@ impl Chat { .find(|&p| p != my_npub) .cloned() } - ChatType::MlsGroup => None, // Groups don't have a single "other" participant + ChatType::MlsGroup => None, } } - /// Check if this is a DM with a specific user pub fn is_dm_with(&self, npub: &str) -> bool { - matches!(self.chat_type, ChatType::DirectMessage) && self.participants.contains(&npub.to_string()) + matches!(self.chat_type, ChatType::DirectMessage) + && self.participants.iter().any(|p| p == npub) } - /// Check if this is an MLS group pub fn is_mls_group(&self) -> bool { matches!(self.chat_type, ChatType::MlsGroup) } - /// Check if user is a participant in this chat pub fn has_participant(&self, npub: &str) -> bool { - self.participants.contains(&npub.to_string()) + self.participants.iter().any(|p| p == npub) } - /// Get active typers (non-expired) for group chats - /// Returns a list of npubs that are currently typing pub fn get_active_typers(&self) -> Vec { let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_secs(); - + self.typing_participants .iter() .filter(|(_, &expires_at)| expires_at > now) @@ -160,13 +247,8 @@ impl Chat { .collect() } - /// Update typing state for a participant in a group chat - /// Automatically cleans up expired entries pub fn update_typing_participant(&mut self, npub: String, expires_at: u64) { - // Add or update the typing participant self.typing_participants.insert(npub, expires_at); - - // Clean up expired entries let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() @@ -174,106 +256,115 @@ impl Chat { self.typing_participants.retain(|_, &mut exp| exp > now); } - // Getter methods for private fields - pub fn id(&self) -> &String { - &self.id - } - - pub fn chat_type(&self) -> &ChatType { - &self.chat_type - } - - pub fn participants(&self) -> &Vec { - &self.participants - } + // Getters + pub fn id(&self) -> &String { &self.id } + pub fn chat_type(&self) -> &ChatType { &self.chat_type } + pub fn participants(&self) -> &Vec { &self.participants } + pub fn last_read(&self) -> &String { &self.last_read } + pub fn created_at(&self) -> u64 { self.created_at } + pub fn metadata(&self) -> &ChatMetadata { &self.metadata } + pub fn muted(&self) -> bool { self.muted } +} - pub fn last_read(&self) -> &String { - &self.last_read - } +// ============================================================================ +// SerializableChat (Frontend Communication) +// ============================================================================ - pub fn created_at(&self) -> u64 { - self.created_at - } +/// Serializable chat format for Tauri commands and emit(). +/// +/// This is what the frontend receives. Create via `chat.to_serializable(interner)`. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct SerializableChat { + pub id: String, + pub chat_type: ChatType, + pub participants: Vec, + pub messages: Vec, + pub last_read: String, + pub created_at: u64, + pub metadata: ChatMetadata, + pub muted: bool, +} - pub fn metadata(&self) -> &ChatMetadata { - &self.metadata - } +impl SerializableChat { + /// Convert to a Chat with compact storage (for loading from DB) + #[allow(clippy::wrong_self_convention)] + pub fn to_chat(self, interner: &mut NpubInterner) -> Chat { + let mut chat = Chat::new(self.id, self.chat_type, self.participants); + chat.last_read = self.last_read; + chat.created_at = self.created_at; + chat.metadata = self.metadata; + chat.muted = self.muted; + + for msg in self.messages { + chat.add_message(msg, interner); + } - pub fn muted(&self) -> bool { - self.muted + chat } } +// ============================================================================ +// Supporting Types +// ============================================================================ + #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] pub enum ChatType { DirectMessage, MlsGroup, - // Future types can be added here } impl ChatType { - /// Convert ChatType to integer for database storage - /// 0 = DirectMessage, 1 = MlsGroup pub fn to_i32(&self) -> i32 { match self { ChatType::DirectMessage => 0, ChatType::MlsGroup => 1, } } - - /// Convert integer from database to ChatType + pub fn from_i32(value: i32) -> Self { match value { 1 => ChatType::MlsGroup, - _ => ChatType::DirectMessage, // Default to DM for safety + _ => ChatType::DirectMessage, } } } #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Default)] pub struct ChatMetadata { - pub custom_fields: HashMap, // For extensibility + pub custom_fields: HashMap, } impl ChatMetadata { pub fn new() -> Self { - Self { - custom_fields: HashMap::new(), - } + Self { custom_fields: HashMap::new() } } - /// Set the group name in custom_fields pub fn set_name(&mut self, name: String) { self.custom_fields.insert("name".to_string(), name); } - /// Get the group name from custom_fields pub fn get_name(&self) -> Option<&str> { self.custom_fields.get("name").map(|s| s.as_str()) } - /// Set the member count in custom_fields pub fn set_member_count(&mut self, count: usize) { self.custom_fields.insert("member_count".to_string(), count.to_string()); } - /// Get the member count from custom_fields pub fn get_member_count(&self) -> Option { self.custom_fields.get("member_count").and_then(|s| s.parse().ok()) } } -//// Marks a specific message as read for a chat. -/// Behavior: -/// - If message_id is Some(id): set chat.last_read = id. -/// - Else: call chat.set_as_read() to pick the last non-mine message. -/// - Persist the chat (outside the STATE lock) and update unread counter on success. +// ============================================================================ +// Tauri Commands +// ============================================================================ + +/// Marks a specific message as read for a chat. #[tauri::command] pub async fn mark_as_read(chat_id: String, message_id: Option) -> bool { - // Apply the read change regardless of window focus; frontend intent is authoritative let handle = crate::TAURI_APP.get().unwrap(); - // Apply the read change to the specified chat let (result, chat_id_for_save) = { let mut state = crate::STATE.lock().await; let mut result = false; @@ -281,12 +372,10 @@ pub async fn mark_as_read(chat_id: String, message_id: Option) -> bool { if let Some(chat) = state.chats.iter_mut().find(|c| c.id == chat_id) { if let Some(msg_id) = &message_id { - // Explicit message -> set that as last_read chat.last_read = msg_id.clone(); result = true; chat_id_for_save = Some(chat.id.clone()); } else { - // No explicit message -> fall back to set_as_read behaviour result = chat.set_as_read(); if result { chat_id_for_save = Some(chat.id.clone()); @@ -297,25 +386,20 @@ pub async fn mark_as_read(chat_id: String, message_id: Option) -> bool { (result, chat_id_for_save) }; - // Update the unread counter and save to DB if the marking was successful if result { - // Update the badge count crate::commands::messaging::update_unread_counter(handle.clone()).await; - // Save the updated chat to the DB if let Some(chat_id) = chat_id_for_save { - // Get the updated chat to save its metadata (including last_read) - let updated_chat = { + let chat_to_save = { let state = crate::STATE.lock().await; state.get_chat(&chat_id).cloned() }; - // Save to DB - if let Some(chat) = updated_chat { + if let Some(chat) = chat_to_save { let _ = crate::db::save_chat(handle.clone(), &chat).await; } } } result -} \ No newline at end of file +} diff --git a/src-tauri/src/commands/account.rs b/src-tauri/src/commands/account.rs index 77c2c75b..3e3e29a1 100644 --- a/src-tauri/src/commands/account.rs +++ b/src-tauri/src/commands/account.rs @@ -66,11 +66,15 @@ pub async fn debug_hot_reload_sync() -> Result { println!("[Debug Hot-Reload] Sending cached state to frontend ({} profiles, {} chats)", state.profiles.len(), state.chats.len()); + // Convert chats to serializable format + let serializable_chats: Vec<_> = state.chats.iter() + .map(|c| c.to_serializable(&state.interner)) + .collect(); Ok(serde_json::json!({ "success": true, "npub": my_npub, "profiles": &state.profiles, - "chats": &state.chats, + "chats": serializable_chats, "is_syncing": state.is_syncing, "sync_mode": format!("{:?}", state.sync_mode) })) @@ -92,7 +96,7 @@ pub async fn login(import_key: String) -> Result { if prev_npub == new_npub { // Simply return the same KeyPair and allow the frontend to continue login as usual return Ok(LoginKeyPair { - public: signer.get_public_key().await.unwrap().to_bech32().unwrap(), + public: prev_npub, private: new_keys.secret_key().to_bech32().unwrap(), }); } else { diff --git a/src-tauri/src/commands/attachments.rs b/src-tauri/src/commands/attachments.rs index 8c15b8b4..582f5181 100644 --- a/src-tauri/src/commands/attachments.rs +++ b/src-tauri/src/commands/attachments.rs @@ -7,9 +7,9 @@ use tauri::{AppHandle, Emitter, Manager, Runtime}; -use crate::{STATE, TAURI_APP, ChatType, Attachment, Message}; -use crate::{util, crypto, net, db, mls}; -use crate::db::save_chat_messages; +use crate::{STATE, TAURI_APP, ChatType, Attachment}; +use crate::{util, crypto, net, db, mls, simd}; +use crate::util::hex_string_to_bytes; // ============================================================================ // Helper Functions @@ -64,7 +64,8 @@ pub async fn decrypt_and_save_attachment( /// Decrypt an MLS attachment using MDK's MIP-04 decryption /// /// This derives the encryption key from the MLS group secret using the original file hash -/// and other metadata stored in the MediaReference. +/// and other metadata stored in the MediaReference. MDK internally handles epoch fallback, +/// trying historical epoch secrets if the current epoch's key doesn't work. async fn decrypt_mls_attachment( handle: &AppHandle, encrypted_data: &[u8], @@ -89,8 +90,7 @@ async fn decrypt_mls_attachment( } // Parse the engine group ID - let engine_gid_bytes = hex::decode(&group_meta.engine_group_id) - .map_err(|e| format!("Invalid engine_group_id hex: {}", e))?; + let engine_gid_bytes = hex_string_to_bytes(&group_meta.engine_group_id); let gid = mdk_core::GroupId::from_slice(&engine_gid_bytes); // Get MDK engine and media manager @@ -101,14 +101,12 @@ async fn decrypt_mls_attachment( // Parse the original_hash from the attachment let original_hash_hex = attachment.original_hash.as_ref() .ok_or("MLS attachment missing original_hash")?; - let original_hash_bytes = hex::decode(original_hash_hex) - .map_err(|e| format!("Invalid original_hash hex: {}", e))?; + let original_hash_bytes = hex_string_to_bytes(original_hash_hex); let original_hash: [u8; 32] = original_hash_bytes.try_into() .map_err(|_| "Invalid original_hash length (expected 32 bytes)")?; // Parse the nonce from the attachment - let nonce_bytes = hex::decode(&attachment.nonce) - .map_err(|e| format!("Invalid nonce hex: {}", e))?; + let nonce_bytes = hex_string_to_bytes(&attachment.nonce); let nonce: [u8; 12] = nonce_bytes.try_into() .map_err(|_| "Invalid nonce length (expected 12 bytes)")?; @@ -136,14 +134,13 @@ async fn decrypt_mls_attachment( let media_ref = MediaReference { url: attachment.url.clone(), original_hash, - mime_type: mime_type.clone(), + mime_type, filename, dimensions: attachment.img_meta.as_ref().map(|m| (m.width, m.height)), scheme_version, nonce, }; - // Decrypt using MDK media_manager.decrypt_from_download(encrypted_data, &media_ref) .map_err(|e| format!("MIP-04 decryption failed: {}", e)) } @@ -171,7 +168,7 @@ pub async fn generate_blurhash_preview(npub: String, msg_id: String) -> Result data, Err(error) => { // Handle download error let mut state = STATE.lock().await; - - // Find and update the attachment status - for chat in &mut state.chats { - let is_target_chat = match &chat.chat_type { - ChatType::MlsGroup => chat.id == npub, - ChatType::DirectMessage => chat.has_participant(&npub), - }; - - if is_target_chat { - if let Some(message) = chat.messages.iter_mut().find(|m| m.id == msg_id) { - if let Some(attachment) = message.attachments.iter_mut().find(|a| a.id == attachment_id) { - attachment.downloading = false; - attachment.downloaded = false; - break; - } - } - } - } + state.update_attachment(&npub, &msg_id, &attachment_id, |att| { + att.set_downloading(false); + att.set_downloaded(false); + }); // Emit the error handle.emit("attachment_download_result", serde_json::json!({ @@ -335,24 +319,11 @@ pub async fn download_attachment(npub: String, msg_id: String, attachment_id: St if encrypted_data.len() < 16 { eprintln!("Downloaded file too small: {} bytes for attachment {}", encrypted_data.len(), attachment_id); let mut state = STATE.lock().await; - - // Find and update the attachment status - for chat in &mut state.chats { - let is_target_chat = match &chat.chat_type { - ChatType::MlsGroup => chat.id == npub, - ChatType::DirectMessage => chat.has_participant(&npub), - }; - - if is_target_chat { - if let Some(message) = chat.messages.iter_mut().find(|m| m.id == msg_id) { - if let Some(attachment) = message.attachments.iter_mut().find(|a| a.id == attachment_id) { - attachment.downloading = false; - attachment.downloaded = false; - break; - } - } - } - } + state.update_attachment(&npub, &msg_id, &attachment_id, |att| { + att.set_downloading(false); + att.set_downloaded(false); + }); + drop(state); // Emit a more helpful error let error_msg = format!("Downloaded file too small ({} bytes). URL may be invalid or expired.", encrypted_data.len()); @@ -366,8 +337,9 @@ pub async fn download_attachment(npub: String, msg_id: String, attachment_id: St return false; } - // Decrypt and save the file - let result = decrypt_and_save_attachment(handle, &encrypted_data, &attachment).await; + // Decrypt and save the file (convert CompactAttachment to Attachment for compatibility) + let attachment_for_decrypt = attachment.to_attachment(); + let result = decrypt_and_save_attachment(handle, &encrypted_data, &attachment_for_decrypt).await; // Process the result match result { @@ -381,65 +353,16 @@ pub async fn download_attachment(npub: String, msg_id: String, attachment_id: St // Handle decryption/saving error let mut state = STATE.lock().await; + state.update_attachment(&npub, &msg_id, &attachment_id, |att| { + att.set_downloading(false); + att.set_downloaded(false); + }); - // Find and update the attachment status - let mut should_remove = false; - for chat in &mut state.chats { - let is_target_chat = match &chat.chat_type { - ChatType::MlsGroup => chat.id == npub, - ChatType::DirectMessage => chat.has_participant(&npub), - }; - - if is_target_chat { - if let Some(message) = chat.messages.iter_mut().find(|m| m.id == msg_id) { - if let Some(attachment) = message.attachments.iter_mut().find(|a| a.id == attachment_id) { - attachment.downloading = false; - attachment.downloaded = false; - - // If it's a decryption error, mark for removal as it's corrupted - if is_decryption_error { - eprintln!("Marking corrupted attachment for removal: {}", attachment_id); - should_remove = true; - } - break; - } - } - } - } - - // Remove corrupted attachment if needed and save - if should_remove { - // Collect chat_id and messages to save - let save_data: Option<(String, Vec)> = { - let mut result = None; - for chat in &mut state.chats { - let is_target_chat = match &chat.chat_type { - ChatType::MlsGroup => chat.id == npub, - ChatType::DirectMessage => chat.has_participant(&npub), - }; - - if is_target_chat { - let chat_id = chat.id().to_string(); - - if let Some(message) = chat.messages.iter_mut().find(|m| m.id == msg_id) { - let original_count = message.attachments.len(); - message.attachments.retain(|a| a.id != attachment_id); - if message.attachments.len() < original_count { - result = Some((chat_id, vec![message.clone()])); - } - break; - } - } - } - result - }; - - // Drop state and save - drop(state); - if let Some((chat_id, messages)) = save_data { - let _ = save_chat_messages(handle.clone(), &chat_id, &messages).await; - } + // Log decryption errors but don't remove the attachment - allow retry + if is_decryption_error { + eprintln!("Decryption error for attachment {} - keeping for retry", attachment_id); } + drop(state); // Emit the error handle.emit("attachment_download_result", serde_json::json!({ @@ -447,8 +370,8 @@ pub async fn download_attachment(npub: String, msg_id: String, attachment_id: St "msg_id": msg_id, "id": attachment_id, "success": false, - "result": if should_remove { - "Corrupted attachment removed. Please re-send the file.".to_string() + "result": if is_decryption_error { + "Decryption failed - file may be corrupted".to_string() } else { error } @@ -464,29 +387,19 @@ pub async fn download_attachment(npub: String, msg_id: String, attachment_id: St .to_string(); // Update state with successful download + let path_str = hash_file_path.to_string_lossy().to_string(); { let mut state = STATE.lock().await; - - // Find and update the attachment - for chat in &mut state.chats { - let is_target_chat = match &chat.chat_type { - ChatType::MlsGroup => chat.id == npub, - ChatType::DirectMessage => chat.has_participant(&npub), - }; - - if is_target_chat { - if let Some(message) = chat.messages.iter_mut().find(|m| m.id == msg_id) { - if let Some(attachment_index) = message.attachments.iter().position(|a| a.id == attachment_id) { - let attachment = &mut message.attachments[attachment_index]; - attachment.id = file_hash.clone(); // Update ID from nonce to hash - attachment.downloading = false; - attachment.downloaded = true; - attachment.path = hash_file_path.to_string_lossy().to_string(); // Update to hash-based path - break; - } - } + state.update_attachment(&npub, &msg_id, &attachment_id, |att| { + // Update ID from nonce to hash + let hash_bytes = hex_string_to_bytes(&file_hash); + if hash_bytes.len() == 32 { + att.id.copy_from_slice(&hash_bytes); } - } + att.set_downloading(false); + att.set_downloaded(true); + att.path = path_str.clone().into_boxed_str(); + }); // Emit the finished download with both old and new IDs handle.emit("attachment_download_result", serde_json::json!({ @@ -499,17 +412,18 @@ pub async fn download_attachment(npub: String, msg_id: String, attachment_id: St // Persist updated message/attachment metadata to the database if let Some(handle) = TAURI_APP.get() { - // Find and save only the updated message + // Find and save only the updated message (convert to Message for serialization) let updated_chat = state.get_chat(&npub).unwrap(); - let updated_message = { - updated_chat.messages.iter().find(|m| m.id == msg_id).cloned() - }.unwrap(); + let chat_id = updated_chat.id().clone(); + let updated_message = updated_chat.messages.find_by_hex_id(&msg_id) + .map(|m| m.to_message(&state.interner)) + .unwrap(); // Update the frontend state handle.emit("message_update", serde_json::json!({ "old_id": &updated_message.id, - "message": updated_message.clone(), - "chat_id": updated_chat.id() + "message": &updated_message, + "chat_id": &chat_id })).unwrap(); // Drop the STATE lock before performing async I/O diff --git a/src-tauri/src/commands/media.rs b/src-tauri/src/commands/media.rs index d582144a..b9cfcf18 100644 --- a/src-tauri/src/commands/media.rs +++ b/src-tauri/src/commands/media.rs @@ -59,11 +59,6 @@ pub async fn transcribe( // Convert the file path to a Path let path = std::path::Path::new(&file_path); - // Check if the file exists - if !path.exists() { - return Err(format!("File does not exist: {}", file_path)); - } - // Decode and resample to 16kHz for Whisper match audio::decode_and_resample(path, 16000) { Ok(audio_data) => { diff --git a/src-tauri/src/commands/messaging.rs b/src-tauri/src/commands/messaging.rs index a7fa7d05..f679f4de 100644 --- a/src-tauri/src/commands/messaging.rs +++ b/src-tauri/src/commands/messaging.rs @@ -29,21 +29,29 @@ pub async fn get_chat_messages_paginated( // Also add these messages to the backend state for cache synchronization // This ensures operations like fetch_msg_metadata can find the messages + // Clone for return, move originals to batch (zero-copy in batch insert) + let messages_for_return = messages.clone(); + if !messages.is_empty() { + #[cfg(debug_assertions)] + let start = std::time::Instant::now(); let mut state = STATE.lock().await; - if let Some(chat) = state.chats.iter_mut().find(|c| c.id == chat_id) { - for msg in &messages { - // Only add if not already present (avoid duplicates) - if !chat.messages.iter().any(|m| m.id == msg.id) { - chat.messages.push(msg.clone()); - } - } - // Sort messages by timestamp to maintain order - chat.messages.sort_by_key(|m| m.at); + + // Use batch insert with zero-copy (moves the messages) + let added = state.add_messages_to_chat_batch(&chat_id, messages); + + #[cfg(debug_assertions)] + if added > 0 { + state.cache_stats.insert_count += added as u64; + state.cache_stats.record_insert(start.elapsed()); + let chats_clone = state.chats.clone(); + state.cache_stats.update_from_chats(&chats_clone); + println!("[CacheStats] paginated load: added {} msgs in {:?}", added, start.elapsed()); + state.cache_stats.log(); } } - Ok(messages) + Ok(messages_for_return) } /// Get the total message count for a chat @@ -70,17 +78,30 @@ pub async fn get_message_views( // Get materialized message views from events let messages = db::get_message_views(&handle, chat_int_id, limit, offset).await?; - // Sync to backend state for cache compatibility (uses binary search for efficient insertion) + // Sync to backend state for cache compatibility (batch insert for efficiency) + // Clone for return, move originals to batch (zero-copy in batch insert) + let messages_for_return = messages.clone(); + if !messages.is_empty() { + #[cfg(debug_assertions)] + let start = std::time::Instant::now(); let mut state = STATE.lock().await; - if let Some(chat) = state.chats.iter_mut().find(|c| c.id == chat_id) { - for msg in messages.iter().cloned() { - chat.internal_add_message(msg); - } + + // Use batch insert with zero-copy (moves the messages) + let added = state.add_messages_to_chat_batch(&chat_id, messages); + + #[cfg(debug_assertions)] + if added > 0 { + state.cache_stats.insert_count += added as u64; + state.cache_stats.record_insert(start.elapsed()); + let chats_clone = state.chats.clone(); + state.cache_stats.update_from_chats(&chats_clone); + println!("[CacheStats] message_views load: added {} msgs in {:?}", added, start.elapsed()); + state.cache_stats.log(); } } - Ok(messages) + Ok(messages_for_return) } /// Get messages around a specific message ID (for scrolling to replied-to messages) @@ -95,16 +116,29 @@ pub async fn get_messages_around_id( let messages = db::get_messages_around_id(&handle, &chat_id, &target_message_id, context_before).await?; // Sync to backend state so fetch_msg_metadata and other functions can find these messages + // Clone for return, move originals to batch (zero-copy in batch insert) + let messages_for_return = messages.clone(); + if !messages.is_empty() { + #[cfg(debug_assertions)] + let start = std::time::Instant::now(); let mut state = STATE.lock().await; - if let Some(chat) = state.chats.iter_mut().find(|c| c.id == chat_id) { - for msg in messages.iter().cloned() { - chat.internal_add_message(msg); - } + + // Use batch insert with zero-copy (moves the messages) + let added = state.add_messages_to_chat_batch(&chat_id, messages); + + #[cfg(debug_assertions)] + if added > 0 { + state.cache_stats.insert_count += added as u64; + state.cache_stats.record_insert(start.elapsed()); + let chats_clone = state.chats.clone(); + state.cache_stats.update_from_chats(&chats_clone); + println!("[CacheStats] messages_around load: added {} msgs in {:?}", added, start.elapsed()); + state.cache_stats.log(); } } - Ok(messages) + Ok(messages_for_return) } // ============================================================================ @@ -156,25 +190,17 @@ pub async fn get_system_events( pub async fn evict_chat_messages(chat_id: String, keep_count: usize) -> Result<(), String> { let mut state = STATE.lock().await; if let Some(chat) = state.chats.iter_mut().find(|c| c.id == chat_id) { - let total = chat.messages.len(); + let total = chat.message_count(); if total > keep_count { // Keep only the last `keep_count` messages (most recent) let drain_count = total - keep_count; chat.messages.drain(0..drain_count); + chat.messages.rebuild_index(); } } Ok(()) } -/// Build and return the file hash index for deduplication -/// Returns a map of file_hash -> attachment reference data -#[tauri::command] -pub async fn get_file_hash_index( - handle: AppHandle, -) -> Result, String> { - db::build_file_hash_index(&handle).await -} - // ============================================================================ // Unread Count Commands // ============================================================================ @@ -232,5 +258,4 @@ pub async fn update_unread_counter(handle: AppHandle) -> u32 { // - get_messages_around_id // - get_system_events // - evict_chat_messages -// - get_file_hash_index // - update_unread_counter diff --git a/src-tauri/src/commands/mls.rs b/src-tauri/src/commands/mls.rs index 2c8c4b89..303976dd 100644 --- a/src-tauri/src/commands/mls.rs +++ b/src-tauri/src/commands/mls.rs @@ -11,6 +11,7 @@ use rand::{thread_rng, Rng}; use rand::distributions::Alphanumeric; use tauri::Emitter; use crate::{db, mls, MlsService, NotificationData, show_notification_generic, NOSTR_CLIENT, NOTIFIED_WELCOMES, STATE, TAURI_APP, TRUSTED_RELAYS}; +use crate::util::{bytes_to_hex_string, hex_string_to_bytes}; // ============================================================================ // Device & KeyPackage Read Commands @@ -342,7 +343,8 @@ pub async fn get_mls_group_members(group_id: String) -> Result = Vec::new(); let mut admins: Vec = Vec::new(); - if let Ok(gid_bytes) = hex::decode(&engine_id) { + let gid_bytes = hex_string_to_bytes(&engine_id); + if !gid_bytes.is_empty() { // Decode engine id to GroupId let gid = GroupId::from_slice(&gid_bytes); @@ -357,7 +359,7 @@ pub async fn get_mls_group_members(group_id: String) -> Result Result, String> { out.push(SimpleWelcome { id: w.id.to_hex(), wrapper_event_id: w.wrapper_event_id.to_hex(), - nostr_group_id: hex::encode(w.nostr_group_id), + nostr_group_id: bytes_to_hex_string(&w.nostr_group_id), group_name: w.group_name.clone(), group_description: Some(w.group_description.clone()), group_image_url: None, // MDK uses group_image_hash/key/nonce instead of URL @@ -919,7 +921,7 @@ pub async fn accept_mls_welcome(welcome_event_id_hex: String) -> Result Result Result>>> = /// Check if a URL is a default relay pub fn is_default_relay(url: &str) -> bool { - let normalized = url.trim().trim_end_matches('/').to_lowercase(); - DEFAULT_RELAYS.iter().any(|r| r.to_lowercase() == normalized) + let normalized = url.trim().trim_end_matches('/'); + DEFAULT_RELAYS.iter().any(|r| r.eq_ignore_ascii_case(normalized)) } /// Validate a relay URL format (must be wss://) @@ -303,10 +303,10 @@ pub async fn get_relays(handle: AppHandle) -> Result "initialized", RelayStatus::Pending => "pending", @@ -323,7 +323,7 @@ pub async fn get_relays(handle: AppHandle) -> Result(handle: AppHandle) -> Result "initialized", RelayStatus::Pending => "pending", @@ -385,9 +385,9 @@ pub async fn toggle_default_relay(handle: AppHandle, url: String, let mut disabled = get_disabled_default_relays(&handle).await?; if enabled { - disabled.retain(|d| d.to_lowercase() != normalized_url.to_lowercase()); + disabled.retain(|d| !d.eq_ignore_ascii_case(&normalized_url)); } else { - if !disabled.iter().any(|d| d.to_lowercase() == normalized_url.to_lowercase()) { + if !disabled.iter().any(|d| d.eq_ignore_ascii_case(&normalized_url)) { disabled.push(normalized_url.clone()); } } @@ -427,8 +427,7 @@ pub async fn add_custom_relay(handle: AppHandle, url: String, mod let mut relays = load_custom_relays(&handle).await?; - let url_lower = normalized_url.to_lowercase(); - if relays.iter().any(|r| r.url.to_lowercase() == url_lower) { + if relays.iter().any(|r| r.url.eq_ignore_ascii_case(&normalized_url)) { return Err("Relay already exists".to_string()); } @@ -467,9 +466,8 @@ pub async fn add_custom_relay(handle: AppHandle, url: String, mod pub async fn remove_custom_relay(handle: AppHandle, url: String) -> Result { let mut relays = load_custom_relays(&handle).await?; - let url_lower = url.to_lowercase(); let original_len = relays.len(); - relays.retain(|r| r.url.to_lowercase() != url_lower); + relays.retain(|r| !r.url.eq_ignore_ascii_case(&url)); if relays.len() == original_len { return Ok(false); @@ -493,12 +491,11 @@ pub async fn remove_custom_relay(handle: AppHandle, url: String) pub async fn toggle_custom_relay(handle: AppHandle, url: String, enabled: bool) -> Result { let mut relays = load_custom_relays(&handle).await?; - let url_lower = url.to_lowercase(); let mut found = false; let mut relay_mode = "both".to_string(); for relay in relays.iter_mut() { - if relay.url.to_lowercase() == url_lower { + if relay.url.eq_ignore_ascii_case(&url) { relay.enabled = enabled; relay_mode = relay.mode.clone(); found = true; @@ -542,12 +539,11 @@ pub async fn update_relay_mode(handle: AppHandle, url: String, mo let mut relays = load_custom_relays(&handle).await?; - let url_lower = url.to_lowercase(); let mut found = false; let mut is_enabled = false; for relay in relays.iter_mut() { - if relay.url.to_lowercase() == url_lower { + if relay.url.eq_ignore_ascii_case(&url) { relay.mode = mode.clone(); is_enabled = relay.enabled; found = true; @@ -758,55 +754,80 @@ pub async fn connect(handle: AppHandle) -> bool { let client = NOSTR_CLIENT.get().expect("Nostr client not initialized"); // If we're already connected to some relays - skip and tell the frontend our client is already online - if client.relays().await.len() > 0 { + if !client.relays().await.is_empty() { return false; } - // Get disabled default relays - let disabled_defaults = get_disabled_default_relays(&handle).await.unwrap_or_default(); + // Get disabled default relays and custom relays concurrently + let (disabled_defaults, custom_relays_result) = tokio::join!( + get_disabled_default_relays(&handle), + get_custom_relays(handle.clone()) + ); + let disabled_defaults = disabled_defaults.unwrap_or_default(); + + // Collect all relays to add (URL, options, is_default, mode_info) + let mut relays_to_add: Vec<(String, RelayOptions, bool, String)> = Vec::new(); // Add default relays (unless disabled) for default_url in DEFAULT_RELAYS { - let is_disabled = disabled_defaults.iter().any(|d| d.to_lowercase() == default_url.to_lowercase()); + let is_disabled = disabled_defaults.iter().any(|d| d.eq_ignore_ascii_case(default_url)); if !is_disabled { - match client.pool().add_relay(*default_url, RelayOptions::new().reconnect(false)).await { - Ok(_) => { - println!("[Relay] Added default relay: {}", default_url); - add_relay_log(default_url, "info", "Added to relay pool"); - } - Err(e) => { - eprintln!("[Relay] Failed to add default relay {}: {}", default_url, e); - add_relay_log(default_url, "error", &format!("Failed to add: {}", e)); - } - } + relays_to_add.push(( + default_url.to_string(), + RelayOptions::new().reconnect(false), + true, + "both".to_string(), + )); } else { println!("[Relay] Skipping disabled default relay: {}", default_url); add_relay_log(default_url, "info", "Skipped (disabled by user)"); } } - // Add user's custom relays (if any) - match get_custom_relays(handle.clone()).await { - Ok(custom_relays) => { - for relay in custom_relays { - if relay.enabled { - match client.pool().add_relay(&relay.url, relay_options_for_mode(&relay.mode)).await { - Ok(_) => { - println!("[Relay] Added custom relay: {} (mode: {})", relay.url, relay.mode); - add_relay_log(&relay.url, "info", &format!("Added to relay pool (mode: {})", relay.mode)); - } - Err(e) => { - eprintln!("[Relay] Failed to add custom relay {}: {}", relay.url, e); - add_relay_log(&relay.url, "error", &format!("Failed to add: {}", e)); - } + // Add custom relays + if let Ok(custom_relays) = custom_relays_result { + for relay in custom_relays { + if relay.enabled { + relays_to_add.push(( + relay.url, + relay_options_for_mode(&relay.mode), + false, + relay.mode, + )); + } + } + } + + // Add all relays in parallel + let pool = client.pool(); + let add_futures: Vec<_> = relays_to_add.into_iter().map(|(url, opts, is_default, mode)| { + let pool = pool.clone(); + async move { + match pool.add_relay(&url, opts).await { + Ok(_) => { + if is_default { + println!("[Relay] Added default relay: {}", url); + add_relay_log(&url, "info", "Added to relay pool"); + } else { + println!("[Relay] Added custom relay: {} (mode: {})", url, mode); + add_relay_log(&url, "info", &format!("Added to relay pool (mode: {})", mode)); } } + Err(e) => { + if is_default { + eprintln!("[Relay] Failed to add default relay {}: {}", url, e); + } else { + eprintln!("[Relay] Failed to add custom relay {}: {}", url, e); + } + add_relay_log(&url, "error", &format!("Failed to add: {}", e)); + } } } - Err(e) => eprintln!("[Relay] Failed to load custom relays: {}", e), - } + }).collect(); + + futures_util::future::join_all(add_futures).await; - // Connect! + // Connect to all added relays client.connect().await; // Post-connect: force-regenerate device KeyPackage if flagged by migration 13 diff --git a/src-tauri/src/commands/sync.rs b/src-tauri/src/commands/sync.rs index 0e3ba8e5..427a9f1c 100644 --- a/src-tauri/src/commands/sync.rs +++ b/src-tauri/src/commands/sync.rs @@ -9,11 +9,10 @@ use nostr_sdk::prelude::*; use tauri::{AppHandle, Emitter, Manager, Runtime}; use crate::{ - db, mls, profile, profile_sync, - ChatState, ChatType, Profile, SyncMode, + db, profile, profile_sync, + ChatType, Profile, SyncMode, NOSTR_CLIENT, STATE, WRAPPER_ID_CACHE, }; -use crate::db::save_chat_messages; // ============================================================================ // Profile Sync Commands @@ -30,7 +29,7 @@ pub async fn queue_profile_sync(npub: String, priority: String, force_refresh: b _ => return Err(format!("Invalid priority: {}", priority)), }; - profile_sync::queue_profile_sync(npub, sync_priority, force_refresh).await; + profile_sync::queue_profile_sync(npub, sync_priority, force_refresh); Ok(()) } @@ -44,7 +43,7 @@ pub async fn queue_chat_profiles_sync(chat_id: String, is_opening: bool) -> Resu /// Immediately refresh a specific profile. #[tauri::command] pub async fn refresh_profile_now(npub: String) -> Result<(), String> { - profile_sync::refresh_profile_now(npub).await; + profile_sync::refresh_profile_now(npub); Ok(()) } @@ -123,8 +122,6 @@ pub async fn fetch_messages( if init { // Set current account for SQL mode if profile database exists // This must be done BEFORE loading chats/messages so SQL mode is active - let signer = client.signer().await.unwrap(); - let my_public_key = signer.get_public_key().await.unwrap(); let npub = my_public_key.to_bech32().unwrap(); let app_data = handle.path().app_data_dir().ok(); @@ -137,124 +134,140 @@ pub async fn fetch_messages( } // Load our DB (if we haven't already; i.e: our profile is the single loaded profile since login) - let mut needs_integrity_check = false; if state.profiles.len() == 1 { - let profiles = db::get_all_profiles(&handle).await.unwrap(); - // Load our Profile Cache into the state - state.merge_db_profiles(profiles).await; + // Load profiles, chats, MLS groups, and last messages in parallel (all are independent reads) + let (profiles_result, slim_chats_result, mls_groups_result, last_messages_result) = tokio::join!( + db::get_all_profiles(&handle), + db::get_all_chats(&handle), + db::load_mls_groups(&handle), + db::get_all_chats_last_messages(&handle) + ); + + // Process profiles + if let Ok(profiles) = profiles_result { + state.merge_db_profiles(profiles).await; + } // Spawn background task to cache profile images for offline support tokio::spawn(async { profile::cache_all_profile_images().await; }); - // Load chats and their messages from database - let slim_chats_result = db::get_all_chats(&handle).await; + // Get the last messages map (single batch query result) + let mut last_messages_map = last_messages_result.unwrap_or_default(); + + // Process chats if let Ok(slim_chats) = slim_chats_result { - // Load MLS groups to check for evicted status - let mls_groups: Option> = - db::load_mls_groups(&handle).await.ok(); + // Build HashSet of evicted MLS group IDs for O(1) lookup + let evicted_groups: std::collections::HashSet<&str> = mls_groups_result + .as_ref() + .map(|groups| groups.iter() + .filter(|g| g.evicted) + .map(|g| g.group_id.as_str()) + .collect()) + .unwrap_or_default(); + + // Build HashSet of existing profile IDs for O(1) lookup + let mut known_profiles: std::collections::HashSet = + state.profiles.iter().map(|p| p.id.clone()).collect(); + + // Pre-allocate capacity for chats (avoids reallocations during push) + state.chats.reserve(slim_chats.len()); + + // Convert slim chats to full chats and merge last messages + #[cfg(debug_assertions)] + let start = std::time::Instant::now(); + let mut total_messages = 0usize; - // Convert slim chats to full chats and load their messages for slim_chat in slim_chats { - let mut chat = slim_chat.to_chat(); - - // Skip MLS group chats that are marked as evicted - // MLS group chat IDs are just the group_id (no prefix) - if chat.chat_type == ChatType::MlsGroup { - if let Some(ref groups) = mls_groups { - if let Some(group) = groups.iter().find(|g| g.group_id.as_str() == chat.id()) { - if group.evicted { - println!("[Startup] Skipping evicted MLS group chat: {}", chat.id()); - continue; // Skip this chat - } - } - } + // Skip evicted MLS groups (O(1) lookup) + if slim_chat.chat_type == ChatType::MlsGroup && evicted_groups.contains(slim_chat.id.as_str()) { + continue; } - // Load only the last message for preview (optimization: full messages loaded on-demand by frontend) - let last_messages_result = db::get_chat_last_messages(&handle, &chat.id(), 1).await; - if let Ok(last_messages) = last_messages_result { - for message in last_messages { - // Check if this message has downloaded attachments (for integrity check) - if !needs_integrity_check && message.attachments.iter().any(|att| att.downloaded) { - needs_integrity_check = true; - } - chat.internal_add_message(message); - } - } else { - eprintln!("Failed to load last message for chat {}: {:?}", chat.id(), last_messages_result); - } + let mut chat = slim_chat.to_chat(); + let chat_id = chat.id().to_string(); - // Ensure profiles exist for all chat participants + // Ensure profiles exist for all chat participants (O(1) lookup) for participant in chat.participants() { - if state.get_profile(participant).is_none() { - // Create a basic profile for the participant + if !known_profiles.contains(participant) { let mut profile = Profile::new(); profile.id = participant.clone(); - profile.mine = false; // It's not our profile + profile.mine = false; state.profiles.push(profile); + known_profiles.insert(participant.clone()); } } - // Add chat to state + // Get messages to add (if any) + let messages_to_add = last_messages_map.remove(&chat_id); + + // Add messages to the chat using interner, then push + // This avoids double borrow by operating on local chat before adding to state + if let Some(messages) = messages_to_add { + total_messages += messages.len(); + for message in messages { + chat.internal_add_message(message, &mut state.interner); + } + } + + // Push the chat (now with messages) to state state.chats.push(chat); + } + + // Sort chats by last message time (do once at the end, not per-chat) + state.chats.sort_by(|a, b| b.last_message_time().cmp(&a.last_message_time())); - // Sort the chats by their last received message - state.chats.sort_by(|a, b| b.last_message_time().cmp(&a.last_message_time())); + // Record startup load timing (debug builds only) + #[cfg(debug_assertions)] + { + let elapsed = start.elapsed(); + if total_messages > 0 { + state.cache_stats.insert_count = total_messages as u64; + state.cache_stats.record_insert(elapsed); + } + let chats_clone = state.chats.clone(); + state.cache_stats.update_from_chats(&chats_clone); + println!("[CacheStats] startup load: {} chats, {} msgs in {:?}", state.chats.len(), total_messages, elapsed); + state.cache_stats.log(); } } else { eprintln!("Failed to load chats from database: {:?}", slim_chats_result); } } - if needs_integrity_check { - // Clean up empty file attachments first - cleanup_empty_file_attachments(&handle, &mut state).await; - - // Check integrity without dropping state - check_attachment_filesystem_integrity(&handle, &mut state).await; - - // Preload ID caches for maximum performance - if let Err(e) = db::preload_id_caches(&handle).await { - eprintln!("[Cache] Failed to preload ID caches: {}", e); - } - - // Preload wrapper_event_ids for fast duplicate detection during sync - // Load last 30 days of wrapper_ids to cover typical sync window - if let Ok(wrapper_ids) = db::load_recent_wrapper_ids(&handle, 30).await { - let mut cache = WRAPPER_ID_CACHE.lock().await; - *cache = wrapper_ids; + // Check filesystem integrity for downloaded attachments (queries DB directly) + let handle_for_integrity = handle.clone(); + tokio::spawn(async move { + if let Err(e) = db::check_downloaded_attachments_integrity(&handle_for_integrity).await { + eprintln!("[Integrity] Check failed: {}", e); } + }); - // Send the state to our frontend to signal finalised init with a full state - handle.emit("init_finished", serde_json::json!({ - "profiles": &state.profiles, - "chats": &state.chats - })).unwrap(); - } else { - // Even if no integrity check needed, still clean up empty files - cleanup_empty_file_attachments(&handle, &mut state).await; - - // Preload ID caches for maximum performance - if let Err(e) = db::preload_id_caches(&handle).await { - eprintln!("[Cache] Failed to preload ID caches: {}", e); - } + // Preload caches in parallel + let (id_cache_result, wrapper_ids_result) = tokio::join!( + db::preload_id_caches(&handle), + db::load_recent_wrapper_ids(&handle, 30) + ); - // Preload wrapper_event_ids for fast duplicate detection during sync - // Load last 30 days of wrapper_ids to cover typical sync window - if let Ok(wrapper_ids) = db::load_recent_wrapper_ids(&handle, 30).await { - let mut cache = WRAPPER_ID_CACHE.lock().await; - *cache = wrapper_ids; - } + if let Err(e) = id_cache_result { + eprintln!("[Cache] Failed to preload ID caches: {}", e); + } - // No integrity check needed, send init immediately - handle.emit("init_finished", serde_json::json!({ - "profiles": &state.profiles, - "chats": &state.chats - })).unwrap(); + if let Ok(wrapper_ids) = wrapper_ids_result { + let mut cache = WRAPPER_ID_CACHE.lock().await; + cache.load(wrapper_ids); } + // Send the state to frontend (convert chats to serializable format) + let serializable_chats: Vec<_> = state.chats.iter() + .map(|c| c.to_serializable(&state.interner)) + .collect(); + handle.emit("init_finished", serde_json::json!({ + "profiles": &state.profiles, + "chats": serializable_chats + })).unwrap(); + // ALWAYS begin with an initial sync of at least the last 2 days let now = Timestamp::now(); @@ -493,11 +506,17 @@ pub async fn fetch_messages( let mut cache = WRAPPER_ID_CACHE.lock().await; let cache_size = cache.len(); cache.clear(); - cache.shrink_to_fit(); - // Each entry: 64-char hex String (~88 bytes) + HashSet overhead (~48 bytes) ≈ 136 bytes - println!("[Startup] Sync Complete - Dumped NIP-59 Decryption Cache (~{} KB Memory freed)", (cache_size * 136) / 1024); + // Each entry: [u8; 32] in Vec (32 bytes) or HashSet (~48 bytes) ≈ 35 bytes average + println!("[Startup] Sync Complete - Dumped NIP-59 Decryption Cache (~{} KB Memory freed)", (cache_size * 35) / 1024); } + // Warm the file hash cache in the background (for attachment deduplication) + // Only builds if there are attachments and cache wasn't already built during sync + let handle_for_cache = handle.clone(); + tokio::task::spawn(async move { + db::warm_file_hash_cache(&handle_for_cache).await; + }); + if relay_url.is_none() { handle.emit("sync_finished", ()).unwrap(); @@ -556,178 +575,6 @@ pub async fn deep_rescan(handle: AppHandle) -> Result( - handle: &AppHandle, - state: &mut ChatState, -) { - const EMPTY_FILE_HASH: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; - let mut cleaned_count = 0; - let mut chats_to_update = Vec::new(); - - for chat in &mut state.chats { - let mut chat_had_changes = false; - - // First pass: remove attachments with empty file hash - for message in &mut chat.messages { - let original_count = message.attachments.len(); - - // Remove attachments with empty file hash in their URL - message.attachments.retain(|attachment| { - !attachment.url.contains(EMPTY_FILE_HASH) - }); - - let removed = original_count - message.attachments.len(); - if removed > 0 { - cleaned_count += removed; - chat_had_changes = true; - } - } - - // Second pass: remove messages that are now empty (no content, no attachments) - let messages_before = chat.messages.len(); - chat.messages.retain(|message| { - !message.content.is_empty() || !message.attachments.is_empty() - }); - - if chat.messages.len() < messages_before { - chat_had_changes = true; - } - - // If this chat had changes, save all its messages - if chat_had_changes { - chats_to_update.push((chat.id(), chat.messages.clone())); - } - } - - // Save updated chats to database - for (chat_id, messages) in chats_to_update { - if let Err(e) = save_chat_messages(handle.clone(), &chat_id, &messages).await { - eprintln!("Failed to save chat after cleaning empty attachments: {}", e); - } - } - - if cleaned_count > 0 { - eprintln!("Cleaned up {} empty file attachments", cleaned_count); - } -} - -/// Checks if downloaded attachments still exist on the filesystem -/// Sets downloaded=false for any missing files and updates the database -async fn check_attachment_filesystem_integrity( - handle: &AppHandle, - state: &mut ChatState, -) { - let mut total_checked = 0; - let mut chats_with_updates = std::collections::HashMap::new(); - - // Capture the starting timestamp - let start_time = std::time::Instant::now(); - - // First pass: count total attachments to check - let mut total_attachments = 0; - for chat in &state.chats { - for message in &chat.messages { - for attachment in &message.attachments { - if attachment.downloaded { - total_attachments += 1; - } - } - } - } - - // Iterate through all chats and their messages with mutable access to update downloaded status - for (chat_idx, chat) in state.chats.iter_mut().enumerate() { - let mut updated_messages = Vec::new(); - - for message in &mut chat.messages { - let mut message_updated = false; - - for attachment in &mut message.attachments { - // Only check attachments that are marked as downloaded - if attachment.downloaded { - total_checked += 1; - - // Emit progress every 2 attachments or on the last one, but only if process has taken >1 second - if (total_checked % 2 == 0 || total_checked == total_attachments) && start_time.elapsed().as_secs() >= 1 { - handle.emit("progress_operation", serde_json::json!({ - "type": "progress", - "current": total_checked, - "total": total_attachments, - "message": "Checking file integrity" - })).unwrap(); - } - - // Check if the file exists on the filesystem - let file_path = std::path::Path::new(&attachment.path); - if !file_path.exists() { - // File is missing, set downloaded to false - attachment.downloaded = false; - message_updated = true; - attachment.path = String::new(); - } - } - } - - // If any attachment in this message was updated, we need to save the message - if message_updated { - updated_messages.push(message.clone()); - } - } - - // If any messages in this chat were updated, store them for database update - if !updated_messages.is_empty() { - chats_with_updates.insert(chat_idx, updated_messages); - } - } - - // Update database for any messages with missing attachments - if !chats_with_updates.is_empty() { - // Only emit progress if process has taken >1 second - if start_time.elapsed().as_secs() >= 1 { - handle.emit("progress_operation", serde_json::json!({ - "type": "progress", - "total": chats_with_updates.len(), - "current": 0, - "message": "Updating database..." - })).unwrap(); - } - - // Save updated messages for each chat that had changes - let mut saved_count = 0; - let total_chats = chats_with_updates.len(); - for (chat_idx, _updated_messages) in chats_with_updates { - // Since we're iterating over existing indices, we know the chat exists - let chat = &state.chats[chat_idx]; - let chat_id = chat.id().clone(); - - // Save - let all_messages = &chat.messages; - if let Err(e) = save_chat_messages(handle.clone(), &chat_id, all_messages).await { - eprintln!("Failed to update messages after filesystem check: {}", e); - } else { - saved_count += 1; - } - - // Emit progress for database updates, but only if process has taken >1 second - if ((saved_count) % 5 == 0 || saved_count == total_chats) && start_time.elapsed().as_secs() >= 1 { - handle.emit("progress_operation", serde_json::json!({ - "type": "progress", - "current": saved_count, - "total": total_chats, - "message": "Updating database" - })).unwrap(); - } - } - } -} - // Handler list for this module (for reference): // - queue_profile_sync // - queue_chat_profiles_sync diff --git a/src-tauri/src/commands/system.rs b/src-tauri/src/commands/system.rs index 51c29d6f..b35e6d44 100644 --- a/src-tauri/src/commands/system.rs +++ b/src-tauri/src/commands/system.rs @@ -187,50 +187,60 @@ pub async fn clear_storage(handle: AppHandle) -> Result = updated_msg_ids.iter() + .filter_map(|msg_id| { + let hex_id = crate::simd::bytes_to_hex_32(msg_id); + state.chats[chat_idx].messages.find_by_hex_id(&hex_id) + .map(|m| m.to_message(&state.interner)) + }) + .collect(); + // Save updated messages to database - db::save_chat_messages(handle.clone(), chat.id(), &messages_to_update).await - .map_err(|e| format!("Failed to save updated messages for chat {}: {}", chat.id(), e))?; + db::save_chat_messages(handle.clone(), &chat_id, &messages_to_update).await + .map_err(|e| format!("Failed to save updated messages for chat {}: {}", chat_id, e))?; // Emit message_update events for each updated message for message in &messages_to_update { handle.emit("message_update", serde_json::json!({ "old_id": &message.id, "message": message, - "chat_id": chat.id() - })).map_err(|e| format!("Failed to emit message_update for chat {}: {}", chat.id(), e))?; + "chat_id": &chat_id + })).map_err(|e| format!("Failed to emit message_update for chat {}: {}", chat_id, e))?; } - updated_chats.insert(chat.id().to_string()); + updated_chats.insert(chat_id); } } diff --git a/src-tauri/src/crypto.rs b/src-tauri/src/crypto.rs index 62b4ccbd..d7e977ef 100644 --- a/src-tauri/src/crypto.rs +++ b/src-tauri/src/crypto.rs @@ -27,16 +27,16 @@ pub fn generate_encryption_params() -> EncryptionParams { let nonce: [u8; 16] = rng.gen(); EncryptionParams { - key: hex::encode(key), - nonce: hex::encode(nonce), + key: bytes_to_hex_string(&key), + nonce: bytes_to_hex_string(&nonce), } } /// Encrypts data using AES-256-GCM with a 16-byte nonce pub fn encrypt_data(data: &[u8], params: &EncryptionParams) -> Result, String> { // Decode key and nonce from hex - let key_bytes = hex::decode(¶ms.key).unwrap(); - let nonce_bytes = hex::decode(¶ms.nonce).unwrap(); + let key_bytes = hex_string_to_bytes(¶ms.key); + let nonce_bytes = hex_string_to_bytes(¶ms.nonce); // Initialize AES-GCM cipher let cipher = AesGcm::::new_from_slice(&key_bytes) @@ -185,8 +185,8 @@ pub fn decrypt_data(encrypted_data: &[u8], key_hex: &str, nonce_hex: &str) -> Re } // Decode key and nonce from hex - let key_bytes = hex::decode(key_hex).unwrap(); - let nonce_bytes = hex::decode(nonce_hex).unwrap(); + let key_bytes = hex_string_to_bytes(key_hex); + let nonce_bytes = hex_string_to_bytes(nonce_hex); // Split input into ciphertext and authentication tag let (ciphertext, tag_bytes) = encrypted_data.split_at(encrypted_data.len() - 16); diff --git a/src-tauri/src/db/attachments.rs b/src-tauri/src/db/attachments.rs index e132c596..ccb680f5 100644 --- a/src-tauri/src/db/attachments.rs +++ b/src-tauri/src/db/attachments.rs @@ -1,29 +1,41 @@ //! Attachment database operations. //! //! This module handles: -//! - AttachmentRef for file deduplication -//! - Building file hash indexes +//! - [`AttachmentRef`] for file deduplication +//! - [`UltraPackedFileHashIndex`] - memory-efficient sorted index with binary search +//! - Lazy singleton caching via [`lookup_attachment_cached`] +//! - Background cache warming via [`warm_file_hash_cache`] //! - Paginated message queries //! - Wrapper event ID tracking for deduplication //! - Attachment download status updates +//! +//! # Performance +//! +//! The file hash index uses several optimizations: +//! - Binary storage (`[u8; 32]`) instead of hex strings (50% memory savings) +//! - String interning for repeated values (chat IDs, URLs, extensions) +//! - Bitpacked indices in a single `u32` +//! - Sorted `Vec` with binary search instead of `HashMap` (no hash overhead) +//! - Lazy singleton pattern - built once, reused for all lookups +//! - NEON SIMD hex encoding on ARM64 (~1000x faster than `format!`) +//! - LUT fallback on other platforms (~43x faster than `format!`) +//! +//! Typical performance: ~10μs per lookup after initial ~250ms build. use serde::{Deserialize, Serialize}; use tauri::{AppHandle, Runtime}; use std::collections::HashMap; use crate::{Message, Attachment}; +use crate::util::{bytes_to_hex_32, bytes_to_hex_string, hex_to_bytes_32, hex_string_to_bytes}; use super::{get_chat_id_by_identifier, get_message_views}; -/// Lightweight attachment reference for file deduplication -/// Contains only the data needed to reuse an existing upload +/// Lightweight attachment reference for file deduplication. +/// Contains only the data needed to reuse an existing upload. #[derive(Serialize, Deserialize, Clone, Debug)] pub struct AttachmentRef { /// The SHA256 hash of the original file (used as ID) pub hash: String, - /// The message ID containing this attachment - pub message_id: String, - /// The chat ID containing this message - pub chat_id: String, /// The encrypted file URL on the server pub url: String, /// The encryption key @@ -36,81 +48,486 @@ pub struct AttachmentRef { pub size: u64, } -/// Build a file hash index from all attachments in the database -/// This is used for file deduplication without loading full message content -/// Returns a HashMap of file_hash -> AttachmentRef -pub async fn build_file_hash_index( - handle: &AppHandle, -) -> Result, String> { - use crate::stored_event::event_kind; - let mut index: HashMap = HashMap::new(); +// ============================================================================ +// Ultra-Packed File Hash Index - Sorted Vec + Binary Search + Bitpacking +// ============================================================================ - // Use guard - connection returned automatically after query, before heavy processing - let attachment_data: Vec<(String, String, String)> = { - let conn = crate::account_manager::get_db_connection_guard(handle)?; +/// Ultra-packed attachment entry optimized for memory efficiency. +/// +/// Uses fixed-size byte arrays instead of heap-allocated strings, and bitpacks +/// indices for interned strings. Total size: 116 bytes per entry. +/// +/// # Memory Layout +/// +/// | Field | Size | Description | +/// |-----------------|---------|------------------------------------------| +/// | `hash` | 32 bytes| Original file hash (binary search key) | +/// | `url_file_hash` | 32 bytes| Encrypted hash from URL | +/// | `key` | 32 bytes| Encryption key | +/// | `nonce` | 12 bytes| AES-GCM nonce (standard 96-bit) | +/// | `packed_indices`| 4 bytes | Bitpacked: base_url(6) + ext(6) | +/// | `size` | 4 bytes | File size (max 4GB) | +#[derive(Clone, Debug)] +#[repr(C)] // Ensure predictable memory layout +pub struct UltraPackedEntry { + /// Original file hash (SHA256) - used as sort key for binary search. + pub hash: [u8; 32], + /// Encrypted file hash extracted from URL (different from original). + /// Required for URL reconstruction since encryption changes the hash. + pub url_file_hash: [u8; 32], + /// Encryption key for decrypting the file. + pub key: [u8; 32], + /// Encryption nonce (12 bytes = 96-bit AES-GCM standard). + pub nonce: [u8; 12], + /// Bitpacked indices into string tables. + /// Layout: `[base_url: 6 bits][extension: 6 bits][unused: 20 bits]` + pub packed_indices: u32, + /// Encrypted file size in bytes (max 4GB per file). + pub size: u32, +} - // Query file attachment events (kind=15) from the events table - // Attachments are stored in the tags field as JSON - let mut stmt = conn.prepare( - "SELECT e.id, c.chat_identifier, e.tags - FROM events e - JOIN chats c ON e.chat_id = c.id - WHERE e.kind = ?1" - ).map_err(|e| format!("Failed to prepare attachment query: {}", e))?; - - let rows = stmt.query_map(rusqlite::params![event_kind::FILE_ATTACHMENT], |row| { - Ok(( - row.get::<_, String>(0)?, // event_id (message_id) - row.get::<_, String>(1)?, // chat_identifier - row.get::<_, String>(2)?, // tags JSON - )) - }).map_err(|e| format!("Failed to query attachments: {}", e))?; +impl UltraPackedEntry { + /// Pack indices into a single u32. + /// Layout: `[base_url: 6 bits][extension: 6 bits][unused: 20 bits]` + #[inline] + pub fn pack_indices(base_url: u16, extension: u8) -> u32 { + ((base_url as u32 & 0x3F) << 26) // 6 bits, max 63 + | ((extension as u32 & 0x3F) << 20) // 6 bits, max 63 + // 20 bits unused (for future use) + } - // Collect immediately to consume the iterator while stmt is still alive - let result: Result, _> = rows.collect(); - result.map_err(|e| format!("Failed to collect attachment rows: {}", e))? - // conn guard dropped here, connection returned to pool - }; + /// Unpack base_url index + #[inline] + pub fn base_url_idx(&self) -> u16 { + ((self.packed_indices >> 26) & 0x3F) as u16 + } - // Process the collected data - const EMPTY_FILE_HASH: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; - for (message_id, chat_id, tags_json) in attachment_data { - // Parse tags to find the "attachments" tag - let tags: Vec> = serde_json::from_str(&tags_json).unwrap_or_default(); - - // Find the attachments tag: ["attachments", ""] - let attachments_json = tags.iter() - .find(|tag| tag.first().map(|s| s.as_str()) == Some("attachments")) - .and_then(|tag| tag.get(1)) - .map(|s| s.as_str()) - .unwrap_or("[]"); - - // Parse the attachments JSON - let attachments: Vec = serde_json::from_str(attachments_json) - .unwrap_or_default(); + /// Unpack extension index + #[inline] + pub fn extension_idx(&self) -> u8 { + ((self.packed_indices >> 20) & 0x3F) as u8 + } +} - // Add each attachment to the index (skip empty hashes and empty URLs) - for attachment in attachments { - if !attachment.id.is_empty() - && attachment.id != EMPTY_FILE_HASH - && !attachment.url.is_empty() - { - index.insert(attachment.id.clone(), AttachmentRef { - hash: attachment.id, - message_id: message_id.clone(), - chat_id: chat_id.clone(), - url: attachment.url, - key: attachment.key, - nonce: attachment.nonce, - extension: attachment.extension, - size: attachment.size, +/// Memory-efficient file hash index using sorted Vec + binary search. +/// +/// This index enables O(log n) attachment lookup by file hash without the +/// memory overhead of a HashMap. String interning further reduces memory +/// by deduplicating repeated values like URLs and extensions. +/// +/// # Memory Usage +/// +/// For 6,800 attachments: ~789 KB total +/// - Entries: 116 bytes × 6,800 = ~789 KB +/// - String tables: ~3 KB (interned, highly deduplicated) +/// +/// Compare to naive HashMap: ~4.2 MB +/// +/// # Lookup Performance +/// +/// Binary search: O(log n) = ~13 comparisons for 6,800 entries +/// Typical lookup time: ~10μs +pub struct UltraPackedFileHashIndex { + /// Interned base URLs (host + API path + uploader). Max 64 unique values (6-bit index). + pub base_urls: Vec, + /// Interned file extensions. Max 64 unique values (6-bit index). + pub extensions: Vec, + /// Entries sorted by `hash` field for binary search. + pub entries: Vec, +} + +impl UltraPackedFileHashIndex { + /// Build the index from all file attachments in the database. + /// + /// Queries all `kind=15` (FILE_ATTACHMENT) events, parses their attachment + /// metadata, and builds a sorted index for binary search lookup. + /// + /// # Performance + /// + /// Build time scales linearly with attachment count: + /// - 6,800 attachments: ~250ms + /// + /// # Note + /// + /// Prefer using [`lookup_attachment_cached`] which manages a singleton + /// cache, rather than calling this directly. + pub async fn build(handle: &AppHandle) -> Result { + use crate::stored_event::event_kind; + + // String interning tables + let mut base_url_map: HashMap = HashMap::new(); + let mut ext_map: HashMap = HashMap::new(); + + let mut base_urls: Vec = Vec::new(); + let mut extensions: Vec = Vec::new(); + let mut entries: Vec = Vec::new(); + + fn intern_u16(s: &str, map: &mut HashMap, vec: &mut Vec) -> u16 { + if let Some(&idx) = map.get(s) { + idx + } else { + let idx = vec.len() as u16; + vec.push(s.to_string()); + map.insert(s.to_string(), idx); + idx + } + } + fn intern_u8(s: &str, map: &mut HashMap, vec: &mut Vec) -> u8 { + if let Some(&idx) = map.get(s) { + idx + } else { + let idx = vec.len() as u8; + vec.push(s.to_string()); + map.insert(s.to_string(), idx); + idx + } + } + + /// Extract the encrypted file hash from a URL + /// URL format: https://host/api/uploader_hash/encrypted_hash.ext + fn extract_url_file_hash(url: &str) -> [u8; 32] { + let without_scheme = url.strip_prefix("https://") + .or_else(|| url.strip_prefix("http://")) + .unwrap_or(url); + + // Get the last path segment (filename) + if let Some(filename) = without_scheme.rsplit('/').next() { + // Remove extension to get the hash + let hash_part = filename.split('.').next().unwrap_or(filename); + return hex_to_bytes_32(hash_part); + } + [0u8; 32] + } + + /// Extract base URL (everything before the encrypted file hash). + /// + /// Returns the URL path prefix including host, API path, and uploader hash. + /// Example: `"host.com/media/uploader123/"` from full URL. + fn extract_base_url(url: &str) -> String { + let without_scheme = url.strip_prefix("https://") + .or_else(|| url.strip_prefix("http://")) + .unwrap_or(url); + + let parts: Vec<&str> = without_scheme.split('/').collect(); + if parts.len() >= 3 { + // host/api/uploader/ or host/api/ + let host = parts[0]; + let api = parts[1]; + if parts.len() >= 4 { + // Has uploader path segment + let uploader = parts[2]; + format!("{}/{}/{}/", host, api, uploader) + } else { + format!("{}/{}/", host, api) + } + } else if parts.len() >= 2 { + format!("{}/", parts[0]) + } else { + without_scheme.to_string() + } + } + + // Query attachment data (only need tags - no chat_id or message_id needed) + let attachment_data: Vec = { + let conn = crate::account_manager::get_db_connection_guard(handle)?; + let mut stmt = conn.prepare( + "SELECT tags FROM events WHERE kind = ?1" + ).map_err(|e| format!("Failed to prepare attachment query: {}", e))?; + + let rows = stmt.query_map(rusqlite::params![event_kind::FILE_ATTACHMENT], |row| { + row.get::<_, String>(0) + }).map_err(|e| format!("Failed to query attachments: {}", e))?; + + rows.collect::, _>>() + .map_err(|e| format!("Failed to collect attachment rows: {}", e))? + }; + + const EMPTY_FILE_HASH: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + + for tags_json in attachment_data { + let tags: Vec> = serde_json::from_str(&tags_json).unwrap_or_default(); + let attachments_json = tags.iter() + .find(|tag| tag.first().map(|s| s.as_str()) == Some("attachments")) + .and_then(|tag| tag.get(1)) + .map(|s| s.as_str()) + .unwrap_or("[]"); + + let parsed: Vec = serde_json::from_str(attachments_json) + .unwrap_or_default(); + + for att in parsed { + if att.id.is_empty() || att.id == EMPTY_FILE_HASH || att.url.is_empty() { + continue; + } + + // Skip MLS attachments - they use rolling keys and can't be reused for deduplication. + // MLS attachments have original_hash set (used for key derivation). + if att.original_hash.is_some() { + continue; + } + + // Extract encrypted file hash from URL (this is different from att.id!) + let url_file_hash = extract_url_file_hash(&att.url); + + // Extract and intern the base URL (includes host/api/uploader/) + let base_url = extract_base_url(&att.url); + let base_url_idx = intern_u16(&base_url, &mut base_url_map, &mut base_urls); + let extension_idx = intern_u8(&att.extension, &mut ext_map, &mut extensions); + + // Clamp size to u32 max (4GB) + let size = if att.size > u32::MAX as u64 { u32::MAX } else { att.size as u32 }; + + // Parse nonce (12 bytes = 24 hex chars for AES-GCM) + let nonce = { + let bytes = hex_string_to_bytes(&att.nonce); + let mut arr = [0u8; 12]; + let len = bytes.len().min(12); + arr[..len].copy_from_slice(&bytes[..len]); + arr + }; + + entries.push(UltraPackedEntry { + hash: hex_to_bytes_32(&att.id), + url_file_hash, + key: hex_to_bytes_32(&att.key), + nonce, + packed_indices: UltraPackedEntry::pack_indices(base_url_idx, extension_idx), + size, }); } } + + // Sort by hash for binary search! + entries.sort_unstable_by(|a, b| a.hash.cmp(&b.hash)); + + let index = Self { base_urls, extensions, entries }; + index.log_memory(); + Ok(index) + } + + /// Find an entry by its original file hash using binary search. + /// + /// Returns a reference to the packed entry if found. Use [`get_full`] + /// if you need the fully reconstructed [`AttachmentRef`]. + /// + /// # Performance + /// + /// O(log n) binary search - ~13 comparisons for 6,800 entries. + #[inline] + pub fn get(&self, hash: &[u8; 32]) -> Option<&UltraPackedEntry> { + self.entries + .binary_search_by(|entry| entry.hash.cmp(hash)) + .ok() + .map(|idx| &self.entries[idx]) + } + + /// Find an entry and reconstruct the full [`AttachmentRef`]. + /// + /// This performs binary search lookup and then reconstructs all string + /// fields from the interned tables, including the full URL. + /// + /// # URL Reconstruction + /// + /// The URL is reconstructed as: `https://{base_url}{url_file_hash}.{extension}` + /// where `base_url` includes the host, API path, and uploader hash. + pub fn get_full(&self, hash: &[u8; 32]) -> Option { + self.get(hash).map(|entry| { + // Use SIMD-accelerated hex conversion (NEON on ARM64, LUT fallback elsewhere) + let hash_hex = bytes_to_hex_32(hash); + let url_hash_hex = bytes_to_hex_32(&entry.url_file_hash); + let ext = self.extensions.get(entry.extension_idx() as usize) + .map(|s| s.as_str()).unwrap_or(""); + + // base_urls now includes the full path up to the filename + // e.g., "host/api/uploader/" so we just append hash.ext + let base = self.base_urls.get(entry.base_url_idx() as usize) + .map(|s| s.as_str()).unwrap_or(""); + + let url = format!("https://{}{}.{}", base, url_hash_hex, ext); + + AttachmentRef { + hash: hash_hex, + url, + key: bytes_to_hex_32(&entry.key), + nonce: bytes_to_hex_string(&entry.nonce), + extension: ext.to_string(), + size: entry.size as u64, + } + }) + } + + /// Log memory usage statistics (debug builds only) + #[cfg(debug_assertions)] + pub fn log_memory(&self) { + let entry_size = std::mem::size_of::(); + let entries_total = self.entries.len() * entry_size; + let string_tables: usize = self.base_urls.iter().map(|s| s.capacity()).sum::() + + self.extensions.iter().map(|s| s.capacity()).sum::(); + let total_bytes = entries_total + string_tables + 16; // +16 for Vec overhead (2 Vecs) + + println!("[FileHashIndex] {} entries, {:.1} KB ({}B/entry, sorted Vec + binary search)", + self.entries.len(), total_bytes as f64 / 1024.0, entry_size); + } + + /// No-op in release builds + #[cfg(not(debug_assertions))] + #[inline] + pub fn log_memory(&self) {} +} + +// ============================================================================ +// Cached File Hash Index (Lazy Singleton) +// ============================================================================ + +use std::sync::OnceLock; +use tokio::sync::RwLock; + +/// Global cached file hash index - built once, reused for all lookups +static CACHED_FILE_HASH_INDEX: OnceLock>> = OnceLock::new(); + +/// Get or build the cached file hash index (lazy singleton). +/// +/// Uses double-checked locking to ensure the index is only built once, +/// even if multiple tasks call this concurrently. +/// +/// # Returns +/// +/// A reference to the global `RwLock` containing the cached index. +/// The index will be built on first access if not already cached. +pub async fn get_cached_file_hash_index( + handle: &AppHandle, +) -> Result<&'static RwLock>, String> { + let lock = CACHED_FILE_HASH_INDEX.get_or_init(|| RwLock::new(None)); + + // Fast path: check if already built (read lock) + { + let read_guard = lock.read().await; + if read_guard.is_some() { + return Ok(lock); + } + } + + // Slow path: acquire write lock and double-check before building + { + let mut write_guard = lock.write().await; + + // Double-check: another task may have built it while we waited for the write lock + if write_guard.is_some() { + return Ok(lock); + } + + // Build the index (we hold the write lock, so only one task builds) + let index = UltraPackedFileHashIndex::build(handle).await?; + *write_guard = Some(index); + } + + Ok(lock) +} + +/// Lookup an attachment by its original file hash using the cached index. +/// +/// This is the primary lookup function for attachment deduplication. +/// Uses the lazy singleton cache, building it on first access if needed. +/// +/// # Arguments +/// +/// * `handle` - Tauri app handle for database access +/// * `file_hash` - The SHA256 hash of the original (unencrypted) file +/// +/// # Returns +/// +/// * `Ok(Some(AttachmentRef))` - Found an existing attachment with this hash +/// * `Ok(None)` - No attachment found with this hash +/// * `Err(String)` - Database or cache error +/// +/// # Performance +/// +/// * First call: ~250ms (builds index from database) +/// * Subsequent calls: ~10μs (binary search in cached index) +pub async fn lookup_attachment_cached( + handle: &AppHandle, + file_hash: &str, +) -> Result, String> { + let lock = get_cached_file_hash_index(handle).await?; + let guard = lock.read().await; + + if let Some(index) = guard.as_ref() { + let hash_bytes = hex_to_bytes_32(file_hash); + Ok(index.get_full(&hash_bytes)) + } else { + Ok(None) + } +} + +/// Invalidate the cached index (call when new attachments are added) +#[allow(dead_code)] // Will be used when we add cache invalidation on new uploads +pub async fn invalidate_file_hash_cache() { + if let Some(lock) = CACHED_FILE_HASH_INDEX.get() { + let mut guard = lock.write().await; + *guard = None; + } +} + +/// Check if the file hash cache is already built. +/// +/// Non-blocking check using `try_read()`. Returns `false` if the lock +/// is currently held by a writer (cache being built). +pub fn is_file_hash_cache_built() -> bool { + CACHED_FILE_HASH_INDEX.get() + .map(|lock| lock.try_read().map(|g| g.is_some()).unwrap_or(false)) + .unwrap_or(false) +} + +/// Pre-warm the file hash cache in the background. +/// +/// Call this after sync completes to ensure fast lookups when the user +/// sends their first attachment. This function is safe to call multiple +/// times - it will skip if the cache is already built or if there are +/// no attachments in the database. +/// +/// # When to call +/// +/// - After initial sync completes +/// - After deep rescan completes +/// +/// # Behavior +/// +/// 1. Checks if cache is already built (skips if so) +/// 2. Checks if any attachments exist in database (skips if none) +/// 3. Builds the cache (~250ms for 6000+ attachments) +pub async fn warm_file_hash_cache(handle: &AppHandle) { + // Skip if already built + if is_file_hash_cache_built() { + println!("[FileHashIndex] Cache already built, skipping warm-up"); + return; + } + + // Check if there are any attachments worth caching (EXISTS is faster than COUNT) + let has_attachments = { + if let Ok(conn) = crate::account_manager::get_db_connection_guard(handle) { + conn.query_row( + "SELECT EXISTS(SELECT 1 FROM events WHERE kind = 15)", + [], + |row| row.get::<_, bool>(0) + ).unwrap_or(false) + } else { + false + } + }; + + if !has_attachments { + println!("[FileHashIndex] No attachments found, skipping cache warm-up"); + return; } - Ok(index) + // Build the cache + println!("[FileHashIndex] Warming cache in background..."); + let start = std::time::Instant::now(); + match get_cached_file_hash_index(handle).await { + Ok(_) => println!("[FileHashIndex] Cache warmed in {:?}", start.elapsed()), + Err(e) => eprintln!("[FileHashIndex] Cache warm-up failed: {}", e), + } } /// Get paginated messages for a chat (newest first, with offset) @@ -162,19 +579,6 @@ pub async fn get_chat_message_count( Ok(count as usize) } -/// Get the last N messages for a chat (for preview purposes) -/// This is optimized for getting just the most recent messages without loading the full history -pub async fn get_chat_last_messages( - handle: &AppHandle, - chat_id: &str, - count: usize, -) -> Result, String> { - // Get integer chat ID - let chat_int_id = get_chat_id_by_identifier(handle, chat_id)?; - // Use the events-based message views - get_message_views(handle, chat_int_id, count, 0).await -} - /// Get messages around a specific message ID /// Returns messages from (target - context_before) to the most recent /// This is used for scrolling to old replied-to messages @@ -319,18 +723,18 @@ pub async fn update_wrapper_event_id( Ok(rows_updated > 0) } -/// Load recent wrapper_event_ids into a HashSet for fast duplicate detection +/// Load recent wrapper_event_ids as raw bytes for the hybrid cache /// This preloads wrapper_ids from the last N days to avoid SQL queries during sync +/// +/// Returns Vec<[u8; 32]> for memory-efficient storage (76% less than HashSet) pub async fn load_recent_wrapper_ids( handle: &AppHandle, days: u64, -) -> Result, String> { - let mut wrapper_ids = std::collections::HashSet::new(); - +) -> Result, String> { // Try to get a database connection - if it fails, we're not using DB mode let conn = match crate::account_manager::get_db_connection(handle) { Ok(c) => c, - Err(_) => return Ok(wrapper_ids), // No DB, return empty set + Err(_) => return Ok(Vec::new()), // No DB, return empty vec }; // Calculate timestamp for N days ago (in seconds, matching events.created_at) @@ -360,14 +764,19 @@ pub async fn load_recent_wrapper_ids( crate::account_manager::return_db_connection(conn); match result { - Ok(ids) => { - for id in ids { - wrapper_ids.insert(id); + Ok(hex_ids) => { + // Convert hex strings to [u8; 32] using SIMD-accelerated decode + let mut wrapper_ids = Vec::with_capacity(hex_ids.len()); + for hex in hex_ids { + if hex.len() == 64 { + let bytes = crate::simd::hex::hex_to_bytes_32(&hex); + wrapper_ids.push(bytes); + } } Ok(wrapper_ids) } Err(_) => { - Ok(wrapper_ids) // Return empty set on error, will fall back to DB queries + Ok(Vec::new()) // Return empty vec on error, will fall back to DB queries } } } @@ -438,3 +847,89 @@ pub fn update_attachment_downloaded_status( Ok(()) } + +/// Check all downloaded attachments in the database for missing files. +/// Updates the database directly for any files that no longer exist. +/// Returns (total_checked, missing_count, elapsed_time). +pub async fn check_downloaded_attachments_integrity( + handle: &AppHandle, +) -> Result<(usize, usize, std::time::Duration), String> { + let start = std::time::Instant::now(); + + // Query all events with file attachments that have downloaded files + // Using JSON extract to filter only events with downloaded attachments + let events_with_downloaded: Vec<(String, String)> = { + let conn = crate::account_manager::get_db_connection_guard(handle)?; + + // Query all file attachment events - we'll filter in Rust for downloaded=true + // This is more reliable than JSON filtering in SQLite + let mut stmt = conn.prepare( + "SELECT id, tags FROM events WHERE kind = 15" + ).map_err(|e| format!("Failed to prepare integrity query: {}", e))?; + + let rows = stmt.query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + }).map_err(|e| format!("Failed to query attachments: {}", e))?; + + rows.filter_map(|r| r.ok()).collect() + }; + + let mut total_checked = 0; + let mut missing_count = 0; + let mut updates: Vec<(String, String)> = Vec::new(); // (event_id, updated_tags_json) + + for (event_id, tags_json) in events_with_downloaded { + let mut tags: Vec> = serde_json::from_str(&tags_json).unwrap_or_default(); + + let attachments_tag_idx = tags.iter().position(|tag| { + tag.first().map(|s| s.as_str()) == Some("attachments") + }); + + let Some(idx) = attachments_tag_idx else { continue }; + let Some(attachments_json) = tags.get(idx).and_then(|t| t.get(1)) else { continue }; + + let mut attachments: Vec = serde_json::from_str(attachments_json) + .unwrap_or_default(); + + let mut modified = false; + for att in &mut attachments { + if att.downloaded && !att.path.is_empty() { + total_checked += 1; + if !std::path::Path::new(&att.path).exists() { + att.downloaded = false; + att.path = String::new(); + modified = true; + missing_count += 1; + } + } + } + + if modified { + let updated_attachments_json = serde_json::to_string(&attachments) + .map_err(|e| format!("Failed to serialize: {}", e))?; + tags[idx] = vec!["attachments".to_string(), updated_attachments_json]; + let updated_tags_json = serde_json::to_string(&tags) + .map_err(|e| format!("Failed to serialize tags: {}", e))?; + updates.push((event_id, updated_tags_json)); + } + } + + // Batch update all modified events + if !updates.is_empty() { + let conn = crate::account_manager::get_db_connection_guard(handle)?; + for (event_id, tags_json) in updates { + conn.execute( + "UPDATE events SET tags = ?1 WHERE id = ?2", + rusqlite::params![tags_json, event_id], + ).ok(); // Ignore individual errors + } + } + + let elapsed = start.elapsed(); + println!( + "[Integrity] Checked {} downloaded attachments in {:?}, {} missing files updated", + total_checked, elapsed, missing_count + ); + + Ok((total_checked, missing_count, elapsed)) +} diff --git a/src-tauri/src/db/events.rs b/src-tauri/src/db/events.rs index 4ff9cecf..fec70d5f 100644 --- a/src-tauri/src/db/events.rs +++ b/src-tauri/src/db/events.rs @@ -48,8 +48,8 @@ pub async fn save_event( r#" INSERT OR REPLACE INTO events ( id, kind, chat_id, user_id, content, tags, reference_id, - created_at, received_at, mine, pending, failed, wrapper_event_id, npub - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14) + created_at, received_at, mine, pending, failed, wrapper_event_id, npub, preview_metadata + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15) "#, rusqlite::params![ event.id, @@ -66,6 +66,7 @@ pub async fn save_event( event.failed as i32, event.wrapper_event_id, event.npub, + event.preview_metadata, ], ).map_err(|e| format!("Failed to save event: {}", e))?; @@ -209,6 +210,7 @@ pub async fn get_pivx_payments_for_chat( failed: row.get::<_, i32>(11)? != 0, wrapper_event_id: row.get(12)?, npub: row.get(13)?, + preview_metadata: None, // PIVX events don't have preview metadata }) } ).map_err(|e| format!("Failed to query events: {}", e))?; @@ -281,6 +283,7 @@ pub async fn get_system_events_for_chat( failed: row.get::<_, i32>(11)? != 0, wrapper_event_id: row.get(12)?, npub: row.get(13)?, + preview_metadata: None, // System events don't have preview metadata }) } ).map_err(|e| format!("Failed to query events: {}", e))?; @@ -337,6 +340,7 @@ pub async fn save_reaction_event( failed: false, wrapper_event_id: None, npub: Some(reaction.author_id.clone()), + preview_metadata: None, }; save_event(handle, &event).await @@ -382,6 +386,7 @@ pub async fn save_edit_event( failed: false, wrapper_event_id: None, npub: Some(npub.to_string()), + preview_metadata: None, }; save_event(handle, &event).await @@ -434,7 +439,7 @@ pub async fn get_events( let sql = format!( r#" SELECT id, kind, chat_id, user_id, content, tags, reference_id, - created_at, received_at, mine, pending, failed, wrapper_event_id, npub + created_at, received_at, mine, pending, failed, wrapper_event_id, npub, preview_metadata FROM events WHERE chat_id = ?1 AND kind IN ({}) ORDER BY created_at DESC @@ -474,7 +479,7 @@ pub async fn get_events( } else { let sql = r#" SELECT id, kind, chat_id, user_id, content, tags, reference_id, - created_at, received_at, mine, pending, failed, wrapper_event_id, npub + created_at, received_at, mine, pending, failed, wrapper_event_id, npub, preview_metadata FROM events WHERE chat_id = ?1 ORDER BY created_at DESC @@ -526,6 +531,7 @@ fn parse_event_row(row: &rusqlite::Row) -> rusqlite::Result { failed: row.get::<_, i32>(11)? != 0, wrapper_event_id: row.get(12)?, npub: row.get(13)?, + preview_metadata: row.get(14)?, }) } @@ -547,7 +553,7 @@ pub async fn get_related_events( let sql = format!( r#" SELECT id, kind, chat_id, user_id, content, tags, reference_id, - created_at, received_at, mine, pending, failed, wrapper_event_id, npub + created_at, received_at, mine, pending, failed, wrapper_event_id, npub, preview_metadata FROM events WHERE reference_id IN ({}) ORDER BY created_at ASC @@ -581,6 +587,7 @@ pub async fn get_related_events( failed: row.get::<_, i32>(11)? != 0, wrapper_event_id: row.get(12)?, npub: row.get(13)?, + preview_metadata: row.get(14)?, }) }) .map_err(|e| format!("Failed to query related events: {}", e))? @@ -852,7 +859,8 @@ pub async fn get_message_views( let reactions = reactions_by_msg.remove(&event.id).unwrap_or_default(); // Get attachments from the lookup map (for kind=15 file messages) - let attachments = attachments_by_msg.remove(&event.id).unwrap_or_default(); + let attachments: Vec = attachments_by_msg.remove(&event.id) + .unwrap_or_default(); // Get original content (already decrypted by get_events()) let original_content = if event.kind == event_kind::FILE_ATTACHMENT { @@ -892,6 +900,10 @@ pub async fn get_message_views( (original_content, false, None) }; + // Deserialize preview_metadata if present + let preview_metadata = event.preview_metadata + .and_then(|json| serde_json::from_str(&json).ok()); + let message = Message { id: event.id, content, @@ -899,7 +911,7 @@ pub async fn get_message_views( replied_to_content: None, // Populated below replied_to_npub: None, // Populated below replied_to_has_attachment: None, // Populated below - preview_metadata: None, // TODO: Parse from tags if needed + preview_metadata, attachments, reactions, at, @@ -939,4 +951,198 @@ pub async fn get_message_views( } Ok(messages) +} + +/// Get the last message for ALL chats in a single batch query. +/// +/// This is optimized for app startup where we need one preview message per chat. +/// Uses ROW_NUMBER() OVER (PARTITION BY chat_id) to get the latest message per chat +/// in a single query, avoiding N separate queries. +/// +/// Returns: HashMap> (Vec will have 0 or 1 message) +pub async fn get_all_chats_last_messages( + handle: &AppHandle, +) -> Result>, String> { + // Step 1: Get the last message event for each chat using window function + let message_events: Vec<(String, StoredEvent)> = { + let conn = crate::account_manager::get_db_connection_guard(handle)?; + + // Use ROW_NUMBER to get the latest message per chat + // Join with chats table to get chat_identifier + let sql = r#" + SELECT c.chat_identifier, + e.id, e.kind, e.chat_id, e.user_id, e.content, e.tags, e.reference_id, + e.created_at, e.received_at, e.mine, e.pending, e.failed, e.wrapper_event_id, e.npub, e.preview_metadata + FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY chat_id ORDER BY created_at DESC) as rn + FROM events + WHERE kind IN (?1, ?2, ?3) + ) e + JOIN chats c ON e.chat_id = c.id + WHERE e.rn = 1 + "#; + + let mut stmt = conn.prepare(sql) + .map_err(|e| format!("Failed to prepare batch last messages query: {}", e))?; + + let rows = stmt.query_map( + rusqlite::params![ + event_kind::MLS_CHAT_MESSAGE as i32, + event_kind::PRIVATE_DIRECT_MESSAGE as i32, + event_kind::FILE_ATTACHMENT as i32 + ], + |row| { + let chat_identifier: String = row.get(0)?; + let tags_json: String = row.get(6)?; + let tags: Vec> = serde_json::from_str(&tags_json).unwrap_or_default(); + + let event = StoredEvent { + id: row.get(1)?, + kind: row.get::<_, i32>(2)? as u16, + chat_id: row.get(3)?, + user_id: row.get(4)?, + content: row.get(5)?, + tags, + reference_id: row.get(7)?, + created_at: row.get::<_, i64>(8)? as u64, + received_at: row.get::<_, i64>(9)? as u64, + mine: row.get::<_, i32>(10)? != 0, + pending: row.get::<_, i32>(11)? != 0, + failed: row.get::<_, i32>(12)? != 0, + wrapper_event_id: row.get(13)?, + npub: row.get(14)?, + preview_metadata: row.get(15)?, + }; + Ok((chat_identifier, event)) + } + ).map_err(|e| format!("Failed to query batch last messages: {}", e))?; + + rows.filter_map(|r| r.ok()).collect() + }; + + if message_events.is_empty() { + return Ok(HashMap::new()); + } + + // Step 2: Get related events (reactions, edits) for all these messages + let message_ids: Vec = message_events.iter().map(|(_, e)| e.id.clone()).collect(); + let related_events = get_related_events(handle, &message_ids).await?; + + // Group reactions and edits by message ID + let mut reactions_by_msg: HashMap> = HashMap::new(); + let mut edits_by_msg: HashMap> = HashMap::new(); + + for event in related_events { + if let Some(ref_id) = &event.reference_id { + match event.kind { + k if k == event_kind::REACTION => { + let reaction = Reaction { + id: event.id.clone(), + reference_id: ref_id.clone(), + author_id: event.npub.clone().unwrap_or_default(), + emoji: event.content.clone(), + }; + reactions_by_msg.entry(ref_id.clone()).or_default().push(reaction); + } + k if k == event_kind::MESSAGE_EDIT => { + let decrypted_content = internal_decrypt(event.content.clone(), None).await + .unwrap_or_else(|_| event.content.clone()); + let timestamp_ms = event.created_at * 1000; + edits_by_msg.entry(ref_id.clone()).or_default().push((timestamp_ms, decrypted_content)); + } + _ => {} + } + } + } + + // Sort edits by timestamp + for edits in edits_by_msg.values_mut() { + edits.sort_by_key(|(ts, _)| *ts); + } + + // Step 3: Parse attachments from event tags + let mut attachments_by_msg: HashMap> = HashMap::new(); + + for (_, event) in &message_events { + if event.kind != event_kind::FILE_ATTACHMENT && event.kind != event_kind::MLS_CHAT_MESSAGE { + continue; + } + + if let Some(attachments_json) = event.get_tag("attachments") { + if let Ok(attachments) = serde_json::from_str::>(attachments_json) { + if !attachments.is_empty() { + attachments_by_msg.insert(event.id.clone(), attachments); + } + } + } + } + + // Step 4: Compose into Message structs, grouped by chat_identifier + let mut result: HashMap> = HashMap::new(); + + for (chat_identifier, event) in message_events { + let reactions = reactions_by_msg.remove(&event.id).unwrap_or_default(); + let attachments = attachments_by_msg.remove(&event.id).unwrap_or_default(); + + // Get replied_to from tags + let replied_to = event.get_tag("e") + .map(|s| s.to_string()) + .unwrap_or_default(); + + // Decrypt content + let original_content = if event.kind == event_kind::MLS_CHAT_MESSAGE + || event.kind == event_kind::PRIVATE_DIRECT_MESSAGE + { + internal_decrypt(event.content.clone(), None).await + .unwrap_or_else(|_| event.content.clone()) + } else { + String::new() // File attachments don't have displayable content + }; + + // Apply edits if any + let (content, edited, edit_history) = if let Some(edits) = edits_by_msg.remove(&event.id) { + let latest_content = edits.last() + .map(|(_, c)| c.clone()) + .unwrap_or_else(|| original_content.clone()); + let history: Vec = std::iter::once(EditEntry { + content: original_content, + edited_at: event.created_at * 1000, + }) + .chain(edits.into_iter().map(|(ts, c)| EditEntry { content: c, edited_at: ts })) + .collect(); + (latest_content, true, Some(history)) + } else { + (original_content, false, None) + }; + + let at = event.created_at * 1000; // Convert to ms + + // Deserialize preview_metadata if present + let preview_metadata = event.preview_metadata + .and_then(|json| serde_json::from_str(&json).ok()); + + let message = Message { + id: event.id, + content, + replied_to, + replied_to_content: None, + replied_to_npub: None, + replied_to_has_attachment: None, + preview_metadata, + attachments, + reactions, + at, + pending: event.pending, + failed: event.failed, + mine: event.mine, + npub: event.npub, + wrapper_event_id: event.wrapper_event_id, + edited, + edit_history, + }; + + result.entry(chat_identifier).or_default().push(message); + } + + Ok(result) } \ No newline at end of file diff --git a/src-tauri/src/db/messages.rs b/src-tauri/src/db/messages.rs index adb1b297..da907fb6 100644 --- a/src-tauri/src/db/messages.rs +++ b/src-tauri/src/db/messages.rs @@ -87,6 +87,10 @@ fn message_to_stored_event(message: &Message, chat_id: i64, user_id: Option } } + // Serialize preview_metadata if present + let preview_metadata = message.preview_metadata.as_ref() + .and_then(|m| serde_json::to_string(m).ok()); + StoredEvent { id: message.id.clone(), kind, @@ -105,6 +109,7 @@ fn message_to_stored_event(message: &Message, chat_id: i64, user_id: Option failed: message.failed, wrapper_event_id: message.wrapper_event_id.clone(), npub: message.npub.clone(), + preview_metadata, } } diff --git a/src-tauri/src/db/mod.rs b/src-tauri/src/db/mod.rs index 4be7d684..868a0a97 100644 --- a/src-tauri/src/db/mod.rs +++ b/src-tauri/src/db/mod.rs @@ -48,17 +48,18 @@ pub(crate) use chats::{get_or_create_chat_id, get_or_create_user_id}; pub use messages::{save_message, save_chat_messages}; // Attachment database functions pub use attachments::{ - AttachmentRef, build_file_hash_index, - get_chat_messages_paginated, get_chat_message_count, get_chat_last_messages, + lookup_attachment_cached, warm_file_hash_cache, + get_chat_messages_paginated, get_chat_message_count, get_messages_around_id, message_exists_in_db, wrapper_event_exists, update_wrapper_event_id, load_recent_wrapper_ids, update_attachment_downloaded_status, + check_downloaded_attachments_integrity, }; // Event database functions pub use events::{ save_event, save_pivx_payment_event, save_system_event_by_id, get_pivx_payments_for_chat, get_system_events_for_chat, save_reaction_event, save_edit_event, event_exists, - populate_reply_context, get_message_views, + populate_reply_context, get_message_views, get_all_chats_last_messages, }; /// In-memory cache for chat_identifier → integer ID mappings diff --git a/src-tauri/src/image_cache.rs b/src-tauri/src/image_cache.rs index a688525d..398ccbda 100644 --- a/src-tauri/src/image_cache.rs +++ b/src-tauri/src/image_cache.rs @@ -25,6 +25,7 @@ use log::{info, warn, debug}; use serde_json::json; use crate::net::{ProgressReporter, download_with_reporter}; +use crate::util::bytes_to_hex_string; use std::collections::HashSet; use tokio::sync::Mutex; @@ -172,7 +173,7 @@ fn url_to_cache_key(url: &str) -> String { hasher.update(url.as_bytes()); let result = hasher.finalize(); // Use first 16 bytes for a shorter but still unique filename - hex::encode(&result[..16]) + bytes_to_hex_string(&result[..16]) } /// Validate image bytes and detect format @@ -195,10 +196,17 @@ fn validate_image(bytes: &[u8]) -> Option<&'static str> { } } - // Also accept SVG (text-based) + // Also accept SVG (text-based) - search bytes directly to avoid allocation if bytes.len() > 5 { - let start = String::from_utf8_lossy(&bytes[..std::cmp::min(256, bytes.len())]); - if start.contains(" bool { + haystack.windows(needle.len()).any(|w| w == needle) + } + + if contains_pattern(header, b"( // Check for any file with this cache key (any extension) if let Ok(entries) = std::fs::read_dir(&cache_dir) { for entry in entries.flatten() { - let filename = entry.file_name().to_string_lossy().to_string(); - if filename.starts_with(&cache_key) { - return Some(entry.path().to_string_lossy().to_string()); + let filename = entry.file_name(); + let filename_str = filename.to_string_lossy(); + if filename_str.starts_with(&cache_key) { + return Some(entry.path().to_string_lossy().into_owned()); } } } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 598f4cc4..33620bf5 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -60,6 +60,9 @@ mod audio; // Shared utilities module (error handling, image encoding, state access) mod shared; +// SIMD-accelerated operations (hex encoding, image alpha, etc.) +mod simd; + // State management module mod state; // Re-export commonly used state items at crate root for backwards compatibility @@ -278,7 +281,6 @@ pub fn run() { commands::messaging::get_messages_around_id, commands::messaging::get_system_events, commands::messaging::get_chat_message_count, - commands::messaging::get_file_hash_index, commands::messaging::evict_chat_messages, // Realtime signaling commands (commands/realtime.rs) commands::realtime::notifs, diff --git a/src-tauri/src/message/compact.rs b/src-tauri/src/message/compact.rs new file mode 100644 index 00000000..ce70259b --- /dev/null +++ b/src-tauri/src/message/compact.rs @@ -0,0 +1,1811 @@ +//! Compact message storage with binary IDs and interned strings. +//! +//! This module provides memory-efficient message storage: +//! - `[u8; 32]` for IDs instead of hex strings (saves ~56 bytes per ID) +//! - Interned npubs via `NpubInterner` (each unique npub stored once) +//! - Bitflags for boolean states (1 byte instead of 4+) +//! - Binary search for O(log n) message lookup +//! - Boxed optional fields (replied_to, wrapper_id) to save inline space +//! - Compact timestamp (u32 seconds since 2020 epoch) + +use crate::message::{Attachment, EditEntry, ImageMetadata, Reaction}; +use crate::net::SiteMetadata; +use crate::simd::{bytes_to_hex_32, hex_to_bytes_32, hex_string_to_bytes}; + +// ============================================================================ +// Pending ID Encoding +// ============================================================================ + +/// Marker byte for pending IDs (first byte = 0x01) +/// Real event IDs are random SHA256 hashes, so this is safe. +const PENDING_ID_MARKER: u8 = 0x01; + +/// Encode an ID string to 32 bytes, handling pending IDs specially. +/// - Pending IDs ("pending-{nanoseconds}") are encoded with marker byte + timestamp +/// - Regular hex IDs are decoded normally +#[inline] +fn encode_message_id(id: &str) -> [u8; 32] { + if let Some(timestamp_str) = id.strip_prefix("pending-") { + // Encode pending ID: marker byte + timestamp as u128 (16 bytes) + let mut bytes = [0u8; 32]; + bytes[0] = PENDING_ID_MARKER; + if let Ok(timestamp) = timestamp_str.parse::() { + bytes[1..17].copy_from_slice(×tamp.to_le_bytes()); + } + bytes + } else { + hex_to_bytes_32(id) + } +} + +/// Decode 32 bytes back to an ID string, handling pending IDs specially. +#[inline] +fn decode_message_id(bytes: &[u8; 32]) -> String { + if bytes[0] == PENDING_ID_MARKER { + // Decode pending ID: extract timestamp from bytes 1-16 + let mut timestamp_bytes = [0u8; 16]; + timestamp_bytes.copy_from_slice(&bytes[1..17]); + let timestamp = u128::from_le_bytes(timestamp_bytes); + format!("pending-{}", timestamp) + } else { + bytes_to_hex_32(bytes) + } +} + +// ============================================================================ +// Compact Timestamp +// ============================================================================ + +/// Custom epoch: 2020-01-01 00:00:00 UTC (in milliseconds) +/// This allows us to use u32 for timestamps until ~2156 +const EPOCH_2020_MS: u64 = 1577836800000; + +/// Convert milliseconds timestamp to compact u32 (seconds since 2020) +#[inline] +pub fn timestamp_to_compact(ms: u64) -> u32 { + ((ms.saturating_sub(EPOCH_2020_MS)) / 1000) as u32 +} + +/// Convert compact u32 back to milliseconds timestamp +#[inline] +pub fn timestamp_from_compact(compact: u32) -> u64 { + EPOCH_2020_MS + (compact as u64 * 1000) +} + +// ============================================================================ +// Message Flags +// ============================================================================ + +/// Bitflags for message state (1 byte instead of 4+ bytes for separate bools) +/// +/// Layout (bits): 0=mine, 1=pending, 2=failed, 3-4=replied_to_has_attachment +/// replied_to_has_attachment: 00=None, 01=Some(false), 10=Some(true) +/// +/// Note: No EDITED flag - check `edit_history.is_some()` instead +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct MessageFlags(u8); + +impl MessageFlags { + pub const NONE: Self = Self(0); + pub const MINE: Self = Self(0b00001); + pub const PENDING: Self = Self(0b00010); + pub const FAILED: Self = Self(0b00100); + // Bits 3-4 for replied_to_has_attachment: + // 00 = None, 01 = Some(false), 10 = Some(true) + const REPLY_ATTACH_MASK: u8 = 0b11000; + const REPLY_ATTACH_SHIFT: u8 = 3; + + #[inline] + pub fn is_mine(self) -> bool { + self.0 & Self::MINE.0 != 0 + } + + #[inline] + pub fn is_pending(self) -> bool { + self.0 & Self::PENDING.0 != 0 + } + + #[inline] + pub fn is_failed(self) -> bool { + self.0 & Self::FAILED.0 != 0 + } + + /// Get replied_to_has_attachment as Option + /// Returns None (unknown), Some(false), or Some(true) + #[inline] + pub fn replied_to_has_attachment(self) -> Option { + match (self.0 & Self::REPLY_ATTACH_MASK) >> Self::REPLY_ATTACH_SHIFT { + 0b00 => None, // Unknown + 0b01 => Some(false), // No attachment + 0b10 => Some(true), // Has attachment + _ => None, // Invalid, treat as unknown + } + } + + #[inline] + pub fn set_mine(&mut self, value: bool) { + if value { + self.0 |= Self::MINE.0; + } else { + self.0 &= !Self::MINE.0; + } + } + + #[inline] + pub fn set_pending(&mut self, value: bool) { + if value { + self.0 |= Self::PENDING.0; + } else { + self.0 &= !Self::PENDING.0; + } + } + + #[inline] + pub fn set_failed(&mut self, value: bool) { + if value { + self.0 |= Self::FAILED.0; + } else { + self.0 &= !Self::FAILED.0; + } + } + + /// Set replied_to_has_attachment from Option + #[inline] + pub fn set_replied_to_has_attachment(&mut self, value: Option) { + // Clear existing bits + self.0 &= !Self::REPLY_ATTACH_MASK; + // Set new value + let bits = match value { + None => 0b00, + Some(false) => 0b01, + Some(true) => 0b10, + }; + self.0 |= bits << Self::REPLY_ATTACH_SHIFT; + } + + /// Create flags from individual booleans + #[inline] + pub fn from_bools(mine: bool, pending: bool, failed: bool) -> Self { + let mut flags = Self::NONE; + flags.set_mine(mine); + flags.set_pending(pending); + flags.set_failed(failed); + flags + } + + /// Create flags from all values including replied_to_has_attachment + #[inline] + pub fn from_all(mine: bool, pending: bool, failed: bool, replied_to_has_attachment: Option) -> Self { + let mut flags = Self::from_bools(mine, pending, failed); + flags.set_replied_to_has_attachment(replied_to_has_attachment); + flags + } +} + +// ============================================================================ +// TinyVec - 8-byte thin pointer for small collections +// ============================================================================ + +use std::alloc::{alloc, dealloc, Layout}; +use std::marker::PhantomData; +use std::ptr::NonNull; + +/// Ultra-compact vector using a thin pointer (8 bytes on stack). +/// +/// Memory layout: +/// - Stack: single pointer (8 bytes) - null for empty +/// - Heap: `[len: u8][items: T...]` - only allocated when non-empty +/// +/// Compared to standard types: +/// - `Vec`: 24 bytes (ptr + len + cap) +/// - `Box<[T]>`: 16 bytes (fat pointer) +/// - `TinyVec`: 8 bytes (thin pointer) +/// +/// Limitations: +/// - Max 255 items (u8 length) +/// - Immutable after creation (no push/pop - recreate to modify) +/// - Perfect for attachments/reactions which rarely change +pub struct TinyVec { + /// Null = empty, otherwise points to: [len: u8][items: T...] + ptr: Option>, + _marker: PhantomData, +} + +impl TinyVec { + /// Create an empty TinyVec (no allocation) + #[inline] + pub const fn new() -> Self { + Self { + ptr: None, + _marker: PhantomData, + } + } + + /// Create from a Vec, consuming it + pub fn from_vec(vec: Vec) -> Self { + if vec.is_empty() { + return Self::new(); + } + + let len = vec.len().min(255) as u8; + + // Calculate layout: 1 byte for length + items + let (layout, items_offset) = Self::layout_for(len as usize); + + unsafe { + // Allocate + let ptr = alloc(layout); + if ptr.is_null() { + std::alloc::handle_alloc_error(layout); + } + + // Write length + *ptr = len; + + // Move items (no clone!) + let items_ptr = ptr.add(items_offset) as *mut T; + for (i, item) in vec.into_iter().take(len as usize).enumerate() { + std::ptr::write(items_ptr.add(i), item); + } + + Self { + ptr: NonNull::new(ptr), + _marker: PhantomData, + } + } + } + + /// Calculate layout for allocation + fn layout_for(len: usize) -> (Layout, usize) { + let header_layout = Layout::new::(); + let items_layout = Layout::array::(len).unwrap(); + header_layout.extend(items_layout).unwrap() + } + + /// Number of items + #[inline] + pub fn len(&self) -> usize { + match self.ptr { + None => 0, + Some(ptr) => unsafe { *ptr.as_ptr() as usize }, + } + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.ptr.is_none() + } + + /// Get items offset within allocation + #[inline] + fn items_offset() -> usize { + let header_layout = Layout::new::(); + let items_layout = Layout::new::(); + header_layout.extend(items_layout).map(|(_, offset)| offset).unwrap_or(1) + } + + /// Get a slice of the items + #[inline] + pub fn as_slice(&self) -> &[T] { + match self.ptr { + None => &[], + Some(ptr) => unsafe { + let base = ptr.as_ptr(); + let len = *base as usize; + let items_ptr = base.add(Self::items_offset()) as *const T; + std::slice::from_raw_parts(items_ptr, len) + }, + } + } + + /// Get a mutable slice of the items + #[inline] + pub fn as_mut_slice(&mut self) -> &mut [T] { + match self.ptr { + None => &mut [], + Some(ptr) => unsafe { + let base = ptr.as_ptr(); + let len = *base as usize; + let items_ptr = base.add(Self::items_offset()) as *mut T; + std::slice::from_raw_parts_mut(items_ptr, len) + }, + } + } + + /// Iterate over items + #[inline] + pub fn iter(&self) -> std::slice::Iter<'_, T> { + self.as_slice().iter() + } + + /// Iterate mutably + #[inline] + pub fn iter_mut(&mut self) -> std::slice::IterMut<'_, T> { + self.as_mut_slice().iter_mut() + } + + /// Convert to Vec (clones items) + pub fn to_vec(&self) -> Vec + where + T: Clone, + { + self.as_slice().to_vec() + } + + /// Get the first item (immutable) + #[inline] + pub fn first(&self) -> Option<&T> { + self.as_slice().first() + } + + /// Get the last item (immutable) + #[inline] + pub fn last(&self) -> Option<&T> { + self.as_slice().last() + } + + /// Get the last item (mutable) + #[inline] + pub fn last_mut(&mut self) -> Option<&mut T> { + self.as_mut_slice().last_mut() + } + + /// Get item by index (immutable) + #[inline] + pub fn get(&self, index: usize) -> Option<&T> { + self.as_slice().get(index) + } + + /// Get item by index (mutable) + #[inline] + pub fn get_mut(&mut self, index: usize) -> Option<&mut T> { + self.as_mut_slice().get_mut(index) + } + + /// Push an item (rebuilds the entire allocation - use sparingly!) + pub fn push(&mut self, item: T) + where + T: Clone, + { + let mut vec = self.to_vec(); + vec.push(item); + *self = Self::from_vec(vec); + } + + /// Retain items matching a predicate (rebuilds the allocation) + pub fn retain(&mut self, f: F) + where + T: Clone, + F: FnMut(&T) -> bool, + { + let mut vec = self.to_vec(); + vec.retain(f); + *self = Self::from_vec(vec); + } + + /// Check if any item matches a predicate + pub fn any(&self, f: F) -> bool + where + F: FnMut(&T) -> bool, + { + self.as_slice().iter().any(f) + } +} + +// Index trait for direct indexing (msg.attachments[0]) +impl std::ops::Index for TinyVec { + type Output = T; + + fn index(&self, index: usize) -> &Self::Output { + &self.as_slice()[index] + } +} + +impl std::ops::IndexMut for TinyVec { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + &mut self.as_mut_slice()[index] + } +} + +// IntoIterator for &TinyVec +impl<'a, T> IntoIterator for &'a TinyVec { + type Item = &'a T; + type IntoIter = std::slice::Iter<'a, T>; + + fn into_iter(self) -> Self::IntoIter { + self.as_slice().iter() + } +} + +// IntoIterator for &mut TinyVec +impl<'a, T> IntoIterator for &'a mut TinyVec { + type Item = &'a mut T; + type IntoIter = std::slice::IterMut<'a, T>; + + fn into_iter(self) -> Self::IntoIter { + self.as_mut_slice().iter_mut() + } +} + +impl Default for TinyVec { + fn default() -> Self { + Self::new() + } +} + +impl Clone for TinyVec { + fn clone(&self) -> Self { + Self::from_vec(self.to_vec()) + } +} + +impl std::fmt::Debug for TinyVec { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_list().entries(self.as_slice()).finish() + } +} + +impl Drop for TinyVec { + fn drop(&mut self) { + if let Some(ptr) = self.ptr { + unsafe { + let base = ptr.as_ptr(); + let len = *base as usize; + let items_ptr = base.add(Self::items_offset()) as *mut T; + + // Drop each item + for i in 0..len { + std::ptr::drop_in_place(items_ptr.add(i)); + } + + // Deallocate + let (layout, _) = Self::layout_for(len); + dealloc(base, layout); + } + } + } +} + +// Safety: TinyVec is Send/Sync if T is +unsafe impl Send for TinyVec {} +unsafe impl Sync for TinyVec {} + +// ============================================================================ +// Compact Reaction +// ============================================================================ + +/// Memory-efficient reaction with binary IDs and interned author. +/// +/// Compared to the regular `Reaction` struct (~292 bytes with heap): +/// - IDs use `[u8; 32]` instead of hex String (saves ~56 bytes each) +/// - Author uses u16 index into interner (saves ~86 bytes) +/// - Emoji uses Box (saves 8 bytes, supports custom emoji like `:cat_heart_eyes:`) +/// - Total: ~82 bytes vs ~292 bytes (72% savings!) +#[derive(Clone, Debug)] +pub struct CompactReaction { + /// Reaction event ID as binary + pub id: [u8; 32], + /// Message being reacted to (binary event ID) + pub reference_id: [u8; 32], + /// Author npub index (interned via NpubInterner) + pub author_idx: u16, + /// Emoji string (supports standard emoji and custom like `:cat_heart_eyes:`) + pub emoji: Box, +} + +impl CompactReaction { + /// Get reaction ID as hex string + #[inline] + pub fn id_hex(&self) -> String { + bytes_to_hex_32(&self.id) + } + + /// Get reference ID as hex string + #[inline] + pub fn reference_id_hex(&self) -> String { + bytes_to_hex_32(&self.reference_id) + } + + /// Convert from regular Reaction, interning author + pub fn from_reaction(reaction: &Reaction, interner: &mut NpubInterner) -> Self { + Self { + id: hex_to_bytes_32(&reaction.id), + reference_id: hex_to_bytes_32(&reaction.reference_id), + author_idx: interner.intern(&reaction.author_id), + emoji: reaction.emoji.clone().into_boxed_str(), + } + } + + /// Convert from regular Reaction (owned), interning author + pub fn from_reaction_owned(reaction: Reaction, interner: &mut NpubInterner) -> Self { + Self { + id: hex_to_bytes_32(&reaction.id), + reference_id: hex_to_bytes_32(&reaction.reference_id), + author_idx: interner.intern(&reaction.author_id), + emoji: reaction.emoji.into_boxed_str(), + } + } + + /// Convert back to regular Reaction, resolving author from interner + pub fn to_reaction(&self, interner: &NpubInterner) -> Reaction { + Reaction { + id: self.id_hex(), + reference_id: self.reference_id_hex(), + author_id: interner.resolve(self.author_idx) + .map(|s| s.to_string()) + .unwrap_or_default(), + emoji: self.emoji.to_string(), + } + } +} + +// ============================================================================ +// Compact Attachment +// ============================================================================ + +/// Packed flags for attachment state (1 byte instead of multiple bools) +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct AttachmentFlags(u8); + +impl AttachmentFlags { + pub const NONE: Self = Self(0); + const DOWNLOADING: u8 = 0b0001; + const DOWNLOADED: u8 = 0b0010; + const SHORT_NONCE: u8 = 0b0100; // 12-byte nonce (MLS) vs 16-byte (DM) + + #[inline] + pub fn is_downloading(self) -> bool { self.0 & Self::DOWNLOADING != 0 } + #[inline] + pub fn is_downloaded(self) -> bool { self.0 & Self::DOWNLOADED != 0 } + #[inline] + pub fn is_short_nonce(self) -> bool { self.0 & Self::SHORT_NONCE != 0 } + + #[inline] + pub fn set_downloading(&mut self, value: bool) { + if value { self.0 |= Self::DOWNLOADING; } else { self.0 &= !Self::DOWNLOADING; } + } + #[inline] + pub fn set_downloaded(&mut self, value: bool) { + if value { self.0 |= Self::DOWNLOADED; } else { self.0 &= !Self::DOWNLOADED; } + } + #[inline] + pub fn set_short_nonce(&mut self, value: bool) { + if value { self.0 |= Self::SHORT_NONCE; } else { self.0 &= !Self::SHORT_NONCE; } + } + + pub fn from_bools(downloading: bool, downloaded: bool) -> Self { + let mut flags = Self::NONE; + flags.set_downloading(downloading); + flags.set_downloaded(downloaded); + flags + } +} + +/// Memory-efficient attachment with binary hashes and compact strings. +/// +/// Compared to the regular `Attachment` struct (~320+ bytes): +/// - id (SHA256): `[u8; 32]` instead of hex String (saves ~56 bytes) +/// - key: `[u8; 32]` instead of String (saves ~56 bytes) +/// - nonce: `[u8; 16]` instead of String (saves ~32 bytes) +/// - Bools packed into AttachmentFlags (saves padding) +/// - Strings use Box (saves 8 bytes each) +/// - Rare fields boxed (saves ~100+ bytes when None) +/// - Total: ~120 bytes vs ~320 bytes (62% savings!) +#[derive(Clone, Debug)] +pub struct CompactAttachment { + // === Fixed binary fields === + /// SHA256 file hash as binary (was hex String) + pub id: [u8; 32], + /// Encryption key - 32 bytes (empty = MLS derived) + pub key: [u8; 32], + /// Encryption nonce - 16 bytes (AES-256-GCM with 0xChat compatibility) + pub nonce: [u8; 16], + /// File size in bytes + pub size: u64, + /// Packed boolean flags (downloading, downloaded) + pub flags: AttachmentFlags, + + // === Variable fields (Box = 16 bytes each vs String's 24) === + /// File extension (e.g., "png", "mp4") + pub extension: Box, + /// Host URL (blossom server, etc.) + pub url: Box, + /// Local file path (empty if not downloaded) + pub path: Box, + + // === Optional fields - boxed to save space when None === + /// Image metadata (only for images/videos) + pub img_meta: Option>, + /// MLS group ID for key derivation + pub group_id: Option>, + /// Original file hash before encryption + pub original_hash: Option>, + /// WebXDC topic (Mini Apps only - very rare) + pub webxdc_topic: Option>, + /// MLS filename for AAD + pub mls_filename: Option>, + /// Scheme version (e.g., "mip04-v1") + pub scheme_version: Option>, +} + +impl CompactAttachment { + // === Convenience accessors for flags === + #[inline] + pub fn downloaded(&self) -> bool { self.flags.is_downloaded() } + #[inline] + pub fn downloading(&self) -> bool { self.flags.is_downloading() } + #[inline] + pub fn set_downloaded(&mut self, value: bool) { self.flags.set_downloaded(value); } + #[inline] + pub fn set_downloading(&mut self, value: bool) { self.flags.set_downloading(value); } + + /// Check if this attachment's ID matches a hex string + #[inline] + pub fn id_eq(&self, hex_id: &str) -> bool { + self.id == hex_to_bytes_32(hex_id) + } + + /// Get file ID as hex string + #[inline] + pub fn id_hex(&self) -> String { + bytes_to_hex_32(&self.id) + } + + /// Get encryption key as hex string (empty if zeros) + pub fn key_hex(&self) -> String { + if self.key == [0u8; 32] { + String::new() + } else { + bytes_to_hex_32(&self.key) + } + } + + /// Get nonce as hex string (empty if zeros, respects original length) + pub fn nonce_hex(&self) -> String { + if self.nonce == [0u8; 16] { + String::new() + } else if self.flags.is_short_nonce() { + // 12-byte nonce (MLS/MIP-04) + crate::simd::bytes_to_hex_string(&self.nonce[..12]) + } else { + // 16-byte nonce (DM/0xChat) + crate::simd::bytes_to_hex_string(&self.nonce) + } + } + + /// Convert from regular Attachment (borrowed) + pub fn from_attachment(att: &Attachment) -> Self { + // Detect short nonce (12 bytes = 24 hex chars) for MLS attachments + let is_short_nonce = att.nonce.len() == 24; + let mut flags = AttachmentFlags::from_bools(att.downloading, att.downloaded); + flags.set_short_nonce(is_short_nonce); + + Self { + id: hex_to_bytes_32(&att.id), + key: if att.key.is_empty() { [0u8; 32] } else { hex_to_bytes_32(&att.key) }, + nonce: if att.nonce.is_empty() { [0u8; 16] } else { parse_nonce(&att.nonce) }, + size: att.size, + flags, + extension: att.extension.clone().into_boxed_str(), + url: att.url.clone().into_boxed_str(), + path: att.path.clone().into_boxed_str(), + img_meta: att.img_meta.clone().map(Box::new), + group_id: att.group_id.as_ref().map(|s| Box::new(hex_to_bytes_32(s))), + original_hash: att.original_hash.as_ref().map(|s| Box::new(hex_to_bytes_32(s))), + webxdc_topic: att.webxdc_topic.clone().map(|s| s.into_boxed_str()), + mls_filename: att.mls_filename.clone().map(|s| s.into_boxed_str()), + scheme_version: att.scheme_version.clone().map(|s| s.into_boxed_str()), + } + } + + /// Convert from regular Attachment (owned) - zero-copy where possible + pub fn from_attachment_owned(att: Attachment) -> Self { + // Detect short nonce (12 bytes = 24 hex chars) for MLS attachments + let is_short_nonce = att.nonce.len() == 24; + let mut flags = AttachmentFlags::from_bools(att.downloading, att.downloaded); + flags.set_short_nonce(is_short_nonce); + + Self { + id: hex_to_bytes_32(&att.id), + key: if att.key.is_empty() { [0u8; 32] } else { hex_to_bytes_32(&att.key) }, + nonce: if att.nonce.is_empty() { [0u8; 16] } else { parse_nonce(&att.nonce) }, + size: att.size, + flags, + extension: att.extension.into_boxed_str(), + url: att.url.into_boxed_str(), + path: att.path.into_boxed_str(), + img_meta: att.img_meta.map(Box::new), + group_id: att.group_id.map(|s| Box::new(hex_to_bytes_32(&s))), + original_hash: att.original_hash.map(|s| Box::new(hex_to_bytes_32(&s))), + webxdc_topic: att.webxdc_topic.map(|s| s.into_boxed_str()), + mls_filename: att.mls_filename.map(|s| s.into_boxed_str()), + scheme_version: att.scheme_version.map(|s| s.into_boxed_str()), + } + } + + /// Convert back to regular Attachment + pub fn to_attachment(&self) -> Attachment { + Attachment { + id: self.id_hex(), + key: self.key_hex(), + nonce: self.nonce_hex(), + extension: self.extension.to_string(), + url: self.url.to_string(), + path: self.path.to_string(), + size: self.size, + img_meta: self.img_meta.as_ref().map(|b| (**b).clone()), + downloading: self.flags.is_downloading(), + downloaded: self.flags.is_downloaded(), + webxdc_topic: self.webxdc_topic.as_ref().map(|s| s.to_string()), + group_id: self.group_id.as_ref().map(|b| bytes_to_hex_32(b)), + original_hash: self.original_hash.as_ref().map(|b| bytes_to_hex_32(b)), + scheme_version: self.scheme_version.as_ref().map(|s| s.to_string()), + mls_filename: self.mls_filename.as_ref().map(|s| s.to_string()), + } + } +} + +/// Parse a hex nonce string into [u8; 16] +fn parse_nonce(hex: &str) -> [u8; 16] { + let mut result = [0u8; 16]; + let bytes = hex_string_to_bytes(hex); + let len = bytes.len().min(16); + result[..len].copy_from_slice(&bytes[..len]); + result +} + +// ============================================================================ +// Npub Interner +// ============================================================================ + +/// String interner for npubs using sorted Vec + binary search. +/// +/// Each unique npub is stored exactly once. Messages reference npubs by u16 index. +/// - `intern()`: O(log n) lookup + O(n) insert for new strings +/// - `resolve()`: O(1) by index +/// +/// Memory: ~2 bytes per npub for the sorted index, plus the strings themselves. +#[derive(Clone, Debug, Default)] +pub struct NpubInterner { + /// npubs in insertion order - index is the stable ID used by messages + npubs: Vec, + /// Indices into npubs, sorted alphabetically for binary search + sorted: Vec, +} + +/// Sentinel value for "no npub" (avoids Option overhead) +pub const NO_NPUB: u16 = u16::MAX; + +impl NpubInterner { + pub fn new() -> Self { + Self { + npubs: Vec::new(), + sorted: Vec::new(), + } + } + + /// Pre-allocate capacity for expected number of unique npubs + pub fn with_capacity(capacity: usize) -> Self { + Self { + npubs: Vec::with_capacity(capacity), + sorted: Vec::with_capacity(capacity), + } + } + + /// Intern an npub string, returning its stable index. + /// + /// If the npub already exists, returns the existing index. + /// If new, stores it and returns a new index. + pub fn intern(&mut self, npub: &str) -> u16 { + // Binary search in sorted order + let result = self.sorted.binary_search_by(|&idx| { + self.npubs[idx as usize].as_str().cmp(npub) + }); + + match result { + Ok(pos) => self.sorted[pos], // Found existing + Err(insert_pos) => { + // New npub - add to both vectors + let new_idx = self.npubs.len() as u16; + self.npubs.push(npub.to_string()); + self.sorted.insert(insert_pos, new_idx); + new_idx + } + } + } + + /// Intern an optional npub, returning NO_NPUB sentinel for None. + #[inline] + pub fn intern_opt(&mut self, npub: Option<&str>) -> u16 { + match npub { + Some(s) if !s.is_empty() => self.intern(s), + _ => NO_NPUB, + } + } + + /// Resolve an index back to the npub string. + /// + /// Returns None for NO_NPUB sentinel or out-of-bounds index. + #[inline] + pub fn resolve(&self, idx: u16) -> Option<&str> { + if idx == NO_NPUB { + return None; + } + self.npubs.get(idx as usize).map(|s| s.as_str()) + } + + /// Number of unique npubs stored + #[inline] + pub fn len(&self) -> usize { + self.npubs.len() + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.npubs.is_empty() + } + + /// Total memory used by the interner (approximate) + pub fn memory_usage(&self) -> usize { + std::mem::size_of::() + + self.npubs.capacity() * std::mem::size_of::() + + self.npubs.iter().map(|s| s.capacity()).sum::() + + self.sorted.capacity() * std::mem::size_of::() + } +} + +// ============================================================================ +// Compact Message +// ============================================================================ + +/// Memory-efficient message with binary IDs and interned npubs. +/// +/// Compared to the regular `Message` struct: +/// - IDs use `[u8; 32]` instead of hex String (saves ~56 bytes each) +/// - npubs use u16 index into interner (saves ~85 bytes each) +/// - Booleans packed into MessageFlags (saves ~24 bytes + 2 for replied_to_has_attachment) +/// - Boxed optional IDs (replied_to, wrapper_id) save ~40 bytes when None +/// - Compact timestamp (u32 seconds since 2020) saves 4 bytes +/// - TinyVec for attachments/reactions (8 bytes vs 24 = saves 32 bytes) +/// - Box for content (8 bytes vs 24 = saves 16 bytes) +/// - Total savings: ~350+ bytes per message +#[derive(Clone, Debug)] +pub struct CompactMessage { + /// Message ID as binary (64 hex chars → 32 bytes) + pub id: [u8; 32], + /// Compact timestamp: seconds since 2020-01-01 (u32 = good until 2156) + pub at: u32, + /// Packed boolean flags (mine, pending, failed, replied_to_has_attachment) + pub flags: MessageFlags, + /// Index into NpubInterner for sender's npub (NO_NPUB if none) + pub npub_idx: u16, + /// Replied-to message ID (boxed - None for ~70% of messages saves 24 bytes) + pub replied_to: Option>, + /// Index into NpubInterner for replied-to author (NO_NPUB if none) + pub replied_to_npub_idx: u16, + /// Wrapper event ID for gift-wrapped messages (boxed - saves 25 bytes when None) + pub wrapper_id: Option>, + + // Variable-length fields - optimized for memory + /// Message content (Box = 16 bytes vs String's 24 bytes) + pub content: Box, + /// Content of replied-to message + pub replied_to_content: Option>, + /// File attachments (CompactAttachment = ~120 bytes vs Attachment's ~320 bytes) + pub attachments: TinyVec, + /// Emoji reactions (CompactReaction = ~82 bytes vs Reaction's ~292 bytes) + pub reactions: TinyVec, + /// Edit history - boxed since <1% of messages are edited (saves 16 bytes inline) + #[allow(clippy::box_collection)] + pub edit_history: Option>>, + /// Link preview metadata - boxed since ~216 bytes but rare (saves ~208 bytes) + pub preview_metadata: Option>, +} + +impl CompactMessage { + /// Check if this message has a replied-to reference + #[inline] + pub fn has_reply(&self) -> bool { + self.replied_to.is_some() + } + + /// Check if this message has been edited + #[inline] + pub fn is_edited(&self) -> bool { + self.edit_history.is_some() + } + + /// Get the message ID as a string (hex for event IDs, "pending-..." for pending) + #[inline] + pub fn id_hex(&self) -> String { + decode_message_id(&self.id) + } + + /// Get the replied-to ID as a hex string, or empty if none + #[inline] + pub fn replied_to_hex(&self) -> String { + match &self.replied_to { + Some(id) => bytes_to_hex_32(id), + None => String::new(), + } + } + + /// Get wrapper ID as hex string if present + #[inline] + pub fn wrapper_id_hex(&self) -> Option { + self.wrapper_id.as_ref().map(|id| bytes_to_hex_32(id)) + } + + /// Get timestamp as milliseconds (for compatibility with frontend) + #[inline] + pub fn timestamp_ms(&self) -> u64 { + timestamp_from_compact(self.at) + } + + /// Apply an edit to this message + pub fn apply_edit(&mut self, new_content: String, edited_at: u64) { + // Initialize edit history with original content if not present + if self.edit_history.is_none() { + self.edit_history = Some(Box::new(vec![EditEntry { + content: self.content.to_string(), + edited_at: self.timestamp_ms(), // Convert compact to ms + }])); + } + + if let Some(ref mut history) = self.edit_history { + // Deduplicate: skip if we already have this edit + if history.iter().any(|e| e.edited_at == edited_at) { + return; + } + + // Add new edit to history + history.push(EditEntry { + content: new_content.clone(), + edited_at, + }); + + // Sort by timestamp + history.sort_by_key(|e| e.edited_at); + } + + // Update current content (convert to Box) + self.content = new_content.into_boxed_str(); + } + + /// Get replied_to_has_attachment from flags + #[inline] + pub fn replied_to_has_attachment(&self) -> Option { + self.flags.replied_to_has_attachment() + } + + /// Add a reaction to this message + /// Note: Since TinyVec is immutable, this rebuilds the entire reactions list + pub fn add_reaction(&mut self, reaction: Reaction, interner: &mut NpubInterner) -> bool { + // Convert to binary ID for comparison + let reaction_id = hex_to_bytes_32(&reaction.id); + + // Check if already exists + if self.reactions.iter().any(|r| r.id == reaction_id) { + return false; + } + + // Convert to compact and rebuild + let compact = CompactReaction::from_reaction_owned(reaction, interner); + let mut reactions = self.reactions.to_vec(); + reactions.push(compact); + self.reactions = TinyVec::from_vec(reactions); + true + } + + // Flag accessors for compatibility + #[inline] + pub fn is_mine(&self) -> bool { self.flags.is_mine() } + #[inline] + pub fn is_pending(&self) -> bool { self.flags.is_pending() } + #[inline] + pub fn is_failed(&self) -> bool { self.flags.is_failed() } + + // Flag setters + #[inline] + pub fn set_pending(&mut self, value: bool) { self.flags.set_pending(value); } + #[inline] + pub fn set_failed(&mut self, value: bool) { self.flags.set_failed(value); } + #[inline] + pub fn set_mine(&mut self, value: bool) { self.flags.set_mine(value); } +} + +// ============================================================================ +// Compact Message Vec with Binary Search +// ============================================================================ + +/// Sorted message storage with O(log n) lookup by ID. +/// +/// Messages are stored sorted by timestamp. A separate index provides +/// O(log n) lookup by message ID using binary search. +#[derive(Clone, Debug, Default)] +pub struct CompactMessageVec { + /// Messages sorted by timestamp (ascending) + messages: Vec, + /// Index for ID lookup: (id, position in messages), sorted by id + id_index: Vec<([u8; 32], u32)>, +} + +impl CompactMessageVec { + pub fn new() -> Self { + Self { + messages: Vec::new(), + id_index: Vec::new(), + } + } + + pub fn with_capacity(capacity: usize) -> Self { + Self { + messages: Vec::with_capacity(capacity), + id_index: Vec::with_capacity(capacity), + } + } + + /// Number of messages + #[inline] + pub fn len(&self) -> usize { + self.messages.len() + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.messages.is_empty() + } + + /// Get all messages (sorted by timestamp) + #[inline] + pub fn messages(&self) -> &[CompactMessage] { + &self.messages + } + + /// Get a mutable reference to all messages + #[inline] + pub fn messages_mut(&mut self) -> &mut Vec { + &mut self.messages + } + + /// Iterate over messages (supports .rev()) + #[inline] + pub fn iter(&self) -> std::slice::Iter<'_, CompactMessage> { + self.messages.iter() + } + + /// Iterate over messages mutably + #[inline] + pub fn iter_mut(&mut self) -> std::slice::IterMut<'_, CompactMessage> { + self.messages.iter_mut() + } + + /// Get the last message + #[inline] + pub fn last(&self) -> Option<&CompactMessage> { + self.messages.last() + } + + /// Get last message timestamp (in milliseconds) + #[inline] + pub fn last_timestamp(&self) -> Option { + self.messages.last().map(|m| timestamp_from_compact(m.at)) + } + + /// Get the first message + #[inline] + pub fn first(&self) -> Option<&CompactMessage> { + self.messages.first() + } + + /// Find a message by ID using binary search - O(log n) + pub fn find_by_id(&self, id: &[u8; 32]) -> Option<&CompactMessage> { + let pos = self.id_index + .binary_search_by(|(idx_id, _)| idx_id.cmp(id)) + .ok()?; + let msg_pos = self.id_index[pos].1 as usize; + self.messages.get(msg_pos) + } + + /// Find a message by ID (mutable) - O(log n) + pub fn find_by_id_mut(&mut self, id: &[u8; 32]) -> Option<&mut CompactMessage> { + let pos = self.id_index + .binary_search_by(|(idx_id, _)| idx_id.cmp(id)) + .ok()?; + let msg_pos = self.id_index[pos].1 as usize; + self.messages.get_mut(msg_pos) + } + + /// Find a message by ID string (hex or pending) - O(log n) + pub fn find_by_hex_id(&self, id_str: &str) -> Option<&CompactMessage> { + if id_str.is_empty() { + return None; + } + let id = encode_message_id(id_str); + self.find_by_id(&id) + } + + /// Find a message by ID string (mutable) - O(log n) + pub fn find_by_hex_id_mut(&mut self, id_str: &str) -> Option<&mut CompactMessage> { + if id_str.is_empty() { + return None; + } + let id = encode_message_id(id_str); + self.find_by_id_mut(&id) + } + + /// Check if a message with the given ID exists - O(log n) + pub fn contains_id(&self, id: &[u8; 32]) -> bool { + self.id_index + .binary_search_by(|(idx_id, _)| idx_id.cmp(id)) + .is_ok() + } + + /// Check if a message with the given ID string exists - O(log n) + pub fn contains_hex_id(&self, id_str: &str) -> bool { + if id_str.is_empty() { + return false; + } + let id = encode_message_id(id_str); + self.contains_id(&id) + } + + /// Insert a message, maintaining sort order by timestamp. + /// + /// Returns true if the message was added, false if duplicate ID. + /// + /// **Performance**: O(log n) for append (common case), O(n) for out-of-order insert. + pub fn insert(&mut self, msg: CompactMessage) -> bool { + // Check for duplicate ID - O(log n) + if self.contains_id(&msg.id) { + return false; + } + + let msg_id = msg.id; + + // Fast path: append if message is newer than or equal to last (common case) + // This is O(log n) for the index insert only + if self.messages.last().is_none_or(|last| msg.at >= last.at) { + let msg_pos = self.messages.len() as u32; + self.messages.push(msg); + + // Insert into id_index (maintain sorted order by ID) - O(log n) search + O(n) shift + // But the shift is typically small since IDs are random/sequential + let idx_pos = self.id_index + .binary_search_by(|(id, _)| id.cmp(&msg_id)) + .unwrap_err(); + self.id_index.insert(idx_pos, (msg_id, msg_pos)); + + return true; + } + + // Slow path: out-of-order insert (rare for real-time chat) + // Find insertion position by timestamp + let msg_pos = match self.messages.binary_search_by(|m| m.at.cmp(&msg.at)) { + Ok(pos) => pos, + Err(pos) => pos, + }; + + // Update id_index positions for messages that will shift - O(n) + for (_, pos) in &mut self.id_index { + if *pos >= msg_pos as u32 { + *pos += 1; + } + } + + // Insert into messages - O(n) + self.messages.insert(msg_pos, msg); + + // Insert into id_index - O(n) + let idx_pos = self.id_index + .binary_search_by(|(id, _)| id.cmp(&msg_id)) + .unwrap_err(); + self.id_index.insert(idx_pos, (msg_id, msg_pos as u32)); + + true + } + + /// Rebuild the ID index (call after bulk modifications) + pub fn rebuild_index(&mut self) { + self.id_index.clear(); + self.id_index.reserve(self.messages.len()); + for (pos, msg) in self.messages.iter().enumerate() { + self.id_index.push((msg.id, pos as u32)); + } + self.id_index.sort_by(|(a, _), (b, _)| a.cmp(b)); + } + + /// Batch insert messages - optimized for different scenarios. + /// + /// Returns the number of messages actually added (excludes duplicates). + /// + /// **Performance**: + /// - Append case (newer msgs): O(k log n) where k = new messages + /// - Prepend case (older msgs): O(k log n + k) + /// - Mixed: O(n log n) full sort + pub fn insert_batch(&mut self, messages: impl IntoIterator) -> usize { + let messages: Vec<_> = messages.into_iter().collect(); + if messages.is_empty() { + return 0; + } + + // Quick dedup check using the index + let mut to_add: Vec = Vec::with_capacity(messages.len()); + for msg in messages { + if !self.contains_id(&msg.id) { + to_add.push(msg); + } + } + + if to_add.is_empty() { + return 0; + } + + let added = to_add.len(); + + // Determine the insertion strategy based on timestamps + let our_first = self.messages.first().map(|m| m.at); + let our_last = self.messages.last().map(|m| m.at); + let their_min = to_add.iter().map(|m| m.at).min().unwrap(); + let their_max = to_add.iter().map(|m| m.at).max().unwrap(); + + if self.messages.is_empty() { + // Empty vec - just add and sort + self.messages = to_add; + self.messages.sort_by_key(|m| m.at); + self.rebuild_index(); + } else if their_min >= our_last.unwrap() { + // All new messages are NEWER - append path (common for real-time) + to_add.sort_by_key(|m| m.at); + let base_pos = self.messages.len() as u32; + for (i, msg) in to_add.into_iter().enumerate() { + let msg_id = msg.id; + self.messages.push(msg); + // Insert into index + let idx_pos = self.id_index + .binary_search_by(|(id, _)| id.cmp(&msg_id)) + .unwrap_err(); + self.id_index.insert(idx_pos, (msg_id, base_pos + i as u32)); + } + } else if their_max <= our_first.unwrap() { + // All new messages are OLDER - prepend path (common for pagination) + to_add.sort_by_key(|m| m.at); + let prepend_count = to_add.len(); + + // Shift all existing index positions + for (_, pos) in &mut self.id_index { + *pos += prepend_count as u32; + } + + // Build new index entries (already sorted by construction since to_add is sorted by timestamp) + let mut new_index_entries: Vec<_> = to_add.iter() + .enumerate() + .map(|(i, msg)| (msg.id, i as u32)) + .collect(); + new_index_entries.sort_by(|(a, _), (b, _)| a.cmp(b)); + + // Merge sorted index entries in O(n + k) instead of O(k * n) + let old_index = std::mem::take(&mut self.id_index); + self.id_index.reserve(old_index.len() + new_index_entries.len()); + + let mut old_iter = old_index.into_iter().peekable(); + let mut new_iter = new_index_entries.into_iter().peekable(); + + while old_iter.peek().is_some() || new_iter.peek().is_some() { + match (old_iter.peek(), new_iter.peek()) { + (Some((old_id, _)), Some((new_id, _))) => { + if old_id < new_id { + self.id_index.push(old_iter.next().unwrap()); + } else { + self.id_index.push(new_iter.next().unwrap()); + } + } + (Some(_), None) => self.id_index.push(old_iter.next().unwrap()), + (None, Some(_)) => self.id_index.push(new_iter.next().unwrap()), + (None, None) => break, + } + } + + // Prepend messages + let mut new_messages = to_add; + new_messages.append(&mut self.messages); + self.messages = new_messages; + } else { + // Mixed timestamps - fall back to full sort + self.messages.extend(to_add); + self.messages.sort_by_key(|m| m.at); + self.rebuild_index(); + } + + added + } + + /// Total memory used (approximate) + pub fn memory_usage(&self) -> usize { + std::mem::size_of::() + + self.messages.capacity() * std::mem::size_of::() + + self.id_index.capacity() * std::mem::size_of::<([u8; 32], u32)>() + // Note: doesn't include heap allocations inside CompactMessage + } + + /// Drain messages from a range (rebuilds index after) + pub fn drain(&mut self, range: std::ops::Range) -> std::vec::Drain<'_, CompactMessage> { + let drain = self.messages.drain(range); + // Note: caller should call rebuild_index() after consuming the drain + drain + } + + /// Sort messages by a key (rebuilds index after) + pub fn sort_by_key(&mut self, f: F) + where + F: FnMut(&CompactMessage) -> K, + K: Ord, + { + self.messages.sort_by_key(f); + self.rebuild_index(); + } + + /// Clear all messages + pub fn clear(&mut self) { + self.messages.clear(); + self.id_index.clear(); + } +} + +// ============================================================================ +// Conversion from/to Message +// ============================================================================ + +use crate::Message; + +impl CompactMessage { + /// Convert from a regular Message (borrowed), interning npubs + pub fn from_message(msg: &Message, interner: &mut NpubInterner) -> Self { + Self { + id: encode_message_id(&msg.id), + at: timestamp_to_compact(msg.at), + flags: MessageFlags::from_all(msg.mine, msg.pending, msg.failed, msg.replied_to_has_attachment), + npub_idx: interner.intern_opt(msg.npub.as_deref()), + // Box replied_to only when present (saves 24 bytes when None) + replied_to: if msg.replied_to.is_empty() { + None + } else { + Some(Box::new(hex_to_bytes_32(&msg.replied_to))) + }, + replied_to_npub_idx: interner.intern_opt(msg.replied_to_npub.as_deref()), + // Box wrapper_id (saves 25 bytes when None) + wrapper_id: msg.wrapper_event_id.as_ref().map(|s| Box::new(hex_to_bytes_32(s))), + // Box for content (saves 8 bytes per field) + content: msg.content.clone().into_boxed_str(), + replied_to_content: msg.replied_to_content.as_ref().map(|s| s.clone().into_boxed_str()), + // Convert attachments to compact format + attachments: TinyVec::from_vec( + msg.attachments.iter() + .map(CompactAttachment::from_attachment) + .collect() + ), + // Convert reactions to compact format + reactions: TinyVec::from_vec( + msg.reactions.iter() + .map(|r| CompactReaction::from_reaction(r, interner)) + .collect() + ), + // Box rare fields to save inline space + edit_history: msg.edit_history.clone().map(Box::new), + preview_metadata: msg.preview_metadata.clone().map(Box::new), + } + } + + /// Convert from a regular Message (owned) - ZERO-COPY for strings! + /// + /// Takes ownership of the Message and moves strings directly. + /// Use this when you don't need the original Message anymore. + pub fn from_message_owned(msg: Message, interner: &mut NpubInterner) -> Self { + Self { + id: encode_message_id(&msg.id), + at: timestamp_to_compact(msg.at), + flags: MessageFlags::from_all(msg.mine, msg.pending, msg.failed, msg.replied_to_has_attachment), + npub_idx: interner.intern_opt(msg.npub.as_deref()), + // Box replied_to only when present (saves 24 bytes when None) + replied_to: if msg.replied_to.is_empty() { + None + } else { + Some(Box::new(hex_to_bytes_32(&msg.replied_to))) + }, + replied_to_npub_idx: interner.intern_opt(msg.replied_to_npub.as_deref()), + // Box wrapper_id (saves 25 bytes when None) + wrapper_id: msg.wrapper_event_id.as_ref().map(|s| Box::new(hex_to_bytes_32(s))), + // Zero-copy: into_boxed_str() reuses the String's buffer! + content: msg.content.into_boxed_str(), + replied_to_content: msg.replied_to_content.map(|s| s.into_boxed_str()), + // Convert attachments to compact format (zero-copy where possible) + attachments: TinyVec::from_vec( + msg.attachments.into_iter() + .map(CompactAttachment::from_attachment_owned) + .collect() + ), + // Convert reactions to compact format (zero-copy for emoji string) + reactions: TinyVec::from_vec( + msg.reactions.into_iter() + .map(|r| CompactReaction::from_reaction_owned(r, interner)) + .collect() + ), + // Box rare fields to save inline space + edit_history: msg.edit_history.map(Box::new), + preview_metadata: msg.preview_metadata.map(Box::new), + } + } + + /// Convert back to a regular Message, resolving npubs from interner + pub fn to_message(&self, interner: &NpubInterner) -> Message { + Message { + id: self.id_hex(), + at: self.timestamp_ms(), // Convert compact back to ms + mine: self.flags.is_mine(), + pending: self.flags.is_pending(), + failed: self.flags.is_failed(), + edited: self.is_edited(), + npub: interner.resolve(self.npub_idx).map(|s| s.to_string()), + replied_to: self.replied_to_hex(), + replied_to_content: self.replied_to_content.as_ref().map(|s| s.to_string()), + replied_to_npub: interner.resolve(self.replied_to_npub_idx).map(|s| s.to_string()), + replied_to_has_attachment: self.flags.replied_to_has_attachment(), + wrapper_event_id: self.wrapper_id_hex(), + content: self.content.to_string(), + // Convert compact attachments back to regular Attachment + attachments: self.attachments.iter() + .map(|a| a.to_attachment()) + .collect(), + // Convert compact reactions back to regular Reaction + reactions: self.reactions.iter() + .map(|r| r.to_reaction(interner)) + .collect(), + // Unbox rare fields + edit_history: self.edit_history.as_ref().map(|b| (**b).clone()), + preview_metadata: self.preview_metadata.as_ref().map(|b| (**b).clone()), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_message_flags() { + let mut flags = MessageFlags::NONE; + assert!(!flags.is_mine()); + assert!(!flags.is_pending()); + assert!(!flags.is_failed()); + + flags.set_mine(true); + assert!(flags.is_mine()); + + flags.set_pending(true); + assert!(flags.is_pending()); + assert!(flags.is_mine()); // Still set + + flags.set_mine(false); + assert!(!flags.is_mine()); + assert!(flags.is_pending()); // Still set + } + + #[test] + fn test_npub_interner() { + let mut interner = NpubInterner::new(); + + let idx1 = interner.intern("npub1alice"); + let idx2 = interner.intern("npub1bob"); + let idx3 = interner.intern("npub1alice"); // Duplicate + + assert_eq!(idx1, idx3); // Same string = same index + assert_ne!(idx1, idx2); + + assert_eq!(interner.resolve(idx1), Some("npub1alice")); + assert_eq!(interner.resolve(idx2), Some("npub1bob")); + assert_eq!(interner.resolve(NO_NPUB), None); + } + + #[test] + fn test_compact_message_vec_insert_and_find() { + let mut vec = CompactMessageVec::new(); + let mut interner = NpubInterner::new(); + + let msg1 = CompactMessage { + id: hex_to_bytes_32("0000000000000000000000000000000000000000000000000000000000000001"), + at: 1000, + flags: MessageFlags::NONE, + npub_idx: interner.intern("npub1test"), + replied_to: None, + replied_to_npub_idx: NO_NPUB, + wrapper_id: None, + content: "First message".to_string().into_boxed_str(), + replied_to_content: None, + attachments: TinyVec::new(), + reactions: TinyVec::new(), + edit_history: None, + preview_metadata: None, // Boxed, but None = 8 bytes + }; + + let msg2 = CompactMessage { + id: hex_to_bytes_32("0000000000000000000000000000000000000000000000000000000000000002"), + at: 2000, + flags: MessageFlags::MINE, + npub_idx: interner.intern("npub1me"), + replied_to: None, + replied_to_npub_idx: NO_NPUB, + wrapper_id: None, + content: "Second message".to_string().into_boxed_str(), + replied_to_content: None, + attachments: TinyVec::new(), + reactions: TinyVec::new(), + edit_history: None, + preview_metadata: None, // Boxed, but None = 8 bytes + }; + + assert!(vec.insert(msg1)); + assert!(vec.insert(msg2)); + assert_eq!(vec.len(), 2); + + // Find by ID + let found = vec.find_by_hex_id("0000000000000000000000000000000000000000000000000000000000000001"); + assert!(found.is_some()); + assert_eq!(&*found.unwrap().content, "First message"); + + // Find non-existent + let not_found = vec.find_by_hex_id("0000000000000000000000000000000000000000000000000000000000000099"); + assert!(not_found.is_none()); + } + + #[test] + fn test_duplicate_insert_rejected() { + let mut vec = CompactMessageVec::new(); + + let msg = CompactMessage { + id: hex_to_bytes_32("abcd000000000000000000000000000000000000000000000000000000000000"), + at: 1000, + flags: MessageFlags::NONE, + npub_idx: NO_NPUB, + replied_to: None, + replied_to_npub_idx: NO_NPUB, + wrapper_id: None, + content: "Test".to_string().into_boxed_str(), + replied_to_content: None, + attachments: TinyVec::new(), + reactions: TinyVec::new(), + edit_history: None, + preview_metadata: None, // Boxed + }; + + assert!(vec.insert(msg.clone())); + assert!(!vec.insert(msg)); // Duplicate rejected + assert_eq!(vec.len(), 1); + } + + /// Comprehensive benchmark test for memory reduction and performance + #[test] + fn benchmark_compact_vs_message() { + use std::time::Instant; + + const NUM_MESSAGES: usize = 10_000; + const NUM_UNIQUE_USERS: usize = 50; // Realistic chat scenario + + println!("\n========================================"); + println!(" COMPACT MESSAGE BENCHMARK"); + println!(" {} messages, {} unique users", NUM_MESSAGES, NUM_UNIQUE_USERS); + println!("========================================\n"); + + // Generate test data + let users: Vec = (0..NUM_UNIQUE_USERS) + .map(|i| format!("npub1{:0>62}", i)) + .collect(); + + // Create regular Messages + let messages: Vec = (0..NUM_MESSAGES) + .map(|i| { + let user_idx = i % NUM_UNIQUE_USERS; + Message { + id: format!("{:0>64x}", i), + at: 1700000000000 + (i as u64 * 1000), + mine: user_idx == 0, + pending: false, + failed: false, + edited: false, + npub: Some(users[user_idx].clone()), + replied_to: if i > 0 && i % 5 == 0 { + format!("{:0>64x}", i - 1) + } else { + String::new() + }, + replied_to_content: if i > 0 && i % 5 == 0 { + Some("Previous message content".to_string()) + } else { + None + }, + replied_to_npub: if i > 0 && i % 5 == 0 { + Some(users[(i - 1) % NUM_UNIQUE_USERS].clone()) + } else { + None + }, + replied_to_has_attachment: None, + wrapper_event_id: Some(format!("{:0>64x}", i + 1000000)), + content: format!("This is message number {} with some typical content length.", i), + attachments: vec![], + reactions: vec![], + edit_history: None, + preview_metadata: None, + } + }) + .collect(); + + // ===== MEMORY COMPARISON ===== + println!("--- STRUCT SIZES ---"); + println!(" Message struct: {} bytes", std::mem::size_of::()); + println!(" CompactMessage struct: {} bytes", std::mem::size_of::()); + println!(" Savings per struct: {} bytes ({:.1}%)", + std::mem::size_of::().saturating_sub(std::mem::size_of::()), + (1.0 - std::mem::size_of::() as f64 / std::mem::size_of::() as f64) * 100.0 + ); + println!(); + + // Measure Message storage (simulating Vec) + let msg_heap_estimate: usize = messages.iter().map(|m| { + m.id.capacity() + + m.npub.as_ref().map(|s| s.capacity()).unwrap_or(0) + + m.replied_to.capacity() + + m.replied_to_content.as_ref().map(|s| s.capacity()).unwrap_or(0) + + m.replied_to_npub.as_ref().map(|s| s.capacity()).unwrap_or(0) + + m.wrapper_event_id.as_ref().map(|s| s.capacity()).unwrap_or(0) + + m.content.capacity() + }).sum(); + let msg_total = messages.len() * std::mem::size_of::() + msg_heap_estimate; + + // ===== CONVERSION + INSERT BENCHMARK ===== + println!("--- INSERT BENCHMARK ---"); + + // Test 1: Sequential inserts (simulates real-time message arrival) + let mut interner = NpubInterner::with_capacity(NUM_UNIQUE_USERS); + let mut compact_vec = CompactMessageVec::with_capacity(NUM_MESSAGES); + + let insert_start = Instant::now(); + for msg in &messages { + let compact = CompactMessage::from_message(msg, &mut interner); + compact_vec.insert(compact); + } + let insert_elapsed = insert_start.elapsed(); + + println!(" Sequential insert (optimized append path):"); + println!(" {} messages in {:?}", NUM_MESSAGES, insert_elapsed); + println!(" Rate: {:.0} msgs/sec", NUM_MESSAGES as f64 / insert_elapsed.as_secs_f64()); + println!(" Per message: {:.3} µs ({} ns)", + insert_elapsed.as_micros() as f64 / NUM_MESSAGES as f64, + insert_elapsed.as_nanos() / NUM_MESSAGES as u128); + println!(); + + // Test 2: Batch insert (simulates pagination/history loading) + let mut interner2 = NpubInterner::with_capacity(NUM_UNIQUE_USERS); + let mut compact_vec2 = CompactMessageVec::with_capacity(NUM_MESSAGES); + + let batch_start = Instant::now(); + let compact_messages: Vec<_> = messages.iter() + .map(|msg| CompactMessage::from_message(msg, &mut interner2)) + .collect(); + let batch_added = compact_vec2.insert_batch(compact_messages); + let batch_elapsed = batch_start.elapsed(); + + println!(" Batch insert (pagination/history load):"); + println!(" {} messages in {:?}", batch_added, batch_elapsed); + println!(" Rate: {:.0} msgs/sec", NUM_MESSAGES as f64 / batch_elapsed.as_secs_f64()); + println!(" Per message: {:.3} µs ({} ns)", + batch_elapsed.as_micros() as f64 / NUM_MESSAGES as f64, + batch_elapsed.as_nanos() / NUM_MESSAGES as u128); + println!(); + + // ===== COMPACT MEMORY USAGE ===== + println!("--- MEMORY COMPARISON ---"); + let compact_heap_estimate: usize = compact_vec.iter().map(|m| { + m.content.len() // Box has no capacity, just len + + m.replied_to_content.as_ref().map(|s| s.len()).unwrap_or(0) + + m.attachments.len() * std::mem::size_of::() + if m.attachments.is_empty() { 0 } else { 1 } + + m.reactions.len() * std::mem::size_of::() + if m.reactions.is_empty() { 0 } else { 1 } + }).sum(); + let compact_struct_mem = compact_vec.len() * std::mem::size_of::(); + let compact_index_mem = compact_vec.len() * std::mem::size_of::<([u8; 32], u32)>(); + let interner_mem = interner.memory_usage(); + let compact_total = compact_struct_mem + compact_heap_estimate + compact_index_mem + interner_mem; + + println!(" Regular Message storage:"); + println!(" Struct memory: {:>10} bytes", messages.len() * std::mem::size_of::()); + println!(" Heap (strings): {:>10} bytes", msg_heap_estimate); + println!(" TOTAL: {:>10} bytes ({:.2} MB)", msg_total, msg_total as f64 / 1_000_000.0); + println!(); + println!(" CompactMessage storage:"); + println!(" Struct memory: {:>10} bytes", compact_struct_mem); + println!(" Heap (strings): {:>10} bytes", compact_heap_estimate); + println!(" ID index: {:>10} bytes", compact_index_mem); + println!(" Interner: {:>10} bytes ({} unique npubs)", interner_mem, interner.len()); + println!(" TOTAL: {:>10} bytes ({:.2} MB)", compact_total, compact_total as f64 / 1_000_000.0); + println!(); + println!(" SAVINGS: {} bytes ({:.1}%)", + msg_total.saturating_sub(compact_total), + (1.0 - compact_total as f64 / msg_total as f64) * 100.0 + ); + println!(" Per message: {} → {} bytes (avg)", + msg_total / NUM_MESSAGES, + compact_total / NUM_MESSAGES + ); + println!(); + + // ===== LOOKUP BENCHMARK ===== + println!("--- LOOKUP BENCHMARK ---"); + + // Generate random lookup IDs (mix of existing and non-existing) + let lookup_ids: Vec = (0..1000) + .map(|i| format!("{:0>64x}", i * 10)) // Every 10th message + .collect(); + + // Benchmark binary search lookup (CompactMessageVec) + let lookup_start = Instant::now(); + let mut found_count = 0; + for _ in 0..100 { // 100 iterations + for id in &lookup_ids { + if compact_vec.find_by_hex_id(id).is_some() { + found_count += 1; + } + } + } + let lookup_elapsed = lookup_start.elapsed(); + let total_lookups = 100 * lookup_ids.len(); + + println!(" Binary search (CompactMessageVec):"); + println!(" {} lookups in {:?}", total_lookups, lookup_elapsed); + println!(" Rate: {:.0} lookups/sec", total_lookups as f64 / lookup_elapsed.as_secs_f64()); + println!(" Per lookup: {:.2} µs", lookup_elapsed.as_micros() as f64 / total_lookups as f64); + println!(" Found: {} / {}", found_count, total_lookups); + println!(); + + // Benchmark linear search (simulating Vec) + let linear_start = Instant::now(); + let mut linear_found = 0; + for _ in 0..100 { + for id in &lookup_ids { + if messages.iter().find(|m| &m.id == id).is_some() { + linear_found += 1; + } + } + } + let linear_elapsed = linear_start.elapsed(); + + println!(" Linear search (Vec):"); + println!(" {} lookups in {:?}", total_lookups, linear_elapsed); + println!(" Rate: {:.0} lookups/sec", total_lookups as f64 / linear_elapsed.as_secs_f64()); + println!(" Per lookup: {:.2} µs", linear_elapsed.as_micros() as f64 / total_lookups as f64); + println!(); + + let speedup = linear_elapsed.as_nanos() as f64 / lookup_elapsed.as_nanos() as f64; + println!(" SPEEDUP: {:.1}x faster with binary search!", speedup); + println!(); + + // ===== INTERNER EFFICIENCY ===== + println!("--- INTERNER EFFICIENCY ---"); + let npub_string_size = 63 + 1; // "npub1" + 58 chars + null + let naive_npub_mem = NUM_MESSAGES * npub_string_size * 2; // npub + replied_to_npub + let actual_npub_mem = interner_mem; + println!(" Naive (every msg stores npubs): {} bytes", naive_npub_mem); + println!(" Interned ({} unique): {} bytes", interner.len(), actual_npub_mem); + println!(" SAVINGS: {} bytes ({:.1}%)", + naive_npub_mem.saturating_sub(actual_npub_mem), + (1.0 - actual_npub_mem as f64 / naive_npub_mem as f64) * 100.0 + ); + println!(); + + println!("========================================"); + println!(" BENCHMARK COMPLETE"); + println!("========================================\n"); + + // Verify correctness + assert_eq!(compact_vec.len(), NUM_MESSAGES); + assert_eq!(interner.len(), NUM_UNIQUE_USERS); + assert_eq!(found_count, linear_found); + } +} diff --git a/src-tauri/src/message/compression.rs b/src-tauri/src/message/compression.rs index 8bb860c1..c8c08d21 100644 --- a/src-tauri/src/message/compression.rs +++ b/src-tauri/src/message/compression.rs @@ -6,6 +6,8 @@ //! - PNG for transparent images, JPEG for opaque //! - Blurhash generation for previews +use std::sync::Arc; + use super::types::{CachedCompressedImage, ImageMetadata}; #[cfg(target_os = "android")] @@ -17,11 +19,11 @@ use crate::android::filesystem; pub const MIN_SAVINGS_PERCENT: u64 = 1; /// Internal function to compress bytes -/// Takes ownership of bytes to avoid cloning. +/// Takes Arc> for zero-copy sharing. /// If `min_savings_percent` is Some and compression doesn't meet threshold, /// returns original bytes with metadata (no wasted clone). pub(super) fn compress_bytes_internal( - bytes: Vec, + bytes: Arc>, extension: &str, min_savings_percent: Option, ) -> Result { @@ -33,19 +35,14 @@ pub(super) fn compress_bytes_internal( .map_err(|e| format!("Failed to decode GIF: {}", e))?; let (width, height) = (img.width(), img.height()); - let rgba_img = img.to_rgba8(); - let img_meta = crate::util::generate_blurhash_from_rgba( - rgba_img.as_raw(), - width, - height - ).map(|blurhash| ImageMetadata { - blurhash, - width, - height, - }); + let img_meta = crate::util::generate_blurhash_from_image(&img) + .map(|blurhash| ImageMetadata { + blurhash, + width, + height, + }); - // Return owned bytes directly - no clone needed return Ok(CachedCompressedImage { bytes, extension: "gif".to_string(), @@ -59,33 +56,10 @@ pub(super) fn compress_bytes_internal( let img = ::image::load_from_memory(&bytes) .map_err(|e| format!("Failed to decode image: {}", e))?; - // Generate metadata from original image (needed for both paths) - let rgba_for_meta = img.to_rgba8(); - let img_meta = crate::util::generate_blurhash_from_rgba( - rgba_for_meta.as_raw(), - img.width(), - img.height() - ).map(|blurhash| ImageMetadata { - blurhash, - width: img.width(), - height: img.height(), - }); - // Determine target dimensions (max 1920px on longest side) + use crate::shared::image::{calculate_resize_dimensions, MAX_DIMENSION}; let (width, height) = (img.width(), img.height()); - let max_dimension = 1920u32; - - let (new_width, new_height) = if width > max_dimension || height > max_dimension { - if width > height { - let ratio = max_dimension as f32 / width as f32; - (max_dimension, (height as f32 * ratio) as u32) - } else { - let ratio = max_dimension as f32 / height as f32; - ((width as f32 * ratio) as u32, max_dimension) - } - } else { - (width, height) - }; + let (new_width, new_height) = calculate_resize_dimensions(width, height, MAX_DIMENSION); // Resize if needed let resized_img = if new_width != width || new_height != height { @@ -94,9 +68,21 @@ pub(super) fn compress_bytes_internal( img }; + let actual_width = resized_img.width(); + let actual_height = resized_img.height(); + + // Generate metadata from final image only (avoid redundant blurhash generation) + let final_meta = crate::util::generate_blurhash_from_image(&resized_img) + .map(|blurhash| ImageMetadata { + blurhash, + width: actual_width, + height: actual_height, + }); + + // Keep reference to original metadata for fallback path + let img_meta = final_meta.clone(); + let rgba_img = resized_img.to_rgba8(); - let actual_width = rgba_img.width(); - let actual_height = rgba_img.height(); // Encode as PNG (alpha/small) or JPEG (standard) use crate::shared::image::{encode_rgba_auto, JPEG_QUALITY_STANDARD}; @@ -115,7 +101,7 @@ pub(super) fn compress_bytes_internal( }; if savings_percent < min_percent { - // Compression not worth it - return original bytes with metadata + // Compression not worth it - return original return Ok(CachedCompressedImage { bytes, extension: extension.to_string(), @@ -126,23 +112,8 @@ pub(super) fn compress_bytes_internal( } } - // Update metadata with resized dimensions if image was resized - let final_meta = if actual_width != width || actual_height != height { - crate::util::generate_blurhash_from_rgba( - rgba_img.as_raw(), - actual_width, - actual_height - ).map(|blurhash| ImageMetadata { - blurhash, - width: actual_width, - height: actual_height, - }) - } else { - img_meta - }; - Ok(CachedCompressedImage { - bytes: compressed_bytes, + bytes: Arc::new(compressed_bytes), extension: new_extension.to_string(), img_meta: final_meta, original_size, @@ -154,12 +125,6 @@ pub(super) fn compress_bytes_internal( pub(super) fn compress_image_internal(file_path: &str) -> Result { #[cfg(not(target_os = "android"))] { - let path = std::path::Path::new(file_path); - - if !path.exists() { - return Err(format!("File does not exist: {}", file_path)); - } - // Get extension early to check if it's a GIF let extension = file_path .rsplit('.') @@ -179,23 +144,19 @@ pub(super) fn compress_image_internal(file_path: &str) -> Result Result max_dimension || height > max_dimension { - if width > height { - let ratio = max_dimension as f32 / width as f32; - (max_dimension, (height as f32 * ratio) as u32) - } else { - let ratio = max_dimension as f32 / height as f32; - ((width as f32 * ratio) as u32, max_dimension) - } - } else { - (width, height) - }; - + let (new_width, new_height) = calculate_resize_dimensions(width, height, MAX_DIMENSION); + // Resize if needed let resized_img = if new_width != width || new_height != height { img.resize(new_width, new_height, ::image::imageops::FilterType::Lanczos3) } else { img }; - + + let actual_width = resized_img.width(); + let actual_height = resized_img.height(); + + let img_meta = crate::util::generate_blurhash_from_image(&resized_img) + .map(|blurhash| ImageMetadata { + blurhash, + width: actual_width, + height: actual_height, + }); + let rgba_img = resized_img.to_rgba8(); - let actual_width = rgba_img.width(); - let actual_height = rgba_img.height(); - - // Encode as PNG (alpha/small) or JPEG (standard) - use crate::shared::image::{encode_rgba_auto, JPEG_QUALITY_STANDARD}; let encoded = encode_rgba_auto(rgba_img.as_raw(), actual_width, actual_height, JPEG_QUALITY_STANDARD)?; let compressed_bytes = encoded.bytes; let extension = encoded.extension; - let img_meta = crate::util::generate_blurhash_from_rgba( - rgba_img.as_raw(), - actual_width, - actual_height - ).map(|blurhash| ImageMetadata { - blurhash, - width: actual_width, - height: actual_height, - }); - let compressed_size = compressed_bytes.len() as u64; Ok(CachedCompressedImage { - bytes: compressed_bytes, + bytes: Arc::new(compressed_bytes), extension: extension.to_string(), img_meta, original_size, @@ -262,7 +207,7 @@ pub(super) fn compress_image_internal(file_path: &str) -> Result Result Result max_dimension || height > max_dimension { - if width > height { - let ratio = max_dimension as f32 / width as f32; - (max_dimension, (height as f32 * ratio) as u32) - } else { - let ratio = max_dimension as f32 / height as f32; - ((width as f32 * ratio) as u32, max_dimension) - } - } else { - (width, height) - }; - + let (new_width, new_height) = calculate_resize_dimensions(width, height, MAX_DIMENSION); + // Resize if needed let resized_img = if new_width != width || new_height != height { img.resize(new_width, new_height, ::image::imageops::FilterType::Lanczos3) } else { img }; - + + let actual_width = resized_img.width(); + let actual_height = resized_img.height(); + + let img_meta = crate::util::generate_blurhash_from_image(&resized_img) + .map(|blurhash| ImageMetadata { + blurhash, + width: actual_width, + height: actual_height, + }); + let rgba_img = resized_img.to_rgba8(); - let actual_width = rgba_img.width(); - let actual_height = rgba_img.height(); - - // Encode as PNG (alpha/small) or JPEG (standard) - use crate::shared::image::{encode_rgba_auto, JPEG_QUALITY_STANDARD}; let encoded = encode_rgba_auto(rgba_img.as_raw(), actual_width, actual_height, JPEG_QUALITY_STANDARD)?; let compressed_bytes = encoded.bytes; let extension = encoded.extension; - let img_meta = crate::util::generate_blurhash_from_rgba( - rgba_img.as_raw(), - actual_width, - actual_height - ).map(|blurhash| ImageMetadata { - blurhash, - width: actual_width, - height: actual_height, - }); - let compressed_size = compressed_bytes.len() as u64; Ok(CachedCompressedImage { - bytes: compressed_bytes, + bytes: Arc::new(compressed_bytes), extension: extension.to_string(), img_meta, original_size, diff --git a/src-tauri/src/message/files.rs b/src-tauri/src/message/files.rs index b8a15d07..e2e16968 100644 --- a/src-tauri/src/message/files.rs +++ b/src-tauri/src/message/files.rs @@ -6,21 +6,23 @@ //! - Image preview generation //! - Android file handling +use std::sync::Arc; use nostr_sdk::prelude::*; use tokio::sync::Mutex as TokioMutex; use once_cell::sync::Lazy; use crate::util; +use crate::shared::image::read_file_checked; use super::types::{CachedCompressedImage, AttachmentFile, ImageMetadata, COMPRESSION_CACHE, ANDROID_FILE_CACHE}; use super::compression::{compress_bytes_internal, compress_image_internal}; -use super::message; +use super::sending::{message, MessageSendResult}; #[cfg(target_os = "android")] use crate::android::filesystem; /// Cache for bytes received from JavaScript (for Android file handling) -static JS_FILE_CACHE: Lazy, String, String)>>> = +static JS_FILE_CACHE: Lazy>, String, String)>>> = Lazy::new(|| std::sync::Mutex::new(None)); /// Cache for compressed bytes from JavaScript file @@ -43,17 +45,19 @@ pub struct CacheFileBytesResult { #[tauri::command] pub fn cache_file_bytes(bytes: Vec, file_name: String, extension: String) -> Result { let size = bytes.len() as u64; - + // Generate preview for supported image types let preview = if matches!(extension.as_str(), "png" | "jpg" | "jpeg" | "gif" | "webp" | "tiff" | "tif" | "ico") { - generate_image_preview_from_bytes(&bytes, 15).ok() + generate_image_preview_from_bytes(&bytes).ok() } else { None }; - + + let bytes = Arc::new(bytes); + let mut cache = JS_FILE_CACHE.lock().unwrap(); *cache = Some((bytes, file_name.clone(), extension.clone())); - + Ok(CacheFileBytesResult { size, name: file_name, @@ -77,68 +81,50 @@ pub fn get_cached_file_info() -> Result, String> { } /// Get base64 preview of cached image bytes -/// Uses ultra-fast nearest-neighbor downsampling for performance #[tauri::command] pub fn get_cached_image_preview(quality: u32) -> Result { - let quality = quality.clamp(1, 100); - + use crate::shared::image::{calculate_preview_dimensions, encode_rgba_auto, JPEG_QUALITY_PREVIEW}; + let cache = JS_FILE_CACHE.lock().unwrap(); let (bytes, _, _) = cache.as_ref().ok_or("No cached file")?; let bytes = bytes.clone(); drop(cache); - + let img = ::image::load_from_memory(&bytes) .map_err(|e| format!("Failed to decode image: {}", e))?; - + let (width, height) = (img.width(), img.height()); - let new_width = ((width * quality) / 100).max(1); - let new_height = ((height * quality) / 100).max(1); - - // Convert to RGBA8 for fast nearest-neighbor downsampling - let rgba = img.to_rgba8(); - let pixels = rgba.as_raw(); - - // Use ultra-fast nearest-neighbor downsampling - let resized_pixels = crate::util::nearest_neighbor_downsample( - pixels, - width, - height, - new_width, - new_height, - ); - - // Encode preview (PNG for alpha, JPEG otherwise) - use crate::shared::image::{encode_rgba_auto, JPEG_QUALITY_PREVIEW}; - let encoded = encode_rgba_auto(&resized_pixels, new_width, new_height, JPEG_QUALITY_PREVIEW)?; + let (new_width, new_height) = calculate_preview_dimensions(width, height, quality); - use base64::Engine; - let base64_str = base64::engine::general_purpose::STANDARD.encode(&encoded.bytes); - let mime = if encoded.extension == "png" { "image/png" } else { "image/jpeg" }; - Ok(format!("data:{};base64,{}", mime, base64_str)) + // Use SIMD-accelerated resize (10-15x faster for large JPEGs) + let (rgba_pixels, out_w, out_h) = crate::simd::image::fast_resize_to_rgba(&img, new_width, new_height); + let encoded = encode_rgba_auto(&rgba_pixels, out_w, out_h, JPEG_QUALITY_PREVIEW)?; + + Ok(encoded.to_data_uri()) } /// Start compression of cached bytes #[tauri::command] pub async fn start_cached_bytes_compression() -> Result<(), String> { - // Get bytes from cache let (bytes, _, extension) = { let cache = JS_FILE_CACHE.lock().unwrap(); - cache.clone().ok_or("No cached file")? + let (b, _, e) = cache.as_ref().ok_or("No cached file")?; + (b.clone(), String::new(), e.clone()) }; - + // Clear any previous compression result { let mut comp_cache = JS_COMPRESSION_CACHE.lock().await; *comp_cache = None; } - + // Spawn compression task (no min_savings - checked later by caller) tokio::spawn(async move { let result = compress_bytes_internal(bytes, &extension, None); let mut comp_cache = JS_COMPRESSION_CACHE.lock().await; *comp_cache = result.ok(); }); - + Ok(()) } @@ -167,7 +153,7 @@ pub async fn get_cached_bytes_compression_status() -> Result Result { +pub async fn send_cached_file(receiver: String, replied_to: String, use_compression: bool) -> Result { use super::compression::MIN_SAVINGS_PERCENT; if use_compression { @@ -202,65 +188,51 @@ pub async fn send_cached_file(receiver: String, replied_to: String, use_compress let mut cache = JS_FILE_CACHE.lock().unwrap(); cache.take().ok_or("No cached file")? }; - + // Clear compression cache *JS_COMPRESSION_CACHE.lock().await = None; - - // Process images: compress if use_compression is true, otherwise just generate metadata - let (bytes, extension, img_meta) = if matches!(original_extension.as_str(), "png" | "jpg" | "jpeg" | "webp" | "tiff" | "tif" | "ico") { + + // Check if this is an image type + let is_image = matches!(original_extension.as_str(), "png" | "jpg" | "jpeg" | "gif" | "webp" | "tiff" | "tif" | "ico"); + + // Process images: generate metadata and optionally compress + let (bytes, extension, img_meta) = if is_image { if let Ok(img) = ::image::load_from_memory(&original_bytes) { - let rgba_img = img.to_rgba8(); - let blurhash_meta = crate::util::generate_blurhash_from_rgba( - rgba_img.as_raw(), - img.width(), - img.height() - ).map(|blurhash| ImageMetadata { - blurhash, - width: img.width(), - height: img.height(), - }); - - if use_compression { + let blurhash_meta = crate::util::generate_blurhash_from_image(&img) + .map(|blurhash| ImageMetadata { + blurhash, + width: img.width(), + height: img.height(), + }); + + // GIFs: never compress, preserve animation + // Other images: compress if requested + if original_extension == "gif" || !use_compression { + // No compression - just use original bytes with metadata + (original_bytes, original_extension, blurhash_meta) + } else { // Compress on-the-fly since pre-compression wasn't ready use crate::shared::image::{encode_rgba_auto, JPEG_QUALITY_STANDARD}; + let rgba_img = img.to_rgba8(); match encode_rgba_auto(rgba_img.as_raw(), img.width(), img.height(), JPEG_QUALITY_STANDARD) { - Ok(encoded) => (encoded.bytes, encoded.extension.to_string(), blurhash_meta), + Ok(encoded) => (Arc::new(encoded.bytes), encoded.extension.to_string(), blurhash_meta), Err(_) => (original_bytes, original_extension, blurhash_meta), } - } else { - // No compression - just use original bytes with metadata - (original_bytes, original_extension, blurhash_meta) } } else { (original_bytes, original_extension, None) } - } else if original_extension == "gif" { - // For GIFs, just generate metadata but keep original bytes - let img_meta = if let Ok(img) = ::image::load_from_memory(&original_bytes) { - let rgba_img = img.to_rgba8(); - crate::util::generate_blurhash_from_rgba( - rgba_img.as_raw(), - img.width(), - img.height() - ).map(|blurhash| ImageMetadata { - blurhash, - width: img.width(), - height: img.height(), - }) - } else { - None - }; - (original_bytes, original_extension, img_meta) } else { + // Non-image file (original_bytes, original_extension, None) }; - + let attachment_file = AttachmentFile { bytes, extension, img_meta, }; - + message(receiver, String::new(), replied_to, Some(attachment_file)).await } @@ -299,7 +271,7 @@ pub async fn send_file_bytes( file_bytes: Vec, file_name: String, use_compression: bool -) -> Result { +) -> Result { use super::compression::MIN_SAVINGS_PERCENT; // Extract extension from filename @@ -319,7 +291,7 @@ pub async fn send_file_bytes( None // GIFs or no compression - just get metadata }; - match compress_bytes_internal(file_bytes, &extension, min_savings) { + match compress_bytes_internal(Arc::new(file_bytes), &extension, min_savings) { Ok(result) => AttachmentFile { bytes: result.bytes, extension: result.extension, @@ -333,7 +305,7 @@ pub async fn send_file_bytes( } else { // Non-image file - send as-is AttachmentFile { - bytes: file_bytes, + bytes: Arc::new(file_bytes), extension, img_meta: None, } @@ -343,28 +315,13 @@ pub async fn send_file_bytes( } #[tauri::command] -pub async fn file_message(receiver: String, replied_to: String, file_path: String) -> Result { +pub async fn file_message(receiver: String, replied_to: String, file_path: String) -> Result { // Load the file as AttachmentFile let mut attachment_file = { #[cfg(not(target_os = "android"))] { - let path = std::path::Path::new(&file_path); - - // Check if file exists first - if !path.exists() { - return Err(format!("File does not exist: {}", file_path)); - } - - // Read file bytes - let bytes = std::fs::read(&file_path) - .map_err(|e| format!("Failed to read file: {}", e))?; + let bytes = read_file_checked(&file_path)?; - // Check if file is empty - if bytes.is_empty() { - return Err(format!("File is empty (0 bytes): {}", file_path)); - } - - // Extract extension from filepath let extension = file_path .rsplit('.') .next() @@ -372,7 +329,7 @@ pub async fn file_message(receiver: String, replied_to: String, file_path: Strin .to_lowercase(); AttachmentFile { - bytes, + bytes: Arc::new(bytes), img_meta: None, extension, } @@ -380,7 +337,7 @@ pub async fn file_message(receiver: String, replied_to: String, file_path: Strin #[cfg(target_os = "android")] { // First check if we have cached bytes for this URI - // Take ownership from cache to avoid clone + // Take ownership from cache to avoid clone - bytes already Arc let mut cache = ANDROID_FILE_CACHE.lock().unwrap(); if let Some((bytes, extension, _, _)) = cache.remove(&file_path) { drop(cache); @@ -397,18 +354,7 @@ pub async fn file_message(receiver: String, replied_to: String, file_path: Strin filesystem::read_android_uri(file_path)? } else { // Regular file path (e.g., marketplace apps) - use standard file I/O - let path = std::path::Path::new(&file_path); - - if !path.exists() { - return Err(format!("File does not exist: {}", file_path)); - } - - let bytes = std::fs::read(&file_path) - .map_err(|e| format!("Failed to read file: {}", e))?; - - if bytes.is_empty() { - return Err(format!("File is empty (0 bytes): {}", file_path)); - } + let bytes = read_file_checked(&file_path)?; let extension = file_path .rsplit('.') @@ -417,7 +363,7 @@ pub async fn file_message(receiver: String, replied_to: String, file_path: Strin .to_lowercase(); AttachmentFile { - bytes, + bytes: Arc::new(bytes), img_meta: None, extension, } @@ -428,18 +374,13 @@ pub async fn file_message(receiver: String, replied_to: String, file_path: Strin // Generate image metadata if the file is an image if matches!(attachment_file.extension.as_str(), "png" | "jpg" | "jpeg" | "gif" | "webp" | "tiff" | "tif" | "ico") { - // Try to load and decode the image if let Ok(img) = ::image::load_from_memory(&attachment_file.bytes) { - let rgba_img = img.to_rgba8(); - attachment_file.img_meta = util::generate_blurhash_from_rgba( - rgba_img.as_raw(), - img.width(), - img.height() - ).map(|blurhash| ImageMetadata { - blurhash, - width: img.width(), - height: img.height(), - }); + attachment_file.img_meta = util::generate_blurhash_from_image(&img) + .map(|blurhash| ImageMetadata { + blurhash, + width: img.width(), + height: img.height(), + }); } } @@ -475,24 +416,20 @@ pub fn cache_android_file(file_path: String) -> Result Result Result Result { +/// Preview is capped to UI display size (300x400 mobile, 512x512 desktop) +/// For files smaller than 5MB or GIFs, returns the original image as base64 +fn generate_image_preview_from_bytes(bytes: &[u8]) -> Result { use base64::Engine; - + use crate::shared::image::{calculate_capped_preview_dimensions, encode_rgba_auto, JPEG_QUALITY_PREVIEW}; + const SKIP_RESIZE_THRESHOLD: usize = 5 * 1024 * 1024; // 5MB - + // Detect if this is a GIF (we never resize GIFs to preserve animation) let is_gif = bytes.starts_with(b"GIF"); - + // For small files or GIFs, just return the original as base64 (skip resizing) if bytes.len() < SKIP_RESIZE_THRESHOLD || is_gif { let base64_str = base64::engine::general_purpose::STANDARD.encode(bytes); - + // Detect image type from magic bytes for correct MIME type let mime_type = if bytes.starts_with(&[0xFF, 0xD8, 0xFF]) { "image/jpeg" @@ -568,39 +505,21 @@ fn generate_image_preview_from_bytes(bytes: &[u8], quality: u32) -> Result Result { #[cfg(not(target_os = "android"))] { let path = std::path::Path::new(&file_path); - - if !path.exists() { - return Err(format!("File does not exist: {}", file_path)); - } - + let metadata = std::fs::metadata(&file_path) .map_err(|e| format!("Failed to get file metadata: {}", e))?; - + let name = path.file_name() .and_then(|n| n.to_str()) .unwrap_or("unknown") .to_string(); - + let extension = path.extension() .and_then(|e| e.to_str()) .unwrap_or("") .to_lowercase(); - + Ok(FileInfo { size: metadata.len(), name, @@ -653,44 +568,26 @@ pub fn get_file_info(file_path: String) -> Result { /// Get a base64 preview of an image (for Android where convertFileSrc doesn't work) /// The quality parameter (1-100) determines the resize percentage -/// Uses ultra-fast nearest-neighbor downsampling for performance #[tauri::command] pub fn get_image_preview_base64(file_path: String, quality: u32) -> Result { - let quality = quality.clamp(1, 100); - + use crate::shared::image::{calculate_preview_dimensions, encode_rgba_auto, JPEG_QUALITY_PREVIEW}; + #[cfg(not(target_os = "android"))] { let bytes = std::fs::read(&file_path) .map_err(|e| format!("Failed to read file: {}", e))?; - + let img = ::image::load_from_memory(&bytes) .map_err(|e| format!("Failed to decode image: {}", e))?; - + let (width, height) = (img.width(), img.height()); - let new_width = ((width * quality) / 100).max(1); - let new_height = ((height * quality) / 100).max(1); - - // Convert to RGBA8 for fast nearest-neighbor downsampling - let rgba = img.to_rgba8(); - let pixels = rgba.as_raw(); - - // Use ultra-fast nearest-neighbor downsampling - let resized_pixels = crate::util::nearest_neighbor_downsample( - pixels, - width, - height, - new_width, - new_height, - ); - - // Encode preview (PNG for alpha/small images, JPEG otherwise) - use crate::shared::image::{encode_rgba_auto, JPEG_QUALITY_PREVIEW}; - let encoded = encode_rgba_auto(&resized_pixels, new_width, new_height, JPEG_QUALITY_PREVIEW)?; - - use base64::Engine; - let base64_str = base64::engine::general_purpose::STANDARD.encode(&encoded.bytes); - let mime = if encoded.extension == "png" { "image/png" } else { "image/jpeg" }; - Ok(format!("data:{};base64,{}", mime, base64_str)) + let (new_width, new_height) = calculate_preview_dimensions(width, height, quality); + + // Use SIMD-accelerated resize (10-15x faster for large JPEGs) + let (rgba_pixels, out_w, out_h) = crate::simd::image::fast_resize_to_rgba(&img, new_width, new_height); + let encoded = encode_rgba_auto(&rgba_pixels, out_w, out_h, JPEG_QUALITY_PREVIEW)?; + + Ok(encoded.to_data_uri()) } #[cfg(target_os = "android")] @@ -703,7 +600,7 @@ pub fn get_image_preview_base64(file_path: String, quality: u32) -> Result Result Result { +pub async fn file_message_compressed(receiver: String, replied_to: String, file_path: String) -> Result { // Load the file as AttachmentFile let mut attachment_file = { #[cfg(not(target_os = "android"))] { - let path = std::path::Path::new(&file_path); + let bytes = read_file_checked(&file_path)?; - // Check if file exists first - if !path.exists() { - return Err(format!("File does not exist: {}", file_path)); - } - - // Read file bytes - let bytes = std::fs::read(&file_path) - .map_err(|e| format!("Failed to read file: {}", e))?; - - // Check if file is empty - if bytes.is_empty() { - return Err(format!("File is empty (0 bytes): {}", file_path)); - } - - // Extract extension from filepath let extension = file_path .rsplit('.') .next() @@ -769,7 +634,7 @@ pub async fn file_message_compressed(receiver: String, replied_to: String, file_ .to_lowercase(); AttachmentFile { - bytes, + bytes: Arc::new(bytes), img_meta: None, extension, } @@ -777,7 +642,7 @@ pub async fn file_message_compressed(receiver: String, replied_to: String, file_ #[cfg(target_os = "android")] { // First check if we have cached bytes for this URI - // Take ownership from cache to avoid clone + // Take ownership from cache to avoid clone - bytes already Arc let mut cache = ANDROID_FILE_CACHE.lock().unwrap(); if let Some((bytes, extension, _, _)) = cache.remove(&file_path) { drop(cache); @@ -794,18 +659,7 @@ pub async fn file_message_compressed(receiver: String, replied_to: String, file_ filesystem::read_android_uri(file_path)? } else { // Regular file path (e.g., marketplace apps) - use standard file I/O - let path = std::path::Path::new(&file_path); - - if !path.exists() { - return Err(format!("File does not exist: {}", file_path)); - } - - let bytes = std::fs::read(&file_path) - .map_err(|e| format!("Failed to read file: {}", e))?; - - if bytes.is_empty() { - return Err(format!("File is empty (0 bytes): {}", file_path)); - } + let bytes = read_file_checked(&file_path)?; let extension = file_path .rsplit('.') @@ -814,7 +668,7 @@ pub async fn file_message_compressed(receiver: String, replied_to: String, file_ .to_lowercase(); AttachmentFile { - bytes, + bytes: Arc::new(bytes), img_meta: None, extension, } @@ -827,32 +681,25 @@ pub async fn file_message_compressed(receiver: String, replied_to: String, file_ if matches!(attachment_file.extension.as_str(), "png" | "jpg" | "jpeg" | "gif" | "webp" | "tiff" | "tif" | "ico") { if let Ok(img) = ::image::load_from_memory(&attachment_file.bytes) { // Determine target dimensions (max 1920px on longest side) + use crate::shared::image::{calculate_resize_dimensions, MAX_DIMENSION, encode_rgba_auto, JPEG_QUALITY_STANDARD}; let (width, height) = (img.width(), img.height()); - let max_dimension = 1920u32; - - let (new_width, new_height) = if width > max_dimension || height > max_dimension { - if width > height { - let ratio = max_dimension as f32 / width as f32; - (max_dimension, (height as f32 * ratio) as u32) - } else { - let ratio = max_dimension as f32 / height as f32; - ((width as f32 * ratio) as u32, max_dimension) - } - } else { - (width, height) - }; - + let (new_width, new_height) = calculate_resize_dimensions(width, height, MAX_DIMENSION); + // Resize if needed let resized_img = if new_width != width || new_height != height { img.resize(new_width, new_height, ::image::imageops::FilterType::Lanczos3) } else { img }; - - // Get RGBA image for alpha check and blurhash + + attachment_file.img_meta = crate::util::generate_blurhash_from_image(&resized_img) + .map(|blurhash| ImageMetadata { + blurhash, + width: resized_img.width(), + height: resized_img.height(), + }); + let rgba_img = resized_img.to_rgba8(); - let actual_width = rgba_img.width(); - let actual_height = rgba_img.height(); let mut compressed_bytes = Vec::new(); @@ -863,30 +710,17 @@ pub async fn file_message_compressed(receiver: String, replied_to: String, file_ let mut encoder = ::image::codecs::gif::GifEncoder::new(&mut cursor); encoder.encode( rgba_img.as_raw(), - actual_width, - actual_height, + rgba_img.width(), + rgba_img.height(), ::image::ExtendedColorType::Rgba8.into() ).map_err(|e| format!("Failed to encode GIF: {}", e))?; } else { - // Encode as PNG (alpha/small) or JPEG (standard) - use crate::shared::image::{encode_rgba_auto, JPEG_QUALITY_STANDARD}; - let encoded = encode_rgba_auto(rgba_img.as_raw(), actual_width, actual_height, JPEG_QUALITY_STANDARD)?; + let encoded = encode_rgba_auto(rgba_img.as_raw(), rgba_img.width(), rgba_img.height(), JPEG_QUALITY_STANDARD)?; compressed_bytes = encoded.bytes; attachment_file.extension = encoded.extension.to_string(); } - - attachment_file.bytes = compressed_bytes; - - // Generate blurhash from the resized image - attachment_file.img_meta = crate::util::generate_blurhash_from_rgba( - rgba_img.as_raw(), - actual_width, - actual_height - ).map(|blurhash| ImageMetadata { - blurhash, - width: actual_width, - height: actual_height, - }); + + attachment_file.bytes = Arc::new(compressed_bytes); } } @@ -975,7 +809,7 @@ pub async fn clear_compression_cache(file_path: String) -> Result<(), String> { /// Send a file using the cached compressed version if available #[tauri::command] -pub async fn send_cached_compressed_file(receiver: String, replied_to: String, file_path: String) -> Result { +pub async fn send_cached_compressed_file(receiver: String, replied_to: String, file_path: String) -> Result { use super::compression::MIN_SAVINGS_PERCENT; // First check if compression is complete or still in progress diff --git a/src-tauri/src/message/mod.rs b/src-tauri/src/message/mod.rs index 1fd7b854..f102b656 100644 --- a/src-tauri/src/message/mod.rs +++ b/src-tauri/src/message/mod.rs @@ -16,6 +16,7 @@ mod types; mod compression; mod sending; mod files; +pub mod compact; // Re-exports (use * for Tauri commands to include generated __cmd__ macros) pub use sending::*; @@ -24,6 +25,11 @@ pub use types::{ Message, ImageMetadata, Attachment, AttachmentFile, Reaction, EditEntry, }; +#[allow(unused_imports)] +pub use compact::{ + CompactMessage, CompactMessageVec, CompactReaction, CompactAttachment, + AttachmentFlags, MessageFlags, NpubInterner, NO_NPUB, +}; /// Protocol-agnostic reaction function that works for both DMs and Group Chats #[tauri::command] @@ -79,26 +85,29 @@ pub async fn react_to_message(reference_id: String, chat_id: String, emoji: Stri emoji, }; - let mut state = STATE.lock().await; - if let Some(chat) = state.chats.iter_mut().find(|c| c.has_participant(&chat_id)) { - if let Some(msg) = chat.messages.iter_mut().find(|m| m.id == reference_id) { - let was_added = msg.add_reaction(reaction, Some(&chat_id)); - + let msg_for_save = { + let mut state = STATE.lock().await; + // Use helper that handles interner access via split borrowing + if let Some((chat_id, was_added)) = state.add_reaction_to_message(&reference_id, reaction) { if was_added { - // Save the updated message to database - if let Some(handle) = TAURI_APP.get() { - let updated_message = msg.clone(); - let chat_id = chat.id.clone(); - drop(state); // Release lock before async operation - let _ = crate::db::save_message(handle.clone(), &chat_id, &updated_message).await; - return Ok(true); - } - } - - return Ok(was_added); + state.find_message(&reference_id) + .map(|(_, msg)| (chat_id, msg)) + } else { None } + } else { None } + }; + + if let Some((chat_id, msg)) = msg_for_save { + if let Some(handle) = TAURI_APP.get() { + let _ = crate::db::save_message(handle.clone(), &chat_id, &msg).await; + let _ = handle.emit("message_update", serde_json::json!({ + "old_id": &reference_id, + "message": &msg, + "chat_id": &chat_id + })); + return Ok(true); } } - + Ok(false) } ChatType::MlsGroup => { @@ -122,26 +131,29 @@ pub async fn react_to_message(reference_id: String, chat_id: String, emoji: Stri emoji, }; - let mut state = STATE.lock().await; - if let Some(chat) = state.chats.iter_mut().find(|c| c.id == chat_id) { - if let Some(msg) = chat.messages.iter_mut().find(|m| m.id == reference_id) { - let was_added = msg.add_reaction(reaction, Some(&chat_id)); - + let msg_for_save = { + let mut state = STATE.lock().await; + // Use helper that handles interner access via split borrowing + if let Some((returned_chat_id, was_added)) = state.add_reaction_to_message(&reference_id, reaction) { if was_added { - // Save the updated message to database - if let Some(handle) = TAURI_APP.get() { - let updated_message = msg.clone(); - let chat_id_clone = chat.id.clone(); - drop(state); // Release lock before async operation - let _ = crate::db::save_message(handle.clone(), &chat_id_clone, &updated_message).await; - return Ok(true); - } - } - - return Ok(was_added); + state.find_message(&reference_id) + .map(|(_, msg)| (returned_chat_id, msg)) + } else { None } + } else { None } + }; + + if let Some((chat_id_clone, msg)) = msg_for_save { + if let Some(handle) = TAURI_APP.get() { + let _ = crate::db::save_message(handle.clone(), &chat_id_clone, &msg).await; + let _ = handle.emit("message_update", serde_json::json!({ + "old_id": &reference_id, + "message": &msg, + "chat_id": &chat_id_clone + })); + return Ok(true); } } - + Ok(false) } } @@ -151,21 +163,23 @@ pub async fn react_to_message(reference_id: String, chat_id: String, emoji: Stri pub async fn fetch_msg_metadata(chat_id: String, msg_id: String) -> bool { // Find the message we're extracting metadata from let text = { - let mut state = STATE.lock().await; - let message = state.chats.iter_mut() - .find(|chat| chat.id == chat_id) - .and_then(|chat| chat.messages.iter_mut().find(|m| m.id == msg_id)); + let state = STATE.lock().await; + let chat_idx = state.chats.iter().position(|c| c.id == chat_id); + if let Some(idx) = chat_idx { + state.chats[idx].messages.find_by_hex_id(&msg_id) + .map(|m| m.content.clone()) + } else { None } + }; - // Message might not be in backend state (e.g., loaded via get_messages_around_id) - match message { - Some(msg) => msg.content.clone(), - None => return false, - } + // Message might not be in backend state (e.g., loaded via get_messages_around_id) + let text = match text { + Some(t) => t, + None => return false, }; // Extract URLs from the message const MAX_URLS_TO_TRY: usize = 3; - let urls = util::extract_https_urls(text.as_str()); + let urls = util::extract_https_urls(&text); if urls.is_empty() { return false; } @@ -182,27 +196,24 @@ pub async fn fetch_msg_metadata(chat_id: String, msg_id: String) -> bool { // Extracted metadata! if has_content { - // Re-fetch the message and add our metadata - let mut state = STATE.lock().await; - let msg = state.chats.iter_mut() - .find(|chat| chat.id == chat_id) - .and_then(|chat| chat.messages.iter_mut().find(|m| m.id == msg_id)) - .unwrap(); - msg.preview_metadata = Some(metadata); - - // Update the renderer - let handle = TAURI_APP.get().unwrap(); - handle.emit("message_update", serde_json::json!({ - "old_id": &msg_id, - "message": &msg, - "chat_id": &chat_id - })).unwrap(); + // Update message with metadata + let msg_for_save = { + let mut state = STATE.lock().await; + state.update_message_in_chat(&chat_id, &msg_id, |msg| { + msg.preview_metadata = Some(Box::new(metadata)); + }) + }; - // Save the updated message with metadata to the DB - let message_to_save = msg.clone(); - drop(state); // Release lock before async DB operation - let _ = crate::db::save_message(handle.clone(), &chat_id, &message_to_save).await; - return true; + if let Some(msg) = msg_for_save { + let handle = TAURI_APP.get().unwrap(); + handle.emit("message_update", serde_json::json!({ + "old_id": &msg_id, + "message": &msg, + "chat_id": &chat_id + })).unwrap(); + let _ = crate::db::save_message(handle.clone(), &chat_id, &msg).await; + return true; + } } } Err(_) => continue, @@ -227,11 +238,11 @@ pub async fn forward_attachment( // Search through all chats to find the message let mut found_path: Option = None; for chat in &state.chats { - if let Some(msg) = chat.messages.iter().find(|m| m.id == source_msg_id) { + if let Some(msg) = chat.messages.find_by_hex_id(&source_msg_id) { // Find the attachment in the message - if let Some(attachment) = msg.attachments.iter().find(|a| a.id == source_attachment_id) { - if !attachment.path.is_empty() && attachment.downloaded { - found_path = Some(attachment.path.clone()); + if let Some(attachment) = msg.attachments.iter().find(|a| a.id_eq(&source_attachment_id)) { + if !attachment.path.is_empty() && attachment.downloaded() { + found_path = Some(attachment.path.to_string()); } } break; @@ -241,11 +252,6 @@ pub async fn forward_attachment( found_path.ok_or_else(|| "Attachment not found or not downloaded".to_string())? }; - // Verify the file exists - if !std::path::Path::new(&attachment_path).exists() { - return Err("Attachment file not found on disk".to_string()); - } - // Send the file to the target chat using the existing file_message function // The hash-based reuse will automatically avoid re-uploading file_message(target_chat_id, String::new(), attachment_path).await?; @@ -329,42 +335,22 @@ pub async fn edit_message( } // Update local state - let mut state = STATE.lock().await; - if let Some(chat) = state.chats.iter_mut().find(|c| c.id == chat_id) { - if let Some(msg) = chat.messages.iter_mut().find(|m| m.id == message_id) { - // Build edit history if not present - if msg.edit_history.is_none() { - // First edit: save original content - msg.edit_history = Some(vec![EditEntry { - content: msg.content.clone(), - edited_at: msg.at, - }]); - } - - // Add new edit to history - let edit_timestamp_ms = created_at * 1000; - if let Some(ref mut history) = msg.edit_history { - history.push(EditEntry { - content: new_content.clone(), - edited_at: edit_timestamp_ms, - }); - } - - // Update the message content and flag - msg.content = new_content; - msg.edited = true; - - // Clear preview metadata (URL may have changed or been removed) + let edit_timestamp_ms = created_at * 1000; + let msg_for_emit = { + let mut state = STATE.lock().await; + state.update_message_in_chat(&chat_id, &message_id, |msg| { + msg.apply_edit(new_content, edit_timestamp_ms); msg.preview_metadata = None; + }) + }; - // Emit update to frontend - if let Some(handle) = TAURI_APP.get() { - handle.emit("message_update", serde_json::json!({ - "old_id": &message_id, - "message": &msg, - "chat_id": &chat_id - })).ok(); - } + if let Some(msg) = msg_for_emit { + if let Some(handle) = TAURI_APP.get() { + handle.emit("message_update", serde_json::json!({ + "old_id": &message_id, + "message": msg, + "chat_id": &chat_id + })).ok(); } } diff --git a/src-tauri/src/message/sending.rs b/src-tauri/src/message/sending.rs index 1ad7e7d1..ac0ef096 100644 --- a/src-tauri/src/message/sending.rs +++ b/src-tauri/src/message/sending.rs @@ -18,6 +18,7 @@ use tauri_plugin_clipboard_manager::ClipboardExt; use crate::crypto; use crate::db::{self, save_chat}; use crate::mls::MlsService; +use crate::util::{bytes_to_hex_string, hex_string_to_bytes}; use crate::util::calculate_file_hash; use crate::net; use crate::STATE; @@ -28,34 +29,33 @@ use crate::miniapps::realtime::{generate_topic_id, encode_topic_id}; use super::types::{AttachmentFile, ImageMetadata, Message, Attachment}; +/// Result of sending a message, returned to frontend for state update +#[derive(serde::Serialize)] +pub struct MessageSendResult { + /// The pending ID that was used while sending + pub pending_id: String, + /// The real event ID after successful send (None if failed) + pub event_id: Option, +} + /// Helper function to mark message as failed and update frontend -async fn mark_message_failed(pending_id: Arc, receiver: &str) { - // Find the message in chats and mark it as failed - let mut state = STATE.lock().await; - - // Search through all chats to find the message with this pending ID - for chat in &mut state.chats { - if chat.has_participant(receiver) { - if let Some(message) = chat.messages.iter_mut().find(|m| m.id == *pending_id) { - // Mark the message as failed - message.failed = true; - message.pending = false; - - // Update the frontend - let handle = TAURI_APP.get().unwrap(); - handle.emit("message_update", serde_json::json!({ - "old_id": pending_id.as_ref(), - "message": message, - "chat_id": receiver - })).unwrap(); +async fn mark_message_failed(pending_id: Arc, _receiver: &str) { + let result = { + let mut state = STATE.lock().await; + state.update_message(&pending_id, |msg| { + msg.set_failed(true); + msg.set_pending(false); + }) + }; - // Save the failed message to our DB - let message_to_save = message.clone(); - drop(state); // Release lock before async DB operation - let _ = crate::db::save_message(handle.clone(), receiver, &message_to_save).await; - break; - } - } + if let Some((chat_id, msg)) = result { + let handle = TAURI_APP.get().unwrap(); + handle.emit("message_update", serde_json::json!({ + "old_id": pending_id.as_ref(), + "message": &msg, + "chat_id": &chat_id + })).unwrap(); + let _ = crate::db::save_message(handle.clone(), &chat_id, &msg).await; } } @@ -105,8 +105,7 @@ async fn encrypt_and_upload_mls_media( } // Parse the engine group ID (this is what MDK uses internally) - let engine_gid_bytes = hex::decode(&group_meta.engine_group_id) - .map_err(|e| format!("Invalid engine_group_id hex: {}", e))?; + let engine_gid_bytes = hex_string_to_bytes(&group_meta.engine_group_id); let gid = mdk_core::GroupId::from_slice(&engine_gid_bytes); // Get MDK engine and media manager @@ -136,7 +135,7 @@ async fn encrypt_and_upload_mls_media( }; // Encrypt the file using MDK's media_manager - let upload = media_manager + let mut upload = media_manager .encrypt_for_upload_with_options(&file.bytes, &mdk_mime_type, filename, &options) .map_err(|e| format!("MIP-04 encryption failed: {}", e))?; @@ -149,7 +148,7 @@ async fn encrypt_and_upload_mls_media( let url = crate::blossom::upload_blob_with_progress_and_failover( signer, servers, - upload.encrypted_data.clone(), + Arc::new(std::mem::take(&mut upload.encrypted_data)), Some(&mime_type), progress_callback, Some(3), @@ -199,14 +198,14 @@ async fn encrypt_and_upload_mls_media( imeta_tag: sdk_imeta_tag, url, encrypted_size: upload.encrypted_size, - original_hash: hex::encode(upload.original_hash), - nonce: hex::encode(upload.nonce), + original_hash: bytes_to_hex_string(&upload.original_hash), + nonce: bytes_to_hex_string(&upload.nonce), scheme_version, }) } #[tauri::command] -pub async fn message(receiver: String, content: String, replied_to: String, file: Option) -> Result { +pub async fn message(receiver: String, content: String, replied_to: String, file: Option) -> Result { // Immediately add the message to our state as "Pending" with an ID derived from the current nanosecond, we'll update it as either Sent (non-pending) or Failed in the future let current_time = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -280,32 +279,10 @@ pub async fn message(receiver: String, content: String, replied_to: String, file // Prepare the rumor let handle = TAURI_APP.get().unwrap(); - let mut rumor = if file.is_none() { - // Send the text message to our frontend with appropriate event - if is_group_chat { - handle.emit("mls_message_new", serde_json::json!({ - "group_id": &receiver, - "message": &msg - })).unwrap(); - } else { - handle.emit("message_new", serde_json::json!({ - "message": &msg, - "chat_id": &receiver - })).unwrap(); - } - - // Text Message - if !is_group_chat { - EventBuilder::private_msg_rumor(receiver_pubkey, msg.content) - } else { - // For MLS groups, use Kind 9 (White Noise compatible) - EventBuilder::new(Kind::from_u16(crate::stored_event::event_kind::MLS_CHAT_MESSAGE), msg.content) - } - } else { - let attached_file = file.unwrap(); + let mut rumor = if let Some(attached_file) = file { // Calculate the file hash first (before encryption) - let file_hash = calculate_file_hash(&attached_file.bytes); + let file_hash = calculate_file_hash(&*attached_file.bytes); // The SHA-256 hash of an empty file - we should never reuse this const EMPTY_FILE_HASH: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; @@ -339,7 +316,7 @@ pub async fn message(receiver: String, content: String, replied_to: String, file let dir = handle.path().resolve("vector", base_directory).unwrap(); std::fs::create_dir_all(&dir).unwrap(); let hash_file_path = dir.join(&filename); - std::fs::write(&hash_file_path, &attached_file.bytes).unwrap(); + std::fs::write(&hash_file_path, &*attached_file.bytes).unwrap(); // Create progress callback for MLS upload let pending_id_for_callback = Arc::clone(&pending_id); @@ -372,15 +349,7 @@ pub async fn message(receiver: String, content: String, replied_to: String, file // Update the pending message with the uploaded attachment { - let pending_id_for_update = Arc::clone(&pending_id); - let mut state = STATE.lock().await; - let message = state.chats.iter_mut() - .find(|chat| chat.id() == &receiver) - .and_then(|chat| chat.messages.iter_mut().find(|m| m.id == *pending_id_for_update)) - .unwrap(); - - // Add the Attachment in-state - message.attachments.push(Attachment { + let attachment = Attachment { id: file_hash.clone(), key: String::new(), // MLS uses derived keys, not explicit nonce: mls_upload_result.nonce.clone(), @@ -396,13 +365,19 @@ pub async fn message(receiver: String, content: String, replied_to: String, file original_hash: Some(mls_upload_result.original_hash.clone()), scheme_version: Some(mls_upload_result.scheme_version.clone()), mls_filename: Some(filename.clone()), - }); + }; + let compact_att = crate::message::CompactAttachment::from_attachment_owned(attachment); + + let mut state = STATE.lock().await; + state.add_attachment_to_message(&receiver, &pending_id, compact_att); // Emit update to frontend - handle.emit("mls_message_new", serde_json::json!({ - "group_id": &receiver, - "message": &message - })).unwrap(); + if let Some(msg) = state.update_message_in_chat(&receiver, &pending_id, |_| {}) { + handle.emit("mls_message_new", serde_json::json!({ + "group_id": &receiver, + "message": msg + })).unwrap(); + } } // Build Kind 9 event with text content and imeta tag @@ -446,32 +421,24 @@ pub async fn message(receiver: String, content: String, replied_to: String, file .map_err(|e| format!("Failed to send MLS message: {}", e))?; // Update message state to non-pending (send_mls_message handles failures) - { + let msg_for_save = { let mut state = STATE.lock().await; - if let Some(message) = state.chats.iter_mut() - .find(|chat| chat.id() == &receiver) - .and_then(|chat| chat.messages.iter_mut().find(|m| m.id == *pending_id)) - { - // Update with actual event ID - let old_id = message.id.clone(); - message.id = event_id.clone(); - message.pending = false; - - // Emit update to frontend - handle.emit("message_update", serde_json::json!({ - "old_id": old_id, - "message": message, - "chat_id": &receiver - })).unwrap(); + state.finalize_pending_message(&receiver, &pending_id, &event_id) + }; - // Save to database - let message_to_save = message.clone(); - drop(state); - let _ = crate::db::save_message(handle.clone(), &receiver, &message_to_save).await; - } + if let Some((old_id, msg)) = msg_for_save { + handle.emit("message_update", serde_json::json!({ + "old_id": old_id, + "message": &msg, + "chat_id": &receiver + })).unwrap(); + let _ = crate::db::save_message(handle.clone(), &receiver, &msg).await; } - return Ok(true); + return Ok(MessageSendResult { + pending_id: pending_id.to_string(), + event_id: Some(event_id), + }); } // ============================================================ @@ -480,81 +447,61 @@ pub async fn message(receiver: String, content: String, replied_to: String, file // Check for existing attachment with same hash across all profiles BEFORE encrypting // BUT: Never reuse empty file hashes - always force a new upload - let existing_attachment = if file_hash == EMPTY_FILE_HASH { + let existing_attachment: Option = if file_hash == EMPTY_FILE_HASH { None } else { - let mut found_attachment: Option<(String, Attachment)> = None; - + let mut found_attachment: Option = None; + // First, search through in-memory state (fastest check) { let state = STATE.lock().await; - for chat in &state.chats { - for message in &chat.messages { + 'outer: for chat in &state.chats { + for message in chat.messages.iter() { for attachment in &message.attachments { - if attachment.id == file_hash && !attachment.url.is_empty() { - // Found a matching attachment with a valid URL - // For DMs, use first participant; for groups, use chat ID - let chat_identifier = if let Some(participant_id) = chat.participants.first() { - participant_id.clone() - } else { - // Group chat - use the chat ID itself - chat.id.clone() - }; - found_attachment = Some((chat_identifier, attachment.clone())); - break; + if attachment.id_eq(&file_hash) && !attachment.url.is_empty() { + found_attachment = Some(attachment.to_attachment()); + break 'outer; } } - if found_attachment.is_some() { - break; - } - } - if found_attachment.is_some() { - break; } } } - - // Fallback: check database index if not found in memory (covers all stored attachments) + + // Fallback: check database if not found in memory (covers all stored attachments) if found_attachment.is_none() { - if let Ok(index) = db::build_file_hash_index(handle).await { - if let Some(attachment_ref) = index.get(&file_hash) { - // Found in database index - convert AttachmentRef to Attachment - found_attachment = Some((attachment_ref.chat_id.clone(), Attachment { - id: attachment_ref.hash.clone(), - url: attachment_ref.url.clone(), - key: attachment_ref.key.clone(), - nonce: attachment_ref.nonce.clone(), - extension: attachment_ref.extension.clone(), - size: attachment_ref.size, - path: String::new(), - img_meta: None, - downloading: false, - downloaded: false, - webxdc_topic: None, // Not stored in attachment index - group_id: None, // DM attachments don't use MLS encryption - original_hash: None, // Not stored in attachment index - scheme_version: None, // DM attachments don't use MIP-04 - mls_filename: None, // DM attachments don't use MIP-04 - })); - } + if let Ok(Some(attachment_ref)) = db::lookup_attachment_cached(handle, &file_hash).await { + found_attachment = Some(Attachment { + id: attachment_ref.hash, + url: attachment_ref.url, + key: attachment_ref.key, + nonce: attachment_ref.nonce, + extension: attachment_ref.extension, + size: attachment_ref.size, + path: String::new(), + img_meta: None, + downloading: false, + downloaded: false, + webxdc_topic: None, + group_id: None, + original_hash: None, + scheme_version: None, + mls_filename: None, + }); } } - + found_attachment }; // Determine if we need to encrypt based on whether we'll reuse an existing attachment - let will_reuse_existing = if let Some((_, ref existing)) = existing_attachment { + let will_reuse_existing = if let Some(ref existing) = existing_attachment { // Check if URL contains empty hash - never reuse those const EMPTY_FILE_HASH: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; if existing.url.contains(EMPTY_FILE_HASH) { false } else { // Check if URL is live - match net::check_url_live(&existing.url).await { - Ok(is_live) => is_live, - Err(_) => false - } + net::check_url_live(&existing.url).await.unwrap_or_default() } } else { false @@ -567,7 +514,7 @@ pub async fn message(receiver: String, content: String, replied_to: String, file } else { // Encrypt the attachment - either it's new or the existing URL is dead let params = crypto::generate_encryption_params(); - let enc_file = crypto::encrypt_data(attached_file.bytes.as_slice(), ¶ms).unwrap(); + let enc_file = crypto::encrypt_data(&*attached_file.bytes, ¶ms).unwrap(); (params, enc_file) }; @@ -584,17 +531,6 @@ pub async fn message(receiver: String, content: String, replied_to: String, file { // Use a clone of the Arc for this block let pending_id_clone = Arc::clone(&pending_id); - - // Retrieve the Pending Message - let mut state = STATE.lock().await; - let message = state.chats.iter_mut() - .find(|chat| { - // For DMs, check if receiver is a participant - // For MLS groups, check if receiver matches the chat ID - chat.id() == &receiver || chat.has_participant(&receiver) - }) - .and_then(|chat| chat.messages.iter_mut().find(|m| m.id == *pending_id_clone)) - .unwrap(); // Choose the appropriate base directory based on platform let base_directory = if cfg!(target_os = "ios") { @@ -613,10 +549,10 @@ pub async fn message(receiver: String, content: String, replied_to: String, file std::fs::create_dir_all(&dir).unwrap(); // Save the hash-named file - std::fs::write(&hash_file_path, &attached_file.bytes).unwrap(); + std::fs::write(&hash_file_path, &*attached_file.bytes).unwrap(); // Determine encryption params and file size based on whether we found an existing attachment - let (attachment_key, attachment_nonce, file_size) = if let Some((_, ref existing)) = existing_attachment { + let (attachment_key, attachment_nonce, file_size) = if let Some(ref existing) = existing_attachment { // Reuse existing encryption params (existing.key.clone(), existing.nonce.clone(), existing.size) } else { @@ -624,9 +560,8 @@ pub async fn message(receiver: String, content: String, replied_to: String, file (params.key.clone(), params.nonce.clone(), enc_file.len() as u64) }; - // Add the Attachment in-state (with our local path, to prevent re-downloading it accidentally from server) - message.attachments.push(Attachment { - // Use SHA256 hash as the ID + // Add the attachment to the pending message + let attachment = Attachment { id: file_hash.clone(), key: attachment_key, nonce: attachment_nonce, @@ -640,22 +575,27 @@ pub async fn message(receiver: String, content: String, replied_to: String, file webxdc_topic: webxdc_topic.clone(), group_id: if is_group_chat { Some(receiver.clone()) } else { None }, original_hash: Some(file_hash.clone()), - scheme_version: None, // DM attachments use explicit key/nonce, not MIP-04 - mls_filename: None, // DM attachments use explicit key/nonce, not MIP-04 - }); + scheme_version: None, + mls_filename: None, + }; + let compact_att = crate::message::CompactAttachment::from_attachment_owned(attachment); - // Send the pending file upload to our frontend with appropriate event - // This provides immediate UI feedback for the sender - if is_group_chat { - handle.emit("mls_message_new", serde_json::json!({ - "group_id": &receiver, - "message": &message - })).unwrap(); - } else { - handle.emit("message_new", serde_json::json!({ - "message": &message, - "chat_id": &receiver - })).unwrap(); + let mut state = STATE.lock().await; + state.add_attachment_to_message(&receiver, &pending_id_clone, compact_att); + + // Emit update to frontend + if let Some(msg) = state.update_message_in_chat(&receiver, &pending_id_clone, |_| {}) { + if is_group_chat { + handle.emit("mls_message_new", serde_json::json!({ + "group_id": &receiver, + "message": msg + })).unwrap(); + } else { + handle.emit("message_new", serde_json::json!({ + "message": msg, + "chat_id": &receiver + })).unwrap(); + } } } @@ -664,52 +604,56 @@ pub async fn message(receiver: String, content: String, replied_to: String, file // Check if we found an existing attachment with the same hash let mut should_upload = true; - let attachment_rumor = if let Some((_found_profile_id, existing_attachment)) = existing_attachment { + let attachment_rumor = if let Some(existing) = existing_attachment { // Never reuse URLs with the empty file hash const EMPTY_FILE_HASH: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; - let is_empty_hash = existing_attachment.url.contains(EMPTY_FILE_HASH); + let is_empty_hash = existing.url.contains(EMPTY_FILE_HASH); // Verify the URL is still live before reusing (but skip if it's an empty hash) let url_is_live = if is_empty_hash { false } else { - match net::check_url_live(&existing_attachment.url).await { - Ok(is_live) => is_live, - Err(_) => false // Treat errors as dead URL - } + net::check_url_live(&existing.url).await.unwrap_or_default() }; - + if url_is_live { should_upload = false; - + // Update our pending message with the existing URL + let reused_url = existing.url.clone(); { - let pending_id_for_update = Arc::clone(&pending_id); + let url = reused_url.clone(); let mut state = STATE.lock().await; - let message = state.chats.iter_mut() - .find(|chat| chat.id() == &receiver || chat.has_participant(&receiver)) - .and_then(|chat| chat.messages.iter_mut().find(|m| m.id == *pending_id_for_update)) - .unwrap(); - if let Some(attachment) = message.attachments.last_mut() { - attachment.url = existing_attachment.url.clone(); - } + state.update_message(&pending_id, |msg| { + if let Some(att) = msg.attachments.last_mut() { + att.url = url.into_boxed_str(); + } + }); } - + + // Emit attachment_update so frontend can update the URL immediately + handle.emit("attachment_update", serde_json::json!({ + "chat_id": &receiver, + "message_id": pending_id.as_ref(), + "attachment_id": &file_hash, + "url": &reused_url, + })).ok(); + // Create the attachment rumor with the existing URL - let mut attachment_rumor = EventBuilder::new(Kind::from_u16(15), existing_attachment.url); - + let mut attachment_rumor = EventBuilder::new(Kind::from_u16(15), existing.url); + // Only add p-tag for DMs, not for MLS groups if !is_group_chat { attachment_rumor = attachment_rumor.tag(Tag::public_key(receiver_pubkey)); } - + // Append decryption keys and file metadata (using existing attachment's params) attachment_rumor = attachment_rumor .tag(Tag::custom(TagKind::custom("file-type"), [mime_type.as_str()])) - .tag(Tag::custom(TagKind::custom("size"), [existing_attachment.size.to_string()])) + .tag(Tag::custom(TagKind::custom("size"), [existing.size.to_string()])) .tag(Tag::custom(TagKind::custom("encryption-algorithm"), ["aes-gcm"])) - .tag(Tag::custom(TagKind::custom("decryption-key"), [existing_attachment.key.as_str()])) - .tag(Tag::custom(TagKind::custom("decryption-nonce"), [existing_attachment.nonce.as_str()])) + .tag(Tag::custom(TagKind::custom("decryption-key"), [existing.key.as_str()])) + .tag(Tag::custom(TagKind::custom("decryption-nonce"), [existing.nonce.as_str()])) .tag(Tag::custom(TagKind::custom("ox"), [file_hash.clone()])); // Append image metadata if available @@ -757,21 +701,27 @@ pub async fn message(receiver: String, content: String, replied_to: String, file }); // Upload the file with progress, retries, and automatic server failover - match crate::blossom::upload_blob_with_progress_and_failover(signer.clone(), servers, enc_file, Some(mime_type.as_str()), progress_callback, Some(3), Some(std::time::Duration::from_secs(2))).await { + match crate::blossom::upload_blob_with_progress_and_failover(signer.clone(), servers, Arc::new(enc_file), Some(mime_type.as_str()), progress_callback, Some(3), Some(std::time::Duration::from_secs(2))).await { Ok(url) => { // Update our pending message with the uploaded URL { - let pending_id_for_url_update = Arc::clone(&pending_id); + let url_clone = url.clone(); let mut state = STATE.lock().await; - let message = state.chats.iter_mut() - .find(|chat| chat.id() == &receiver || chat.has_participant(&receiver)) - .and_then(|chat| chat.messages.iter_mut().find(|m| m.id == *pending_id_for_url_update)) - .unwrap(); - if let Some(attachment) = message.attachments.last_mut() { - attachment.url = url.clone(); - } + state.update_message(&pending_id, |msg| { + if let Some(att) = msg.attachments.last_mut() { + att.url = url_clone.into_boxed_str(); + } + }); } - + + // Emit attachment_update so frontend can update the URL immediately + handle.emit("attachment_update", serde_json::json!({ + "chat_id": &receiver, + "message_id": pending_id.as_ref(), + "attachment_id": &file_hash, + "url": &url, + })).ok(); + // Create the attachment rumor let mut attachment_rumor = EventBuilder::new(Kind::from_u16(15), url); @@ -819,6 +769,26 @@ pub async fn message(receiver: String, content: String, replied_to: String, file // Return the final attachment rumor as the main rumor final_attachment_rumor + } else { + // Text message (no attachment) + // Send the text message to our frontend with appropriate event + if is_group_chat { + handle.emit("mls_message_new", serde_json::json!({ + "group_id": &receiver, + "message": &msg + })).unwrap(); + } else { + handle.emit("message_new", serde_json::json!({ + "message": &msg, + "chat_id": &receiver + })).unwrap(); + } + + if !is_group_chat { + EventBuilder::private_msg_rumor(receiver_pubkey, msg.content) + } else { + EventBuilder::new(Kind::from_u16(crate::stored_event::event_kind::MLS_CHAT_MESSAGE), msg.content) + } }; // If a reply reference is included, add the tag @@ -853,13 +823,19 @@ pub async fn message(receiver: String, content: String, replied_to: String, file // - Updates message ID when processed // - Marks as success/failure after network confirmation // - Saves to database - match crate::mls::send_mls_message(&receiver, built_rumor.clone(), Some(pending_id.to_string())).await { - Ok(_) => return Ok(true), + return match crate::mls::send_mls_message(&receiver, built_rumor.clone(), Some(pending_id.to_string())).await { + Ok(_) => Ok(MessageSendResult { + pending_id: pending_id.to_string(), + event_id: Some(rumor_id.to_hex()), + }), Err(e) => { eprintln!("Failed to send MLS message: {:?}", e); - return Ok(false); + Ok(MessageSendResult { + pending_id: pending_id.to_string(), + event_id: None, + }) } - } + }; } else { // DM - use NIP-17 giftwrap // Send message to the real receiver with retry logic @@ -871,7 +847,7 @@ pub async fn message(receiver: String, content: String, replied_to: String, file while send_attempts < MAX_ATTEMPTS { send_attempts += 1; - + match client .gift_wrap(&receiver_pubkey, built_rumor.clone(), []) .await @@ -886,36 +862,40 @@ pub async fn message(receiver: String, content: String, replied_to: String, file // Immediately update frontend and save to DB // This provides faster visual feedback without waiting for the self-send - { + let msg_for_save = { let pending_id_for_early_update = Arc::clone(&pending_id); let mut state = STATE.lock().await; - if let Some(msg) = state.chats.iter_mut() - .find(|chat| chat.id() == &receiver || chat.has_participant(&receiver)) - .and_then(|chat| chat.messages.iter_mut().find(|m| m.id == *pending_id_for_early_update)) - { - // Update the message ID and clear pending state - msg.id = rumor_id.to_hex(); - msg.pending = false; - msg.wrapper_event_id = Some(wrapper_id); - - // Emit update to frontend for immediate visual feedback - let handle = TAURI_APP.get().unwrap(); - let _ = handle.emit("message_update", serde_json::json!({ - "old_id": pending_id_for_early_update.as_ref(), - "message": &msg, - "chat_id": &receiver - })); - - // Save to DB immediately (don't wait for self-send) - let message_to_save = msg.clone(); - let chat_to_save = state.get_chat(&receiver).cloned(); - drop(state); // Release lock before async DB operations - - if let Some(chat) = chat_to_save { - let _ = save_chat(handle.clone(), &chat).await; - let _ = crate::db::save_message(handle.clone(), &receiver, &message_to_save).await; + let chat_idx = state.chats.iter().position(|c| c.id() == &receiver || c.has_participant(&receiver)); + if let Some(idx) = chat_idx { + let real_id_hex = rumor_id.to_hex(); + if let Some(msg) = state.chats[idx].messages.find_by_hex_id_mut(&pending_id_for_early_update) { + // Update the message ID and clear pending state + msg.id = crate::simd::hex_to_bytes_32(&real_id_hex); + msg.set_pending(false); + msg.wrapper_id = Some(Box::new(crate::simd::hex_to_bytes_32(&wrapper_id))); } + // Rebuild index since ID changed + state.chats[idx].messages.rebuild_index(); + // Get data for emit and save - clone Chat for db, convert message for emit + let chat_for_save = state.chats[idx].clone(); + let msg_for_emit = state.chats[idx].messages.find_by_hex_id(&real_id_hex) + .map(|m| (pending_id_for_early_update.to_string(), m.to_message(&state.interner))); + (Some(chat_for_save), msg_for_emit) + } else { + (None, None) } + }; + + if let (Some(chat), Some((old_id, msg))) = msg_for_save { + let handle = TAURI_APP.get().unwrap(); + // Emit full message_update for backwards compatibility + let _ = handle.emit("message_update", serde_json::json!({ + "old_id": old_id, + "message": &msg, + "chat_id": &receiver + })); + let _ = save_chat(handle.clone(), &chat).await; + let _ = crate::db::save_message(handle.clone(), &receiver, &msg).await; } break; @@ -927,7 +907,10 @@ pub async fn message(receiver: String, content: String, replied_to: String, file if send_attempts == MAX_ATTEMPTS { // Final attempt failed mark_message_failed(Arc::clone(&pending_id), &receiver).await; - return Ok(false); + return Ok(MessageSendResult { + pending_id: pending_id.to_string(), + event_id: None, + }); } } @@ -939,13 +922,16 @@ pub async fn message(receiver: String, content: String, replied_to: String, file Err(e) => { // Network or other error - log and retry if we haven't exceeded attempts eprintln!("Failed to send message (attempt {}/{}): {:?}", send_attempts, MAX_ATTEMPTS, e); - + if send_attempts == MAX_ATTEMPTS { // Final attempt failed mark_message_failed(Arc::clone(&pending_id), &receiver).await; - return Ok(false); + return Ok(MessageSendResult { + pending_id: pending_id.to_string(), + event_id: None, + }); } - + // Wait before retrying tokio::time::sleep(tokio::time::Duration::from_secs(RETRY_DELAY)).await; } @@ -955,7 +941,10 @@ pub async fn message(receiver: String, content: String, replied_to: String, file // If we get here without final_output, all attempts failed if final_output.is_none() { mark_message_failed(Arc::clone(&pending_id), &receiver).await; - return Ok(false); + return Ok(MessageSendResult { + pending_id: pending_id.to_string(), + event_id: None, + }); } // Send message to our own public key, to allow for message recovering @@ -963,12 +952,15 @@ pub async fn message(receiver: String, content: String, replied_to: String, file .gift_wrap(&my_public_key, built_rumor, []) .await; - Ok(true) + Ok(MessageSendResult { + pending_id: pending_id.to_string(), + event_id: Some(rumor_id.to_hex()), + }) } } #[tauri::command] -pub async fn paste_message(handle: AppHandle, receiver: String, replied_to: String, transparent: bool) -> Result { +pub async fn paste_message(handle: AppHandle, receiver: String, replied_to: String, transparent: bool) -> Result { // Platform-specific clipboard reading #[cfg(target_os = "android")] let img = { @@ -996,21 +988,19 @@ pub async fn paste_message(handle: AppHandle, receiver: String, r // Get original pixels let original_pixels = img.as_raw(); - // Windows: check that every image has a non-zero-ish Alpha channel + // Windows: check if clipboard corrupted alpha channel (all values near zero) let mut _transparency_bug_search = false; #[cfg(target_os = "windows")] { - _transparency_bug_search = original_pixels.iter().skip(3).step_by(4).all(|&a| a <= 2); + _transparency_bug_search = util::has_all_alpha_near_zero(original_pixels); } - // For non-transparent images: manually account for the zero'ing out of the Alpha channel + // For non-transparent images: set alpha to opaque let pixels = if !transparent || _transparency_bug_search { - // Only clone if we need to modify let mut modified = original_pixels.to_vec(); - modified.iter_mut().skip(3).step_by(4).for_each(|a| *a = 255); + util::set_all_alpha_opaque(&mut modified); std::borrow::Cow::Owned(modified) } else { - // No modification needed, use the original data std::borrow::Cow::Borrowed(original_pixels) }; @@ -1031,7 +1021,7 @@ pub async fn paste_message(handle: AppHandle, receiver: String, r // Generate an Attachment File let attachment_file = AttachmentFile { - bytes: encoded_bytes, + bytes: Arc::new(encoded_bytes), img_meta, extension: extension.to_string(), }; @@ -1041,10 +1031,10 @@ pub async fn paste_message(handle: AppHandle, receiver: String, r } #[tauri::command] -pub async fn voice_message(receiver: String, replied_to: String, bytes: Vec) -> Result { +pub async fn voice_message(receiver: String, replied_to: String, bytes: Vec) -> Result { // Generate an Attachment File let attachment_file = AttachmentFile { - bytes, + bytes: Arc::new(bytes), img_meta: None, extension: String::from("wav") }; diff --git a/src-tauri/src/message/types.rs b/src-tauri/src/message/types.rs index 80adf59f..d693a2e4 100644 --- a/src-tauri/src/message/types.rs +++ b/src-tauri/src/message/types.rs @@ -5,6 +5,7 @@ //! - Image metadata and compression cache types use std::collections::HashMap; +use std::sync::Arc; use tauri::Emitter; use tokio::sync::Mutex as TokioMutex; use once_cell::sync::Lazy; @@ -15,7 +16,7 @@ use crate::TAURI_APP; /// Cached compressed image data #[derive(Clone)] pub struct CachedCompressedImage { - pub bytes: Vec, + pub bytes: Arc>, pub extension: String, pub img_meta: Option, pub original_size: u64, @@ -29,7 +30,7 @@ pub static COMPRESSION_CACHE: Lazy (bytes, extension, name, size) /// This is used to cache file bytes immediately after file selection on Android, /// before the temporary content URI permission expires. -pub static ANDROID_FILE_CACHE: Lazy, String, String, u64)>>> = +pub static ANDROID_FILE_CACHE: Lazy>, String, String, u64)>>> = Lazy::new(|| std::sync::Mutex::new(HashMap::new())); #[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq)] @@ -238,14 +239,21 @@ impl Default for Attachment { } /// A simple pre-upload format to associate a byte stream with a file extension -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] +/// Note: This type is used internally - the bytes field is not serialized. +/// For Tauri commands, AttachmentFile is constructed in Rust, not deserialized from JS. +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct AttachmentFile { - pub bytes: Vec, + #[serde(skip, default = "default_arc_bytes")] + pub bytes: Arc>, /// Image metadata (for images only) pub img_meta: Option, pub extension: String, } +fn default_arc_bytes() -> Arc> { + Arc::new(Vec::new()) +} + #[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq)] pub struct Reaction { pub id: String, diff --git a/src-tauri/src/miniapps/commands.rs b/src-tauri/src/miniapps/commands.rs index 4932346b..9ea2d836 100644 --- a/src-tauri/src/miniapps/commands.rs +++ b/src-tauri/src/miniapps/commands.rs @@ -17,6 +17,7 @@ use serde::{Deserialize, Serialize}; use super::error::Error; use super::state::{MiniAppInstance, MiniAppsState, MiniAppPackage, RealtimeChannelState}; use super::realtime::{RealtimeEvent, encode_topic_id, encode_node_addr}; +use crate::util::bytes_to_hex_string; // Network isolation proxy - only used on Linux (not macOS due to version requirements, not Windows due to WebView2 freeze, not Android) #[cfg(all(not(target_os = "macos"), not(target_os = "windows"), not(target_os = "android")))] @@ -520,7 +521,7 @@ pub async fn miniapp_load_info_from_bytes( use sha2::{Sha256, Digest}; let mut hasher = Sha256::new(); hasher.update(&bytes); - let file_hash = hex::encode(hasher.finalize()); + let file_hash = bytes_to_hex_string(hasher.finalize().as_slice()); let (manifest, icon_bytes) = MiniAppPackage::load_info_from_bytes(&bytes, &fallback_name)?; diff --git a/src-tauri/src/miniapps/marketplace.rs b/src-tauri/src/miniapps/marketplace.rs index 79560316..adb9006a 100644 --- a/src-tauri/src/miniapps/marketplace.rs +++ b/src-tauri/src/miniapps/marketplace.rs @@ -52,6 +52,7 @@ use log::{info, warn}; use crate::blossom; use crate::NOSTR_CLIENT; use crate::image_cache::{self, CacheResult, ImageType}; +use crate::util::bytes_to_hex_string; /// The event kind for Mini App marketplace listings /// Kind 30078 = Parameterized Replaceable Application-Specific Data @@ -224,7 +225,7 @@ impl MarketplaceState { /// Check if a publisher is trusted #[allow(dead_code)] pub fn is_trusted_publisher(&self, npub: &str) -> bool { - self.trusted_publishers.contains(&npub.to_string()) + self.trusted_publishers.iter().any(|p| p == npub) } /// Add a trusted publisher @@ -590,7 +591,7 @@ pub async fn install_marketplace_app( use sha2::{Sha256, Digest}; let mut hasher = Sha256::new(); hasher.update(&bytes); - let hash = hex::encode(hasher.finalize()); + let hash = bytes_to_hex_string(hasher.finalize().as_slice()); if hash != app.blossom_hash { let mut state = MARKETPLACE_STATE.write().await; @@ -746,7 +747,7 @@ pub async fn update_marketplace_app( use sha2::{Sha256, Digest}; let mut hasher = Sha256::new(); hasher.update(&old_data); - Some(hex::encode(hasher.finalize())) + Some(bytes_to_hex_string(hasher.finalize().as_slice())) } Err(e) => { warn!("Failed to read old app file for hash: {}", e); @@ -778,7 +779,7 @@ pub async fn update_marketplace_app( use sha2::{Sha256, Digest}; let mut hasher = Sha256::new(); hasher.update(&bytes_vec); - let new_file_hash = hex::encode(hasher.finalize()); + let new_file_hash = bytes_to_hex_string(hasher.finalize().as_slice()); if new_file_hash != app.blossom_hash { // Clean up temp file if it exists @@ -894,7 +895,7 @@ pub async fn publish_to_marketplace( use sha2::{Sha256, Digest}; let mut hasher = Sha256::new(); hasher.update(&file_data); - let blossom_hash = hex::encode(hasher.finalize()); + let blossom_hash = bytes_to_hex_string(hasher.finalize().as_slice()); // Extract icon from the .xdc file let icon_bytes = extract_icon_from_xdc(&file_data); @@ -909,7 +910,7 @@ pub async fn publish_to_marketplace( match blossom::upload_blob_with_failover( signer.clone(), blossom_servers.clone(), - icon_data, + Arc::new(icon_data), Some(mime_type), ) .await @@ -935,7 +936,7 @@ pub async fn publish_to_marketplace( let download_url = blossom::upload_blob_with_failover( signer.clone(), blossom_servers, - file_data, + Arc::new(file_data), Some("application/octet-stream"), ) .await?; diff --git a/src-tauri/src/miniapps/state.rs b/src-tauri/src/miniapps/state.rs index 8ba72c93..6b734408 100644 --- a/src-tauri/src/miniapps/state.rs +++ b/src-tauri/src/miniapps/state.rs @@ -11,6 +11,7 @@ use tauri::ipc::Channel; use super::error::Error; use super::realtime::{RealtimeEvent, RealtimeManager}; +use crate::util::bytes_to_hex_string; /// Metadata from manifest.toml in the Mini App package #[derive(Debug, Clone, Serialize, Deserialize, Default)] @@ -54,7 +55,7 @@ impl MiniAppPackage { use sha2::{Sha256, Digest}; let mut hasher = Sha256::new(); hasher.update(&file_data); - let file_hash = hex::encode(hasher.finalize()); + let file_hash = bytes_to_hex_string(hasher.finalize().as_slice()); // Open archive from the data we already read use std::io::Cursor; diff --git a/src-tauri/src/mls/messaging.rs b/src-tauri/src/mls/messaging.rs index 3bf1662a..1218e439 100644 --- a/src-tauri/src/mls/messaging.rs +++ b/src-tauri/src/mls/messaging.rs @@ -7,6 +7,7 @@ use mdk_core::prelude::GroupId; use tauri::Emitter; use crate::{TAURI_APP, NOSTR_CLIENT, TRUSTED_RELAYS}; +use crate::util::hex_string_to_bytes; use super::{MlsService, MlsGroupMetadata}; /// Send an MLS message (rumor) to a group @@ -49,10 +50,7 @@ pub async fn send_mls_message(group_id: &str, rumor: nostr_sdk::UnsignedEvent, p let engine_group_id = if group_meta.engine_group_id.is_empty() { return Err("Group has no engine_group_id".to_string()); } else { - GroupId::from_slice( - &hex::decode(&group_meta.engine_group_id) - .map_err(|e| format!("Invalid engine_group_id hex: {}", e))? - ) + GroupId::from_slice(&hex_string_to_bytes(&group_meta.engine_group_id)) }; // Now get the MLS engine and create message (no await while engine is in scope) @@ -84,20 +82,20 @@ pub async fn send_mls_message(group_id: &str, rumor: nostr_sdk::UnsignedEvent, p // Mark pending message as failed if we have a pending_id if let Some(ref pid) = pending_id { - let mut state = crate::STATE.lock().await; - if let Some(chat) = state.chats.iter_mut().find(|c| c.id == group_id) { - if let Some(msg) = chat.messages.iter_mut().find(|m| m.id == *pid) { - msg.failed = true; - msg.pending = false; - - if let Some(handle) = TAURI_APP.get() { - handle.emit("message_update", serde_json::json!({ - "old_id": pid, - "message": msg, - "chat_id": &group_id - })).ok(); - } - } + let msg_for_emit = { + let mut state = crate::STATE.lock().await; + state.update_message_in_chat(&group_id, pid, |msg| { + msg.set_failed(true); + msg.set_pending(false); + }) + }; + + if let (Some(handle), Some(msg)) = (TAURI_APP.get(), msg_for_emit) { + handle.emit("message_update", serde_json::json!({ + "old_id": pid, + "message": msg, + "chat_id": &group_id + })).ok(); } } @@ -158,48 +156,38 @@ pub async fn send_mls_message(group_id: &str, rumor: nostr_sdk::UnsignedEvent, p match send_result { Ok(_) => { // Mark message as successfully sent and update ID - let mut state = crate::STATE.lock().await; - if let Some(chat) = state.chats.iter_mut().find(|c| c.id == group_id) { - if let Some(msg) = chat.messages.iter_mut().find(|m| m.id == *pid) { - // Update to real ID and mark as sent - msg.id = real_id.clone(); - msg.pending = false; - - // Emit update - if let Some(handle) = TAURI_APP.get() { - handle.emit("message_update", serde_json::json!({ - "old_id": pid, - "message": msg, - "chat_id": &group_id - })).ok(); - } - - // Save to database - let msg_clone = msg.clone(); - drop(state); - if let Some(handle) = TAURI_APP.get() { - let _ = crate::db::save_message(handle.clone(), &group_id, &msg_clone).await; - } + let result = { + let mut state = crate::STATE.lock().await; + state.finalize_pending_message(&group_id, pid, real_id) + }; + + if let Some((old_id, msg)) = result { + if let Some(handle) = TAURI_APP.get() { + handle.emit("message_update", serde_json::json!({ + "old_id": &old_id, + "message": &msg, + "chat_id": &group_id + })).ok(); + let _ = crate::db::save_message(handle.clone(), &group_id, &msg).await; } } } Err(e) => { // Mark message as failed (keep pending ID) - let mut state = crate::STATE.lock().await; - if let Some(chat) = state.chats.iter_mut().find(|c| c.id == group_id) { - if let Some(msg) = chat.messages.iter_mut().find(|m| m.id == *pid) { - msg.failed = true; - msg.pending = false; - - // Emit update - if let Some(handle) = TAURI_APP.get() { - handle.emit("message_update", serde_json::json!({ - "old_id": pid, - "message": msg, - "chat_id": &group_id - })).ok(); - } - } + let msg_for_emit = { + let mut state = crate::STATE.lock().await; + state.update_message_in_chat(&group_id, pid, |msg| { + msg.set_failed(true); + msg.set_pending(false); + }) + }; + + if let (Some(handle), Some(msg)) = (TAURI_APP.get(), msg_for_emit) { + handle.emit("message_update", serde_json::json!({ + "old_id": pid, + "message": msg, + "chat_id": &group_id + })).ok(); } return Err(format!("Failed to send MLS wrapper: {}", e)); } diff --git a/src-tauri/src/mls/mod.rs b/src-tauri/src/mls/mod.rs index 0418d315..4c4dfb33 100644 --- a/src-tauri/src/mls/mod.rs +++ b/src-tauri/src/mls/mod.rs @@ -6,9 +6,10 @@ use std::collections::HashMap; use mdk_core::prelude::*; use mdk_sqlite_storage::MdkSqliteStorage; use tauri::{AppHandle, Runtime, Emitter}; -use crate::{TAURI_APP, NOSTR_CLIENT, TRUSTED_RELAYS, STATE}; +use crate::{TAURI_APP, NOSTR_CLIENT, TRUSTED_RELAYS, STATE, Message}; use crate::rumor::{RumorEvent, RumorContext, ConversationType, process_rumor, RumorProcessingResult}; use crate::db::{save_chat, save_chat_messages}; +use crate::util::{bytes_to_hex_string, hex_string_to_bytes}; // Submodules mod types; @@ -305,7 +306,7 @@ impl MlsService { // GroupId is already a GroupId type in MDK (no conversion needed) let gid_bytes = create_out.group.mls_group_id.as_slice(); - let engine_gid_hex = hex::encode(gid_bytes); + let engine_gid_hex = bytes_to_hex_string(gid_bytes); // Attempt to derive wire id (wrapper 'h' tag, 64-hex) using a non-published dummy wrapper. // If unavailable, fall back to engine id. @@ -547,10 +548,7 @@ impl MlsService { .ok_or(MlsError::GroupNotFound)?; // Convert engine_group_id hex to GroupId - let mls_group_id = GroupId::from_slice( - &hex::decode(&group_meta.engine_group_id) - .map_err(|e| MlsError::CryptoError(format!("Invalid group ID hex: {}", e)))? - ); + let mls_group_id = GroupId::from_slice(&hex_string_to_bytes(&group_meta.engine_group_id)); // Perform engine operations: add member and merge commit BEFORE publishing // This ensures our local state is correct before announcing to the network @@ -647,8 +645,7 @@ impl MlsService { // Send a "leave request" application message so admins auto-remove us // This is more reliable than the MLS proposal which needs explicit commit if let Some(ref meta) = group_meta { - if let Ok(mls_group_id) = hex::decode(&meta.engine_group_id) - .map(|bytes| GroupId::from_slice(&bytes)) + let mls_group_id = GroupId::from_slice(&hex_string_to_bytes(&meta.engine_group_id)); { // Get our pubkey for building the rumor if let Ok(signer) = client.signer().await { @@ -750,10 +747,7 @@ impl MlsService { .ok_or(MlsError::GroupNotFound)?; // Convert engine_group_id hex to GroupId - let mls_group_id = GroupId::from_slice( - &hex::decode(&group_meta.engine_group_id) - .map_err(|e| MlsError::CryptoError(format!("Invalid group ID hex: {}", e)))? - ); + let mls_group_id = GroupId::from_slice(&hex_string_to_bytes(&group_meta.engine_group_id)); // Perform engine operation: remove member and merge commit BEFORE publishing // This ensures our local state is correct before announcing to the network @@ -1075,14 +1069,8 @@ impl MlsService { if let Some(ref check_id) = group_check_id { // Try to verify if the engine knows about this group // We'll attempt to create a dummy message to see if the group exists - let check_gid_bytes = match hex::decode(&check_id) { - Ok(bytes) => bytes, - Err(_) => { - eprintln!("[MLS] Invalid group_id hex for engine check: {}", check_id); - vec![] - } - }; - + let check_gid_bytes = hex_string_to_bytes(check_id); + if !check_gid_bytes.is_empty() { use nostr_sdk::prelude::*; let check_gid = GroupId::from_slice(&check_gid_bytes); @@ -1166,7 +1154,7 @@ impl MlsService { // Check if we're still a member of this group // Use group_check_id (engine's group_id) instead of gid_for_fetch (wrapper id) if let Some(ref check_id) = group_check_id { - let check_gid_bytes = hex::decode(check_id).unwrap_or_default(); + let check_gid_bytes = hex_string_to_bytes(check_id); if !check_gid_bytes.is_empty() { let check_gid = GroupId::from_slice(&check_gid_bytes); let my_pk = nostr_sdk::PublicKey::from_hex(&my_pubkey_hex).ok(); @@ -1251,13 +1239,21 @@ impl MlsService { // Log details to help diagnose. Note: ev.pubkey is ephemeral, not real sender. println!("[MLS] Unprocessable event: group={}, mls_gid={}, id={}, created_at={}", gid_for_fetch, - hex::encode(mls_group_id.as_slice()), + bytes_to_hex_string(mls_group_id.as_slice()), ev.id.to_hex(), ev.created_at.as_secs()); // Queue for retry - might succeed after other commits are processed pending_retry.push(ev.clone()); } + MessageProcessingResult::PreviouslyFailed => { + // MDK detected this event previously failed processing - + // skip without retrying to avoid repeated failures + processed = processed.saturating_add(1); + last_seen_id = Some(ev.id); + last_seen_at = ev.created_at.as_secs(); + events_to_track.push((ev.id.to_hex(), ev.created_at.as_secs())); + } } } Err(e) => { @@ -1366,7 +1362,7 @@ impl MlsService { MessageProcessingResult::Commit { mls_group_id: _ } => { // Check membership after commit if let Some(ref check_id) = group_check_id { - let check_gid_bytes = hex::decode(check_id).unwrap_or_default(); + let check_gid_bytes = hex_string_to_bytes(check_id); if !check_gid_bytes.is_empty() { let check_gid = GroupId::from_slice(&check_gid_bytes); let my_pk = nostr_sdk::PublicKey::from_hex(&my_pubkey_hex).ok(); @@ -1445,9 +1441,13 @@ impl MlsService { MessageProcessingResult::Unprocessable { mls_group_id } => { // Still can't process - keep for next retry round println!("[MLS] ✗ Retry still unprocessable: id={}, mls_gid={}", - ev.id.to_hex(), hex::encode(mls_group_id.as_slice())); + ev.id.to_hex(), bytes_to_hex_string(mls_group_id.as_slice())); still_pending.push(ev.clone()); } + MessageProcessingResult::PreviouslyFailed => { + // Previously failed - don't retry + println!("[MLS] ✗ Retry skipped (previously failed): id={}", ev.id.to_hex()); + } } } Err(e) => { @@ -1592,21 +1592,12 @@ impl MlsService { // Reactions now work with unified storage! let (was_added, chat_id_for_save) = { let mut state = STATE.lock().await; - let added = if let Some((chat_id, msg)) = state.find_chat_and_message_mut(&reaction.reference_id) { - msg.add_reaction(reaction.clone(), Some(chat_id)) - } else { - false - }; - - // Get chat_id for saving if reaction was added - let chat_id_for_save = if added { - state.find_message(&reaction.reference_id) - .map(|(chat, _)| chat.id().clone()) + // Use helper that handles interner access via split borrowing + if let Some((chat_id, added)) = state.add_reaction_to_message(&reaction.reference_id, reaction.clone()) { + (added, if added { Some(chat_id) } else { None }) } else { - None - }; - - (added, chat_id_for_save) + (false, None) + } }; // Save the updated message to database immediately (like DM reactions) @@ -1781,15 +1772,19 @@ impl MlsService { // Update message in state and emit to frontend let mut state = STATE.lock().await; - if let Some(chat) = state.get_chat_mut(&chat_id) { - if let Some(msg) = chat.get_message_mut(&message_id) { + let chat_idx = state.chats.iter().position(|c| c.id == chat_id); + if let Some(idx) = chat_idx { + // Mutate the message + if let Some(msg) = state.chats[idx].get_message_mut(&message_id) { msg.apply_edit(new_content, edited_at); - - // Emit update to frontend + } + // Convert to Message for emit + if let Some(msg) = state.chats[idx].get_compact_message(&message_id) { + let msg_for_emit = msg.to_message(&state.interner); if let Some(handle) = TAURI_APP.get() { let _ = handle.emit("message_update", serde_json::json!({ "old_id": &message_id, - "message": &msg, + "message": msg_for_emit, "chat_id": &chat_id })); } @@ -1814,12 +1809,16 @@ impl MlsService { // Only save the newly added messages (much more efficient!) // Get the last N messages where N = number of new messages processed if new_msgs > 0 { - let messages_to_save: Vec<_> = chat.messages.iter() - .rev() - .take(new_msgs as usize) - .cloned() - .collect(); - + // Need to get interner from state to convert CompactMessage to Message + let messages_to_save: Vec = { + let state = STATE.lock().await; + chat.messages.iter() + .rev() + .take(new_msgs as usize) + .map(|m| m.to_message(&state.interner)) + .collect() + }; + if !messages_to_save.is_empty() { let _ = save_chat_messages(handle.clone(), &chat_id, &messages_to_save).await; } diff --git a/src-tauri/src/pivx.rs b/src-tauri/src/pivx.rs index e9cd58c1..77a17c0e 100644 --- a/src-tauri/src/pivx.rs +++ b/src-tauri/src/pivx.rs @@ -9,6 +9,7 @@ use serde::{Deserialize, Serialize}; use sha2::{Sha256, Digest}; use ripemd::Ripemd160; +use crate::util::{bytes_to_hex_string, hex_string_to_bytes}; use secp256k1::{Secp256k1, SecretKey, PublicKey, Message}; use tauri::{AppHandle, Runtime, Emitter}; use once_cell::sync::Lazy; @@ -513,8 +514,7 @@ pub async fn build_sweep_transaction( // Build inputs (initially unsigned) for utxo in utxos { // Previous txid (32 bytes, reversed) - let txid_bytes = hex::decode(&utxo.txid) - .map_err(|_| format!("Invalid txid: {}", utxo.txid))?; + let txid_bytes = hex_string_to_bytes(&utxo.txid); tx_bytes.extend(txid_bytes.iter().rev()); // Previous output index (4 bytes, little-endian) @@ -558,8 +558,7 @@ pub async fn build_sweep_transaction( preimage.push(utxos.len() as u8); // Input count for (j, other_utxo) in utxos.iter().enumerate() { - let other_txid = hex::decode(&other_utxo.txid) - .map_err(|e| format!("Invalid txid hex '{}': {}", &other_utxo.txid, e))?; + let other_txid = hex_string_to_bytes(&other_utxo.txid); preimage.extend(other_txid.iter().rev()); preimage.extend_from_slice(&other_utxo.vout.to_le_bytes()); @@ -602,8 +601,7 @@ pub async fn build_sweep_transaction( let script_sig_len = 1 + sig_bytes.len() + 1 + pubkey_bytes.len(); // Write input to signed tx - let txid_bytes = hex::decode(&utxo.txid) - .map_err(|e| format!("Invalid txid hex '{}': {}", &utxo.txid, e))?; + let txid_bytes = hex_string_to_bytes(&utxo.txid); signed_tx.extend(txid_bytes.iter().rev()); signed_tx.extend_from_slice(&utxo.vout.to_le_bytes()); @@ -626,7 +624,7 @@ pub async fn build_sweep_transaction( // Locktime signed_tx.extend_from_slice(&0u32.to_le_bytes()); - Ok(hex::encode(&signed_tx)) + Ok(bytes_to_hex_string(&signed_tx)) } // ============================================================================ @@ -652,7 +650,7 @@ async fn save_promo( privkey: &[u8; 32], ) -> Result<(), String> { // Encrypt private key - let privkey_hex = hex::encode(privkey); + let privkey_hex = bytes_to_hex_string(privkey); let privkey_encrypted = crate::crypto::internal_encrypt(privkey_hex, None).await; let created_at = std::time::SystemTime::now() @@ -677,10 +675,9 @@ async fn decrypt_privkey_bytes(privkey_encrypted: String) -> Result<[u8; 32], St .await .map_err(|_| "Failed to decrypt private key".to_string())?; - hex::decode(&privkey_hex) - .map_err(|_| "Invalid private key format".to_string())? + hex_string_to_bytes(&privkey_hex) .try_into() - .map_err(|_| "Invalid private key length".to_string()) + .map_err(|_| "Invalid private key format/length".to_string()) } /// Promo with decrypted private key ready for signing @@ -1678,8 +1675,7 @@ async fn build_withdrawal_transaction( preimage.push(all_inputs.len() as u8); // Input count for (j, (other_utxo, _, other_script)) in all_inputs.iter().enumerate() { - let other_txid = hex::decode(&other_utxo.txid) - .map_err(|_| format!("Invalid txid: {}", other_utxo.txid))?; + let other_txid = hex_string_to_bytes(&other_utxo.txid); preimage.extend(other_txid.iter().rev()); preimage.extend_from_slice(&other_utxo.vout.to_le_bytes()); @@ -1733,8 +1729,7 @@ async fn build_withdrawal_transaction( let script_sig_len = 1 + sig_bytes.len() + 1 + pubkey_bytes.len(); // Write input to signed tx - let txid_bytes = hex::decode(&utxo.txid) - .map_err(|_| format!("Invalid txid: {}", utxo.txid))?; + let txid_bytes = hex_string_to_bytes(&utxo.txid); signed_tx.extend(txid_bytes.iter().rev()); signed_tx.extend_from_slice(&utxo.vout.to_le_bytes()); @@ -1768,7 +1763,7 @@ async fn build_withdrawal_transaction( // Locktime signed_tx.extend_from_slice(&0u32.to_le_bytes()); - Ok((hex::encode(signed_tx), change_amount)) + Ok((bytes_to_hex_string(&signed_tx), change_amount)) } /// Withdraw PIVX to an external address using coin control diff --git a/src-tauri/src/profile.rs b/src-tauri/src/profile.rs index 636cb83f..c4007f5c 100644 --- a/src-tauri/src/profile.rs +++ b/src-tauri/src/profile.rs @@ -1,6 +1,8 @@ use nostr_sdk::prelude::*; use tauri::Emitter; +#[cfg(not(target_os = "android"))] +use std::sync::Arc; #[cfg(not(target_os = "android"))] use tauri_plugin_fs::FsExt; @@ -642,7 +644,7 @@ pub async fn upload_avatar(filepath: String, upload_type: Option) -> Res .to_lowercase(); AttachmentFile { - bytes, + bytes: Arc::new(bytes), img_meta: None, extension, } diff --git a/src-tauri/src/profile_sync.rs b/src-tauri/src/profile_sync.rs index 8603a998..f0ba3d28 100644 --- a/src-tauri/src/profile_sync.rs +++ b/src-tauri/src/profile_sync.rs @@ -1,8 +1,7 @@ use std::collections::{HashMap, HashSet, VecDeque}; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use lazy_static::lazy_static; -use tokio::sync::Mutex; use crate::{profile, STATE}; /// Priority levels for profile syncing @@ -198,39 +197,42 @@ pub async fn start_profile_sync_processor() { let npub = own_profile.id.clone(); drop(state); - let mut queue = PROFILE_SYNC_QUEUE.lock().await; + let mut queue = PROFILE_SYNC_QUEUE.lock().unwrap(); queue.add(npub.clone(), SyncPriority::Low, false); drop(queue); } last_own_profile_sync = std::time::Instant::now(); } - // Check if we should process - let batch = { - let mut queue = PROFILE_SYNC_QUEUE.lock().await; - + // Check if we should process (lock scoped to avoid holding across await) + let (should_wait, batch) = { + let mut queue = PROFILE_SYNC_QUEUE.lock().unwrap(); + // Prevent multiple processors if queue.is_processing { - drop(queue); - tokio::time::sleep(Duration::from_secs(1)).await; - continue; - } - - queue.is_processing = true; - let batch = queue.get_next_batch(); - - // Mark all as processing - for entry in &batch { - queue.mark_processing(&entry.npub); + (true, vec![]) + } else { + queue.is_processing = true; + let batch = queue.get_next_batch(); + + // Mark all as processing + for entry in &batch { + queue.mark_processing(&entry.npub); + } + + (false, batch) } - - batch }; + if should_wait { + tokio::time::sleep(Duration::from_secs(1)).await; + continue; + } + if batch.is_empty() { // No work to do, release lock and sleep { - let mut queue = PROFILE_SYNC_QUEUE.lock().await; + let mut queue = PROFILE_SYNC_QUEUE.lock().unwrap(); queue.is_processing = false; } tokio::time::sleep(Duration::from_secs(1)).await; @@ -244,7 +246,7 @@ pub async fn start_profile_sync_processor() { // Mark as done { - let mut queue = PROFILE_SYNC_QUEUE.lock().await; + let mut queue = PROFILE_SYNC_QUEUE.lock().unwrap(); queue.mark_done(&entry.npub); } @@ -254,7 +256,7 @@ pub async fn start_profile_sync_processor() { // Release processing lock { - let mut queue = PROFILE_SYNC_QUEUE.lock().await; + let mut queue = PROFILE_SYNC_QUEUE.lock().unwrap(); queue.is_processing = false; } @@ -264,9 +266,9 @@ pub async fn start_profile_sync_processor() { } /// Queue a single profile for syncing -pub async fn queue_profile_sync(npub: String, priority: SyncPriority, force_refresh: bool) { - let mut queue = PROFILE_SYNC_QUEUE.lock().await; - queue.add(npub.clone(), priority, force_refresh); +pub fn queue_profile_sync(npub: String, priority: SyncPriority, force_refresh: bool) { + let mut queue = PROFILE_SYNC_QUEUE.lock().unwrap(); + queue.add(npub, priority, force_refresh); } /// Queue all profiles for a chat @@ -313,7 +315,7 @@ pub async fn queue_chat_profiles(chat_id: String, is_opening: bool) { drop(state); // Release state lock before queuing // Queue all profiles - let mut queue = PROFILE_SYNC_QUEUE.lock().await; + let mut queue = PROFILE_SYNC_QUEUE.lock().unwrap(); for (npub, priority) in profiles_to_queue { queue.add(npub.to_string(), priority, false); @@ -321,9 +323,9 @@ pub async fn queue_chat_profiles(chat_id: String, is_opening: bool) { } /// Force immediate refresh of a profile (for user clicks) -pub async fn refresh_profile_now(npub: String) { - let mut queue = PROFILE_SYNC_QUEUE.lock().await; - queue.add(npub.clone(), SyncPriority::Critical, true); +pub fn refresh_profile_now(npub: String) { + let mut queue = PROFILE_SYNC_QUEUE.lock().unwrap(); + queue.add(npub, SyncPriority::Critical, true); } /// Sync all profiles in the system (replaces old fetchProfiles) @@ -350,7 +352,7 @@ pub async fn sync_all_profiles() { drop(state); // Release state lock // Queue all profiles - let mut queue = PROFILE_SYNC_QUEUE.lock().await; + let mut queue = PROFILE_SYNC_QUEUE.lock().unwrap(); for (npub, priority) in profiles_to_queue { queue.add(npub, priority, false); diff --git a/src-tauri/src/rumor.rs b/src-tauri/src/rumor.rs index 4e0ed048..76cbcd43 100644 --- a/src-tauri/src/rumor.rs +++ b/src-tauri/src/rumor.rs @@ -30,6 +30,7 @@ use nostr_sdk::prelude::*; use tauri::Manager; use crate::{Message, Attachment, Reaction, TAURI_APP, StoredEvent, StoredEventBuilder}; use crate::message::ImageMetadata; +use crate::util::{bytes_to_hex_string, hex_string_to_bytes}; use crate::mls::MlsService; /// Protocol-agnostic rumor event representation @@ -349,13 +350,7 @@ async fn parse_mls_imeta_attachments( } // Parse the engine group ID - let engine_gid_bytes = match hex::decode(&group_meta.engine_group_id) { - Ok(b) => b, - Err(e) => { - eprintln!("[MIP-04] Invalid engine_group_id hex: {}", e); - return Vec::new(); - } - }; + let engine_gid_bytes = hex_string_to_bytes(&group_meta.engine_group_id); let gid = mdk_core::GroupId::from_slice(&engine_gid_bytes); // Get MDK engine and media manager @@ -399,7 +394,7 @@ async fn parse_mls_imeta_attachments( // Extract hash from URL (Blossom URLs typically have hash as the path) let encrypted_hash = extract_hash_from_blossom_url(&media_ref.url) - .unwrap_or_else(|| hex::encode(media_ref.original_hash)); + .unwrap_or_else(|| bytes_to_hex_string(&media_ref.original_hash)); // Build image metadata from dimensions if available let img_meta = media_ref.dimensions.and_then(|(width, height)| { @@ -451,7 +446,7 @@ async fn parse_mls_imeta_attachments( attachments.push(Attachment { id: encrypted_hash, key: String::new(), // MLS uses derived keys - nonce: hex::encode(media_ref.nonce), + nonce: bytes_to_hex_string(&media_ref.nonce), extension, url: media_ref.url.clone(), path: file_path, @@ -461,7 +456,7 @@ async fn parse_mls_imeta_attachments( downloaded, webxdc_topic: None, // Set by caller if present group_id: Some(context.conversation_id.clone()), - original_hash: Some(hex::encode(media_ref.original_hash)), + original_hash: Some(bytes_to_hex_string(&media_ref.original_hash)), scheme_version: Some(media_ref.scheme_version.clone()), mls_filename: Some(media_ref.filename.clone()), }); diff --git a/src-tauri/src/services/event_handler.rs b/src-tauri/src/services/event_handler.rs index ddea9a5c..2675c010 100644 --- a/src-tauri/src/services/event_handler.rs +++ b/src-tauri/src/services/event_handler.rs @@ -23,8 +23,10 @@ use crate::{ // Not exposed as a Tauri command to frontend pub(crate) async fn handle_event(event: Event, is_new: bool) -> bool { // Get the wrapper (giftwrap) event ID for duplicate detection + // Use bytes for cache (memory efficient), hex string for DB operations + let wrapper_event_id_bytes: [u8; 32] = event.id.to_bytes(); let wrapper_event_id = event.id.to_hex(); - + // For historical sync events (is_new = false), use the wrapper_id cache for fast duplicate detection // For real-time new events (is_new = true), skip cache checks - they're guaranteed to be new if !is_new { @@ -32,7 +34,7 @@ pub(crate) async fn handle_event(event: Event, is_new: bool) -> bool { // This cache is only populated during init and cleared after sync finishes { let cache = WRAPPER_ID_CACHE.lock().await; - if cache.contains(&wrapper_event_id) { + if cache.contains(&wrapper_event_id_bytes) { // Already processed this giftwrap, skip (cache hit) return false; } @@ -95,7 +97,7 @@ pub(crate) async fn handle_event(event: Event, is_new: bool) -> bool { // Dedup: the same welcome can arrive from multiple relays simultaneously { let cache = WRAPPER_ID_CACHE.lock().await; - if cache.contains(&wrapper_event_id) { + if cache.contains(&wrapper_event_id_bytes) { return false; } } @@ -137,7 +139,7 @@ pub(crate) async fn handle_event(event: Event, is_new: bool) -> bool { // Mark this wrapper as processed to prevent duplicates from other relays { let mut cache = WRAPPER_ID_CACHE.lock().await; - cache.insert(wrapper_event_id.clone()); + cache.insert(wrapper_event_id_bytes); } // Only notify UI after initial sync is complete // During initial sync, invites are processed but not emitted to avoid UI updates before chats are loaded @@ -164,11 +166,12 @@ pub(crate) async fn handle_event(event: Event, is_new: bool) -> bool { } // Convert rumor to RumorEvent for protocol-agnostic processing + // Move content and tags instead of cloning (rumor is owned and not used after this) let rumor_event = RumorEvent { id: rumor.id.unwrap(), kind: rumor.kind, - content: rumor.content.clone(), - tags: rumor.tags.clone(), + content: rumor.content, + tags: rumor.tags, created_at: rumor.created_at, pubkey: rumor.pubkey, }; @@ -187,12 +190,12 @@ pub(crate) async fn handle_event(event: Event, is_new: bool) -> bool { RumorProcessingResult::TextMessage(mut msg) => { // Set the wrapper event ID for database storage msg.wrapper_event_id = Some(wrapper_event_id.clone()); - handle_text_message(msg, &contact, is_mine, is_new, &wrapper_event_id).await + handle_text_message(msg, &contact, is_mine, is_new, &wrapper_event_id, wrapper_event_id_bytes).await } RumorProcessingResult::FileAttachment(mut msg) => { // Set the wrapper event ID for database storage msg.wrapper_event_id = Some(wrapper_event_id.clone()); - handle_file_attachment(msg, &contact, is_mine, is_new, &wrapper_event_id).await + handle_file_attachment(msg, &contact, is_mine, is_new, &wrapper_event_id, wrapper_event_id_bytes).await } RumorProcessingResult::Reaction(reaction) => { handle_reaction(reaction, &contact).await @@ -270,19 +273,20 @@ pub(crate) async fn handle_event(event: Event, is_new: bool) -> bool { } // Update message in state and emit to frontend - let mut state = STATE.lock().await; - if let Some(chat) = state.get_chat_mut(&contact) { - if let Some(msg) = chat.get_message_mut(&message_id) { + let msg_for_emit = { + let mut state = STATE.lock().await; + state.update_message_in_chat(&contact, &message_id, |msg| { msg.apply_edit(new_content, edited_at); + }) + }; - // Emit update to frontend - if let Some(handle) = TAURI_APP.get() { - let _ = handle.emit("message_update", serde_json::json!({ - "old_id": &message_id, - "message": &msg, - "chat_id": &contact - })); - } + if let Some(msg) = msg_for_emit { + if let Some(handle) = TAURI_APP.get() { + let _ = handle.emit("message_update", serde_json::json!({ + "old_id": &message_id, + "message": msg, + "chat_id": &contact + })); } } true @@ -300,7 +304,7 @@ pub(crate) async fn handle_event(event: Event, is_new: bool) -> bool { } /// Handle a processed text message -async fn handle_text_message(mut msg: Message, contact: &str, is_mine: bool, is_new: bool, wrapper_event_id: &str) -> bool { +async fn handle_text_message(mut msg: Message, contact: &str, is_mine: bool, is_new: bool, wrapper_event_id: &str, wrapper_event_id_bytes: [u8; 32]) -> bool { // Check if message already exists in database (important for sync with partial message loading) if let Some(handle) = TAURI_APP.get() { if let Ok(exists) = db::message_exists_in_db(&handle, &msg.id).await { @@ -313,7 +317,7 @@ async fn handle_text_message(mut msg: Message, contact: &str, is_mine: bool, is_ if !updated { // Message has a different wrapper_id - add this duplicate wrapper to cache let mut cache = WRAPPER_ID_CACHE.lock().await; - cache.insert(wrapper_event_id.to_string()); + cache.insert(wrapper_event_id_bytes); } } return false; @@ -387,7 +391,7 @@ async fn handle_text_message(mut msg: Message, contact: &str, is_mine: bool, is_ } /// Handle a processed file attachment -async fn handle_file_attachment(mut msg: Message, contact: &str, is_mine: bool, is_new: bool, wrapper_event_id: &str) -> bool { +async fn handle_file_attachment(mut msg: Message, contact: &str, is_mine: bool, is_new: bool, wrapper_event_id: &str, wrapper_event_id_bytes: [u8; 32]) -> bool { // Check if message already exists in database (important for sync with partial message loading) if let Some(handle) = TAURI_APP.get() { if let Ok(exists) = db::message_exists_in_db(&handle, &msg.id).await { @@ -400,7 +404,7 @@ async fn handle_file_attachment(mut msg: Message, contact: &str, is_mine: bool, if !updated { // Message has a different wrapper_id - add this duplicate wrapper to cache let mut cache = WRAPPER_ID_CACHE.lock().await; - cache.insert(wrapper_event_id.to_string()); + cache.insert(wrapper_event_id_bytes); } } return false; @@ -495,23 +499,14 @@ async fn handle_reaction(reaction: Reaction, _contact: &str) -> bool { // Use a single lock scope to avoid nested locks let (reaction_added, chat_id_for_save) = { let mut state = STATE.lock().await; - let reaction_added = if let Some((chat_id, msg_mut)) = state.find_chat_and_message_mut(&reaction.reference_id) { - msg_mut.add_reaction(reaction.clone(), Some(chat_id)) + // Use helper that handles interner access via split borrowing + if let Some((chat_id, added)) = state.add_reaction_to_message(&reaction.reference_id, reaction.clone()) { + (added, if added { Some(chat_id) } else { None }) } else { // Message not found in any chat - this can happen during sync // TODO: track these "ahead" reactions and re-apply them once sync has finished - false - }; - - // If reaction was added, get the chat_id for saving - let chat_id_for_save = if reaction_added { - state.find_message(&reaction.reference_id) - .map(|(chat, _)| chat.id().clone()) - } else { - None - }; - - (reaction_added, chat_id_for_save) + (false, None) + } }; // Save the updated message with the new reaction to our DB (outside of state lock) @@ -526,6 +521,11 @@ async fn handle_reaction(reaction: Reaction, _contact: &str) -> bool { if let Some(msg) = updated_message { let _ = db::save_message(handle.clone(), &chat_id, &msg).await; + let _ = handle.emit("message_update", serde_json::json!({ + "old_id": &reaction.reference_id, + "message": &msg, + "chat_id": &chat_id + })); } } } diff --git a/src-tauri/src/services/subscription_handler.rs b/src-tauri/src/services/subscription_handler.rs index 07a116be..95e163ed 100644 --- a/src-tauri/src/services/subscription_handler.rs +++ b/src-tauri/src/services/subscription_handler.rs @@ -247,16 +247,23 @@ pub(crate) async fn start_subscriptions() -> Result { // Save to database if message was added if was_added { if let Some(handle) = TAURI_APP.get() { - // Get chat and save it - let chat_to_save = { + // Get chat and convert messages with interner access + let (chat_to_save, messages_to_save) = { let state = crate::STATE.lock().await; - state.get_chat(&group_id_for_persist).cloned() + if let Some(chat) = state.get_chat(&group_id_for_persist) { + let msgs: Vec = chat.messages.iter() + .map(|m| m.to_message(&state.interner)) + .collect(); + (Some(chat.clone()), msgs) + } else { + (None, vec![]) + } }; - + if let Some(chat) = chat_to_save { use crate::db::{save_chat, save_chat_messages}; let _ = save_chat(handle.clone(), &chat).await; - let _ = save_chat_messages(handle.clone(), &group_id_for_persist, &chat.messages).await; + let _ = save_chat_messages(handle.clone(), &group_id_for_persist, &messages_to_save).await; } } Some(message) @@ -365,21 +372,12 @@ pub(crate) async fn start_subscriptions() -> Result { // Handle reactions in real-time let (was_added, chat_id_for_save) = { let mut state = crate::STATE.lock().await; - let added = if let Some((chat_id, msg)) = state.find_chat_and_message_mut(&reaction.reference_id) { - msg.add_reaction(reaction.clone(), Some(chat_id)) - } else { - false - }; - - // Get chat_id for saving if reaction was added - let chat_id_for_save = if added { - state.find_message(&reaction.reference_id) - .map(|(chat, _)| chat.id().clone()) + // Use helper that handles interner access via split borrowing + if let Some((chat_id, added)) = state.add_reaction_to_message(&reaction.reference_id, reaction.clone()) { + (added, if added { Some(chat_id) } else { None }) } else { - None - }; - - (added, chat_id_for_save) + (false, None) + } }; // Save the updated message to database immediately (like DM reactions) @@ -394,11 +392,16 @@ pub(crate) async fn start_subscriptions() -> Result { if let Some(msg) = updated_message { let _ = db::save_message(handle.clone(), &chat_id, &msg).await; + let _ = handle.emit("message_update", serde_json::json!({ + "old_id": &reaction.reference_id, + "message": &msg, + "chat_id": &chat_id + })); } } } } - + None // Don't emit as message } RumorProcessingResult::TypingIndicator { profile_id, until } => { @@ -572,19 +575,20 @@ pub(crate) async fn start_subscriptions() -> Result { } // Update message in state and emit to frontend - let mut state = crate::STATE.lock().await; - if let Some(chat) = state.get_chat_mut(&group_id_for_persist) { - if let Some(msg) = chat.get_message_mut(&message_id) { + let msg_for_emit = { + let mut state = crate::STATE.lock().await; + state.update_message_in_chat(&group_id_for_persist, &message_id, |msg| { msg.apply_edit(new_content, edited_at); + }) + }; - // Emit update to frontend - if let Some(handle) = TAURI_APP.get() { - let _ = handle.emit("message_update", serde_json::json!({ - "old_id": &message_id, - "message": &msg, - "chat_id": &group_id_for_persist - })); - } + if let Some(msg) = msg_for_emit { + if let Some(handle) = TAURI_APP.get() { + let _ = handle.emit("message_update", serde_json::json!({ + "old_id": &message_id, + "message": &msg, + "chat_id": &group_id_for_persist + })); } } None // Don't emit as message diff --git a/src-tauri/src/shared/image.rs b/src-tauri/src/shared/image.rs index 8e7e9322..f8efc7f7 100644 --- a/src-tauri/src/shared/image.rs +++ b/src-tauri/src/shared/image.rs @@ -9,11 +9,18 @@ use image::codecs::jpeg::JpegEncoder; use image::ImageEncoder; use std::io::Cursor; +/// Maximum dimension for image compression (1920px on longest side) +pub const MAX_DIMENSION: u32 = 1920; + /// Default JPEG quality for standard compression (0-100) pub const JPEG_QUALITY_STANDARD: u8 = 85; /// JPEG quality for higher compression (smaller files) pub const JPEG_QUALITY_COMPRESSED: u8 = 70; /// JPEG quality for UI previews (fast encoding, small size) +/// Mobile uses lower quality (25) since screens are smaller - faster encode + smaller base64 +#[cfg(target_os = "android")] +pub const JPEG_QUALITY_PREVIEW: u8 = 25; +#[cfg(not(target_os = "android"))] pub const JPEG_QUALITY_PREVIEW: u8 = 50; /// Result of image encoding with format metadata @@ -24,6 +31,32 @@ pub struct EncodedImage { pub extension: &'static str, } +impl EncodedImage { + /// Convert to a base64 data URI (e.g., "data:image/png;base64,...") + /// + /// Pre-allocates exact capacity and encodes directly into the result string, + /// avoiding an intermediate base64 string allocation. + #[inline] + pub fn to_data_uri(&self) -> String { + use base64::Engine; + + let prefix = if self.extension == "png" { + "data:image/png;base64," + } else { + "data:image/jpeg;base64," + }; + + // Base64 output is 4/3 input size, rounded up to nearest 4 (padding) + let base64_len = (self.bytes.len() + 2) / 3 * 4; + let mut result = String::with_capacity(prefix.len() + base64_len); + + result.push_str(prefix); + base64::engine::general_purpose::STANDARD.encode_string(&self.bytes, &mut result); + + result + } +} + /// Minimum dimension threshold for JPEG encoding. /// Images smaller than this (in both width AND height) use PNG to avoid artifacts. /// This preserves quality for pixel art and small icons. @@ -61,16 +94,11 @@ pub fn encode_png(pixels: &[u8], width: u32, height: u32) -> Result, Str /// Convert RGBA pixel data to RGB by dropping the alpha channel. /// -/// This is more efficient than going through DynamicImage when you already -/// have raw RGBA bytes - avoids an extra buffer clone. +/// Convert RGBA to RGB, stripping the alpha channel. +/// Uses SIMD acceleration on ARM64 (NEON vld4/vst3). #[inline] fn rgba_to_rgb(rgba: &[u8]) -> Vec { - let pixel_count = rgba.len() / 4; - let mut rgb = Vec::with_capacity(pixel_count * 3); - for chunk in rgba.chunks_exact(4) { - rgb.extend_from_slice(&chunk[..3]); - } - rgb + crate::simd::image::rgba_to_rgb(rgba) } /// Encode RGB pixel data as JPEG with specified quality. @@ -214,3 +242,107 @@ pub fn encode_rgba_auto(pixels: &[u8], width: u32, height: u32, jpeg_quality: u8 }) } } + +/// Calculate target dimensions to fit within max_dimension while preserving aspect ratio. +/// +/// Returns the original dimensions if both are already within the limit. +/// Otherwise, scales down proportionally so the longest side equals max_dimension. +/// +/// # Arguments +/// * `width` - Original image width +/// * `height` - Original image height +/// * `max_dimension` - Maximum allowed size for either dimension +/// +/// # Returns +/// Tuple of (new_width, new_height) +#[inline] +pub fn calculate_resize_dimensions(width: u32, height: u32, max_dimension: u32) -> (u32, u32) { + if width <= max_dimension && height <= max_dimension { + (width, height) + } else if width > height { + let ratio = max_dimension as f32 / width as f32; + (max_dimension, (height as f32 * ratio) as u32) + } else { + let ratio = max_dimension as f32 / height as f32; + ((width as f32 * ratio) as u32, max_dimension) + } +} + +/// Calculate preview dimensions based on a quality percentage. +/// +/// # Arguments +/// * `width` - Original image width +/// * `height` - Original image height +/// * `quality` - Percentage (1-100) of original size +/// +/// # Returns +/// Tuple of (new_width, new_height), both at least 1 +#[inline] +pub fn calculate_preview_dimensions(width: u32, height: u32, quality: u32) -> (u32, u32) { + let quality = quality.clamp(1, 100); + ( + ((width * quality) / 100).max(1), + ((height * quality) / 100).max(1), + ) +} + +/// Maximum preview dimensions for UI display +/// Mobile: 300x400 (chat bubbles are small) +/// Desktop: 512x512 (larger display area) +#[cfg(target_os = "android")] +pub const PREVIEW_MAX_WIDTH: u32 = 300; +#[cfg(target_os = "android")] +pub const PREVIEW_MAX_HEIGHT: u32 = 400; + +#[cfg(not(target_os = "android"))] +pub const PREVIEW_MAX_WIDTH: u32 = 800; +#[cfg(not(target_os = "android"))] +pub const PREVIEW_MAX_HEIGHT: u32 = 800; + +/// Calculate preview dimensions capped to UI display size. +/// +/// Only downscales, never upscales: +/// - Large photos are scaled to fit within max bounds +/// - Small photos keep original dimensions +/// +/// Maintains aspect ratio. +#[inline] +pub fn calculate_capped_preview_dimensions(width: u32, height: u32) -> (u32, u32) { + // If already smaller than max, keep original (never upscale) + if width <= PREVIEW_MAX_WIDTH && height <= PREVIEW_MAX_HEIGHT { + return (width, height); + } + + // Scale down to fit within bounds while maintaining aspect ratio + let width_ratio = PREVIEW_MAX_WIDTH as f32 / width as f32; + let height_ratio = PREVIEW_MAX_HEIGHT as f32 / height as f32; + // Use smaller ratio to fit within both bounds, cap at 1.0 to never upscale + let ratio = width_ratio.min(height_ratio).min(1.0); + + ( + ((width as f32 * ratio) as u32).max(1), + ((height as f32 * ratio) as u32).max(1), + ) +} + +/// Read a file, checking if it's empty via metadata first to avoid reading 0 bytes. +/// +/// This is more efficient than reading then checking length, especially for +/// large files that would waste I/O bandwidth on empty file detection. +/// +/// # Arguments +/// * `path` - Path to the file +/// +/// # Returns +/// File bytes or an error string +pub fn read_file_checked(path: &str) -> Result, String> { + let metadata = std::fs::metadata(path) + .map_err(|e| format!("Failed to read file metadata: {}", e))?; + + if metadata.len() == 0 { + return Err(format!("File is empty (0 bytes): {}", path)); + } + + std::fs::read(path) + .map_err(|e| format!("Failed to read file: {}", e)) +} diff --git a/src-tauri/src/simd/hex.rs b/src-tauri/src/simd/hex.rs new file mode 100644 index 00000000..0381bff6 --- /dev/null +++ b/src-tauri/src/simd/hex.rs @@ -0,0 +1,992 @@ +//! SIMD-accelerated hex encoding and decoding +//! +//! # Performance +//! +//! **Encoding (32 bytes → 64 hex chars):** +//! - `format!()`: ~1630 ns +//! - Scalar LUT: ~35 ns (47x faster) +//! - NEON SIMD (ARM64): ~26 ns (62x faster) +//! - SSE2 SIMD (x86_64): ~30 ns (estimated) +//! - AVX2 SIMD (x86_64): ~25 ns (estimated) +//! +//! **Decoding (64 hex chars → 32 bytes):** +//! - NEON SIMD (ARM64): ~3 ns (7x faster than LUT) +//! - SSE2 SIMD (x86_64): ~5 ns (estimated) +//! - Scalar LUT fallback: ~19 ns +//! +//! # Algorithm +//! +//! **NEON (ARM64):** Uses TBL instruction for 16-byte lookup table +//! +//! **SSE2/AVX2 (x86_64):** Uses arithmetic approach: +//! 1. Split bytes into nibbles (high = byte >> 4, low = byte & 0x0F) +//! 2. Compare nibbles > 9 to identify hex letters (a-f) +//! 3. Add '0' (0x30) to all, then add 0x27 for letters (a-f) +//! 4. Interleave and store + +#[cfg(target_arch = "aarch64")] +use std::arch::aarch64::*; + +#[cfg(target_arch = "x86_64")] +use std::arch::x86_64::*; + +// ============================================================================ +// Lookup Tables +// ============================================================================ + +/// Nibble-to-hex lookup table for NEON SIMD (16 bytes fits in one register). +#[cfg(target_arch = "aarch64")] +const HEX_NIBBLE: &[u8; 16] = b"0123456789abcdef"; + +/// Lookup table for scalar hex encoding (non-SIMD platforms). +/// Each byte maps to its 2-char hex representation. +#[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))] +const HEX_ENCODE_LUT: &[u8; 512] = b"000102030405060708090a0b0c0d0e0f\ +101112131415161718191a1b1c1d1e1f\ +202122232425262728292a2b2c2d2e2f\ +303132333435363738393a3b3c3d3e3f\ +404142434445464748494a4b4c4d4e4f\ +505152535455565758595a5b5c5d5e5f\ +606162636465666768696a6b6c6d6e6f\ +707172737475767778797a7b7c7d7e7f\ +808182838485868788898a8b8c8d8e8f\ +909192939495969798999a9b9c9d9e9f\ +a0a1a2a3a4a5a6a7a8a9aaabacadaeaf\ +b0b1b2b3b4b5b6b7b8b9babbbcbdbebf\ +c0c1c2c3c4c5c6c7c8c9cacbcccdcecf\ +d0d1d2d3d4d5d6d7d8d9dadbdcdddedf\ +e0e1e2e3e4e5e6e7e8e9eaebecedeeef\ +f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff"; + +/// Compile-time lookup table for hex character to nibble conversion. +/// Maps ASCII byte values to their nibble value (0-15), invalid chars map to 0. +const HEX_DECODE_LUT: [u8; 256] = { + let mut table = [0u8; 256]; + let mut i = 0; + while i < 256 { + table[i] = match i as u8 { + b'0'..=b'9' => (i as u8) - b'0', + b'a'..=b'f' => (i as u8) - b'a' + 10, + b'A'..=b'F' => (i as u8) - b'A' + 10, + _ => 0, + }; + i += 1; + } + table +}; + +// ============================================================================ +// Hex Encoding - NEON (ARM64) +// ============================================================================ + +/// Convert 32-byte array to hex string using NEON SIMD (ARM64). +/// +/// # Performance +/// - ~26 ns total (including String allocation) +/// - Zero-copy: writes directly into String buffer +/// - 62x faster than format!() +#[cfg(target_arch = "aarch64")] +#[inline] +pub fn bytes_to_hex_32(bytes: &[u8; 32]) -> String { + unsafe { + // Allocate String directly - no intermediate buffer, no copy + let mut s = String::with_capacity(64); + let buf = s.as_mut_vec().as_mut_ptr(); + let hex_lut = vld1q_u8(HEX_NIBBLE.as_ptr()); + + for chunk_idx in 0..2 { + let offset = chunk_idx * 16; + let out_offset = chunk_idx * 32; + + let input = vld1q_u8(bytes.as_ptr().add(offset)); + let hi_nibbles = vshrq_n_u8(input, 4); + let lo_nibbles = vandq_u8(input, vdupq_n_u8(0x0f)); + let hi_hex = vqtbl1q_u8(hex_lut, hi_nibbles); + let lo_hex = vqtbl1q_u8(hex_lut, lo_nibbles); + let result_lo = vzip1q_u8(hi_hex, lo_hex); + let result_hi = vzip2q_u8(hi_hex, lo_hex); + + vst1q_u8(buf.add(out_offset), result_lo); + vst1q_u8(buf.add(out_offset + 16), result_hi); + } + + // SAFETY: We wrote exactly 64 ASCII hex chars (0-9, a-f) + s.as_mut_vec().set_len(64); + s + } +} + +/// Convert 16-byte array to hex string using NEON SIMD (ARM64). +/// Zero-copy: writes directly into String buffer. +#[cfg(target_arch = "aarch64")] +#[inline] +pub fn bytes_to_hex_16(bytes: &[u8; 16]) -> String { + unsafe { + // Allocate String directly - no intermediate buffer, no copy + let mut s = String::with_capacity(32); + let buf = s.as_mut_vec().as_mut_ptr(); + let hex_lut = vld1q_u8(HEX_NIBBLE.as_ptr()); + let input = vld1q_u8(bytes.as_ptr()); + + let hi_nibbles = vshrq_n_u8(input, 4); + let lo_nibbles = vandq_u8(input, vdupq_n_u8(0x0f)); + let hi_hex = vqtbl1q_u8(hex_lut, hi_nibbles); + let lo_hex = vqtbl1q_u8(hex_lut, lo_nibbles); + let result_lo = vzip1q_u8(hi_hex, lo_hex); + let result_hi = vzip2q_u8(hi_hex, lo_hex); + + vst1q_u8(buf, result_lo); + vst1q_u8(buf.add(16), result_hi); + + // SAFETY: We wrote exactly 32 ASCII hex chars (0-9, a-f) + s.as_mut_vec().set_len(32); + s + } +} + +// ============================================================================ +// Hex Encoding - x86_64 SIMD (SSE2 + AVX2) +// ============================================================================ + +/// Internal: AVX2 implementation for 32-byte hex encoding. +/// Processes all 32 bytes in a single operation using 256-bit registers. +/// +/// # Safety +/// Caller must ensure AVX2 is available (use `is_x86_feature_detected!`). +/// +/// # Reference +/// Algorithm based on faster-hex crate (MIT license): +/// https://github.com/nervosnetwork/faster-hex +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +#[inline] +unsafe fn hex_encode_32_avx2(bytes: &[u8; 32], buf: *mut u8) { + // Constants for hex conversion + let and4bits = _mm256_set1_epi8(0x0f); + let nines = _mm256_set1_epi8(9); + let ascii_zero = _mm256_set1_epi8(b'0' as i8); + // 'a' - 9 - 1 = 87, so nibble + 87 = 'a' for nibble 10 + let ascii_a_offset = _mm256_set1_epi8((b'a' - 9 - 1) as i8); + + // Load all 32 bytes at once + let invec = _mm256_loadu_si256(bytes.as_ptr() as *const __m256i); + + // Extract nibbles: low = byte & 0x0F, high = (byte >> 4) & 0x0F + // Note: srli_epi64 shifts 64-bit lanes, but we mask afterward so it's fine + let lo_nibbles = _mm256_and_si256(invec, and4bits); + let hi_nibbles = _mm256_and_si256(_mm256_srli_epi64(invec, 4), and4bits); + + // Compare > 9 to identify hex letters (a-f) + let lo_gt9 = _mm256_cmpgt_epi8(lo_nibbles, nines); + let hi_gt9 = _mm256_cmpgt_epi8(hi_nibbles, nines); + + // Convert to ASCII using blendv for conditional offset: + // if nibble <= 9: nibble + '0' + // if nibble > 9: nibble + ('a' - 10) = nibble + 87 + let lo_hex = _mm256_add_epi8( + lo_nibbles, + _mm256_blendv_epi8(ascii_zero, ascii_a_offset, lo_gt9), + ); + let hi_hex = _mm256_add_epi8( + hi_nibbles, + _mm256_blendv_epi8(ascii_zero, ascii_a_offset, hi_gt9), + ); + + // Interleave high and low nibbles: [H0,L0,H1,L1,...] + // Note: AVX2 unpack operates within 128-bit lanes, so output is: + // res1: [H0,L0..H7,L7 | H16,L16..H23,L23] (bytes 0-7, 16-23) + // res2: [H8,L8..H15,L15 | H24,L24..H31,L31] (bytes 8-15, 24-31) + let res1 = _mm256_unpacklo_epi8(hi_hex, lo_hex); + let res2 = _mm256_unpackhi_epi8(hi_hex, lo_hex); + + // Store with lane correction using storeu2_m128i: + // res1 low 128 bits -> positions 0-15 (bytes 0-7 interleaved) + // res2 low 128 bits -> positions 16-31 (bytes 8-15 interleaved) + // res1 high 128 bits -> positions 32-47 (bytes 16-23 interleaved) + // res2 high 128 bits -> positions 48-63 (bytes 24-31 interleaved) + _mm256_storeu2_m128i( + buf.add(32) as *mut __m128i, // high 128 bits + buf as *mut __m128i, // low 128 bits + res1, + ); + _mm256_storeu2_m128i( + buf.add(48) as *mut __m128i, // high 128 bits + buf.add(16) as *mut __m128i, // low 128 bits + res2, + ); +} + +/// Internal: SSE2 implementation for 32-byte hex encoding. +/// Processes 16 bytes at a time using 128-bit registers. +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse2")] +#[inline] +unsafe fn hex_encode_32_sse2(bytes: &[u8; 32], buf: *mut u8) { + let mask_lo = _mm_set1_epi8(0x0f); + let nine = _mm_set1_epi8(9); + let ascii_zero = _mm_set1_epi8(b'0' as i8); + let letter_offset = _mm_set1_epi8(0x27); // 'a' - '0' - 10 = 0x27 + + for chunk_idx in 0..2 { + let offset = chunk_idx * 16; + let out_offset = chunk_idx * 32; + + let input = _mm_loadu_si128(bytes.as_ptr().add(offset) as *const __m128i); + + // Extract nibbles (use epi16 shift then mask) + let hi_nibbles = _mm_and_si128(_mm_srli_epi16(input, 4), mask_lo); + let lo_nibbles = _mm_and_si128(input, mask_lo); + + // Convert: nibble + '0' + ((nibble > 9) ? 0x27 : 0) + let hi_gt9 = _mm_cmpgt_epi8(hi_nibbles, nine); + let lo_gt9 = _mm_cmpgt_epi8(lo_nibbles, nine); + + let hi_hex = _mm_add_epi8( + _mm_add_epi8(hi_nibbles, ascii_zero), + _mm_and_si128(hi_gt9, letter_offset), + ); + let lo_hex = _mm_add_epi8( + _mm_add_epi8(lo_nibbles, ascii_zero), + _mm_and_si128(lo_gt9, letter_offset), + ); + + // Interleave and store + let result_lo = _mm_unpacklo_epi8(hi_hex, lo_hex); + let result_hi = _mm_unpackhi_epi8(hi_hex, lo_hex); + + _mm_storeu_si128(buf.add(out_offset) as *mut __m128i, result_lo); + _mm_storeu_si128(buf.add(out_offset + 16) as *mut __m128i, result_hi); + } +} + +/// Convert 32-byte array to hex string using SIMD (x86_64). +/// +/// Automatically uses AVX2 if available, otherwise falls back to SSE2. +/// Zero-copy: writes directly into String buffer. +#[cfg(target_arch = "x86_64")] +#[inline] +pub fn bytes_to_hex_32(bytes: &[u8; 32]) -> String { + unsafe { + let mut s = String::with_capacity(64); + let buf = s.as_mut_vec().as_mut_ptr(); + + // Runtime feature detection (cached after first call) + if is_x86_feature_detected!("avx2") { + hex_encode_32_avx2(bytes, buf); + } else { + hex_encode_32_sse2(bytes, buf); + } + + s.as_mut_vec().set_len(64); + s + } +} + +/// Internal: SSE2 implementation for 16-byte hex encoding. +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse2")] +#[inline] +unsafe fn hex_encode_16_sse2(bytes: &[u8; 16], buf: *mut u8) { + let mask_lo = _mm_set1_epi8(0x0f); + let nine = _mm_set1_epi8(9); + let ascii_zero = _mm_set1_epi8(b'0' as i8); + let letter_offset = _mm_set1_epi8(0x27); + + let input = _mm_loadu_si128(bytes.as_ptr() as *const __m128i); + + let hi_nibbles = _mm_and_si128(_mm_srli_epi16(input, 4), mask_lo); + let lo_nibbles = _mm_and_si128(input, mask_lo); + + let hi_gt9 = _mm_cmpgt_epi8(hi_nibbles, nine); + let lo_gt9 = _mm_cmpgt_epi8(lo_nibbles, nine); + + let hi_hex = _mm_add_epi8( + _mm_add_epi8(hi_nibbles, ascii_zero), + _mm_and_si128(hi_gt9, letter_offset), + ); + let lo_hex = _mm_add_epi8( + _mm_add_epi8(lo_nibbles, ascii_zero), + _mm_and_si128(lo_gt9, letter_offset), + ); + + let result_lo = _mm_unpacklo_epi8(hi_hex, lo_hex); + let result_hi = _mm_unpackhi_epi8(hi_hex, lo_hex); + + _mm_storeu_si128(buf as *mut __m128i, result_lo); + _mm_storeu_si128(buf.add(16) as *mut __m128i, result_hi); +} + +/// Convert 16-byte array to hex string using SSE2 SIMD (x86_64). +#[cfg(target_arch = "x86_64")] +#[inline] +pub fn bytes_to_hex_16(bytes: &[u8; 16]) -> String { + unsafe { + let mut s = String::with_capacity(32); + let buf = s.as_mut_vec().as_mut_ptr(); + hex_encode_16_sse2(bytes, buf); + s.as_mut_vec().set_len(32); + s + } +} + +// ============================================================================ +// Hex Encoding - Scalar Fallback (other architectures) +// ============================================================================ + +/// Fallback: Convert 32-byte array to hex using scalar LUT. +#[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))] +#[inline] +pub fn bytes_to_hex_32(bytes: &[u8; 32]) -> String { + unsafe { + let mut s = String::with_capacity(64); + let buf = s.as_mut_vec().as_mut_ptr(); + for (i, &b) in bytes.iter().enumerate() { + let idx = (b as usize) * 2; + *buf.add(i * 2) = HEX_ENCODE_LUT[idx]; + *buf.add(i * 2 + 1) = HEX_ENCODE_LUT[idx + 1]; + } + s.as_mut_vec().set_len(64); + s + } +} + +/// Fallback: Convert 16-byte array to hex using scalar LUT. +#[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))] +#[inline] +pub fn bytes_to_hex_16(bytes: &[u8; 16]) -> String { + unsafe { + let mut s = String::with_capacity(32); + let buf = s.as_mut_vec().as_mut_ptr(); + for (i, &b) in bytes.iter().enumerate() { + let idx = (b as usize) * 2; + *buf.add(i * 2) = HEX_ENCODE_LUT[idx]; + *buf.add(i * 2 + 1) = HEX_ENCODE_LUT[idx + 1]; + } + s.as_mut_vec().set_len(32); + s + } +} + +// ============================================================================ +// Hex Encoding - Variable Length +// ============================================================================ + +/// Convert a byte slice to a hex string. +/// +/// For fixed-size arrays, prefer [`bytes_to_hex_32`] or [`bytes_to_hex_16`] +/// which use SIMD acceleration: +/// - **ARM64**: NEON with TBL lookup +/// - **x86_64**: AVX2 (if available) or SSE2 fallback +/// +/// Zero-copy: writes directly into String buffer. +pub fn bytes_to_hex_string(bytes: &[u8]) -> String { + // Use optimized paths for common fixed sizes + if bytes.len() == 32 { + return bytes_to_hex_32(bytes.try_into().unwrap()); + } + if bytes.len() == 16 { + return bytes_to_hex_16(bytes.try_into().unwrap()); + } + + let out_len = bytes.len().checked_mul(2).expect("hex string length overflow"); + + #[cfg(target_arch = "aarch64")] + unsafe { + // Allocate once, write directly - no intermediate buffers + let mut s = String::with_capacity(out_len); + let out_ptr = s.as_mut_vec().as_mut_ptr(); + let chunks = bytes.len() / 16; + let hex_lut = vld1q_u8(HEX_NIBBLE.as_ptr()); + + // SIMD: process 16 input bytes -> 32 output bytes per iteration + for i in 0..chunks { + let input = vld1q_u8(bytes.as_ptr().add(i * 16)); + let hi = vshrq_n_u8(input, 4); + let lo = vandq_u8(input, vdupq_n_u8(0x0f)); + let hi_hex = vqtbl1q_u8(hex_lut, hi); + let lo_hex = vqtbl1q_u8(hex_lut, lo); + vst1q_u8(out_ptr.add(i * 32), vzip1q_u8(hi_hex, lo_hex)); + vst1q_u8(out_ptr.add(i * 32 + 16), vzip2q_u8(hi_hex, lo_hex)); + } + + // Scalar for remaining bytes (0-15 bytes) + let remainder_start = chunks * 16; + let mut out_idx = chunks * 32; + for &b in &bytes[remainder_start..] { + *out_ptr.add(out_idx) = HEX_NIBBLE[(b >> 4) as usize]; + *out_ptr.add(out_idx + 1) = HEX_NIBBLE[(b & 0xf) as usize]; + out_idx += 2; + } + + s.as_mut_vec().set_len(out_len); + s + } + + #[cfg(target_arch = "x86_64")] + unsafe { + // Allocate once, write directly - no intermediate buffers + let mut s = String::with_capacity(out_len); + let out_ptr = s.as_mut_vec().as_mut_ptr(); + let chunks = bytes.len() / 16; + + // SSE2 constants + let mask_lo = _mm_set1_epi8(0x0f); + let nine = _mm_set1_epi8(9); + let ascii_zero = _mm_set1_epi8(b'0' as i8); + let letter_offset = _mm_set1_epi8(0x27); + + // SIMD: process 16 input bytes -> 32 output bytes per iteration + for i in 0..chunks { + let input = _mm_loadu_si128(bytes.as_ptr().add(i * 16) as *const __m128i); + + let hi = _mm_and_si128(_mm_srli_epi16(input, 4), mask_lo); + let lo = _mm_and_si128(input, mask_lo); + + let hi_gt9 = _mm_cmpgt_epi8(hi, nine); + let lo_gt9 = _mm_cmpgt_epi8(lo, nine); + + let hi_hex = _mm_add_epi8( + _mm_add_epi8(hi, ascii_zero), + _mm_and_si128(hi_gt9, letter_offset), + ); + let lo_hex = _mm_add_epi8( + _mm_add_epi8(lo, ascii_zero), + _mm_and_si128(lo_gt9, letter_offset), + ); + + _mm_storeu_si128(out_ptr.add(i * 32) as *mut __m128i, _mm_unpacklo_epi8(hi_hex, lo_hex)); + _mm_storeu_si128(out_ptr.add(i * 32 + 16) as *mut __m128i, _mm_unpackhi_epi8(hi_hex, lo_hex)); + } + + // Scalar for remaining bytes (0-15 bytes) + const HEX_CHARS: &[u8; 16] = b"0123456789abcdef"; + let remainder_start = chunks * 16; + let mut out_idx = chunks * 32; + for &b in &bytes[remainder_start..] { + *out_ptr.add(out_idx) = HEX_CHARS[(b >> 4) as usize]; + *out_ptr.add(out_idx + 1) = HEX_CHARS[(b & 0xf) as usize]; + out_idx += 2; + } + + s.as_mut_vec().set_len(out_len); + s + } + + #[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))] + unsafe { + // Allocate once, write directly + let mut s = String::with_capacity(out_len); + let out_ptr = s.as_mut_vec().as_mut_ptr(); + for (i, &b) in bytes.iter().enumerate() { + let idx = (b as usize) * 2; + *out_ptr.add(i * 2) = HEX_ENCODE_LUT[idx]; + *out_ptr.add(i * 2 + 1) = HEX_ENCODE_LUT[idx + 1]; + } + s.as_mut_vec().set_len(out_len); + s + } +} + +// ============================================================================ +// Hex Decoding - SIMD Accelerated +// ============================================================================ + +/// Convert hex string to fixed 32-byte array. +/// +/// # Performance +/// - NEON (ARM64): ~2.5 ns / 8 cycles (7.7x faster than LUT) +/// - SSE2 (x86_64): ~5 ns (estimated) +/// - Scalar fallback: ~19 ns +/// +/// # Note +/// Invalid hex characters are treated as 0x00. Short strings are zero-padded. +#[cfg(target_arch = "aarch64")] +#[inline] +pub fn hex_to_bytes_32(hex: &str) -> [u8; 32] { + let h = hex.as_bytes(); + + // Fast path: exactly 64 chars, use SIMD + if h.len() >= 64 { + return unsafe { hex_decode_32_neon(h) }; + } + + // Slow path for short strings (zero-pad on left) + hex_to_bytes_32_scalar_padded(h) +} + +/// NEON implementation: decode 64 hex chars to 32 bytes +/// +/// Optimized algorithm: +/// 1. Simplified nibble conversion: (char & 0x0F) + 9*(char has bit 0x40 set) +/// - For '0'-'9': (0x30-0x39 & 0x0F) = 0-9, bit 0x40 not set, so +0 +/// - For 'A'-'F': (0x41-0x46 & 0x0F) = 1-6, bit 0x40 set, so +9 = 10-15 +/// - For 'a'-'f': (0x61-0x66 & 0x0F) = 1-6, bit 0x40 set, so +9 = 10-15 +/// 2. Uses SLI (Shift Left and Insert) to combine nibbles in one instruction +/// 3. Fully unrolled for maximum throughput +#[cfg(target_arch = "aarch64")] +#[inline] +unsafe fn hex_decode_32_neon(h: &[u8]) -> [u8; 32] { + let mut result = [0u8; 32]; + + let mask_0f = vdupq_n_u8(0x0F); + let mask_40 = vdupq_n_u8(0x40); + let nine = vdupq_n_u8(9); + + // Load all 64 hex chars at once + let hex_0 = vld1q_u8(h.as_ptr()); + let hex_1 = vld1q_u8(h.as_ptr().add(16)); + let hex_2 = vld1q_u8(h.as_ptr().add(32)); + let hex_3 = vld1q_u8(h.as_ptr().add(48)); + + // Convert ASCII to nibbles using simplified algorithm + // (char & 0x0F) + 9 if letter (bit 0x40 set) + let lo0 = vandq_u8(hex_0, mask_0f); + let lo1 = vandq_u8(hex_1, mask_0f); + let lo2 = vandq_u8(hex_2, mask_0f); + let lo3 = vandq_u8(hex_3, mask_0f); + + let is_letter0 = vtstq_u8(hex_0, mask_40); + let is_letter1 = vtstq_u8(hex_1, mask_40); + let is_letter2 = vtstq_u8(hex_2, mask_40); + let is_letter3 = vtstq_u8(hex_3, mask_40); + + let n0 = vaddq_u8(lo0, vandq_u8(is_letter0, nine)); + let n1 = vaddq_u8(lo1, vandq_u8(is_letter1, nine)); + let n2 = vaddq_u8(lo2, vandq_u8(is_letter2, nine)); + let n3 = vaddq_u8(lo3, vandq_u8(is_letter3, nine)); + + // Pack nibbles to bytes using UZP + SLI + // SLI (Shift Left and Insert) combines shift+or into one instruction + let evens_a = vuzp1q_u8(n0, n1); + let odds_a = vuzp2q_u8(n0, n1); + let bytes_a = vsliq_n_u8(odds_a, evens_a, 4); + + let evens_b = vuzp1q_u8(n2, n3); + let odds_b = vuzp2q_u8(n2, n3); + let bytes_b = vsliq_n_u8(odds_b, evens_b, 4); + + vst1q_u8(result.as_mut_ptr(), bytes_a); + vst1q_u8(result.as_mut_ptr().add(16), bytes_b); + + result +} + +/// x86_64 SIMD implementation +#[cfg(target_arch = "x86_64")] +#[inline] +pub fn hex_to_bytes_32(hex: &str) -> [u8; 32] { + let h = hex.as_bytes(); + + if h.len() >= 64 { + // SAFETY: We verified length >= 64, and hex_decode_32_sse2 only reads first 64 bytes + let arr: &[u8; 64] = h[..64].try_into().unwrap(); + return unsafe { hex_decode_32_sse2(arr) }; + } + + hex_to_bytes_32_scalar_padded(h) +} + +/// SSE2 implementation: decode 64 hex chars to 32 bytes +/// +/// Uses the same algorithm as NEON: `(char & 0x0F) + 9*(char has bit 0x40 set)` +/// This correctly handles '0'-'9', 'A'-'F', and 'a'-'f'. +/// +/// # Safety +/// Caller must ensure input contains only valid hex characters. +/// Invalid input produces garbage output (no validation performed). +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse2")] +#[inline] +unsafe fn hex_decode_32_sse2(h: &[u8; 64]) -> [u8; 32] { + let mut result = [0u8; 32]; + + // Same algorithm as NEON: (char & 0x0F) + 9 if letter + let mask_0f = _mm_set1_epi8(0x0F); + let mask_40 = _mm_set1_epi8(0x40); + let nine = _mm_set1_epi8(9); + let hi_mask = _mm_set1_epi16(0x00F0u16 as i16); + let lo_mask = _mm_set1_epi16(0x000Fu16 as i16); + let zero = _mm_setzero_si128(); + + // Process 16 hex chars -> 8 bytes at a time (4 iterations) + for chunk in 0..4 { + let in_offset = chunk * 16; + let out_offset = chunk * 8; + + let hex_chars = _mm_loadu_si128(h.as_ptr().add(in_offset) as *const __m128i); + + // Convert ASCII to nibbles using NEON-style algorithm: + // nibble = (char & 0x0F) + ((char & 0x40) == 0x40 ? 9 : 0) + let lo = _mm_and_si128(hex_chars, mask_0f); + let masked = _mm_and_si128(hex_chars, mask_40); + let is_letter = _mm_cmpeq_epi8(masked, mask_40); + let nine_if_letter = _mm_and_si128(is_letter, nine); + let nibbles = _mm_add_epi8(lo, nine_if_letter); + + // Pack pairs of nibbles into bytes + let hi_nibbles = _mm_slli_epi16(nibbles, 4); + let hi = _mm_and_si128(hi_nibbles, hi_mask); + let lo_shifted = _mm_and_si128(_mm_srli_epi16(nibbles, 8), lo_mask); + let combined = _mm_or_si128(hi, lo_shifted); + + let packed = _mm_packus_epi16(combined, zero); + _mm_storel_epi64(result.as_mut_ptr().add(out_offset) as *mut __m128i, packed); + } + + result +} + +/// SSE2 implementation: decode 32 hex chars to 16 bytes +/// +/// Uses the same algorithm as NEON: `(char & 0x0F) + 9*(char has bit 0x40 set)` +/// This correctly handles '0'-'9', 'A'-'F', and 'a'-'f'. +/// +/// # Safety +/// Caller must ensure input contains only valid hex characters. +/// Invalid input produces garbage output (no validation performed). +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse2")] +#[inline] +unsafe fn hex_decode_16_sse2(h: &[u8; 32]) -> [u8; 16] { + let mut result = [0u8; 16]; + + // Same algorithm as NEON: (char & 0x0F) + 9 if letter + let mask_0f = _mm_set1_epi8(0x0F); + let mask_40 = _mm_set1_epi8(0x40); + let nine = _mm_set1_epi8(9); + let hi_mask = _mm_set1_epi16(0x00F0u16 as i16); + let lo_mask = _mm_set1_epi16(0x000Fu16 as i16); + let zero = _mm_setzero_si128(); + + // Process 16 hex chars -> 8 bytes at a time (2 iterations for 16 bytes) + for chunk in 0..2 { + let in_offset = chunk * 16; + let out_offset = chunk * 8; + + let hex_chars = _mm_loadu_si128(h.as_ptr().add(in_offset) as *const __m128i); + + // Convert ASCII to nibbles using NEON-style algorithm: + // nibble = (char & 0x0F) + ((char & 0x40) == 0x40 ? 9 : 0) + let lo = _mm_and_si128(hex_chars, mask_0f); + let masked = _mm_and_si128(hex_chars, mask_40); + let is_letter = _mm_cmpeq_epi8(masked, mask_40); + let nine_if_letter = _mm_and_si128(is_letter, nine); + let nibbles = _mm_add_epi8(lo, nine_if_letter); + + // Pack pairs of nibbles into bytes + let hi_nibbles = _mm_slli_epi16(nibbles, 4); + let hi = _mm_and_si128(hi_nibbles, hi_mask); + let lo_shifted = _mm_and_si128(_mm_srli_epi16(nibbles, 8), lo_mask); + let combined = _mm_or_si128(hi, lo_shifted); + + let packed = _mm_packus_epi16(combined, zero); + _mm_storel_epi64(result.as_mut_ptr().add(out_offset) as *mut __m128i, packed); + } + + result +} + +/// Scalar fallback for other platforms +#[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))] +#[inline] +pub fn hex_to_bytes_32(hex: &str) -> [u8; 32] { + let h = hex.as_bytes(); + + if h.len() >= 64 { + let mut bytes = [0u8; 32]; + for i in 0..32 { + bytes[i] = (HEX_DECODE_LUT[h[i * 2] as usize] << 4) + | HEX_DECODE_LUT[h[i * 2 + 1] as usize]; + } + return bytes; + } + + hex_to_bytes_32_scalar_padded(h) +} + +/// Scalar helper for short/padded hex strings (all platforms) +#[inline] +fn hex_to_bytes_32_scalar_padded(h: &[u8]) -> [u8; 32] { + let mut bytes = [0u8; 32]; + let hex_len = h.len(); + let start_idx = (64 - hex_len) / 2; + let mut out_idx = start_idx / 2; + + let mut i = 0; + while i + 1 < hex_len && out_idx < 32 { + bytes[out_idx] = (HEX_DECODE_LUT[h[i] as usize] << 4) + | HEX_DECODE_LUT[h[i + 1] as usize]; + out_idx += 1; + i += 2; + } + bytes +} + +// ============================================================================ +// Hex Decoding - 16 bytes +// ============================================================================ + +/// Convert hex string to fixed 16-byte array. +/// +/// # Performance +/// - NEON (ARM64): ~2 ns +/// - Scalar fallback: ~10 ns +#[cfg(target_arch = "aarch64")] +#[inline] +pub fn hex_to_bytes_16(hex: &str) -> [u8; 16] { + let h = hex.as_bytes(); + + if h.len() >= 32 { + return unsafe { hex_decode_16_neon(h) }; + } + + hex_to_bytes_16_scalar_padded(h) +} + +/// NEON implementation: decode 32 hex chars to 16 bytes +/// +/// Uses the same optimized algorithm as hex_decode_32_neon: +/// - Simplified nibble conversion: (char & 0x0F) + 9*(char has bit 0x40 set) +/// - SLI instruction to combine nibbles in one operation +#[cfg(target_arch = "aarch64")] +#[inline] +unsafe fn hex_decode_16_neon(h: &[u8]) -> [u8; 16] { + let mut result = [0u8; 16]; + + let mask_0f = vdupq_n_u8(0x0F); + let mask_40 = vdupq_n_u8(0x40); + let nine = vdupq_n_u8(9); + + // Load 32 hex characters + let hex_0 = vld1q_u8(h.as_ptr()); + let hex_1 = vld1q_u8(h.as_ptr().add(16)); + + // Convert ASCII to nibbles: (char & 0x0F) + 9 if letter + let lo0 = vandq_u8(hex_0, mask_0f); + let lo1 = vandq_u8(hex_1, mask_0f); + + let is_letter0 = vtstq_u8(hex_0, mask_40); + let is_letter1 = vtstq_u8(hex_1, mask_40); + + let n0 = vaddq_u8(lo0, vandq_u8(is_letter0, nine)); + let n1 = vaddq_u8(lo1, vandq_u8(is_letter1, nine)); + + // Pack nibbles using UZP + SLI + let evens = vuzp1q_u8(n0, n1); + let odds = vuzp2q_u8(n0, n1); + let bytes = vsliq_n_u8(odds, evens, 4); + + vst1q_u8(result.as_mut_ptr(), bytes); + result +} + +#[cfg(target_arch = "x86_64")] +#[inline] +pub fn hex_to_bytes_16(hex: &str) -> [u8; 16] { + let h = hex.as_bytes(); + + if h.len() >= 32 { + // SAFETY: We verified length >= 32, and hex_decode_16_sse2 only reads first 32 bytes + let arr: &[u8; 32] = h[..32].try_into().unwrap(); + return unsafe { hex_decode_16_sse2(arr) }; + } + + hex_to_bytes_16_scalar_padded(h) +} + +#[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))] +#[inline] +pub fn hex_to_bytes_16(hex: &str) -> [u8; 16] { + let h = hex.as_bytes(); + + if h.len() >= 32 { + let mut bytes = [0u8; 16]; + for i in 0..16 { + bytes[i] = (HEX_DECODE_LUT[h[i * 2] as usize] << 4) + | HEX_DECODE_LUT[h[i * 2 + 1] as usize]; + } + return bytes; + } + + hex_to_bytes_16_scalar_padded(h) +} + +#[inline] +fn hex_to_bytes_16_scalar_padded(h: &[u8]) -> [u8; 16] { + let mut bytes = [0u8; 16]; + let hex_len = h.len(); + let start_idx = (32 - hex_len) / 2; + let mut out_idx = start_idx / 2; + + let mut i = 0; + while i + 1 < hex_len && out_idx < 16 { + bytes[out_idx] = (HEX_DECODE_LUT[h[i] as usize] << 4) + | HEX_DECODE_LUT[h[i + 1] as usize]; + out_idx += 1; + i += 2; + } + bytes +} + +// ============================================================================ +// Hex Decoding - Variable Length +// ============================================================================ + +/// Convert hex string to bytes (arbitrary length). +/// +/// Uses SIMD for the bulk of the conversion when input is large enough. +pub fn hex_string_to_bytes(s: &str) -> Vec { + let h = s.as_bytes(); + let out_len = h.len() / 2; + let mut result = Vec::with_capacity(out_len); + + #[cfg(target_arch = "aarch64")] + unsafe { + result.set_len(out_len); + let out_ptr: *mut u8 = result.as_mut_ptr(); + + let mask_0f = vdupq_n_u8(0x0F); + let mask_40 = vdupq_n_u8(0x40); + let nine = vdupq_n_u8(9); + + let chunks = out_len / 16; // 32 hex chars -> 16 bytes per chunk + for chunk in 0..chunks { + let in_offset = chunk * 32; + let out_offset = chunk * 16; + + // Load 32 hex characters + let hex_0 = vld1q_u8(h.as_ptr().add(in_offset)); + let hex_1 = vld1q_u8(h.as_ptr().add(in_offset + 16)); + + // Convert ASCII to nibbles: (char & 0x0F) + 9 if letter + let lo0 = vandq_u8(hex_0, mask_0f); + let lo1 = vandq_u8(hex_1, mask_0f); + + let is_letter0 = vtstq_u8(hex_0, mask_40); + let is_letter1 = vtstq_u8(hex_1, mask_40); + + let n0 = vaddq_u8(lo0, vandq_u8(is_letter0, nine)); + let n1 = vaddq_u8(lo1, vandq_u8(is_letter1, nine)); + + // Pack nibbles using UZP + SLI + let evens = vuzp1q_u8(n0, n1); + let odds = vuzp2q_u8(n0, n1); + let bytes = vsliq_n_u8(odds, evens, 4); + + vst1q_u8(out_ptr.add(out_offset), bytes); + } + + // Scalar remainder + let remainder_start = chunks * 32; + let mut out_idx = chunks * 16; + let mut i = remainder_start; + while i + 1 < h.len() { + *out_ptr.add(out_idx) = (HEX_DECODE_LUT[h[i] as usize] << 4) + | HEX_DECODE_LUT[h[i + 1] as usize]; + out_idx += 1; + i += 2; + } + } + + #[cfg(target_arch = "x86_64")] + { + // For x86, use scalar for now (SSE2 decode is more complex for variable length) + for chunk in h.chunks(2) { + if chunk.len() == 2 { + result.push( + (HEX_DECODE_LUT[chunk[0] as usize] << 4) | HEX_DECODE_LUT[chunk[1] as usize] + ); + } + } + } + + #[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))] + { + for chunk in h.chunks(2) { + if chunk.len() == 2 { + result.push( + (HEX_DECODE_LUT[chunk[0] as usize] << 4) | HEX_DECODE_LUT[chunk[1] as usize] + ); + } + } + } + + result +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_hex_encode_32() { + let bytes: [u8; 32] = [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, + 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, + 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, + 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10, + ]; + let hex = bytes_to_hex_32(&bytes); + assert_eq!(hex, "00112233445566778899aabbccddeeff0123456789abcdeffedcba9876543210"); + } + + #[test] + fn test_hex_decode_32() { + let hex = "00112233445566778899aabbccddeeff0123456789abcdeffedcba9876543210"; + let bytes = hex_to_bytes_32(hex); + assert_eq!(bytes[0], 0x00); + assert_eq!(bytes[15], 0xff); + assert_eq!(bytes[31], 0x10); + } + + #[test] + fn test_roundtrip() { + let original: [u8; 32] = [42; 32]; + let hex = bytes_to_hex_32(&original); + let decoded = hex_to_bytes_32(&hex); + assert_eq!(original, decoded); + } + + #[test] + fn test_hex_decode_16() { + let hex = "00112233445566778899aabbccddeeff"; + let bytes = hex_to_bytes_16(hex); + assert_eq!(bytes[0], 0x00); + assert_eq!(bytes[7], 0x77); + assert_eq!(bytes[15], 0xff); + } + + #[test] + fn test_hex_decode_uppercase() { + // Test that uppercase hex is decoded correctly + let lowercase = "00112233445566778899aabbccddeeff0123456789abcdeffedcba9876543210"; + let uppercase = "00112233445566778899AABBCCDDEEFF0123456789ABCDEFFEDCBA9876543210"; + assert_eq!(hex_to_bytes_32(lowercase), hex_to_bytes_32(uppercase)); + } + + #[test] + fn test_hex_string_to_bytes() { + let hex = "deadbeef"; + let bytes = hex_string_to_bytes(hex); + assert_eq!(bytes, vec![0xde, 0xad, 0xbe, 0xef]); + } + + #[test] + fn test_hex_string_to_bytes_long() { + // Test variable-length decode with longer input (uses SIMD path) + let hex = "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"; + let bytes = hex_string_to_bytes(hex); + assert_eq!(bytes.len(), 32); + assert_eq!(bytes[0], 0x00); + assert_eq!(bytes[15], 0xff); + assert_eq!(bytes[31], 0xff); + } + + #[test] + fn test_roundtrip_16() { + let original: [u8; 16] = [0xab; 16]; + let hex = bytes_to_hex_16(&original); + let decoded = hex_to_bytes_16(&hex); + assert_eq!(original, decoded); + } +} diff --git a/src-tauri/src/simd/image.rs b/src-tauri/src/simd/image.rs new file mode 100644 index 00000000..a7175ed3 --- /dev/null +++ b/src-tauri/src/simd/image.rs @@ -0,0 +1,1157 @@ +//! SIMD-accelerated image operations +//! +//! # Performance (27 MP image, 109 MB RGBA) +//! +//! | Function | Scalar | SIMD + Parallel | Speedup | +//! |----------|--------|-----------------|---------| +//! | `has_alpha_transparency` | 5.37ms | 0.59ms | 9.1x | +//! | `set_all_alpha_opaque` | 3.08ms | 0.67ms | 4.6x | +//! +//! Theoretical minimum at 200 GB/s memory bandwidth: 0.55ms +//! +//! # Platform Support +//! +//! - **ARM64**: NEON (vld3/vst4 for RGB↔RGBA, TBL for lookups) +//! - **x86_64**: AVX2/SSSE3 (pshufb for byte rearrangement) or SSE2 fallback +//! +//! # Algorithms +//! +//! **Alpha check/set** - Processes 128 bytes (32 RGBA pixels) per iteration: +//! 1. Load 8 x 16-byte chunks into SIMD registers +//! 2. AND/OR all chunks to combine alpha checks/sets +//! 3. Check alpha bytes at positions 3, 7, 11, 15 (every 4th byte) +//! 4. For large images (>4MB), parallelize across CPU cores with rayon +//! +//! **RGB→RGBA conversion** - Uses architecture-specific deinterleaving: +//! - NEON: vld3q loads RGB planes directly, vst4q stores RGBA +//! - SSSE3: pshufb rearranges 12 RGB bytes → 16 RGBA bytes per iteration + +#[cfg(target_arch = "aarch64")] +use std::arch::aarch64::*; + +#[cfg(target_arch = "x86_64")] +use std::arch::x86_64::*; + +use rayon::prelude::*; + +// ============================================================================ +// Alpha Transparency Check +// ============================================================================ + +/// NEON-optimized alpha check - processes 128 bytes (32 pixels) per iteration +#[cfg(target_arch = "aarch64")] +#[inline] +fn has_alpha_neon(rgba_pixels: &[u8]) -> bool { + unsafe { + let len = rgba_pixels.len(); + let ptr = rgba_pixels.as_ptr(); + let mut i = 0; + + // Process 128 bytes (32 pixels) at a time + while i + 128 <= len { + // Load 8 x 16-byte chunks + let c0 = vld1q_u8(ptr.add(i)); + let c1 = vld1q_u8(ptr.add(i + 16)); + let c2 = vld1q_u8(ptr.add(i + 32)); + let c3 = vld1q_u8(ptr.add(i + 48)); + let c4 = vld1q_u8(ptr.add(i + 64)); + let c5 = vld1q_u8(ptr.add(i + 80)); + let c6 = vld1q_u8(ptr.add(i + 96)); + let c7 = vld1q_u8(ptr.add(i + 112)); + + // AND all chunks together - if any alpha was < 255, it will show + let and01 = vandq_u8(c0, c1); + let and23 = vandq_u8(c2, c3); + let and45 = vandq_u8(c4, c5); + let and67 = vandq_u8(c6, c7); + let and0123 = vandq_u8(and01, and23); + let and4567 = vandq_u8(and45, and67); + let and_all = vandq_u8(and0123, and4567); + + // Check alpha positions (bytes 3, 7, 11, 15 in each 16-byte chunk) + let a3 = vgetq_lane_u8(and_all, 3); + let a7 = vgetq_lane_u8(and_all, 7); + let a11 = vgetq_lane_u8(and_all, 11); + let a15 = vgetq_lane_u8(and_all, 15); + + if (a3 & a7 & a11 & a15) != 255 { + return true; + } + i += 128; + } + + // Process remaining 16 bytes at a time + while i + 16 <= len { + let c = vld1q_u8(ptr.add(i)); + let a3 = vgetq_lane_u8(c, 3); + let a7 = vgetq_lane_u8(c, 7); + let a11 = vgetq_lane_u8(c, 11); + let a15 = vgetq_lane_u8(c, 15); + if (a3 & a7 & a11 & a15) != 255 { + return true; + } + i += 16; + } + + // Handle remainder with scalar + while i + 4 <= len { + if rgba_pixels[i + 3] < 255 { + return true; + } + i += 4; + } + false + } +} + +// ============================================================================ +// Alpha Transparency Check - x86_64 SIMD (SSE2 + AVX2) +// ============================================================================ + +/// AVX2-optimized alpha check - processes 128 bytes (32 pixels) per iteration +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +#[inline] +unsafe fn has_alpha_avx2(rgba_pixels: &[u8]) -> bool { + let len = rgba_pixels.len(); + let ptr = rgba_pixels.as_ptr(); + let mut i = 0; + + // All 0xFF for comparison + let all_255 = _mm256_set1_epi8(-1i8); // 0xFF + + // Process 128 bytes (32 pixels) at a time using 4 x 256-bit loads + while i + 128 <= len { + // Load and AND 4 x 32-byte chunks + let c0 = _mm256_loadu_si256(ptr.add(i) as *const __m256i); + let c1 = _mm256_loadu_si256(ptr.add(i + 32) as *const __m256i); + let c2 = _mm256_loadu_si256(ptr.add(i + 64) as *const __m256i); + let c3 = _mm256_loadu_si256(ptr.add(i + 96) as *const __m256i); + + let and01 = _mm256_and_si256(c0, c1); + let and23 = _mm256_and_si256(c2, c3); + let and_all = _mm256_and_si256(and01, and23); + + // Compare with 255 - if any byte < 255, comparison fails for that byte + let cmp = _mm256_cmpeq_epi8(and_all, all_255); + let mask = _mm256_movemask_epi8(cmp); + + // Alpha bytes are at positions 3,7,11,15,19,23,27,31 (every 4th byte starting at 3) + // In the mask, these are bits: 3,7,11,15,19,23,27,31 = 0x88888888 + if (mask as u32 & 0x88888888) != 0x88888888 { + return true; + } + i += 128; + } + + // Fall back to SSE2 for remainder + has_alpha_sse2_remainder(&rgba_pixels[i..]) +} + +/// SSE2-optimized alpha check - processes 64 bytes (16 pixels) per iteration +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse2")] +#[inline] +unsafe fn has_alpha_sse2(rgba_pixels: &[u8]) -> bool { + let len = rgba_pixels.len(); + let ptr = rgba_pixels.as_ptr(); + let mut i = 0; + + let all_255 = _mm_set1_epi8(-1i8); // 0xFF + + // Process 64 bytes (16 pixels) at a time using 4 x 128-bit loads + while i + 64 <= len { + let c0 = _mm_loadu_si128(ptr.add(i) as *const __m128i); + let c1 = _mm_loadu_si128(ptr.add(i + 16) as *const __m128i); + let c2 = _mm_loadu_si128(ptr.add(i + 32) as *const __m128i); + let c3 = _mm_loadu_si128(ptr.add(i + 48) as *const __m128i); + + let and01 = _mm_and_si128(c0, c1); + let and23 = _mm_and_si128(c2, c3); + let and_all = _mm_and_si128(and01, and23); + + let cmp = _mm_cmpeq_epi8(and_all, all_255); + let mask = _mm_movemask_epi8(cmp); + + // Alpha bytes at positions 3,7,11,15 = bits 3,7,11,15 = 0x8888 + if (mask & 0x8888) != 0x8888 { + return true; + } + i += 64; + } + + // Handle remainder + has_alpha_sse2_remainder(&rgba_pixels[i..]) +} + +/// SSE2 remainder handler (also used by AVX2) +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse2")] +#[inline] +unsafe fn has_alpha_sse2_remainder(rgba_pixels: &[u8]) -> bool { + let len = rgba_pixels.len(); + let ptr = rgba_pixels.as_ptr(); + let mut i = 0; + + let all_255 = _mm_set1_epi8(-1i8); + + // Process 16 bytes at a time + while i + 16 <= len { + let c = _mm_loadu_si128(ptr.add(i) as *const __m128i); + let cmp = _mm_cmpeq_epi8(c, all_255); + let mask = _mm_movemask_epi8(cmp); + + if (mask & 0x8888) != 0x8888 { + return true; + } + i += 16; + } + + // Scalar for final pixels + while i + 4 <= len { + if rgba_pixels[i + 3] < 255 { + return true; + } + i += 4; + } + false +} + +/// x86_64 dispatcher - uses AVX2 if available, otherwise SSE2 +#[cfg(target_arch = "x86_64")] +#[inline] +fn has_alpha_simd(rgba_pixels: &[u8]) -> bool { + unsafe { + if is_x86_feature_detected!("avx2") { + has_alpha_avx2(rgba_pixels) + } else { + has_alpha_sse2(rgba_pixels) + } + } +} + +/// Scalar fallback for non-SIMD platforms +#[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))] +#[inline] +fn has_alpha_simd(rgba_pixels: &[u8]) -> bool { + // Fast path for little-endian (WASM, RISC-V, most platforms) + #[cfg(target_endian = "little")] + { + let mut chunks = rgba_pixels.chunks_exact(8); + // On little-endian, alpha bytes are at positions 3 and 7 within each 8-byte chunk + // which correspond to bits 24-31 and 56-63 in the u64 + const ALPHA_MASK: u64 = 0xFF000000_FF000000; + + for chunk in chunks.by_ref() { + let val = u64::from_ne_bytes(chunk.try_into().unwrap()); + if (val & ALPHA_MASK) != ALPHA_MASK { + return true; + } + } + chunks.remainder().chunks_exact(4).any(|px| px[3] < 255) + } + + // Byte-by-byte fallback for big-endian (rare) + #[cfg(target_endian = "big")] + { + for pixel in rgba_pixels.chunks_exact(4) { + if pixel[3] < 255 { + return true; + } + } + false + } +} + +/// Check if RGBA pixel data contains any meaningful transparency (alpha < 255) +/// +/// Uses SIMD acceleration with parallel processing for maximum performance: +/// - **ARM64**: NEON +/// - **x86_64**: AVX2 (if available) or SSE2 +/// +/// Achieves ~0.6ms on 27 MP images (near memory bandwidth limit). +/// +/// # Parallelization Strategy +/// - Images < 4 MB: Single-threaded (parallel overhead > benefit) +/// - Images >= 4 MB: 256 KB chunks (optimal for L2 cache + core utilization) +/// +/// # Example +/// ```ignore +/// let has_transparency = has_alpha_transparency(&rgba_pixels); +/// if has_transparency { +/// // Image has transparent pixels, encode as PNG +/// } else { +/// // Image is fully opaque, can use JPEG +/// } +/// ``` +#[inline] +pub fn has_alpha_transparency(rgba_pixels: &[u8]) -> bool { + // 256 KB chunks: fits L2 cache, good core utilization + // Benchmarked: 2-3x faster than 1 MB chunks for large images + const CHUNK_SIZE: usize = 256 * 1024; + const PARALLEL_THRESHOLD: usize = 4 * 1024 * 1024; // 4 MB (~1 MP) + + #[cfg(target_arch = "aarch64")] + let check_fn = has_alpha_neon; + + #[cfg(target_arch = "x86_64")] + let check_fn = has_alpha_simd; + + #[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))] + let check_fn = has_alpha_simd; + + // Only parallelize for large images where benefit > overhead + if rgba_pixels.len() > PARALLEL_THRESHOLD { + rgba_pixels + .par_chunks(CHUNK_SIZE) + .any(|chunk| check_fn(chunk)) + } else { + check_fn(rgba_pixels) + } +} + +// ============================================================================ +// Alpha Near-Zero Check (Windows clipboard bug) +// ============================================================================ + +/// Check if all alpha values are nearly zero (Windows clipboard bug detection) +/// +/// Returns true if ALL pixels have alpha <= 2. This detects a Windows clipboard +/// bug where RGBA images have their alpha channel corrupted to near-zero values. +#[inline] +#[cfg(target_os = "windows")] +pub fn has_all_alpha_near_zero(rgba_pixels: &[u8]) -> bool { + let mut chunks = rgba_pixels.chunks_exact(8); + // Mask to check if alpha bytes are > 2: if (alpha & 0xFC) != 0, then alpha > 3 + const ALPHA_HIGH_BITS: u64 = 0xFC000000_FC000000; + + for chunk in chunks.by_ref() { + let val = u64::from_ne_bytes(chunk.try_into().unwrap()); + if (val & ALPHA_HIGH_BITS) != 0 { + return false; // Found alpha > 2 + } + } + + chunks.remainder().chunks_exact(4).all(|px| px[3] <= 2) +} + +// ============================================================================ +// Set Alpha Opaque +// ============================================================================ + +/// NEON-optimized alpha set - processes 128 bytes (32 pixels) per iteration +#[cfg(target_arch = "aarch64")] +#[inline] +fn set_alpha_neon(rgba_pixels: &mut [u8]) { + unsafe { + let len = rgba_pixels.len(); + let ptr = rgba_pixels.as_mut_ptr(); + let mut i = 0; + + // Alpha mask: 0x00 for RGB, 0xFF for alpha channel + let mask = vld1q_u8([0, 0, 0, 0xFF, 0, 0, 0, 0xFF, 0, 0, 0, 0xFF, 0, 0, 0, 0xFF].as_ptr()); + + // Process 128 bytes (32 pixels) at a time + while i + 128 <= len { + vst1q_u8(ptr.add(i), vorrq_u8(vld1q_u8(ptr.add(i)), mask)); + vst1q_u8(ptr.add(i + 16), vorrq_u8(vld1q_u8(ptr.add(i + 16)), mask)); + vst1q_u8(ptr.add(i + 32), vorrq_u8(vld1q_u8(ptr.add(i + 32)), mask)); + vst1q_u8(ptr.add(i + 48), vorrq_u8(vld1q_u8(ptr.add(i + 48)), mask)); + vst1q_u8(ptr.add(i + 64), vorrq_u8(vld1q_u8(ptr.add(i + 64)), mask)); + vst1q_u8(ptr.add(i + 80), vorrq_u8(vld1q_u8(ptr.add(i + 80)), mask)); + vst1q_u8(ptr.add(i + 96), vorrq_u8(vld1q_u8(ptr.add(i + 96)), mask)); + vst1q_u8(ptr.add(i + 112), vorrq_u8(vld1q_u8(ptr.add(i + 112)), mask)); + i += 128; + } + + // Process remaining 16 bytes at a time + while i + 16 <= len { + vst1q_u8(ptr.add(i), vorrq_u8(vld1q_u8(ptr.add(i)), mask)); + i += 16; + } + + // Handle remainder with scalar + while i + 4 <= len { + rgba_pixels[i + 3] = 255; + i += 4; + } + } +} + +// ============================================================================ +// Set Alpha Opaque - x86_64 SIMD (SSE2 + AVX2) +// ============================================================================ + +/// AVX2-optimized alpha set - processes 128 bytes (32 pixels) per iteration +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +#[inline] +unsafe fn set_alpha_avx2(rgba_pixels: &mut [u8]) { + let len = rgba_pixels.len(); + let ptr = rgba_pixels.as_mut_ptr(); + let mut i = 0; + + // Alpha mask: 0xFF at positions 3,7,11,15,19,23,27,31 (alpha bytes) + let alpha_mask = _mm256_set_epi8( + -1, 0, 0, 0, -1, 0, 0, 0, -1, 0, 0, 0, -1, 0, 0, 0, + -1, 0, 0, 0, -1, 0, 0, 0, -1, 0, 0, 0, -1, 0, 0, 0, + ); + + // Process 128 bytes (32 pixels) at a time + while i + 128 <= len { + let p0 = ptr.add(i) as *mut __m256i; + let p1 = ptr.add(i + 32) as *mut __m256i; + let p2 = ptr.add(i + 64) as *mut __m256i; + let p3 = ptr.add(i + 96) as *mut __m256i; + + _mm256_storeu_si256(p0, _mm256_or_si256(_mm256_loadu_si256(p0), alpha_mask)); + _mm256_storeu_si256(p1, _mm256_or_si256(_mm256_loadu_si256(p1), alpha_mask)); + _mm256_storeu_si256(p2, _mm256_or_si256(_mm256_loadu_si256(p2), alpha_mask)); + _mm256_storeu_si256(p3, _mm256_or_si256(_mm256_loadu_si256(p3), alpha_mask)); + + i += 128; + } + + // Fall back to SSE2 for remainder + set_alpha_sse2_remainder(&mut rgba_pixels[i..]); +} + +/// SSE2-optimized alpha set - processes 64 bytes (16 pixels) per iteration +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse2")] +#[inline] +unsafe fn set_alpha_sse2(rgba_pixels: &mut [u8]) { + let len = rgba_pixels.len(); + let ptr = rgba_pixels.as_mut_ptr(); + let mut i = 0; + + // Alpha mask: 0xFF at positions 3,7,11,15 + let alpha_mask = _mm_set_epi8(-1, 0, 0, 0, -1, 0, 0, 0, -1, 0, 0, 0, -1, 0, 0, 0); + + // Process 64 bytes (16 pixels) at a time + while i + 64 <= len { + let p0 = ptr.add(i) as *mut __m128i; + let p1 = ptr.add(i + 16) as *mut __m128i; + let p2 = ptr.add(i + 32) as *mut __m128i; + let p3 = ptr.add(i + 48) as *mut __m128i; + + _mm_storeu_si128(p0, _mm_or_si128(_mm_loadu_si128(p0), alpha_mask)); + _mm_storeu_si128(p1, _mm_or_si128(_mm_loadu_si128(p1), alpha_mask)); + _mm_storeu_si128(p2, _mm_or_si128(_mm_loadu_si128(p2), alpha_mask)); + _mm_storeu_si128(p3, _mm_or_si128(_mm_loadu_si128(p3), alpha_mask)); + + i += 64; + } + + // Handle remainder + set_alpha_sse2_remainder(&mut rgba_pixels[i..]); +} + +/// SSE2 remainder handler (also used by AVX2) +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse2")] +#[inline] +unsafe fn set_alpha_sse2_remainder(rgba_pixels: &mut [u8]) { + let len = rgba_pixels.len(); + let ptr = rgba_pixels.as_mut_ptr(); + let mut i = 0; + + let alpha_mask = _mm_set_epi8(-1, 0, 0, 0, -1, 0, 0, 0, -1, 0, 0, 0, -1, 0, 0, 0); + + // Process 16 bytes at a time + while i + 16 <= len { + let p = ptr.add(i) as *mut __m128i; + _mm_storeu_si128(p, _mm_or_si128(_mm_loadu_si128(p), alpha_mask)); + i += 16; + } + + // Scalar for final pixels + while i + 4 <= len { + rgba_pixels[i + 3] = 255; + i += 4; + } +} + +/// x86_64 dispatcher - uses AVX2 if available, otherwise SSE2 +#[cfg(target_arch = "x86_64")] +#[inline] +fn set_alpha_simd(rgba_pixels: &mut [u8]) { + unsafe { + if is_x86_feature_detected!("avx2") { + set_alpha_avx2(rgba_pixels); + } else { + set_alpha_sse2(rgba_pixels); + } + } +} + +/// Scalar fallback for non-SIMD platforms +#[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))] +#[inline] +fn set_alpha_simd(rgba_pixels: &mut [u8]) { + // Fast path for little-endian (WASM, RISC-V, most platforms) + #[cfg(target_endian = "little")] + { + let mut chunks = rgba_pixels.chunks_exact_mut(8); + const ALPHA_MASK: u64 = 0xFF000000_FF000000; + + for chunk in chunks.by_ref() { + let val = u64::from_ne_bytes(chunk.try_into().unwrap()); + chunk.copy_from_slice(&(val | ALPHA_MASK).to_ne_bytes()); + } + for px in chunks.into_remainder().chunks_exact_mut(4) { + px[3] = 255; + } + } + + // Byte-by-byte fallback for big-endian (rare) + #[cfg(target_endian = "big")] + { + for pixel in rgba_pixels.chunks_exact_mut(4) { + pixel[3] = 255; + } + } +} + +/// Set all alpha values to 255 (opaque) in-place +/// +/// Uses SIMD acceleration with parallel processing for maximum performance: +/// - **ARM64**: NEON +/// - **x86_64**: AVX2 (if available) or SSE2 +/// +/// Achieves ~0.7ms on 27 MP images (near memory bandwidth limit). +/// +/// # Parallelization Strategy +/// - Images < 4 MB: Single-threaded (parallel overhead > benefit) +/// - Images >= 4 MB: 256 KB chunks (optimal for L2 cache + core utilization) +/// +/// # Example +/// ```ignore +/// // Fix Windows clipboard alpha bug +/// set_all_alpha_opaque(&mut rgba_pixels); +/// ``` +#[inline] +pub fn set_all_alpha_opaque(rgba_pixels: &mut [u8]) { + // 256 KB chunks: fits L2 cache, good core utilization + const CHUNK_SIZE: usize = 256 * 1024; + const PARALLEL_THRESHOLD: usize = 4 * 1024 * 1024; // 4 MB (~1 MP) + + #[cfg(target_arch = "aarch64")] + let set_fn = set_alpha_neon; + + #[cfg(target_arch = "x86_64")] + let set_fn = set_alpha_simd; + + #[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))] + let set_fn = set_alpha_simd; + + // Only parallelize for large images where benefit > overhead + if rgba_pixels.len() > PARALLEL_THRESHOLD { + rgba_pixels + .par_chunks_mut(CHUNK_SIZE) + .for_each(|chunk| set_fn(chunk)); + } else { + set_fn(rgba_pixels); + } +} + +// ============================================================================ +// Nearest Neighbor Downsampling - Optimized +// ============================================================================ + +/// Fast nearest-neighbor downsampling for RGBA images. +/// +/// Uses integer arithmetic and direct u32 pixel copies for efficiency. +/// Performance is comparable to float-based implementation but with more +/// predictable results due to integer math. +/// +/// # Arguments +/// * `pixels` - Source RGBA pixel data (4 bytes per pixel) +/// * `src_width`, `src_height` - Source image dimensions +/// * `dst_width`, `dst_height` - Target image dimensions +/// +/// # Returns +/// Downsampled RGBA pixel data +/// +/// # Panics +/// - If `pixels` is smaller than `src_width * src_height * 4` +/// - If output size would overflow +pub fn nearest_neighbor_downsample( + pixels: &[u8], + src_width: u32, + src_height: u32, + dst_width: u32, + dst_height: u32, +) -> Vec { + // Validate input dimensions + let src_size = (src_width as usize) + .checked_mul(src_height as usize) + .and_then(|n| n.checked_mul(4)) + .expect("source dimensions overflow"); + assert!(pixels.len() >= src_size, "pixels buffer too small for source dimensions"); + + // Calculate output size with overflow protection + let dst_size = (dst_width as usize) + .checked_mul(dst_height as usize) + .and_then(|n| n.checked_mul(4)) + .expect("destination dimensions overflow"); + + let mut result: Vec = Vec::with_capacity(dst_size); + let src_stride = src_width as usize * 4; + + unsafe { + result.set_len(dst_size); + let dst_ptr = result.as_mut_ptr() as *mut u32; + let src_ptr = pixels.as_ptr(); + let mut dst_idx = 0usize; + + for ty in 0..dst_height { + // Integer division for y coordinate + let sy = (ty as u64 * src_height as u64 / dst_height as u64) as usize; + let row_ptr = src_ptr.add(sy * src_stride); + + for tx in 0..dst_width { + // Integer division for x coordinate + let sx = (tx as u64 * src_width as u64 / dst_width as u64) as usize; + // Copy pixel as u32 (4 bytes at once) + *dst_ptr.add(dst_idx) = *(row_ptr.add(sx * 4) as *const u32); + dst_idx += 1; + } + } + } + + result +} + +/// Downsample RGB image to RGBA using nearest-neighbor interpolation. +/// +/// This is optimized for the common case of resizing JPEG images (RGB) for preview. +/// Instead of converting the entire source image to RGBA first, this function: +/// 1. Samples only the needed source pixels (nearest-neighbor selection) +/// 2. Converts RGB→RGBA only for output pixels +/// +/// For a 48MP image downsampled to 15%, this processes ~1M pixels instead of 48M. +/// +/// # Arguments +/// * `rgb_pixels` - Source RGB pixel data (3 bytes per pixel) +/// * `src_width`, `src_height` - Source image dimensions +/// * `dst_width`, `dst_height` - Target image dimensions +/// +/// # Returns +/// Downsampled RGBA pixel data (4 bytes per pixel, alpha=255) +pub fn nearest_neighbor_downsample_rgb_to_rgba( + rgb_pixels: &[u8], + src_width: u32, + src_height: u32, + dst_width: u32, + dst_height: u32, +) -> Vec { + // Validate input dimensions + let src_size = (src_width as usize) + .checked_mul(src_height as usize) + .and_then(|n| n.checked_mul(3)) + .expect("source dimensions overflow"); + assert!(rgb_pixels.len() >= src_size, "pixels buffer too small for source dimensions"); + + // Calculate output size with overflow protection + let dst_size = (dst_width as usize) + .checked_mul(dst_height as usize) + .and_then(|n| n.checked_mul(4)) + .expect("destination dimensions overflow"); + + let mut result: Vec = Vec::with_capacity(dst_size); + let src_stride = src_width as usize * 3; + + unsafe { + result.set_len(dst_size); + let dst_ptr = result.as_mut_ptr(); + let src_ptr = rgb_pixels.as_ptr(); + let mut dst_idx = 0usize; + + for ty in 0..dst_height { + // Integer division for y coordinate + let sy = (ty as u64 * src_height as u64 / dst_height as u64) as usize; + let row_ptr = src_ptr.add(sy * src_stride); + + for tx in 0..dst_width { + // Integer division for x coordinate + let sx = (tx as u64 * src_width as u64 / dst_width as u64) as usize; + let src_pixel = row_ptr.add(sx * 3); + + // Copy RGB and add alpha=255 + let dst_pixel = dst_ptr.add(dst_idx); + *dst_pixel = *src_pixel; // R + *dst_pixel.add(1) = *src_pixel.add(1); // G + *dst_pixel.add(2) = *src_pixel.add(2); // B + *dst_pixel.add(3) = 255; // A + + dst_idx += 4; + } + } + } + + result +} + +/// Fast nearest-neighbor resize for DynamicImage, returning RGBA pixels. +/// +/// Automatically selects the optimal path based on input color type: +/// - **RGBA/BGRA**: Direct RGBA downsample (fastest for PNG with alpha) +/// - **RGB/BGR**: Fused RGB→RGBA downsample (skips full-image conversion) +/// - **Other formats**: Falls back to image crate conversion first +/// +/// For large JPEG images, this is 10-15x faster than: +/// ```ignore +/// let resized = img.resize(..., FilterType::Nearest); +/// let rgba = resized.to_rgba8(); +/// ``` +/// +/// # Returns +/// Tuple of (rgba_pixels, width, height) +pub fn fast_resize_to_rgba( + img: &image::DynamicImage, + dst_width: u32, + dst_height: u32, +) -> (Vec, u32, u32) { + use image::DynamicImage; + + let src_width = img.width(); + let src_height = img.height(); + + // If no resize needed, just convert to RGBA + if dst_width >= src_width && dst_height >= src_height { + let rgba = img.to_rgba8(); + return (rgba.into_raw(), src_width, src_height); + } + + match img { + // RGBA - use direct RGBA downsample + DynamicImage::ImageRgba8(rgba_img) => { + let pixels = nearest_neighbor_downsample( + rgba_img.as_raw(), + src_width, + src_height, + dst_width, + dst_height, + ); + (pixels, dst_width, dst_height) + } + + // RGB - use fused RGB→RGBA downsample (10-15x faster for large JPEGs) + DynamicImage::ImageRgb8(rgb_img) => { + let pixels = nearest_neighbor_downsample_rgb_to_rgba( + rgb_img.as_raw(), + src_width, + src_height, + dst_width, + dst_height, + ); + (pixels, dst_width, dst_height) + } + + // Other formats - convert to RGBA first, then downsample + // This handles grayscale, BGRA, 16-bit, etc. + _ => { + let rgba = img.to_rgba8(); + let pixels = nearest_neighbor_downsample( + rgba.as_raw(), + src_width, + src_height, + dst_width, + dst_height, + ); + (pixels, dst_width, dst_height) + } + } +} + +// ============================================================================ +// RGBA to RGB Conversion - SIMD Optimized (strips alpha channel) +// ============================================================================ + +/// Convert RGBA pixel data to RGB, stripping the alpha channel. +/// +/// Uses SIMD acceleration where available: +/// - **ARM64**: NEON with vld4/vst3 deinterleave +/// - **x86_64**: Scalar (SSSE3 shuffle is complex for this pattern) +/// +/// This is the inverse of `rgb_to_rgba`. +#[inline] +pub fn rgba_to_rgb(rgba_data: &[u8]) -> Vec { + let pixel_count = rgba_data.len() / 4; + let mut rgb_data = Vec::with_capacity(pixel_count * 3); + + #[cfg(target_arch = "aarch64")] + unsafe { + rgba_to_rgb_neon(rgba_data, &mut rgb_data); + } + + #[cfg(not(target_arch = "aarch64"))] + { + rgba_to_rgb_scalar(rgba_data, &mut rgb_data); + } + + rgb_data +} + +#[cfg(target_arch = "aarch64")] +unsafe fn rgba_to_rgb_neon(rgba_data: &[u8], rgb_data: &mut Vec) { + let pixel_count = rgba_data.len() / 4; + let chunks = pixel_count / 16; // Process 16 pixels at a time + let remainder = pixel_count % 16; + + let src = rgba_data.as_ptr(); + rgb_data.reserve(pixel_count * 3); + let dst = rgb_data.as_mut_ptr(); + + // Process 16 pixels (64 RGBA bytes → 48 RGB bytes) per iteration + for i in 0..chunks { + let src_offset = i * 64; + let dst_offset = i * 48; + + // vld4q_u8 loads 64 bytes and deinterleaves into 4 registers of 16 bytes each + let rgba = vld4q_u8(src.add(src_offset)); + + // rgba.0 = all R values (16 bytes) + // rgba.1 = all G values (16 bytes) + // rgba.2 = all B values (16 bytes) + // rgba.3 = all A values (16 bytes) - discarded + + // vst3q_u8 interleaves 3 registers and stores 48 bytes + let rgb = uint8x16x3_t(rgba.0, rgba.1, rgba.2); + vst3q_u8(dst.add(dst_offset), rgb); + } + + // Handle remaining pixels with scalar code + let processed = chunks * 16; + let src_remainder = src.add(processed * 4); + let dst_remainder = dst.add(processed * 3); + + for i in 0..remainder { + *dst_remainder.add(i * 3) = *src_remainder.add(i * 4); + *dst_remainder.add(i * 3 + 1) = *src_remainder.add(i * 4 + 1); + *dst_remainder.add(i * 3 + 2) = *src_remainder.add(i * 4 + 2); + } + + rgb_data.set_len(pixel_count * 3); +} + +#[cfg(not(target_arch = "aarch64"))] +fn rgba_to_rgb_scalar(rgba_data: &[u8], rgb_data: &mut Vec) { + rgb_data.reserve(rgba_data.len() / 4 * 3); + for chunk in rgba_data.chunks_exact(4) { + rgb_data.extend_from_slice(&chunk[..3]); + } +} + +// ============================================================================ +// RGB to RGBA Conversion - SIMD Optimized +// ============================================================================ + +/// Convert RGB pixel data to RGBA, setting alpha to 255. +/// +/// Uses SIMD acceleration where available: +/// - **ARM64**: NEON with vld3/vst4 deinterleave +/// - **x86_64**: SSSE3 pshufb (with scalar fallback) +/// +/// ~4x speedup on large images compared to naive scalar. +#[inline] +pub fn rgb_to_rgba(rgb_data: &[u8]) -> Vec { + let pixel_count = rgb_data.len() / 3; + let mut rgba_data = Vec::with_capacity(pixel_count * 4); + + #[cfg(target_arch = "aarch64")] + unsafe { + rgb_to_rgba_neon(rgb_data, &mut rgba_data); + } + + #[cfg(target_arch = "x86_64")] + unsafe { + // SSSE3 is available on all x86_64 CPUs since 2006, but check anyway + if is_x86_feature_detected!("ssse3") { + rgb_to_rgba_ssse3(rgb_data, &mut rgba_data); + } else { + rgb_to_rgba_scalar_x86(rgb_data, &mut rgba_data); + } + } + + #[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))] + { + rgb_to_rgba_scalar(rgb_data, &mut rgba_data); + } + + rgba_data +} + +/// NEON-optimized RGB to RGBA conversion +/// +/// Uses vld3q to load RGB data deinterleaved, then vst4q to store as RGBA. +/// Unlike SSE/SSSE3, NEON's vld3q loads exactly 48 bytes, so no OOB risk. +#[cfg(target_arch = "aarch64")] +#[inline] +unsafe fn rgb_to_rgba_neon(rgb_data: &[u8], rgba_data: &mut Vec) { + let pixel_count = rgb_data.len() / 3; + let rgba_size = pixel_count.checked_mul(4).expect("RGB to RGBA size overflow"); + + rgba_data.clear(); + rgba_data.reserve_exact(rgba_size); + rgba_data.set_len(rgba_size); + + let src = rgb_data.as_ptr(); + let dst = rgba_data.as_mut_ptr(); + + let mut i = 0usize; + let mut o = 0usize; + + // Process 16 pixels at a time (48 RGB bytes -> 64 RGBA bytes) + // Using vld3 to deinterleave RGB channels + while i + 48 <= rgb_data.len() { + // Load 48 bytes as 16 RGB pixels (deinterleaved into R, G, B planes) + let rgb = vld3q_u8(src.add(i)); + + // Create alpha channel (all 255) + let alpha = vdupq_n_u8(255); + + // Interleave as RGBA and store + let rgba = uint8x16x4_t(rgb.0, rgb.1, rgb.2, alpha); + vst4q_u8(dst.add(o), rgba); + + i += 48; + o += 64; + } + + // Scalar remainder + while i + 3 <= rgb_data.len() { + *dst.add(o) = *src.add(i); + *dst.add(o + 1) = *src.add(i + 1); + *dst.add(o + 2) = *src.add(i + 2); + *dst.add(o + 3) = 255; + i += 3; + o += 4; + } +} + +/// SSSE3-optimized RGB to RGBA conversion using pshufb for byte rearrangement +/// +/// Processes 16 pixels (48 RGB bytes → 64 RGBA bytes) per iteration. +/// Uses pshufb to rearrange RGB bytes and insert alpha in one operation. +/// +/// # Safety +/// - Caller must ensure SSSE3 is available (use `is_x86_feature_detected!`) +/// - Input length should be a multiple of 3 (trailing 1-2 bytes are ignored) +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "ssse3")] +#[inline] +unsafe fn rgb_to_rgba_ssse3(rgb_data: &[u8], rgba_data: &mut Vec) { + let pixel_count = rgb_data.len() / 3; + // Use checked_mul to prevent overflow on very large inputs + let rgba_size = pixel_count.checked_mul(4).expect("RGB to RGBA size overflow"); + + // Clear and reserve exact capacity to avoid over-allocation on reuse + rgba_data.clear(); + rgba_data.reserve_exact(rgba_size); + rgba_data.set_len(rgba_size); + + let src = rgb_data.as_ptr(); + let dst = rgba_data.as_mut_ptr(); + + let mut i = 0usize; + let mut o = 0usize; + + // Shuffle mask: rearranges 12 RGB bytes to 16 RGBA bytes + // Input positions 0-11 map to output, -1 (0x80) produces zero for alpha slots + let shuffle = _mm_setr_epi8( + 0, 1, 2, -1, // pixel 0: R G B 0 + 3, 4, 5, -1, // pixel 1: R G B 0 + 6, 7, 8, -1, // pixel 2: R G B 0 + 9, 10, 11, -1 // pixel 3: R G B 0 + ); + // Alpha mask: 0xFF at alpha positions (bytes 3, 7, 11, 15) + let alpha = _mm_set_epi8(-1, 0, 0, 0, -1, 0, 0, 0, -1, 0, 0, 0, -1, 0, 0, 0); + + // Process 16 pixels at a time (48 RGB bytes -> 64 RGBA bytes) + // Loop bound: last load is at i+36, reads 16 bytes -> needs i+52 <= len + while i + 52 <= rgb_data.len() { + let rgb0 = _mm_loadu_si128(src.add(i) as *const __m128i); + let rgb1 = _mm_loadu_si128(src.add(i + 12) as *const __m128i); + let rgb2 = _mm_loadu_si128(src.add(i + 24) as *const __m128i); + let rgb3 = _mm_loadu_si128(src.add(i + 36) as *const __m128i); + + let rgba0 = _mm_or_si128(_mm_shuffle_epi8(rgb0, shuffle), alpha); + let rgba1 = _mm_or_si128(_mm_shuffle_epi8(rgb1, shuffle), alpha); + let rgba2 = _mm_or_si128(_mm_shuffle_epi8(rgb2, shuffle), alpha); + let rgba3 = _mm_or_si128(_mm_shuffle_epi8(rgb3, shuffle), alpha); + + _mm_storeu_si128(dst.add(o) as *mut __m128i, rgba0); + _mm_storeu_si128(dst.add(o + 16) as *mut __m128i, rgba1); + _mm_storeu_si128(dst.add(o + 32) as *mut __m128i, rgba2); + _mm_storeu_si128(dst.add(o + 48) as *mut __m128i, rgba3); + + i += 48; + o += 64; + } + + // Process 4 pixels at a time (12 RGB bytes -> 16 RGBA bytes) + // Loop bound: load at i reads 16 bytes -> needs i+16 <= len + while i + 16 <= rgb_data.len() { + let rgb = _mm_loadu_si128(src.add(i) as *const __m128i); + let rgba = _mm_or_si128(_mm_shuffle_epi8(rgb, shuffle), alpha); + _mm_storeu_si128(dst.add(o) as *mut __m128i, rgba); + i += 12; + o += 16; + } + + // Scalar remainder (handles final pixels where SIMD would read OOB) + while i + 3 <= rgb_data.len() { + *dst.add(o) = *src.add(i); + *dst.add(o + 1) = *src.add(i + 1); + *dst.add(o + 2) = *src.add(i + 2); + *dst.add(o + 3) = 255; + i += 3; + o += 4; + } +} + +/// Scalar fallback for RGB to RGBA (used when SSSE3 not available) +#[cfg(target_arch = "x86_64")] +#[inline] +unsafe fn rgb_to_rgba_scalar_x86(rgb_data: &[u8], rgba_data: &mut Vec) { + let pixel_count = rgb_data.len() / 3; + let rgba_size = pixel_count.checked_mul(4).expect("RGB to RGBA size overflow"); + + rgba_data.clear(); + rgba_data.reserve_exact(rgba_size); + rgba_data.set_len(rgba_size); + + let src = rgb_data.as_ptr(); + let dst = rgba_data.as_mut_ptr(); + + let mut i = 0usize; + let mut o = 0usize; + + // Process 4 pixels at a time using u32 operations + while i + 12 <= rgb_data.len() { + let p0 = *src.add(i) as u32 | (*src.add(i+1) as u32) << 8 | (*src.add(i+2) as u32) << 16 | 0xFF000000; + let p1 = *src.add(i+3) as u32 | (*src.add(i+4) as u32) << 8 | (*src.add(i+5) as u32) << 16 | 0xFF000000; + let p2 = *src.add(i+6) as u32 | (*src.add(i+7) as u32) << 8 | (*src.add(i+8) as u32) << 16 | 0xFF000000; + let p3 = *src.add(i+9) as u32 | (*src.add(i+10) as u32) << 8 | (*src.add(i+11) as u32) << 16 | 0xFF000000; + + *(dst.add(o) as *mut u32) = p0; + *(dst.add(o+4) as *mut u32) = p1; + *(dst.add(o+8) as *mut u32) = p2; + *(dst.add(o+12) as *mut u32) = p3; + + i += 12; + o += 16; + } + + // Scalar remainder + while i + 3 <= rgb_data.len() { + *dst.add(o) = *src.add(i); + *dst.add(o + 1) = *src.add(i + 1); + *dst.add(o + 2) = *src.add(i + 2); + *dst.add(o + 3) = 255; + i += 3; + o += 4; + } +} + +/// Scalar RGB to RGBA conversion (fallback for non-SIMD platforms) +#[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))] +#[inline] +fn rgb_to_rgba_scalar(rgb_data: &[u8], rgba_data: &mut Vec) { + let pixel_count = rgb_data.len() / 3; + let rgba_size = pixel_count.checked_mul(4).expect("RGB to RGBA size overflow"); + + rgba_data.clear(); + rgba_data.reserve_exact(rgba_size); + + for chunk in rgb_data.chunks_exact(3) { + rgba_data.extend_from_slice(&[chunk[0], chunk[1], chunk[2], 255]); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_has_alpha_opaque() { + // All pixels opaque (alpha = 255) + let pixels = vec![255u8; 1024]; // 256 pixels + assert!(!has_alpha_transparency(&pixels)); + } + + #[test] + fn test_has_alpha_transparent() { + // One transparent pixel + let mut pixels = vec![255u8; 1024]; + pixels[3] = 128; // First pixel has alpha = 128 + assert!(has_alpha_transparency(&pixels)); + } + + #[test] + fn test_set_alpha_opaque() { + let mut pixels = vec![0u8; 16]; // 4 pixels, all zero + set_all_alpha_opaque(&mut pixels); + + // Check alpha channels are now 255 + assert_eq!(pixels[3], 255); + assert_eq!(pixels[7], 255); + assert_eq!(pixels[11], 255); + assert_eq!(pixels[15], 255); + + // RGB should still be 0 + assert_eq!(pixels[0], 0); + assert_eq!(pixels[1], 0); + assert_eq!(pixels[2], 0); + } + + #[test] + fn test_rgb_to_rgba_basic() { + // 4 RGB pixels: red, green, blue, white + let rgb = vec![ + 255, 0, 0, // red + 0, 255, 0, // green + 0, 0, 255, // blue + 255, 255, 255, // white + ]; + let rgba = rgb_to_rgba(&rgb); + + assert_eq!(rgba.len(), 16); + // Red pixel + assert_eq!(&rgba[0..4], &[255, 0, 0, 255]); + // Green pixel + assert_eq!(&rgba[4..8], &[0, 255, 0, 255]); + // Blue pixel + assert_eq!(&rgba[8..12], &[0, 0, 255, 255]); + // White pixel + assert_eq!(&rgba[12..16], &[255, 255, 255, 255]); + } + + #[test] + fn test_rgb_to_rgba_large() { + // Test with enough pixels to trigger SIMD path (48+ bytes = 16+ pixels) + let pixel_count = 64; + let mut rgb = Vec::with_capacity(pixel_count * 3); + for i in 0..pixel_count { + rgb.push((i * 4) as u8); // R + rgb.push((i * 4 + 1) as u8); // G + rgb.push((i * 4 + 2) as u8); // B + } + + let rgba = rgb_to_rgba(&rgb); + assert_eq!(rgba.len(), pixel_count * 4); + + // Verify each pixel + for i in 0..pixel_count { + let r = (i * 4) as u8; + let g = (i * 4 + 1) as u8; + let b = (i * 4 + 2) as u8; + assert_eq!(rgba[i * 4], r, "pixel {} R mismatch", i); + assert_eq!(rgba[i * 4 + 1], g, "pixel {} G mismatch", i); + assert_eq!(rgba[i * 4 + 2], b, "pixel {} B mismatch", i); + assert_eq!(rgba[i * 4 + 3], 255, "pixel {} A mismatch", i); + } + } +} diff --git a/src-tauri/src/simd/mod.rs b/src-tauri/src/simd/mod.rs new file mode 100644 index 00000000..ea112f6d --- /dev/null +++ b/src-tauri/src/simd/mod.rs @@ -0,0 +1,30 @@ +//! SIMD-accelerated operations for Vector +//! +//! This module provides high-performance implementations using: +//! - **ARM64 (Apple Silicon, Android)**: NEON SIMD intrinsics +//! - **x86_64 (Windows, Linux)**: SSE2/AVX2 SIMD intrinsics +//! +//! All public functions automatically select the best implementation +//! for the current platform at compile time (with runtime AVX2 detection on x86_64). +//! +//! # Modules +//! +//! - [`hex`] - Hex encoding/decoding (~62x faster than format!) +//! - [`image`] - Image operations (~9x faster with parallel SIMD) + +pub mod hex; +pub mod image; + +// Re-export commonly used functions at the simd level +pub use hex::{ + bytes_to_hex_16, bytes_to_hex_32, bytes_to_hex_string, + hex_string_to_bytes, hex_to_bytes_16, hex_to_bytes_32, +}; +pub use image::{has_alpha_transparency, set_all_alpha_opaque}; + +#[cfg(target_os = "windows")] +pub use image::has_all_alpha_near_zero; + +// Also available via crate::simd::image::* +// - nearest_neighbor_downsample (used in util.rs for blurhash) +// - rgb_to_rgba (used in util.rs for base64 image decoding) diff --git a/src-tauri/src/state/chat_state.rs b/src-tauri/src/state/chat_state.rs index 3b00bab6..a3adb742 100644 --- a/src-tauri/src/state/chat_state.rs +++ b/src-tauri/src/state/chat_state.rs @@ -5,22 +5,31 @@ use nostr_sdk::prelude::*; use tauri::Emitter; +use crate::message::compact::{CompactMessage, CompactAttachment, NpubInterner}; use crate::{Profile, Chat, ChatType, Message}; +use crate::chat::SerializableChat; use crate::db::SlimProfile; use super::globals::{TAURI_APP, NOSTR_CLIENT}; use super::SyncMode; +#[cfg(debug_assertions)] +use super::stats::CacheStats; /// Core application state containing profiles, chats, and sync status. -#[derive(serde::Serialize, Clone, Debug)] +#[derive(Clone, Debug)] pub struct ChatState { - pub(crate) profiles: Vec, - pub(crate) chats: Vec, - pub(crate) is_syncing: bool, - pub(crate) sync_window_start: u64, - pub(crate) sync_window_end: u64, - pub(crate) sync_mode: SyncMode, - pub(crate) sync_empty_iterations: u8, - pub(crate) sync_total_iterations: u8, + pub profiles: Vec, + pub chats: Vec, + /// Global npub interner - stores each unique npub string once + pub interner: NpubInterner, + pub is_syncing: bool, + pub sync_window_start: u64, + pub sync_window_end: u64, + pub sync_mode: SyncMode, + pub sync_empty_iterations: u8, + pub sync_total_iterations: u8, + /// Cache statistics for benchmarking (debug builds only) + #[cfg(debug_assertions)] + pub cache_stats: CacheStats, } impl ChatState { @@ -29,223 +38,477 @@ impl ChatState { Self { profiles: Vec::new(), chats: Vec::new(), + interner: NpubInterner::new(), is_syncing: false, sync_window_start: 0, sync_window_end: 0, sync_mode: SyncMode::Finished, sync_empty_iterations: 0, sync_total_iterations: 0, + #[cfg(debug_assertions)] + cache_stats: CacheStats::new(), } } - /// Load a Vector Profile into the state from our SlimProfile database format - pub async fn from_db_profile(&mut self, slim: SlimProfile) { - // Check if profile already exists - if let Some(position) = self.profiles.iter().position(|profile| profile.id == slim.id) { - // Replace existing profile - let mut full_profile = slim.to_profile(); + // ======================================================================== + // Profile Management + // ======================================================================== - // Check if this is our profile: we need to mark it as such + /// Merge multiple Vector Profiles from SlimProfile format into the state + pub async fn merge_db_profiles(&mut self, slim_profiles: Vec) { + let my_public_key = { let client = NOSTR_CLIENT.get().expect("Nostr client not initialized"); let signer = client.signer().await.unwrap(); - let my_public_key = signer.get_public_key().await.unwrap(); - let profile_pubkey = PublicKey::from_bech32(&full_profile.id).unwrap(); - full_profile.mine = my_public_key == profile_pubkey; - - self.profiles[position] = full_profile; - } else { - // Add new profile - self.profiles.push(slim.to_profile()); - } - } + signer.get_public_key().await.unwrap() + }; - /// Merge multiple Vector Profiles from SlimProfile format into the state at once - pub async fn merge_db_profiles(&mut self, slim_profiles: Vec) { for slim in slim_profiles { - self.from_db_profile(slim).await; + if let Some(position) = self.profiles.iter().position(|profile| profile.id == slim.id) { + let mut full_profile = slim.to_profile(); + if let Ok(profile_pubkey) = PublicKey::from_bech32(&full_profile.id) { + full_profile.mine = my_public_key == profile_pubkey; + } + self.profiles[position] = full_profile; + } else { + self.profiles.push(slim.to_profile()); + } } } - /// Get a profile by ID pub fn get_profile(&self, id: &str) -> Option<&Profile> { self.profiles.iter().find(|p| p.id == id) } - /// Get a mutable profile by ID pub fn get_profile_mut(&mut self, id: &str) -> Option<&mut Profile> { self.profiles.iter_mut().find(|p| p.id == id) } - /// Get a chat by ID + // ======================================================================== + // Chat Management + // ======================================================================== + pub fn get_chat(&self, id: &str) -> Option<&Chat> { self.chats.iter().find(|c| c.id == id) } - /// Get a mutable chat by ID pub fn get_chat_mut(&mut self, id: &str) -> Option<&mut Chat> { self.chats.iter_mut().find(|c| c.id == id) } - /// Create a new chat for a DM with a specific user + /// Get a serializable version of a chat (for frontend) + #[allow(dead_code)] + pub fn get_chat_serializable(&self, id: &str) -> Option { + self.get_chat(id).map(|c| c.to_serializable(&self.interner)) + } + + /// Create a new DM chat if it doesn't exist pub fn create_dm_chat(&mut self, their_npub: &str) -> String { - // Check if chat already exists if self.get_chat(their_npub).is_none() { let chat = Chat::new_dm(their_npub.to_string()); self.chats.push(chat); } - their_npub.to_string() } /// Create or get an MLS group chat pub fn create_or_get_mls_group_chat(&mut self, group_id: &str, participants: Vec) -> String { - // Check if chat already exists if self.get_chat(group_id).is_none() { let chat = Chat::new_mls_group(group_id.to_string(), participants); self.chats.push(chat); } - group_id.to_string() } + // ======================================================================== + // Message Management + // ======================================================================== + /// Add a message to a chat via its ID pub fn add_message_to_chat(&mut self, chat_id: &str, message: Message) -> bool { - let is_msg_added = match self.get_chat_mut(chat_id) { - Some(chat) => { - // Add the message to the existing chat - chat.internal_add_message(message) - }, - None => { - // Chat doesn't exist, create it and add the message - // Determine chat type based on chat_id format - let chat = if chat_id.starts_with("npub1") { - // DM chat: use the chat_id as the participant - Chat::new_dm(chat_id.to_string()) - } else { - // MLS group: participants will be set later - Chat::new(chat_id.to_string(), ChatType::MlsGroup, vec![]) - }; - let mut chat = chat; - let was_added = chat.internal_add_message(message); - self.chats.push(chat); - was_added - } + #[cfg(debug_assertions)] + let start = std::time::Instant::now(); + + // Convert to compact using our interner + let compact = CompactMessage::from_message(&message, &mut self.interner); + + let is_msg_added = if let Some(chat) = self.chats.iter_mut().find(|c| c.id == chat_id) { + chat.add_compact_message(compact) + } else { + // Chat doesn't exist, create it + let mut chat = if chat_id.starts_with("npub1") { + Chat::new_dm(chat_id.to_string()) + } else { + Chat::new(chat_id.to_string(), ChatType::MlsGroup, vec![]) + }; + let was_added = chat.add_compact_message(compact); + self.chats.push(chat); + was_added }; - // Sort our chat positions based on last message time + // Sort chats by last message time (newest first) self.chats.sort_by(|a, b| { - // Get last message time for both chats - let a_time = a.last_message_time(); - let b_time = b.last_message_time(); - - // Compare timestamps in reverse order (newest first) - b_time.cmp(&a_time) + b.last_message_time().cmp(&a.last_message_time()) }); + // Track stats (debug builds only) + #[cfg(debug_assertions)] + if is_msg_added { + self.cache_stats.record_insert(start.elapsed()); + if self.cache_stats.should_log(100) { + self.update_and_log_stats(); + } + } + is_msg_added } - /// Add a message to a chat via its participant npub + /// Batch add messages to a chat - much faster for pagination/history loads. + /// + /// Sorts chats only once at the end instead of per-message. + /// Returns the number of messages actually added. + pub fn add_messages_to_chat_batch(&mut self, chat_id: &str, messages: Vec) -> usize { + if messages.is_empty() { + return 0; + } + + // Convert all messages to compact format (zero-copy - moves strings!) + let compact_messages: Vec<_> = messages.into_iter() + .map(|msg| CompactMessage::from_message_owned(msg, &mut self.interner)) + .collect(); + + // Find or create the chat + let chat = if let Some(chat) = self.chats.iter_mut().find(|c| c.id == chat_id) { + chat + } else { + // Chat doesn't exist, create it + let chat = if chat_id.starts_with("npub1") { + Chat::new_dm(chat_id.to_string()) + } else { + Chat::new(chat_id.to_string(), ChatType::MlsGroup, vec![]) + }; + self.chats.push(chat); + self.chats.last_mut().unwrap() + }; + + // Track if last message time changes (only happens when adding newer messages) + let old_last_time = chat.messages.last_timestamp(); + + // Batch insert all messages + let added = chat.messages.insert_batch(compact_messages); + + // Only sort chats if the last message time changed (i.e., we added newer messages) + // Prepending older messages (pagination) doesn't change chat order + if added > 0 && chat.messages.last_timestamp() != old_last_time { + self.chats.sort_by(|a, b| { + b.last_message_time().cmp(&a.last_message_time()) + }); + } + + added + } + + /// Add a message to a chat via participant npub pub fn add_message_to_participant(&mut self, their_npub: &str, message: Message) -> bool { - // Ensure profiles exist for the participant + // Ensure profile exists if self.get_profile(their_npub).is_none() { - // Create a basic profile for the participant let mut profile = Profile::new(); profile.id = their_npub.to_string(); - profile.mine = false; // It's not our profile + profile.mine = false; - // Update the frontend about the new profile if let Some(handle) = TAURI_APP.get() { handle.emit("profile_update", &profile).unwrap(); } - // Add to our profiles list self.profiles.push(profile); } - // Create or get the chat ID let chat_id = self.create_dm_chat(their_npub); - - // Add the message to the chat self.add_message_to_chat(&chat_id, message) } - /// Count unread messages across all profiles + // ======================================================================== + // Message Lookup (O(n chats × log m messages)) + // ======================================================================== + + /// Find a message by ID across all chats - O(n × log m) + pub fn find_message(&self, message_id: &str) -> Option<(&Chat, Message)> { + if message_id.is_empty() { + return None; + } + + for chat in &self.chats { + if let Some(compact) = chat.get_compact_message(message_id) { + return Some((chat, compact.to_message(&self.interner))); + } + } + None + } + + /// Find which chat contains a message - O(n × log m) + #[allow(dead_code)] + pub fn find_chat_for_message(&self, message_id: &str) -> Option<(usize, String)> { + if message_id.is_empty() { + return None; + } + + for (idx, chat) in self.chats.iter().enumerate() { + if chat.has_message(message_id) { + return Some((idx, chat.id.clone())); + } + } + None + } + + /// Find a chat and message by message ID (mutable) - O(n × log m) + #[allow(dead_code)] + pub fn find_chat_and_message_mut(&mut self, message_id: &str) -> Option<(String, &mut CompactMessage)> { + if message_id.is_empty() { + return None; + } + + // First find which chat has it + let chat_idx = self.chats.iter() + .position(|chat| chat.has_message(message_id))?; + + let chat_id = self.chats[chat_idx].id.clone(); + let msg = self.chats[chat_idx].get_compact_message_mut(message_id)?; + + Some((chat_id, msg)) + } + + /// Update a message by ID and return (chat_id, Message) for save/emit. + /// + /// This is the preferred pattern for mutating messages - it handles: + /// 1. Finding the chat containing the message + /// 2. Calling your mutation closure + /// 3. Converting back to Message format for database save + /// + /// Example: + /// ``` + /// let result = state.update_message(&msg_id, |msg| { + /// msg.preview_metadata = Some(Box::new(metadata)); + /// }); + /// if let Some((chat_id, message)) = result { + /// db::save_message(handle, &chat_id, &message).await; + /// } + /// ``` + pub fn update_message(&mut self, message_id: &str, f: F) -> Option<(String, Message)> + where + F: FnOnce(&mut CompactMessage), + { + if message_id.is_empty() { + return None; + } + + // Find which chat has this message + let chat_idx = self.chats.iter() + .position(|chat| chat.has_message(message_id))?; + + // Mutate the message + if let Some(msg) = self.chats[chat_idx].get_compact_message_mut(message_id) { + f(msg); + } + + // Get chat_id and convert to Message (reborrow is fine now) + let chat_id = self.chats[chat_idx].id.clone(); + self.chats[chat_idx].get_compact_message(message_id) + .map(|m| (chat_id, m.to_message(&self.interner))) + } + + /// Update a message in a specific chat and return Message for save/emit. + /// + /// Use this when you already know the chat_id (avoids searching all chats). + pub fn update_message_in_chat(&mut self, chat_id: &str, message_id: &str, f: F) -> Option + where + F: FnOnce(&mut CompactMessage), + { + let chat_idx = self.chats.iter().position(|c| c.id == chat_id)?; + + if let Some(msg) = self.chats[chat_idx].get_compact_message_mut(message_id) { + f(msg); + } + + self.chats[chat_idx].get_compact_message(message_id) + .map(|m| m.to_message(&self.interner)) + } + + /// Finalize a pending message by updating its ID to the real event ID. + /// + /// This is used when a message is confirmed sent and receives its final ID. + /// Rebuilds the message index since the ID changed. + /// Returns (old_id, Message) for emit/save. + pub fn finalize_pending_message(&mut self, chat_id: &str, pending_id: &str, real_id: &str) -> Option<(String, Message)> { + let chat_idx = self.chats.iter().position(|c| c.id == chat_id)?; + + // Update the message ID + if let Some(msg) = self.chats[chat_idx].get_compact_message_mut(pending_id) { + msg.id = crate::simd::hex_to_bytes_32(real_id); + msg.set_pending(false); + } + + // Rebuild index since ID changed + self.chats[chat_idx].messages.rebuild_index(); + + // Get the message with new ID for emit/save + self.chats[chat_idx].get_compact_message(real_id) + .map(|m| (pending_id.to_string(), m.to_message(&self.interner))) + } + + /// Update an attachment within a message. + /// + /// Searches for the chat by hint (group_id for MLS, participant npub for DMs), + /// finds the message, finds the attachment, and applies the mutation. + /// Returns true if the attachment was found and updated. + /// + /// Example: + /// ``` + /// state.update_attachment(&npub, &msg_id, &attachment_id, |att| { + /// att.set_downloaded(true); + /// att.path = path.into_boxed_str(); + /// }); + /// ``` + pub fn update_attachment(&mut self, chat_hint: &str, msg_id: &str, attachment_id: &str, f: F) -> bool + where + F: FnOnce(&mut CompactAttachment), + { + for chat in &mut self.chats { + let is_target = match &chat.chat_type { + ChatType::MlsGroup => chat.id == chat_hint, + ChatType::DirectMessage => chat.has_participant(chat_hint), + }; + + if is_target { + if let Some(msg) = chat.messages.find_by_hex_id_mut(msg_id) { + if let Some(att) = msg.attachments.iter_mut().find(|a| a.id_eq(attachment_id)) { + f(att); + return true; + } + } + } + } + false + } + + /// Add an attachment to a message. + /// + /// Finds the message by ID in the specified chat and appends the attachment. + /// Returns true if the message was found and attachment added. + pub fn add_attachment_to_message(&mut self, chat_id: &str, msg_id: &str, attachment: CompactAttachment) -> bool { + let chat_idx = match self.chats.iter().position(|c| c.id == chat_id || c.has_participant(chat_id)) { + Some(idx) => idx, + None => return false, + }; + + if let Some(msg) = self.chats[chat_idx].messages.find_by_hex_id_mut(msg_id) { + msg.attachments.push(attachment); + true + } else { + false + } + } + + /// Add a reaction to a message by ID - handles interner access via split borrowing + /// Returns (chat_id, was_added) if the message was found + pub fn add_reaction_to_message(&mut self, message_id: &str, reaction: crate::message::Reaction) -> Option<(String, bool)> { + if message_id.is_empty() { + return None; + } + + // Find which chat has it + let chat_idx = self.chats.iter() + .position(|chat| chat.has_message(message_id))?; + + let chat_id = self.chats[chat_idx].id.clone(); + + // Split borrow: access chats[chat_idx] and interner separately + let msg = self.chats[chat_idx].get_compact_message_mut(message_id)?; + let added = msg.add_reaction(reaction, &mut self.interner); + + Some((chat_id, added)) + } + + /// Check if a message exists - O(n × log m) + #[allow(dead_code)] + pub fn message_exists(&self, message_id: &str) -> bool { + if message_id.is_empty() { + return false; + } + self.chats.iter().any(|chat| chat.has_message(message_id)) + } + + // ======================================================================== + // Unread Count + // ======================================================================== + + /// Count unread messages across all chats pub fn count_unread_messages(&self) -> u32 { let mut total_unread = 0; - // Count unread messages in all chats for chat in &self.chats { - // Skip muted chats entirely if chat.muted { continue; } - // Skip chats where the corresponding profile is muted (for DMs) - let mut skip_for_profile_mute = false; - match chat.chat_type { - ChatType::DirectMessage => { - // For DMs, chat.id is the other participant's npub - if let Some(profile) = self.get_profile(&chat.id) { - if profile.muted { - skip_for_profile_mute = true; - } + // Skip if profile is muted (for DMs) + if matches!(chat.chat_type, ChatType::DirectMessage) { + if let Some(profile) = self.get_profile(&chat.id) { + if profile.muted { + continue; } } - ChatType::MlsGroup => { - // For MLS groups, muting is handled at the chat level (already checked above) - // No additional profile-level muting needed - } - } - if skip_for_profile_mute { - continue; } - // Find the last read message ID for this chat let last_read_id = &chat.last_read; - // Walk backwards from the end to count unread messages - // Stop when we hit: 1) our own message, or 2) the last_read message let mut unread_count = 0; - for msg in chat.messages.iter().rev() { - // If we hit our own message, stop - we clearly read everything before it - if msg.mine { + for msg in chat.iter_compact().rev() { + if msg.flags.is_mine() { break; } - - // If we hit the last_read message, stop - everything at and before this is read - if !last_read_id.is_empty() && msg.id == *last_read_id { + if !last_read_id.is_empty() && msg.id_hex() == *last_read_id { break; } - - // Count this message as unread unread_count += 1; } - total_unread += unread_count as u32; + total_unread += unread_count; } total_unread } - /// Find a message by its ID across all chats - pub fn find_message(&self, message_id: &str) -> Option<(&Chat, &Message)> { - for chat in &self.chats { - if let Some(message) = chat.messages.iter().find(|m| m.id == message_id) { - return Some((chat, message)); - } - } - None + // ======================================================================== + // Statistics (debug builds only) + // ======================================================================== + + #[cfg(debug_assertions)] + fn update_and_log_stats(&mut self) { + self.cache_stats.chat_count = self.chats.len(); + self.cache_stats.message_count = self.chats.iter() + .map(|c| c.message_count()) + .sum(); + self.cache_stats.total_memory_bytes = + self.cache_stats.message_count * 500 + self.interner.memory_usage(); + self.cache_stats.log(); + println!("[Interner] {} unique npubs, {} bytes", + self.interner.len(), self.interner.memory_usage()); } - /// Find a chat and message by message ID across all chats (mutable) - pub fn find_chat_and_message_mut(&mut self, message_id: &str) -> Option<(&str, &mut Message)> { - for chat in &mut self.chats { - if let Some(message) = chat.messages.iter_mut().find(|m| m.id == message_id) { - return Some((&chat.id, message)); - } - } - None + #[cfg(debug_assertions)] + pub fn log_cache_stats(&mut self) { + self.update_and_log_stats(); + } + + #[cfg(debug_assertions)] + pub fn cache_stats_summary(&mut self) -> String { + self.cache_stats.chat_count = self.chats.len(); + self.cache_stats.message_count = self.chats.iter() + .map(|c| c.message_count()) + .sum(); + format!("{} interner={} npubs", + self.cache_stats.summary(), + self.interner.len() + ) } } diff --git a/src-tauri/src/state/globals.rs b/src-tauri/src/state/globals.rs index 6635bd1b..2039bce4 100644 --- a/src-tauri/src/state/globals.rs +++ b/src-tauri/src/state/globals.rs @@ -11,6 +11,73 @@ use std::collections::HashSet; use super::ChatState; +/// Hybrid cache for wrapper event IDs: sorted Vec for historical + HashSet for pending +/// +/// Memory efficient (24% of HashSet) with fast lookups: +/// - Historical data: sorted Vec with binary search O(log n) +/// - New inserts during sync: small HashSet O(1) +/// +/// Load time is 5x faster than HashSet due to simple sort vs hash table construction. +pub struct WrapperIdCache { + /// Sorted array of historical wrapper IDs loaded from DB + historical: Vec<[u8; 32]>, + /// New wrapper IDs added during sync (not yet in DB) + pending: HashSet<[u8; 32]>, +} + +impl WrapperIdCache { + pub fn new() -> Self { + Self { + historical: Vec::new(), + pending: HashSet::new(), + } + } + + /// Load historical wrapper IDs from database (call once at init) + pub fn load(&mut self, mut ids: Vec<[u8; 32]>) { + ids.sort_unstable(); + self.historical = ids; + self.pending.clear(); + } + + /// Check if a wrapper ID exists in the cache + #[inline] + pub fn contains(&self, id: &[u8; 32]) -> bool { + // Binary search historical first (likely hit for recent events) + self.historical.binary_search(id).is_ok() || self.pending.contains(id) + } + + /// Insert a new wrapper ID (goes to pending set) + #[inline] + pub fn insert(&mut self, id: [u8; 32]) { + self.pending.insert(id); + } + + /// Clear all cached data (call when sync finishes) + pub fn clear(&mut self) { + self.historical.clear(); + self.historical.shrink_to_fit(); + self.pending.clear(); + self.pending.shrink_to_fit(); + } + + /// Get number of cached entries + pub fn len(&self) -> usize { + self.historical.len() + self.pending.len() + } + + /// Check if cache is empty + pub fn is_empty(&self) -> bool { + self.historical.is_empty() && self.pending.is_empty() + } +} + +impl Default for WrapperIdCache { + fn default() -> Self { + Self::new() + } +} + /// # Trusted Relays /// /// The 'Trusted Relays' handle events that MAY have a small amount of public-facing metadata attached (i.e: Expiration tags). @@ -77,7 +144,9 @@ lazy_static! { /// - Populated at init with recent wrapper_ids (last 30 days) to avoid SQL queries for each historical event /// - Only used for historical sync events (is_new = false), NOT for real-time new events /// - Cleared when sync finishes to free memory - pub static ref WRAPPER_ID_CACHE: Mutex> = Mutex::new(HashSet::new()); + /// + /// Uses hybrid approach: sorted Vec (historical) + HashSet (pending) for 76% memory reduction + pub static ref WRAPPER_ID_CACHE: Mutex = Mutex::new(WrapperIdCache::new()); } lazy_static! { diff --git a/src-tauri/src/state/mod.rs b/src-tauri/src/state/mod.rs index 62cbafe3..2bb69bf2 100644 --- a/src-tauri/src/state/mod.rs +++ b/src-tauri/src/state/mod.rs @@ -4,10 +4,13 @@ //! - `globals`: Global static variables (TAURI_APP, NOSTR_CLIENT, STATE, etc.) //! - `chat_state`: The ChatState struct and its methods //! - `sync`: SyncMode enum for sync state management +//! - `stats`: Cache statistics and memory benchmarking mod globals; mod chat_state; mod sync; +#[cfg(debug_assertions)] +pub mod stats; pub use globals::{ TAURI_APP, NOSTR_CLIENT, STATE, @@ -20,3 +23,5 @@ pub use globals::{ pub use chat_state::ChatState; pub use sync::SyncMode; +#[cfg(debug_assertions)] +pub use stats::{CacheStats, DeepSize}; diff --git a/src-tauri/src/state/stats.rs b/src-tauri/src/state/stats.rs new file mode 100644 index 00000000..d1496c93 --- /dev/null +++ b/src-tauri/src/state/stats.rs @@ -0,0 +1,303 @@ +//! Cache statistics and memory measurement for benchmarking. +//! +//! This module provides tools for measuring memory usage and performance +//! of the message cache, enabling before/after comparison during optimization. + +use std::time::Duration; + +use crate::message::{Message, Attachment, Reaction, EditEntry, ImageMetadata}; +use crate::message::compact::{CompactMessage, CompactMessageVec, CompactReaction, CompactAttachment, MessageFlags, NpubInterner}; +use crate::net::SiteMetadata; +use crate::Chat; + +/// Statistics for message cache operations +#[derive(Debug, Default, Clone)] +pub struct CacheStats { + /// Total number of messages across all chats + pub message_count: usize, + /// Total number of chats + pub chat_count: usize, + /// Estimated total memory in bytes + pub total_memory_bytes: usize, + /// Duration of last insert operation + pub last_insert_duration: Duration, + /// Average insert duration in nanoseconds + pub avg_insert_duration_ns: u64, + /// Number of insert operations recorded + pub insert_count: u64, + /// Total nanoseconds spent inserting + insert_total_ns: u64, +} + +impl CacheStats { + pub fn new() -> Self { + Self::default() + } + + /// Record an insert operation's duration + pub fn record_insert(&mut self, duration: Duration) { + self.last_insert_duration = duration; + self.insert_count += 1; + self.insert_total_ns += duration.as_nanos() as u64; + self.avg_insert_duration_ns = self.insert_total_ns / self.insert_count; + } + + /// Update memory stats from current state + pub fn update_from_chats(&mut self, chats: &[Chat]) { + self.chat_count = chats.len(); + self.message_count = chats.iter().map(|c| c.messages.len()).sum(); + self.total_memory_bytes = chats.iter().map(|c| c.deep_size()).sum(); + } + + /// Print current stats + pub fn log(&self) { + println!( + "[CacheStats] chats={} messages={} memory={} last_insert={:?} avg_insert={}ns inserts={}", + self.chat_count, + self.message_count, + format_bytes(self.total_memory_bytes), + self.last_insert_duration, + self.avg_insert_duration_ns, + self.insert_count, + ); + } + + /// Get a summary string + #[allow(dead_code)] + pub fn summary(&self) -> String { + format!( + "chats={} msgs={} mem={} avg_insert={}ns", + self.chat_count, + self.message_count, + format_bytes(self.total_memory_bytes), + self.avg_insert_duration_ns, + ) + } + + /// Check if we should log (every N inserts) + pub fn should_log(&self, interval: u64) -> bool { + self.insert_count > 0 && self.insert_count.is_multiple_of(interval) + } +} + +/// Format bytes as human-readable string +fn format_bytes(bytes: usize) -> String { + if bytes >= 1024 * 1024 { + format!("{:.2}MB", bytes as f64 / (1024.0 * 1024.0)) + } else if bytes >= 1024 { + format!("{:.2}KB", bytes as f64 / 1024.0) + } else { + format!("{}B", bytes) + } +} + +/// Trait for calculating deep/heap memory size of a value +pub trait DeepSize { + fn deep_size(&self) -> usize; +} + +// === Primitive implementations === + +impl DeepSize for String { + #[inline] + fn deep_size(&self) -> usize { + std::mem::size_of::() + self.capacity() + } +} + +impl DeepSize for str { + #[inline] + fn deep_size(&self) -> usize { + self.len() + } +} + +impl DeepSize for u64 { + #[inline] + fn deep_size(&self) -> usize { + std::mem::size_of::() + } +} + +impl DeepSize for u32 { + #[inline] + fn deep_size(&self) -> usize { + std::mem::size_of::() + } +} + +impl DeepSize for bool { + #[inline] + fn deep_size(&self) -> usize { + std::mem::size_of::() + } +} + +// === Generic implementations === + +impl DeepSize for Vec { + fn deep_size(&self) -> usize { + std::mem::size_of::>() + + self.capacity() * std::mem::size_of::() + + self.iter().map(|item| item.deep_size() - std::mem::size_of::()).sum::() + } +} + +impl DeepSize for Option { + fn deep_size(&self) -> usize { + std::mem::size_of::>() + + self.as_ref().map(|v| v.deep_size() - std::mem::size_of::()).unwrap_or(0) + } +} + +// === Domain type implementations === + +impl DeepSize for ImageMetadata { + fn deep_size(&self) -> usize { + std::mem::size_of::() + self.blurhash.capacity() + } +} + +impl DeepSize for Attachment { + fn deep_size(&self) -> usize { + std::mem::size_of::() + + self.id.capacity() + + self.key.capacity() + + self.nonce.capacity() + + self.extension.capacity() + + self.url.capacity() + + self.path.capacity() + + self.img_meta.as_ref().map(|m| m.blurhash.capacity()).unwrap_or(0) + + self.webxdc_topic.as_ref().map(|s| s.capacity()).unwrap_or(0) + + self.group_id.as_ref().map(|s| s.capacity()).unwrap_or(0) + + self.original_hash.as_ref().map(|s| s.capacity()).unwrap_or(0) + + self.scheme_version.as_ref().map(|s| s.capacity()).unwrap_or(0) + + self.mls_filename.as_ref().map(|s| s.capacity()).unwrap_or(0) + } +} + +impl DeepSize for Reaction { + fn deep_size(&self) -> usize { + std::mem::size_of::() + + self.id.capacity() + + self.reference_id.capacity() + + self.author_id.capacity() + + self.emoji.capacity() + } +} + +impl DeepSize for EditEntry { + fn deep_size(&self) -> usize { + std::mem::size_of::() + self.content.capacity() + } +} + +impl DeepSize for SiteMetadata { + fn deep_size(&self) -> usize { + std::mem::size_of::() + + self.domain.capacity() + + self.og_title.as_ref().map(|s| s.capacity()).unwrap_or(0) + + self.og_description.as_ref().map(|s| s.capacity()).unwrap_or(0) + + self.og_image.as_ref().map(|s| s.capacity()).unwrap_or(0) + + self.og_url.as_ref().map(|s| s.capacity()).unwrap_or(0) + + self.og_type.as_ref().map(|s| s.capacity()).unwrap_or(0) + + self.title.as_ref().map(|s| s.capacity()).unwrap_or(0) + + self.description.as_ref().map(|s| s.capacity()).unwrap_or(0) + + self.favicon.as_ref().map(|s| s.capacity()).unwrap_or(0) + } +} + +impl DeepSize for Message { + fn deep_size(&self) -> usize { + std::mem::size_of::() + + self.id.capacity() + + self.content.capacity() + + self.replied_to.capacity() + + self.replied_to_content.as_ref().map(|s| s.capacity()).unwrap_or(0) + + self.replied_to_npub.as_ref().map(|s| s.capacity()).unwrap_or(0) + + self.npub.as_ref().map(|s| s.capacity()).unwrap_or(0) + + self.wrapper_event_id.as_ref().map(|s| s.capacity()).unwrap_or(0) + + self.preview_metadata.as_ref().map(|m| m.deep_size()).unwrap_or(0) + + self.attachments.iter().map(|a| a.deep_size()).sum::() + + self.reactions.iter().map(|r| r.deep_size()).sum::() + + self.edit_history.as_ref().map(|h| h.iter().map(|e| e.deep_size()).sum::()).unwrap_or(0) + } +} + +impl DeepSize for Chat { + fn deep_size(&self) -> usize { + std::mem::size_of::() + + self.id.capacity() + + self.last_read.capacity() + + self.participants.iter().map(|s| s.capacity()).sum::() + + self.messages.deep_size() + // metadata.custom_fields HashMap + + self.metadata.custom_fields.iter() + .map(|(k, v)| k.capacity() + v.capacity()) + .sum::() + // typing_participants HashMap + + self.typing_participants.keys() + .map(|k| k.capacity() + std::mem::size_of::()) + .sum::() + } +} + +// === Compact message type implementations === + +impl DeepSize for MessageFlags { + #[inline] + fn deep_size(&self) -> usize { + std::mem::size_of::() + } +} + +impl DeepSize for CompactReaction { + fn deep_size(&self) -> usize { + std::mem::size_of::() + + self.emoji.len() // Box heap allocation + } +} + +impl DeepSize for CompactAttachment { + fn deep_size(&self) -> usize { + std::mem::size_of::() + + self.extension.len() // Box heap allocation + + self.url.len() + + self.path.len() + + self.img_meta.as_ref().map(|m| m.deep_size()).unwrap_or(0) + + self.group_id.as_ref().map(|_| 32).unwrap_or(0) + + self.original_hash.as_ref().map(|_| 32).unwrap_or(0) + + self.webxdc_topic.as_ref().map(|s| s.len()).unwrap_or(0) + + self.mls_filename.as_ref().map(|s| s.len()).unwrap_or(0) + + self.scheme_version.as_ref().map(|s| s.len()).unwrap_or(0) + } +} + +impl DeepSize for CompactMessage { + fn deep_size(&self) -> usize { + std::mem::size_of::() + + self.content.len() // Box has no over-allocation, len == allocated + + self.replied_to_content.as_ref().map(|s| s.len()).unwrap_or(0) + + self.preview_metadata.as_ref().map(|m| m.deep_size()).unwrap_or(0) + + self.attachments.iter().map(|a| a.deep_size()).sum::() + + self.reactions.iter().map(|r| r.deep_size()).sum::() + + self.edit_history.as_ref().map(|h| h.iter().map(|e| e.deep_size()).sum::()).unwrap_or(0) + } +} + +impl DeepSize for CompactMessageVec { + fn deep_size(&self) -> usize { + std::mem::size_of::() + + std::mem::size_of_val(self.messages()) + + self.iter().map(|m| m.deep_size() - std::mem::size_of::()).sum::() + // id_index: each entry is ([u8; 32], u32) = 36 bytes + + self.len() * std::mem::size_of::<([u8; 32], u32)>() + } +} + +impl DeepSize for NpubInterner { + fn deep_size(&self) -> usize { + self.memory_usage() + } +} + diff --git a/src-tauri/src/stored_event.rs b/src-tauri/src/stored_event.rs index b61299df..0916512f 100644 --- a/src-tauri/src/stored_event.rs +++ b/src-tauri/src/stored_event.rs @@ -124,6 +124,10 @@ pub struct StoredEvent { /// Sender's npub (for group chats where sender varies) #[serde(skip_serializing_if = "Option::is_none")] pub npub: Option, + + /// Cached link preview metadata (JSON serialized SiteMetadata) + #[serde(skip_serializing_if = "Option::is_none")] + pub preview_metadata: Option, } impl StoredEvent { @@ -144,6 +148,7 @@ impl StoredEvent { failed: false, wrapper_event_id: None, npub: None, + preview_metadata: None, } } @@ -328,6 +333,7 @@ impl StoredEventBuilder { failed: self.failed, wrapper_event_id: self.wrapper_event_id, npub: self.npub, + preview_metadata: None, } } } @@ -393,7 +399,7 @@ mod tests { fn test_unknown_kind() { let event = StoredEvent::new( "abc123".to_string(), - 99999, // Unknown kind + 65535, // Unknown kind (max u16 value) 1, "Unknown content".to_string(), 1234567890, diff --git a/src-tauri/src/util.rs b/src-tauri/src/util.rs index 335efbe5..7c5631ab 100644 --- a/src-tauri/src/util.rs +++ b/src-tauri/src/util.rs @@ -4,6 +4,16 @@ use sha2::{Sha256, Digest}; use blurhash::decode; use base64::{Engine as _, engine::general_purpose}; +// Re-export SIMD-accelerated functions for backwards compatibility +pub use crate::simd::{ + bytes_to_hex_16, bytes_to_hex_32, bytes_to_hex_string, + hex_string_to_bytes, hex_to_bytes_16, hex_to_bytes_32, + has_alpha_transparency, set_all_alpha_opaque, +}; + +#[cfg(target_os = "windows")] +pub use crate::simd::has_all_alpha_near_zero; + /// Extract all HTTPS URLs from a string pub fn extract_https_urls(text: &str) -> Vec { let mut urls = Vec::new(); @@ -27,10 +37,10 @@ pub fn extract_https_urls(text: &str) -> Vec { }) .unwrap_or(url_text.len()); - // Trim trailing punctuation + // Trim trailing punctuation (ASCII-only, so byte access is safe) while end_idx > 0 { - let last_char = url_text[..end_idx].chars().last().unwrap(); - if last_char == '.' || last_char == ',' || last_char == ':' || last_char == ';' { + let last_byte = url_text.as_bytes()[end_idx - 1]; + if last_byte == b'.' || last_byte == b',' || last_byte == b':' || last_byte == b';' { end_idx -= 1; } else { break; @@ -327,61 +337,11 @@ pub fn format_bytes(bytes: u64) -> String { } } -/// Convert a byte slice to a hex string -pub fn bytes_to_hex_string(bytes: &[u8]) -> String { - // Pre-allocate the exact size needed (2 hex chars per byte) - let mut result = String::with_capacity(bytes.len() * 2); - - // Use a lookup table for hex conversion - const HEX_CHARS: &[u8; 16] = b"0123456789abcdef"; - - for &b in bytes { - // Extract high and low nibbles - let high = b >> 4; - let low = b & 0xF; - result.push(HEX_CHARS[high as usize] as char); - result.push(HEX_CHARS[low as usize] as char); - } - - result -} - -/// Convert hex string back to bytes for decryption -pub fn hex_string_to_bytes(s: &str) -> Vec { - // Pre-allocate the result vector to avoid resize operations - let mut result = Vec::with_capacity(s.len() / 2); - let bytes = s.as_bytes(); - - // Process bytes directly to avoid UTF-8 decoding overhead - let mut i = 0; - while i + 1 < bytes.len() { - // Convert two hex characters to a single byte - let high = match bytes[i] { - b'0'..=b'9' => bytes[i] - b'0', - b'a'..=b'f' => bytes[i] - b'a' + 10, - b'A'..=b'F' => bytes[i] - b'A' + 10, - _ => 0, - }; - - let low = match bytes[i + 1] { - b'0'..=b'9' => bytes[i + 1] - b'0', - b'a'..=b'f' => bytes[i + 1] - b'a' + 10, - b'A'..=b'F' => bytes[i + 1] - b'A' + 10, - _ => 0, - }; - - result.push((high << 4) | low); - i += 2; - } - - result -} - /// Calculate SHA-256 hash of file data pub fn calculate_file_hash(data: &[u8]) -> String { let mut hasher = Sha256::new(); hasher.update(data); - hex::encode(hasher.finalize()) + bytes_to_hex_string(hasher.finalize().as_slice()) } /// Ultra-fast nearest-neighbor downsampling for RGBA8 pixel data @@ -400,6 +360,10 @@ pub fn calculate_file_hash(data: &[u8]) -> String { /// /// # Returns /// Downsampled RGBA8 pixel data +/// Fast nearest-neighbor downsampling for RGBA images. +/// +/// Delegates to SIMD-optimized implementation in `crate::simd::image`. +#[inline] pub fn nearest_neighbor_downsample( pixels: &[u8], src_width: u32, @@ -407,21 +371,7 @@ pub fn nearest_neighbor_downsample( dst_width: u32, dst_height: u32, ) -> Vec { - let mut result = Vec::with_capacity((dst_width * dst_height * 4) as usize); - - let x_ratio = src_width as f32 / dst_width as f32; - let y_ratio = src_height as f32 / dst_height as f32; - - for ty in 0..dst_height { - let sy = (ty as f32 * y_ratio) as u32; - for tx in 0..dst_width { - let sx = (tx as f32 * x_ratio) as u32; - let src_idx = ((sy * src_width + sx) * 4) as usize; - result.extend_from_slice(&pixels[src_idx..src_idx + 4]); - } - } - - result + crate::simd::image::nearest_neighbor_downsample(pixels, src_width, src_height, dst_width, dst_height) } /// Generate a blurhash from RGBA8 image data with adaptive downscaling for optimal performance @@ -455,6 +405,32 @@ pub fn generate_blurhash_from_rgba(pixels: &[u8], width: u32, height: u32) -> Op blurhash::encode(4, 3, thumbnail_width, thumbnail_height, &thumbnail_pixels).ok() } +/// Generate a blurhash from a DynamicImage with minimal memory allocation. +/// +/// Generate a blurhash from a DynamicImage. +/// +/// Resizes to a small thumbnail before converting to RGBA, +/// avoiding large temporary allocations for high-resolution images. +#[inline] +pub fn generate_blurhash_from_image(img: &image::DynamicImage) -> Option { + // Target size for blurhash - 64x64 is more than enough for a 4x3 component hash + // Small thumbnail gives good quality blurhash with minimal allocation + const BLURHASH_SIZE: u32 = 64; + + let (width, height) = (img.width(), img.height()); + + // Calculate thumbnail dimensions maintaining aspect ratio + let (thumb_w, thumb_h) = if width > height { + (BLURHASH_SIZE, (BLURHASH_SIZE * height / width).max(1)) + } else { + ((BLURHASH_SIZE * width / height).max(1), BLURHASH_SIZE) + }; + + let thumbnail = img.thumbnail(thumb_w, thumb_h); + let rgba = thumbnail.to_rgba8(); + blurhash::encode(4, 3, rgba.width(), rgba.height(), rgba.as_raw()).ok() +} + /// Decode a blurhash string to a Base64-encoded PNG data URL /// Returns a data URL string that can be used directly in an src attribute pub fn decode_blurhash_to_base64(blurhash: &str, width: u32, height: u32, punch: f32) -> String { @@ -479,17 +455,10 @@ pub fn decode_blurhash_to_base64(blurhash: &str, width: u32, height: u32, punch: // Fast path for RGBA data if bytes_per_pixel == 4 { encode_rgba_to_png_base64(&decoded_data, width, height) - } - // Convert RGB to RGBA + } + // Convert RGB to RGBA using SIMD-optimized function else if bytes_per_pixel == 3 { - // Pre-allocate exact size needed - let mut rgba_data = Vec::with_capacity(pixel_count * 4); - - // Use chunks_exact for safe and efficient iteration - for rgb_chunk in decoded_data.chunks_exact(3) { - rgba_data.extend_from_slice(&[rgb_chunk[0], rgb_chunk[1], rgb_chunk[2], 255]); - } - + let rgba_data = crate::simd::image::rgb_to_rgba(&decoded_data); encode_rgba_to_png_base64(&rgba_data, width, height) } else { eprintln!("Unexpected decoded data length: {} bytes for {} pixels", @@ -522,19 +491,6 @@ fn encode_rgba_to_png_base64(rgba_data: &[u8], width: u32, height: u32) -> Strin result } -/// Check if RGBA pixel data contains any meaningful transparency (alpha < 255) -/// Returns true if any pixel has alpha less than 255, indicating the image uses transparency -#[inline] -pub fn has_alpha_transparency(rgba_pixels: &[u8]) -> bool { - // Check every 4th byte (alpha channel) for any value less than 255 - rgba_pixels - .iter() - .skip(3) - .step_by(4) - .any(|&alpha| alpha < 255) -} - - // ===== MIME & Extension Conversion Utilities ===== static EXT_TO_MIME: Lazy> = Lazy::new(|| { let mut m = HashMap::new(); diff --git a/src/js/event-cache.js b/src/js/event-cache.js index 7d766f5a..032672c9 100644 --- a/src/js/event-cache.js +++ b/src/js/event-cache.js @@ -195,10 +195,6 @@ class EventCache { // Map of conversationId -> ConversationCacheEntry // Using Map to maintain insertion order for LRU this.cache = new Map(); - - // File hash index for deduplication (loaded once at init) - this.fileHashIndex = new Map(); - this.fileHashIndexLoaded = false; } /** @@ -463,8 +459,6 @@ class EventCache { */ clear() { this.cache.clear(); - this.fileHashIndex.clear(); - this.fileHashIndexLoaded = false; } /** @@ -487,44 +481,6 @@ class EventCache { } } - /** - * Load the file hash index for deduplication - * This should be called once at app init - * @returns {Promise} - */ - async loadFileHashIndex() { - if (this.fileHashIndexLoaded) { - return this.fileHashIndex; - } - - try { - const index = await invoke('get_file_hash_index'); - this.fileHashIndex = new Map(Object.entries(index)); - this.fileHashIndexLoaded = true; - return this.fileHashIndex; - } catch (error) { - return this.fileHashIndex; - } - } - - /** - * Check if a file hash exists in the index - * @param {string} hash - The SHA256 hash of the file - * @returns {Object|null} - The attachment reference if found - */ - getExistingAttachment(hash) { - return this.fileHashIndex.get(hash) || null; - } - - /** - * Add a new file hash to the index (after upload) - * @param {string} hash - The SHA256 hash - * @param {Object} attachmentRef - The attachment reference data - */ - addFileHash(hash, attachmentRef) { - this.fileHashIndex.set(hash, attachmentRef); - } - /** * Get overall cache statistics * @returns {Object} @@ -540,8 +496,7 @@ class EventCache { return { cachedConversations: totalConversations, maxConversations: EVENT_CACHE_CONFIG.maxCachedConversations, - totalCachedEvents: totalEvents, - fileHashCount: this.fileHashIndex.size + totalCachedEvents: totalEvents }; } } diff --git a/src/js/file-preview.js b/src/js/file-preview.js index 57a6f307..a702ef99 100644 --- a/src/js/file-preview.js +++ b/src/js/file-preview.js @@ -1060,6 +1060,8 @@ async function sendPreviewedFile() { const isGroup = receiver.startsWith('group:'); // Send file in background + const chatId = isGroup ? receiver.replace('group:', '') : receiver; + let result; try { if (fileObject) { // Android File object flow @@ -1067,8 +1069,8 @@ async function sendPreviewedFile() { // Otherwise, read from File object directly if (shouldCompress || compressionWasStarted) { // Use cached bytes (compression was started, so bytes are cached) - await invoke("send_cached_file", { - receiver: isGroup ? receiver.replace('group:', '') : receiver, + result = await invoke("send_cached_file", { + receiver: chatId, repliedTo: replyRef, useCompression: shouldCompress }); @@ -1076,8 +1078,8 @@ async function sendPreviewedFile() { // No compression needed and bytes weren't cached, read directly const arrayBuffer = await fileObject.arrayBuffer(); const bytes = Array.from(new Uint8Array(arrayBuffer)); - await invoke("send_file_bytes", { - receiver: isGroup ? receiver.replace('group:', '') : receiver, + result = await invoke("send_file_bytes", { + receiver: chatId, repliedTo: replyRef, fileBytes: bytes, fileName: fileObject.name, @@ -1086,27 +1088,32 @@ async function sendPreviewedFile() { } } else if (usingBytes) { // Legacy flow: use cached bytes from JS (clipboard paste) - await invoke("send_cached_file", { - receiver: isGroup ? receiver.replace('group:', '') : receiver, + result = await invoke("send_cached_file", { + receiver: chatId, repliedTo: replyRef, useCompression: shouldCompress }); } else if (shouldCompress) { // Desktop: use cached compressed file (will wait if still compressing) - await invoke("send_cached_compressed_file", { - receiver: isGroup ? receiver.replace('group:', '') : receiver, + result = await invoke("send_cached_compressed_file", { + receiver: chatId, repliedTo: replyRef, filePath: filePath }); } else { // Desktop: send without compression, but clear the cache first await invoke("clear_compression_cache", { filePath: filePath }); - await invoke("file_message", { - receiver: isGroup ? receiver.replace('group:', '') : receiver, + result = await invoke("file_message", { + receiver: chatId, repliedTo: replyRef, filePath: filePath }); } + + // Finalize the pending message with the real event ID + if (result && result.event_id) { + finalizePendingMessage(chatId, result.pending_id, result.event_id); + } } catch (e) { console.error('Failed to send file:', e); popupConfirm(e.toString(), '', true, '', 'vector_warning.svg'); diff --git a/src/main.js b/src/main.js index 6ce7a501..ef20331e 100644 --- a/src/main.js +++ b/src/main.js @@ -2333,12 +2333,17 @@ async function playMiniAppAndInvite() { try { // Send the Mini App file to the current chat - await invoke('file_message', { + const result = await invoke('file_message', { receiver: targetChatId, repliedTo: '', filePath: app.src_url, }); - + + // Finalize the pending message + if (result && result.event_id) { + finalizePendingMessage(targetChatId, result.pending_id, result.event_id); + } + console.log('Mini App sent to chat successfully'); } catch (e) { console.error('Failed to send Mini App to chat:', e); @@ -3760,6 +3765,48 @@ function getOrCreateDMChat(npub) { return getOrCreateChat(npub, 'DirectMessage'); } +/** + * Finalize a pending message after successful send. + * Updates the message ID and clears the pending state. + * @param {string} chatId - The chat ID + * @param {string} pendingId - The temporary pending ID + * @param {string} eventId - The real event ID from the backend + */ +function finalizePendingMessage(chatId, pendingId, eventId) { + const chat = getChat(chatId); + if (!chat) return; + + const msgIdx = chat.messages.findIndex(m => m.id === pendingId); + if (msgIdx === -1) return; + + const msg = chat.messages[msgIdx]; + const oldId = msg.id; + msg.id = eventId; + msg.pending = false; + + // Update event cache + if (eventCache.has(chatId)) { + const cachedEvents = eventCache.getEvents(chatId); + if (cachedEvents) { + const cacheIdx = cachedEvents.findIndex(m => m.id === oldId); + if (cacheIdx !== -1) { + cachedEvents[cacheIdx] = msg; + } + } + } + + // Re-render if this chat is open + if (strOpenChat === chatId) { + const domMsg = document.getElementById(oldId); + if (domMsg) { + const profile = getProfile(chatId); + domMsg.replaceWith(renderMessage(msg, profile, oldId)); + } + strLastMsgID = eventId; + softChatScroll(); + } +} + /** * Compute a timestamp for sorting chats, falling back to metadata for empty groups. * @param {Chat} chat @@ -4782,7 +4829,10 @@ function markAsRead(chat, message) { * @param {string?} replied_to - The reference of the message, if any */ async function message(pubkey, content, replied_to) { - await invoke("message", { receiver: pubkey, content: content, repliedTo: replied_to }); + const result = await invoke("message", { receiver: pubkey, content: content, repliedTo: replied_to }); + if (result && result.event_id) { + finalizePendingMessage(pubkey, result.pending_id, result.event_id); + } } /** @@ -4794,7 +4844,10 @@ async function message(pubkey, content, replied_to) { async function sendFile(pubkey, replied_to, filepath) { try { // Use the protocol-agnostic file_message command for both DMs and MLS groups - await invoke("file_message", { receiver: pubkey, repliedTo: replied_to, filePath: filepath }); + const result = await invoke("file_message", { receiver: pubkey, repliedTo: replied_to, filePath: filepath }); + if (result && result.event_id) { + finalizePendingMessage(pubkey, result.pending_id, result.event_id); + } } catch (e) { // Notify of an attachment send failure popupConfirm(e, '', true, '', 'vector_warning.svg'); @@ -5540,6 +5593,31 @@ async function setupRustListeners() { } }); + // Listen for attachment URL updates (for file uploads and reuse) + await listen('attachment_update', (evt) => { + const { chat_id, message_id, attachment_id, url } = evt.payload; + const cChat = getChat(chat_id); + if (!cChat) return; + + // Find the message + const msg = cChat.messages.find(m => m.id === message_id); + if (!msg || !msg.attachments) return; + + // Find and update the attachment + const att = msg.attachments.find(a => a.id === attachment_id); + if (att) { + att.url = url; + // Re-render if this chat is open + if (strOpenChat === chat_id) { + const domMsg = document.getElementById(message_id); + if (domMsg) { + const profile = getProfile(chat_id); + domMsg.replaceWith(renderMessage(msg, profile, message_id)); + } + } + } + }); + // Listen for Vector Voice AI (Whisper) model download progression updates await listen('whisper_download_progress', async (evt) => { // Update the progression UI @@ -6212,10 +6290,6 @@ async function login() { await hydrateMLSGroupMetadata(); - // Load the file hash index for attachment deduplication - // This is done asynchronously and doesn't block the UI - eventCache.loadFileHashIndex().catch(() => {}); - // Fadeout the login and encryption UI domLogin.classList.add('fadeout-anim'); domLogin.addEventListener('animationend', async () => {