From bfdbf0109ed361b632179a97f304764939f565ec Mon Sep 17 00:00:00 2001 From: Hocuri Date: Thu, 14 Aug 2025 18:14:25 +0200 Subject: [PATCH 01/21] feat: Opt-in weekly sending of statistics This way, the statistics / self-reporting bot will be made into an opt-in regular sending of statistics, where you enable the setting once and then they will be sent automatically. The statistics will be sent to a bot, so that the user can see exactly which data is being sent, and how often. The chat will be archived and muted by default, so that it doesn't disturb the user. --- ...f-reporting-bot.vcf => statistics-bot.vcf} | 0 deltachat-jsonrpc/src/api.rs | 19 +- deltachat-jsonrpc/src/api/types/chat.rs | 4 +- deltachat-jsonrpc/src/api/types/chat_list.rs | 2 +- deltachat-jsonrpc/src/api/types/http.rs | 2 +- src/config.rs | 26 +- src/context.rs | 176 +---- src/context/context_tests.rs | 27 +- src/lib.rs | 1 + src/receive_imf.rs | 7 +- src/scheduler.rs | 2 + src/securejoin.rs | 36 +- src/sql/migrations.rs | 21 + src/statistics.rs | 710 ++++++++++++++++++ src/statistics/statistics_tests.rs | 511 +++++++++++++ 15 files changed, 1351 insertions(+), 193 deletions(-) rename assets/{self-reporting-bot.vcf => statistics-bot.vcf} (100%) create mode 100644 src/statistics.rs create mode 100644 src/statistics/statistics_tests.rs diff --git a/assets/self-reporting-bot.vcf b/assets/statistics-bot.vcf similarity index 100% rename from assets/self-reporting-bot.vcf rename to assets/statistics-bot.vcf diff --git a/deltachat-jsonrpc/src/api.rs b/deltachat-jsonrpc/src/api.rs index fe6d2e7444..01322f51e7 100644 --- a/deltachat-jsonrpc/src/api.rs +++ b/deltachat-jsonrpc/src/api.rs @@ -383,11 +383,6 @@ impl CommandApi { Ok(BlobObject::create_and_deduplicate(&ctx, file, file)?.to_abs_path()) } - async fn draft_self_report(&self, account_id: u32) -> Result { - let ctx = self.get_context(account_id).await?; - Ok(ctx.draft_self_report().await?.to_u32()) - } - /// Sets the given configuration key. async fn set_config(&self, account_id: u32, key: String, value: Option) -> Result<()> { let ctx = self.get_context(account_id).await?; @@ -877,12 +872,22 @@ impl CommandApi { /// **qr**: The text of the scanned QR code. Typically, the same string as given /// to `check_qr()`. /// + /// **source** and **uipath** are for statistics-sending, + /// if the user enabled it in the settings; + /// if you don't have statistics-sending implemented, just pass `None` here. + /// /// **returns**: The chat ID of the joined chat, the UI may redirect to the this chat. /// A returned chat ID does not guarantee that the chat is protected or the belonging contact is verified. /// - async fn secure_join(&self, account_id: u32, qr: String) -> Result { + async fn secure_join( + &self, + account_id: u32, + qr: String, + source: Option, + uipath: Option, + ) -> Result { let ctx = self.get_context(account_id).await?; - let chat_id = securejoin::join_securejoin(&ctx, &qr).await?; + let chat_id = securejoin::join_securejoin_with_source(&ctx, &qr, source, uipath).await?; Ok(chat_id.to_u32()) } diff --git a/deltachat-jsonrpc/src/api/types/chat.rs b/deltachat-jsonrpc/src/api/types/chat.rs index 96388c27be..26e7b4293d 100644 --- a/deltachat-jsonrpc/src/api/types/chat.rs +++ b/deltachat-jsonrpc/src/api/types/chat.rs @@ -1,7 +1,7 @@ use std::time::{Duration, SystemTime}; -use anyhow::{bail, Context as _, Result}; -use deltachat::chat::{self, get_chat_contacts, get_past_chat_contacts, ChatVisibility}; +use anyhow::{Context as _, Result, bail}; +use deltachat::chat::{self, ChatVisibility, get_chat_contacts, get_past_chat_contacts}; use deltachat::chat::{Chat, ChatId}; use deltachat::constants::Chattype; use deltachat::contact::{Contact, ContactId}; diff --git a/deltachat-jsonrpc/src/api/types/chat_list.rs b/deltachat-jsonrpc/src/api/types/chat_list.rs index b5d31a7913..1a9e851086 100644 --- a/deltachat-jsonrpc/src/api/types/chat_list.rs +++ b/deltachat-jsonrpc/src/api/types/chat_list.rs @@ -4,7 +4,7 @@ use deltachat::chatlist::get_last_message_for_chat; use deltachat::constants::*; use deltachat::contact::{Contact, ContactId}; use deltachat::{ - chat::{get_chat_contacts, ChatVisibility}, + chat::{ChatVisibility, get_chat_contacts}, chatlist::Chatlist, }; use num_traits::cast::ToPrimitive; diff --git a/deltachat-jsonrpc/src/api/types/http.rs b/deltachat-jsonrpc/src/api/types/http.rs index 9121a677ec..d370ba8f7c 100644 --- a/deltachat-jsonrpc/src/api/types/http.rs +++ b/deltachat-jsonrpc/src/api/types/http.rs @@ -16,7 +16,7 @@ pub struct HttpResponse { impl From for HttpResponse { fn from(response: CoreHttpResponse) -> Self { - use base64::{engine::general_purpose, Engine as _}; + use base64::{Engine as _, engine::general_purpose}; let blob = general_purpose::STANDARD_NO_PAD.encode(response.blob); let mimetype = response.mimetype; let encoding = response.encoding; diff --git a/src/config.rs b/src/config.rs index 7bfebd868e..f92bf14ad5 100644 --- a/src/config.rs +++ b/src/config.rs @@ -431,9 +431,26 @@ pub enum Config { /// used for signatures, encryption to self and included in `Autocrypt` header. KeyId, - /// This key is sent to the self_reporting bot so that the bot can recognize the user + /// Send statistics to Delta Chat's developers. + /// Can be exposed to the user as a setting. + SendStatistics, + + /// Last time statistics were sent to Delta Chat's developers + LastStatisticsSent, + + /// This key is sent to the statistics bot so that the bot can recognize the user /// without storing the email address - SelfReportingId, + StatisticsId, + + /// The last message id that was already included in the previously sent statistics, + /// or that already existed before the user opted in. + /// Only messages with an id larger than this + /// will be counted in the next statistics. + StatsLastExcludedMsgId, + + /// The last contact id that already existed when statistics-sending was enabled. + /// All newer contacts get the `"new": true` attribute. + StatsLastOldContactId, /// MsgId of webxdc map integration. WebxdcIntegration, @@ -827,6 +844,11 @@ impl Context { .await?; } } + Config::SendStatistics => { + self.sql.set_raw_config(key.as_ref(), value).await?; + crate::statistics::set_last_excluded_msg_id(self).await?; + crate::statistics::set_last_old_contact_id(self).await?; + } _ => { self.sql.set_raw_config(key.as_ref(), value).await?; } diff --git a/src/context.rs b/src/context.rs index 0272a74c06..29f7aa82ab 100644 --- a/src/context.rs +++ b/src/context.rs @@ -10,27 +10,22 @@ use std::time::Duration; use anyhow::{Context as _, Result, bail, ensure}; use async_channel::{self as channel, Receiver, Sender}; -use pgp::types::PublicKeyTrait; use ratelimit::Ratelimit; use tokio::sync::{Mutex, Notify, RwLock}; -use crate::chat::{ChatId, ProtectionStatus, get_chat_cnt}; +use crate::chat::{ChatId, get_chat_cnt}; use crate::chatlist_events; use crate::config::Config; -use crate::constants::{ - self, DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT, DC_CHAT_ID_TRASH, DC_VERSION_STR, -}; -use crate::contact::{Contact, ContactId, import_vcard, mark_contact_id_as_verified}; +use crate::constants::{self, DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT, DC_VERSION_STR}; +use crate::contact::{Contact, ContactId}; use crate::debug_logging::DebugLogging; -use crate::download::DownloadState; use crate::events::{Event, EventEmitter, EventType, Events}; use crate::imap::{FolderMeaning, Imap, ServerMetadata}; -use crate::key::{load_self_secret_key, self_fingerprint}; +use crate::key::self_fingerprint; use crate::log::{info, warn}; use crate::logged_debug_assert; use crate::login_param::{ConfiguredLoginParam, EnteredLoginParam}; -use crate::message::{self, Message, MessageState, MsgId}; -use crate::param::{Param, Params}; +use crate::message::{self, MessageState, MsgId}; use crate::peer_channels::Iroh; use crate::push::PushSubscriber; use crate::quota::QuotaInfo; @@ -38,7 +33,7 @@ use crate::scheduler::{ConnectivityStore, SchedulerState, convert_folder_meaning use crate::sql::Sql; use crate::stock_str::StockStrings; use crate::timesmearing::SmearedTimestamp; -use crate::tools::{self, create_id, duration_to_str, time, time_elapsed}; +use crate::tools::{self, duration_to_str, time, time_elapsed}; /// Builder for the [`Context`]. /// @@ -1081,6 +1076,24 @@ impl Context { .await? .unwrap_or_default(), ); + res.insert( + "statistics_id", + self.get_config_bool(Config::StatisticsId) + .await? + .to_string(), + ); + res.insert( + "send_statistics", + self.get_config_bool(Config::SendStatistics) + .await? + .to_string(), + ); + res.insert( + "last_statistics_sent", + self.get_config_i64(Config::LastStatisticsSent) + .await? + .to_string(), + ); let elapsed = time_elapsed(&self.creation_time); res.insert("uptime", duration_to_str(elapsed)); @@ -1088,147 +1101,6 @@ impl Context { Ok(res) } - async fn get_self_report(&self) -> Result { - #[derive(Default)] - struct ChatNumbers { - protected: u32, - opportunistic_dc: u32, - opportunistic_mua: u32, - unencrypted_dc: u32, - unencrypted_mua: u32, - } - - let mut res = String::new(); - res += &format!("core_version {}\n", get_version_str()); - - let num_msgs: u32 = self - .sql - .query_get_value( - "SELECT COUNT(*) FROM msgs WHERE hidden=0 AND chat_id!=?", - (DC_CHAT_ID_TRASH,), - ) - .await? - .unwrap_or_default(); - res += &format!("num_msgs {num_msgs}\n"); - - let num_chats: u32 = self - .sql - .query_get_value("SELECT COUNT(*) FROM chats WHERE id>9 AND blocked!=1", ()) - .await? - .unwrap_or_default(); - res += &format!("num_chats {num_chats}\n"); - - let db_size = tokio::fs::metadata(&self.sql.dbfile).await?.len(); - res += &format!("db_size_bytes {db_size}\n"); - - let secret_key = &load_self_secret_key(self).await?.primary_key; - let key_created = secret_key.public_key().created_at().timestamp(); - res += &format!("key_created {key_created}\n"); - - // how many of the chats active in the last months are: - // - protected - // - opportunistic-encrypted and the contact uses Delta Chat - // - opportunistic-encrypted and the contact uses a classical MUA - // - unencrypted and the contact uses Delta Chat - // - unencrypted and the contact uses a classical MUA - let three_months_ago = time().saturating_sub(3600 * 24 * 30 * 3); - let chats = self - .sql - .query_map( - "SELECT c.protected, m.param, m.msgrmsg - FROM chats c - JOIN msgs m - ON c.id=m.chat_id - AND m.id=( - SELECT id - FROM msgs - WHERE chat_id=c.id - AND hidden=0 - AND download_state=? - AND to_id!=? - ORDER BY timestamp DESC, id DESC LIMIT 1) - WHERE c.id>9 - AND (c.blocked=0 OR c.blocked=2) - AND IFNULL(m.timestamp,c.created_timestamp) > ? - GROUP BY c.id", - (DownloadState::Done, ContactId::INFO, three_months_ago), - |row| { - let protected: ProtectionStatus = row.get(0)?; - let message_param: Params = - row.get::<_, String>(1)?.parse().unwrap_or_default(); - let is_dc_message: bool = row.get(2)?; - Ok((protected, message_param, is_dc_message)) - }, - |rows| { - let mut chats = ChatNumbers::default(); - for row in rows { - let (protected, message_param, is_dc_message) = row?; - let encrypted = message_param - .get_bool(Param::GuaranteeE2ee) - .unwrap_or(false); - - if protected == ProtectionStatus::Protected { - chats.protected += 1; - } else if encrypted { - if is_dc_message { - chats.opportunistic_dc += 1; - } else { - chats.opportunistic_mua += 1; - } - } else if is_dc_message { - chats.unencrypted_dc += 1; - } else { - chats.unencrypted_mua += 1; - } - } - Ok(chats) - }, - ) - .await?; - res += &format!("chats_protected {}\n", chats.protected); - res += &format!("chats_opportunistic_dc {}\n", chats.opportunistic_dc); - res += &format!("chats_opportunistic_mua {}\n", chats.opportunistic_mua); - res += &format!("chats_unencrypted_dc {}\n", chats.unencrypted_dc); - res += &format!("chats_unencrypted_mua {}\n", chats.unencrypted_mua); - - let self_reporting_id = match self.get_config(Config::SelfReportingId).await? { - Some(id) => id, - None => { - let id = create_id(); - self.set_config(Config::SelfReportingId, Some(&id)).await?; - id - } - }; - res += &format!("self_reporting_id {self_reporting_id}"); - - Ok(res) - } - - /// Drafts a message with statistics about the usage of Delta Chat. - /// The user can inspect the message if they want, and then hit "Send". - /// - /// On the other end, a bot will receive the message and make it available - /// to Delta Chat's developers. - pub async fn draft_self_report(&self) -> Result { - const SELF_REPORTING_BOT_VCARD: &str = include_str!("../assets/self-reporting-bot.vcf"); - let contact_id: ContactId = *import_vcard(self, SELF_REPORTING_BOT_VCARD) - .await? - .first() - .context("Self reporting bot vCard does not contain a contact")?; - mark_contact_id_as_verified(self, contact_id, ContactId::SELF).await?; - - let chat_id = ChatId::create_for_contact(self, contact_id).await?; - chat_id - .set_protection(self, ProtectionStatus::Protected, time(), Some(contact_id)) - .await?; - - let mut msg = Message::new_text(self.get_self_report().await?); - - chat_id.set_draft(self, Some(&mut msg)).await?; - - Ok(chat_id) - } - /// Get a list of fresh, unmuted messages in unblocked chats. /// /// The list starts with the most recent message diff --git a/src/context/context_tests.rs b/src/context/context_tests.rs index e80c17448f..2613e29d85 100644 --- a/src/context/context_tests.rs +++ b/src/context/context_tests.rs @@ -6,9 +6,9 @@ use super::*; use crate::chat::{Chat, MuteDuration, get_chat_contacts, get_chat_msgs, send_msg, set_muted}; use crate::chatlist::Chatlist; use crate::constants::Chattype; -use crate::mimeparser::SystemMessage; +use crate::message::Message; use crate::receive_imf::receive_imf; -use crate::test_utils::{E2EE_INFO_MSGS, TestContext, get_chat_msg}; +use crate::test_utils::{E2EE_INFO_MSGS, TestContext}; use crate::tools::{SystemTime, create_outgoing_rfc724_mid}; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -276,7 +276,6 @@ async fn test_get_info_completeness() { "mail_port", "mail_security", "notify_about_wrong_pw", - "self_reporting_id", "selfstatus", "send_server", "send_user", @@ -296,6 +295,8 @@ async fn test_get_info_completeness() { "webxdc_integration", "device_token", "encrypted_device_token", + "stats_last_excluded_msg_id", + "stats_last_old_contact_id", ]; let t = TestContext::new().await; let info = t.get_info().await.unwrap(); @@ -598,26 +599,6 @@ async fn test_get_next_msgs() -> Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_draft_self_report() -> Result<()> { - let alice = TestContext::new_alice().await; - - let chat_id = alice.draft_self_report().await?; - let msg = get_chat_msg(&alice, chat_id, 0, 1).await; - assert_eq!(msg.get_info_type(), SystemMessage::ChatProtectionEnabled); - - let chat = Chat::load_from_db(&alice, chat_id).await?; - assert!(chat.is_protected()); - - let mut draft = chat_id.get_draft(&alice).await?.unwrap(); - assert!(draft.text.starts_with("core_version")); - - // Test that sending into the protected chat works: - let _sent = alice.send_msg(chat_id, &mut draft).await; - - Ok(()) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_cache_is_cleared_when_io_is_started() -> Result<()> { let alice = TestContext::new_alice().await; diff --git a/src/lib.rs b/src/lib.rs index 3c6402cbfe..b3c26484fc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -98,6 +98,7 @@ pub mod html; pub mod net; pub mod plaintext; mod push; +pub mod statistics; pub mod summary; mod debug_logging; diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 03375862f4..aef0fea7eb 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -42,6 +42,7 @@ use crate::reaction::{Reaction, set_msg_reaction}; use crate::rusqlite::OptionalExtension; use crate::securejoin::{self, handle_securejoin_handshake, observe_securejoin_on_other_device}; use crate::simplify; +use crate::statistics::STATISTICS_BOT_EMAIL; use crate::stock_str; use crate::sync::Sync::*; use crate::tools::{self, buf_compress, remove_subject_prefix}; @@ -1743,7 +1744,11 @@ async fn add_parts( let state = if !mime_parser.incoming { MessageState::OutDelivered - } else if seen || is_mdn || chat_id_blocked == Blocked::Yes || group_changes.silent + } else if seen + || is_mdn + || chat_id_blocked == Blocked::Yes + || group_changes.silent + || mime_parser.from.addr == STATISTICS_BOT_EMAIL // No check for `hidden` because only reactions are such and they should be `InFresh`. { MessageState::InSeen diff --git a/src/scheduler.rs b/src/scheduler.rs index 621a799eb0..c81dca3937 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -27,6 +27,7 @@ use crate::log::{LogExt, error, info, warn}; use crate::message::MsgId; use crate::smtp::{Smtp, send_smtp_messages}; use crate::sql; +use crate::statistics::maybe_send_statistics; use crate::tools::{self, duration_to_str, maybe_add_time_based_warnings, time, time_elapsed}; pub(crate) mod connectivity; @@ -513,6 +514,7 @@ async fn inbox_fetch_idle(ctx: &Context, imap: &mut Imap, mut session: Session) } }; + maybe_send_statistics(ctx).await.log_err(ctx).ok(); match ctx.get_config_bool(Config::FetchedExistingMsgs).await { Ok(fetched_existing_msgs) => { if !fetched_existing_msgs { diff --git a/src/securejoin.rs b/src/securejoin.rs index 4fff0eca28..4b2bcc9db9 100644 --- a/src/securejoin.rs +++ b/src/securejoin.rs @@ -5,7 +5,6 @@ use deltachat_contact_tools::ContactAddress; use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode}; use crate::chat::{self, Chat, ChatId, ChatIdBlocked, ProtectionStatus, get_chat_id_by_grpid}; -use crate::chatlist_events; use crate::config::Config; use crate::constants::{Blocked, Chattype, NON_ALPHANUMERIC_WITHOUT_DOT}; use crate::contact::mark_contact_id_as_verified; @@ -15,6 +14,7 @@ use crate::e2ee::ensure_secret_key_exists; use crate::events::EventType; use crate::headerdef::HeaderDef; use crate::key::{DcKey, Fingerprint, load_self_public_key}; +use crate::log::LogExt as _; use crate::log::{error, info, warn}; use crate::logged_debug_assert; use crate::message::{Message, Viewtype}; @@ -24,11 +24,12 @@ use crate::qr::check_qr; use crate::securejoin::bob::JoinerProgress; use crate::sync::Sync::*; use crate::token; +use crate::{chatlist_events, statistics}; mod bob; mod qrinvite; -use qrinvite::QrInvite; +pub(crate) use qrinvite::QrInvite; use crate::token::Namespace; @@ -146,12 +147,34 @@ async fn get_self_fingerprint(context: &Context) -> Result { /// /// The function returns immediately and the handshake will run in background. pub async fn join_securejoin(context: &Context, qr: &str) -> Result { - securejoin(context, qr).await.map_err(|err| { + join_securejoin_with_source(context, qr, None, None).await +} + +/// Take a scanned QR-code and do the setup-contact/join-group/invite handshake. +/// +/// This is the start of the process for the joiner. See the module and ffi documentation +/// for more details. +/// +/// The function returns immediately and the handshake will run in background. +pub async fn join_securejoin_with_source( + context: &Context, + qr: &str, + source: Option, + uipath: Option, +) -> Result { + let res = securejoin(context, qr).await.map_err(|err| { warn!(context, "Fatal joiner error: {:#}", err); // The user just scanned this QR code so has context on what failed. error!(context, "QR process failed"); err - }) + })?; + + statistics::count_securejoin_source(context, source, uipath) + .await + .log_err(context) + .ok(); + + Ok(res) } async fn securejoin(context: &Context, qr: &str) -> Result { @@ -165,6 +188,11 @@ async fn securejoin(context: &Context, qr: &str) -> Result { let invite = QrInvite::try_from(qr_scan)?; + statistics::count_securejoin_invite(context, &invite) + .await + .log_err(context) + .ok(); + bob::start_protocol(context, invite).await } diff --git a/src/sql/migrations.rs b/src/sql/migrations.rs index 4a7c40e911..39b134c0b7 100644 --- a/src/sql/migrations.rs +++ b/src/sql/migrations.rs @@ -1261,6 +1261,27 @@ CREATE INDEX gossip_timestamp_index ON gossip_timestamp (chat_id, fingerprint); .await?; } + inc_and_check(&mut migration_version, 134)?; + if dbversion < migration_version { + sql.execute_migration( + "CREATE TABLE stats_securejoin_sources( + source INTEGER PRIMARY KEY, + count INTEGER NOT NULL DEFAULT 0 + ) STRICT; + CREATE TABLE stats_securejoin_uipaths( + uipath INTEGER PRIMARY KEY, + count INTEGER NOT NULL DEFAULT 0 + ) STRICT; + CREATE TABLE stats_securejoin_invites( + contact_created INTEGER NOT NULL, + already_verified INTEGER NOT NULL, + type TEXT NOT NULL + ) STRICT;", + migration_version, + ) + .await?; + } + let new_version = sql .get_raw_config_int(VERSION_CFG) .await? diff --git a/src/statistics.rs b/src/statistics.rs new file mode 100644 index 0000000000..5b5593ef2a --- /dev/null +++ b/src/statistics.rs @@ -0,0 +1,710 @@ +//! Delta Chat has an advanced option +//! "Send statistics to the developers of Delta Chat". +//! If this is enabled, a JSON file with some anonymous statistics +//! will be sent to a bot once a week. + +use std::collections::{BTreeMap, BTreeSet}; + +use anyhow::{Context as _, Result, ensure}; +use deltachat_derive::FromSql; +use pgp::types::PublicKeyTrait; +use serde::Serialize; + +use crate::chat::{self, ChatId, ChatVisibility, MuteDuration, ProtectionStatus}; +use crate::config::Config; +use crate::constants::Chattype; +use crate::contact::{Contact, ContactId, Origin, import_vcard, mark_contact_id_as_verified}; +use crate::context::{Context, get_version_str}; +use crate::key::load_self_public_keyring; +use crate::log::LogExt; +use crate::message::{Message, Viewtype}; +use crate::securejoin::QrInvite; +use crate::tools::{create_id, time}; + +pub(crate) const STATISTICS_BOT_EMAIL: &str = "self_reporting@testrun.org"; +const STATISTICS_BOT_VCARD: &str = include_str!("../assets/statistics-bot.vcf"); + +#[derive(Serialize)] +struct Statistics { + core_version: String, + key_created: Vec, + statistics_id: String, + is_chatmail: bool, + contact_stats: Vec, + message_stats_one_one: MessageStats, + message_stats_multi_user: MessageStats, + securejoin_source_stats: SecurejoinSourceStats, + securejoin_uipath_stats: SecurejoinUIPathStats, + securejoin_invites_stats: Vec, +} + +#[derive(Serialize, PartialEq)] +enum VerifiedStatus { + Direct, + Transitive, + TransitiveViaBot, + Opportunistic, + Unencrypted, +} + +#[derive(Serialize)] +struct ContactStat { + #[serde(skip_serializing)] + id: ContactId, + + verified: VerifiedStatus, + bot: bool, + direct_chat: bool, + last_seen: u64, + + #[serde(skip_serializing_if = "Option::is_none")] + transitive_chain: Option, + + /// Whether the contact was established after stats-sending was enabled + new: bool, +} + +#[derive(Serialize)] +struct MessageStats { + to_verified: u32, + unverified_encrypted: u32, + unencrypted: u32, + only_to_self: u32, +} + +#[repr(u32)] +#[derive(Debug, Clone, Copy, FromPrimitive, FromSql, PartialEq, Eq, PartialOrd, Ord)] +enum SecurejoinSource { + Unknown = 0, + ExternalLink = 1, + InternalLink = 2, + Clipboard = 3, + ImageLoaded = 4, + Scan = 5, +} + +#[derive(Serialize)] +struct SecurejoinSourceStats { + unknown: u32, + external_link: u32, + internal_link: u32, + clipboard: u32, + image_loaded: u32, + scan: u32, +} + +#[derive(Debug, Clone, Copy, FromPrimitive, FromSql, PartialEq, Eq, PartialOrd, Ord)] +enum SecurejoinUIPath { + Unknown = 0, + QrIcon = 1, + NewContact = 2, +} + +#[derive(Serialize)] +struct SecurejoinUIPathStats { + other: u32, + qr_icon: u32, + new_contact: u32, +} + +/// Sends a message with statistics about the usage of Delta Chat, +/// if the last time such a message was sent +/// was more than a week ago. +/// +/// On the other end, a bot will receive the message and make it available +/// to Delta Chat's developers. +pub async fn maybe_send_statistics(context: &Context) -> Result> { + if context.get_config_bool(Config::SendStatistics).await? { + let last_sending_time = context.get_config_i64(Config::LastStatisticsSent).await?; + let next_sending_time = last_sending_time.saturating_add(30); // TODO increase to 1 day or 1 week + if next_sending_time <= time() { + return Ok(Some(send_statistics(context).await?)); + } + } + Ok(None) +} + +async fn send_statistics(context: &Context) -> Result { + info!(context, "Sending statistics."); + + // Setting this config at the beginning avoids endless loops when things do not + // work out for whatever reason. + context + .set_config_internal(Config::LastStatisticsSent, Some(&time().to_string())) + .await + .log_err(context) + .ok(); + + let chat_id = get_statistics_bot(context).await?; + + let mut msg = Message::new(Viewtype::File); + msg.set_text( + "The attachment contains anonymous usage statistics, \ +because you enabled this in the settings. \ +This helps us improve the security of Delta Chat. \ +See TODO[blog post] for more information." + .to_string(), + ); + + let statistics = get_statistics(context).await?; + + msg.set_file_from_bytes( + context, + "statistics.txt", + statistics.as_bytes(), + Some("text/plain"), + )?; + + crate::chat::send_msg(context, chat_id, &mut msg) + .await + .context("Failed to send statistics message") + .log_err(context) + .ok(); + + set_last_excluded_msg_id(context).await?; + + Ok(chat_id) +} + +pub(crate) async fn set_last_excluded_msg_id(context: &Context) -> Result<()> { + let last_msgid: u64 = context + .sql + .query_get_value("SELECT MAX(id) FROM msgs", ()) + .await? + .unwrap_or(0); + + context + .sql + .set_raw_config( + Config::StatsLastExcludedMsgId.as_ref(), + Some(&last_msgid.to_string()), + ) + .await?; + + Ok(()) +} + +pub(crate) async fn set_last_old_contact_id(context: &Context) -> Result<()> { + let last_contact_id: u64 = context + .sql + .query_get_value("SELECT MAX(id) FROM contacts", ()) + .await? + .unwrap_or(0); + + context + .sql + .set_raw_config( + Config::StatsLastOldContactId.as_ref(), + Some(&last_contact_id.to_string()), + ) + .await?; + + Ok(()) +} + +async fn get_statistics(context: &Context) -> Result { + // The ID of the last msg that was already counted in the previously sent statistics. + // Only newer messages will be counted in the current statistics. + let last_excluded_msg = context + .get_config_u32(Config::StatsLastExcludedMsgId) + .await?; + + // The Id of the last contact that already existed when the user enabled the setting. + // Newer contacts will get the `new` flag set. + let last_old_contact = context + .get_config_u32(Config::StatsLastOldContactId) + .await?; + + let key_created: Vec = load_self_public_keyring(context) + .await? + .iter() + .map(|k| k.created_at().timestamp()) + .collect(); + + let statistics_id = match context.get_config(Config::StatisticsId).await? { + Some(id) => id, + None => { + let id = create_id(); + context + .set_config_internal(Config::StatisticsId, Some(&id)) + .await?; + id + } + }; + + let statistics = Statistics { + core_version: get_version_str().to_string(), + key_created, + statistics_id, + is_chatmail: context.is_chatmail().await?, + contact_stats: get_contact_stats(context, last_old_contact).await?, + message_stats_one_one: get_message_stats(context, last_excluded_msg, true).await?, + message_stats_multi_user: get_message_stats(context, last_excluded_msg, false).await?, + securejoin_source_stats: get_securejoin_source_stats(context).await?, + securejoin_uipath_stats: get_securejoin_uipath_stats(context).await?, + securejoin_invites_stats: get_securejoin_invite_stats(context).await?, + }; + + Ok(serde_json::to_string_pretty(&statistics)?) +} + +async fn get_statistics_bot(context: &Context) -> Result { + let contact_id: ContactId = *import_vcard(context, STATISTICS_BOT_VCARD) + .await? + .first() + .context("Statistics bot vCard does not contain a contact")?; + mark_contact_id_as_verified(context, contact_id, ContactId::SELF).await?; + + let chat_id = if let Some(res) = ChatId::lookup_by_contact(context, contact_id).await? { + // Already exists, no need to create. + res + } else { + let chat_id = ChatId::get_for_contact(context, contact_id).await?; + chat_id + .set_visibility(context, ChatVisibility::Archived) + .await?; + chat::set_muted(context, chat_id, MuteDuration::Forever).await?; + chat_id + }; + + chat_id + .set_protection( + context, + ProtectionStatus::Protected, + time(), + Some(contact_id), + ) + .await?; + + Ok(chat_id) +} + +async fn get_contact_stats(context: &Context, last_old_contact: u32) -> Result> { + let mut verified_by_map: BTreeMap = BTreeMap::new(); + let mut bot_ids: BTreeSet = BTreeSet::new(); + + let mut contacts: Vec = context + .sql + .query_map( + "SELECT id, fingerprint<>'', verifier, last_seen, is_bot FROM contacts c + WHERE id>9 AND origin>? AND addr<>?", + (Origin::Hidden, STATISTICS_BOT_EMAIL), + |row| { + let id = row.get(0)?; + let is_encrypted: bool = row.get(1)?; + let verifier: ContactId = row.get(2)?; + let last_seen: u64 = row.get(3)?; + let bot: bool = row.get(4)?; + + let verified = match (is_encrypted, verifier) { + (true, ContactId::SELF) => VerifiedStatus::Direct, + (true, ContactId::UNDEFINED) => VerifiedStatus::Opportunistic, + (true, _) => VerifiedStatus::Transitive, // TransitiveViaBot will be filled later + (false, _) => VerifiedStatus::Unencrypted, + }; + + if verifier != ContactId::UNDEFINED { + verified_by_map.insert(id, verifier); + } + + if bot { + bot_ids.insert(id); + } + + Ok(ContactStat { + id, + verified, + bot, + direct_chat: false, // will be filled later + last_seen, + transitive_chain: None, // will be filled later + new: id.to_u32() > last_old_contact, + }) + }, + |rows| { + rows.collect::, _>>() + .map_err(Into::into) + }, + ) + .await?; + + // Fill TransitiveViaBot and transitive_chain + for contact in &mut contacts { + if contact.verified == VerifiedStatus::Transitive { + let mut transitive_chain: u32 = 0; + let mut has_bot = false; + let mut current_verifier_id = contact.id; + + while current_verifier_id != ContactId::SELF && transitive_chain < 100 { + current_verifier_id = match verified_by_map.get(¤t_verifier_id) { + Some(id) => *id, + None => { + // The chain ends here, probably because some verification was done + // before we started recording verifiers. + // It's unclear how long the chain really is. + transitive_chain = 0; + break; + } + }; + if bot_ids.contains(¤t_verifier_id) { + has_bot = true; + } + transitive_chain = transitive_chain.saturating_add(1); + } + + if transitive_chain > 0 { + contact.transitive_chain = Some(transitive_chain); + } + + if has_bot { + contact.verified = VerifiedStatus::TransitiveViaBot; + } + } + } + + // Fill direct_chat + for contact in &mut contacts { + let direct_chat = context + .sql + .exists( + "SELECT COUNT(*) + FROM chats_contacts cc INNER JOIN chats + WHERE cc.contact_id=? AND chats.type=?", + (contact.id, Chattype::Single), + ) + .await?; + contact.direct_chat = direct_chat; + } + + Ok(contacts) +} + +/// - `last_msg_id`: The last msg_id that was already counted in the previous stats. +/// Only messages newer than that will be counted. +/// - `one_one_chats`: If true, only messages in 1:1 chats are counted. +/// If false, only messages in other chats (groups and broadcast channels) are counted. +async fn get_message_stats( + context: &Context, + last_excluded_msg: u32, + one_one_chats: bool, +) -> Result { + ensure!( + last_excluded_msg >= 9, + "Last_msgid < 9 would mean including 'special' messages in the statistics" + ); + + let statistics_bot_chat_id = get_statistics_bot(context).await?; + + let trans_fn = |t: &mut rusqlite::Transaction| { + t.pragma_update(None, "query_only", "0")?; + + // This table will hold all empty chats, + // i.e. all chats that do not contain any members except for self. + // Messages in these chats are not actually sent out. + t.execute( + "CREATE TEMP TABLE temp.empty_chats ( + id INTEGER PRIMARY KEY + ) STRICT", + (), + )?; + + // id>9 because chat ids 0..9 are "special" chats like the trash chat, + // and contact ids 0..9 are "special" contact ids like the 'device'. + t.execute( + "INSERT INTO temp.empty_chats + SELECT id FROM chats + WHERE id>9 AND NOT EXISTS( + SELECT * + FROM contacts, chats_contacts + WHERE chats_contacts.contact_id=contacts.id AND chats_contacts.chat_id=chats.id + AND contacts.id>9 + )", + (), + )?; + + // This table will hold all verified chats, + // i.e. all chats that only contain verified contacts. + t.execute( + "CREATE TEMP TABLE temp.verified_chats ( + id INTEGER PRIMARY KEY + ) STRICT", + (), + )?; + + // Verified chats are chats that are not empty, + // and do not contain any unverified contacts + t.execute( + "INSERT INTO temp.verified_chats + SELECT id FROM chats + WHERE id>9 + AND id NOT IN (SELECT id FROM temp.empty_chats) + AND NOT EXISTS( + SELECT * + FROM contacts, chats_contacts + WHERE chats_contacts.contact_id=contacts.id AND chats_contacts.chat_id=chats.id + AND contacts.id>9 + AND contacts.verifier=0 + )", + (), + )?; + + // This table will hold all 1:1 chats. + t.execute( + "CREATE TEMP TABLE temp.one_one_chats ( + id INTEGER PRIMARY KEY + ) STRICT", + (), + )?; + + t.execute( + "INSERT INTO temp.one_one_chats + SELECT id FROM chats + WHERE type=?;", + (Chattype::Single,), + )?; + + // - `from_id=?` is to count only outgoing messages. + // - `chat_id<>?` excludes the chat with the statistics bot itself, + // - `id>?` excludes messages that were already counted in the previously sent statistics, or messages sent before the config was enabled + // - `hidden=0` excludes hidden system messages, which are not actually shown to the user + // - `chat_id>9` excludes messages in the 'Trash' chat, which is an internal chat assigned to messages that are not shown to the user + let mut general_requirements = + "from_id=? AND chat_id<>? AND id>? AND hidden=0 AND chat_id>9".to_string(); + if one_one_chats { + general_requirements += " AND chat_id IN temp.one_one_chats"; + } else { + general_requirements += " AND chat_id NOT IN temp.one_one_chats"; + } + let params = (ContactId::SELF, statistics_bot_chat_id, last_excluded_msg); + + let to_verified = t.query_row( + &format!( + "SELECT COUNT(*) FROM msgs + WHERE chat_id IN temp.verified_chats + AND {general_requirements}" + ), + params, + |row| row.get(0), + )?; + + let unverified_encrypted = t.query_row( + &format!( + // (param GLOB '*\nc=1*' OR param GLOB 'c=1*')` + // matches all messages that are end-to-end encrypted + "SELECT COUNT(*) FROM msgs + WHERE chat_id NOT IN temp.verified_chats AND chat_id NOT IN temp.empty_chats + AND (param GLOB '*\nc=1*' OR param GLOB 'c=1*') + AND {general_requirements}" + ), + params, + |row| row.get(0), + )?; + + let unencrypted = t.query_row( + &format!( + "SELECT COUNT(*) FROM msgs + WHERE chat_id NOT IN temp.verified_chats AND chat_id NOT IN temp.empty_chats + AND NOT (param GLOB '*\nc=1*' OR param GLOB 'c=1*') + AND {general_requirements}" + ), + params, + |row| row.get(0), + )?; + + let only_to_self = t.query_row( + &format!( + "SELECT COUNT(*) FROM msgs + WHERE chat_id IN temp.empty_chats + AND {general_requirements}" + ), + params, + |row| row.get(0), + )?; + + t.execute("DROP TABLE temp.verified_chats", ())?; + t.execute("DROP TABLE temp.empty_chats", ())?; + t.execute("DROP TABLE temp.one_one_chats", ())?; + + Ok(MessageStats { + to_verified, + unverified_encrypted, + unencrypted, + only_to_self, + }) + }; + + let query_only = true; + let message_stats: MessageStats = context.sql.transaction_ex(query_only, trans_fn).await?; + + Ok(message_stats) +} + +pub(crate) async fn count_securejoin_source( + context: &Context, + source: Option, + uipath: Option, +) -> Result<()> { + if !context.get_config_bool(Config::SendStatistics).await? { + return Ok(()); + } + + let source = source + .context("Missing securejoin source") + .log_err(context) + .unwrap_or(0); + + context + .sql + .execute( + "INSERT INTO stats_securejoin_sources VALUES (?, 1) + ON CONFLICT (source) DO UPDATE SET count=count+1;", + (source,), + ) + .await?; + + // We only get a UI path if the source is a QR code scan, + // a loaded image, or a link pasted from the QR code, + // so, no need to log an error if `uipath` is None: + let uipath = uipath.unwrap_or(0); + context + .sql + .execute( + "INSERT INTO stats_securejoin_uipaths VALUES (?, 1) + ON CONFLICT (uipath) DO UPDATE SET count=count+1;", + (uipath,), + ) + .await?; + Ok(()) +} + +async fn get_securejoin_source_stats(context: &Context) -> Result { + let map = context + .sql + .query_map( + "SELECT source, count FROM stats_securejoin_sources", + (), + |row| { + let source: SecurejoinSource = row.get(0)?; + let count: u32 = row.get(1)?; + Ok((source, count)) + }, + |rows| Ok(rows.collect::>>()?), + ) + .await?; + + let stats = SecurejoinSourceStats { + unknown: *map.get(&SecurejoinSource::Unknown).unwrap_or(&0), + external_link: *map.get(&SecurejoinSource::ExternalLink).unwrap_or(&0), + internal_link: *map.get(&SecurejoinSource::InternalLink).unwrap_or(&0), + clipboard: *map.get(&SecurejoinSource::Clipboard).unwrap_or(&0), + image_loaded: *map.get(&SecurejoinSource::ImageLoaded).unwrap_or(&0), + scan: *map.get(&SecurejoinSource::Scan).unwrap_or(&0), + }; + + Ok(stats) +} + +async fn get_securejoin_uipath_stats(context: &Context) -> Result { + let map = context + .sql + .query_map( + "SELECT uipath, count FROM stats_securejoin_uipaths", + (), + |row| { + let uipath: SecurejoinUIPath = row.get(0)?; + let count: u32 = row.get(1)?; + Ok((uipath, count)) + }, + |rows| Ok(rows.collect::>>()?), + ) + .await?; + + let stats = SecurejoinUIPathStats { + other: *map.get(&SecurejoinUIPath::Unknown).unwrap_or(&0), + qr_icon: *map.get(&SecurejoinUIPath::QrIcon).unwrap_or(&0), + new_contact: *map.get(&SecurejoinUIPath::NewContact).unwrap_or(&0), + }; + + Ok(stats) +} + +pub(crate) async fn count_securejoin_invite(context: &Context, invite: &QrInvite) -> Result<()> { + if !context.get_config_bool(Config::SendStatistics).await? { + return Ok(()); + } + + let contact = Contact::get_by_id(context, invite.contact_id()).await?; + + // If the contact was created just now by the QR code scan, + // (or if a contact existed in the database + // but it was not visible in the contacts list in the UI + // e.g. because it's a past contact of a group we're in), + // then its origin is UnhandledSecurejoinQrScan. + let contact_created = contact.origin == Origin::UnhandledSecurejoinQrScan; + + // Check whether the contact was verified already before the QR scan. + let already_verified = contact.is_verified(context).await?; + + let typ = match invite { + QrInvite::Contact { .. } => "contact", + QrInvite::Group { .. } => "group", + }; + + context + .sql + .execute( + "INSERT INTO stats_securejoin_invites (contact_created, already_verified, type) + VALUES (?, ?, ?)", + (contact_created, already_verified, typ), + ) + .await?; + + Ok(()) +} + +/// Some information on an invite-joining event +/// (i.e. a qr scan or a clicked link). +#[derive(Serialize)] +struct JoinedInvite { + /// Whether the contact was newly created right now. + /// If this is false, then a contact existed already before. + contact_created: bool, + /// If a contact already existed, + /// this tells us whether the contact was verified already. + already_verified: bool, + /// The type of the invite: + /// "contact" for 1:1 invites that setup a verified contact, + /// "group" for invites that invite to a group + /// and also perform the contact verification 'along the way'. + typ: String, +} + +async fn get_securejoin_invite_stats(context: &Context) -> Result> { + let qr_scans: Vec = context + .sql + .query_map( + "SELECT contact_created, already_verified, type FROM stats_securejoin_invites", + (), + |row| { + let contact_created: bool = row.get(0)?; + let already_verified: bool = row.get(1)?; + let typ: String = row.get(2)?; + + Ok(JoinedInvite { + contact_created, + already_verified, + typ, + }) + }, + |rows| { + rows.collect::, _>>() + .map_err(Into::into) + }, + ) + .await?; + + Ok(qr_scans) +} + +#[cfg(test)] +mod statistics_tests; diff --git a/src/statistics/statistics_tests.rs b/src/statistics/statistics_tests.rs new file mode 100644 index 0000000000..d35af2d7ec --- /dev/null +++ b/src/statistics/statistics_tests.rs @@ -0,0 +1,511 @@ +use std::time::Duration; + +use super::*; +use crate::chat::{Chat, create_broadcast, create_group_chat, create_group_ex}; +use crate::mimeparser::SystemMessage; +use crate::qr::check_qr; +use crate::securejoin::{get_securejoin_qr, join_securejoin, join_securejoin_with_source}; +use crate::test_utils::{TestContext, TestContextManager, get_chat_msg}; +use crate::tools::SystemTime; +use pretty_assertions::assert_eq; +use serde_json::{Number, Value}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_maybe_send_statistics() -> Result<()> { + let alice = &TestContext::new_alice().await; + + alice.set_config_bool(Config::SendStatistics, true).await?; + + let chat_id = maybe_send_statistics(&alice).await?.unwrap(); + let msg = get_chat_msg(&alice, chat_id, 0, 2).await; + assert_eq!(msg.get_info_type(), SystemMessage::ChatProtectionEnabled); + + let chat = Chat::load_from_db(&alice, chat_id).await?; + assert!(chat.is_protected()); + + let msg = get_chat_msg(&alice, chat_id, 1, 2).await; + assert_eq!(msg.get_filename().unwrap(), "statistics.txt"); + + let stats = tokio::fs::read(msg.get_file(&alice).unwrap()).await?; + let stats = std::str::from_utf8(&stats)?; + println!("\nEmpty account:\n{}\n", stats); + assert!(stats.contains(r#""contact_stats": []"#)); + + let r: serde_json::Value = serde_json::from_str(&stats)?; + assert_eq!( + r.get("contact_stats").unwrap(), + &serde_json::Value::Array(vec![]) + ); + assert_eq!(r.get("core_version").unwrap(), get_version_str()); + + assert_eq!(maybe_send_statistics(alice).await?, None); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_statistics_one_contact() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + alice.set_config_bool(Config::SendStatistics, true).await?; + + let stats = get_statistics(alice).await?; + let r: serde_json::Value = serde_json::from_str(&stats)?; + + tcm.send_recv_accept(bob, alice, "Hi!").await; + + let stats = get_statistics(alice).await?; + println!("\nWith Bob:\n{stats}\n"); + let r2: serde_json::Value = serde_json::from_str(&stats)?; + + assert_eq!( + r.get("key_created").unwrap(), + r2.get("key_created").unwrap() + ); + assert_eq!( + r.get("statistics_id").unwrap(), + r2.get("statistics_id").unwrap() + ); + let contact_stats = r2.get("contact_stats").unwrap().as_array().unwrap(); + assert_eq!(contact_stats.len(), 1); + let contact_info = &contact_stats[0]; + assert_eq!( + contact_info.get("bot").unwrap(), + &serde_json::Value::Bool(false) + ); + assert_eq!( + contact_info.get("direct_chat").unwrap(), + &serde_json::Value::Bool(true) + ); + assert!(contact_info.get("transitive_chain").is_none(),); + assert_eq!( + contact_info.get("verified").unwrap(), + &serde_json::Value::String("Opportunistic".to_string()) + ); + assert_eq!( + contact_info.get("new").unwrap(), + &serde_json::Value::Bool(true) + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_message_stats() -> Result<()> { + #[track_caller] + fn check_statistics( + stats: &str, + expected_one_one: &MessageStats, + expected_multi_user: &MessageStats, + ) { + let actual: serde_json::Value = serde_json::from_str(&stats).unwrap(); + + for (expected, key) in [ + (expected_one_one, "message_stats_one_one"), + (expected_multi_user, "message_stats_multi_user"), + ] { + let actual = &actual[key]; + + let expected = serde_json::to_string_pretty(&expected).unwrap(); + let expected: serde_json::Value = serde_json::from_str(&expected).unwrap(); + + assert_eq!(actual, &expected, "Wrong {key}"); + } + } + + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + alice.set_config_bool(Config::SendStatistics, true).await?; + let email_chat = alice.create_email_chat(bob).await; + let encrypted_chat = alice.create_chat(bob).await; + + let mut one_one = MessageStats { + to_verified: 0, + unverified_encrypted: 0, + unencrypted: 0, + only_to_self: 0, + }; + let mut multi_user = MessageStats { + to_verified: 0, + unverified_encrypted: 0, + unencrypted: 0, + only_to_self: 0, + }; + + check_statistics(&get_statistics(alice).await?, &one_one, &multi_user); + + alice.send_text(email_chat.id, "foo").await; + one_one.unencrypted += 1; + check_statistics(&get_statistics(alice).await?, &one_one, &multi_user); + + alice.send_text(encrypted_chat.id, "foo").await; + one_one.unverified_encrypted += 1; + check_statistics(&get_statistics(alice).await?, &one_one, &multi_user); + + alice.send_text(encrypted_chat.id, "foo").await; + one_one.unverified_encrypted += 1; + check_statistics(&get_statistics(alice).await?, &one_one, &multi_user); + + let group = alice + .create_group_with_members(ProtectionStatus::Unprotected, "Pizza", &[bob]) + .await; + alice.send_text(group, "foo").await; + multi_user.unverified_encrypted += 1; + check_statistics(&get_statistics(alice).await?, &one_one, &multi_user); + + tcm.execute_securejoin(alice, bob).await; + one_one.to_verified = one_one.unverified_encrypted; + one_one.unverified_encrypted = 0; + multi_user.to_verified = multi_user.unverified_encrypted; + multi_user.unverified_encrypted = 0; + check_statistics(&get_statistics(alice).await?, &one_one, &multi_user); + + alice.send_text(alice.get_self_chat().await.id, "foo").await; + one_one.only_to_self += 1; + check_statistics(&get_statistics(alice).await?, &one_one, &multi_user); + + let empty_group = create_group_chat(alice, ProtectionStatus::Unprotected, "Notes").await?; + alice.send_text(empty_group, "foo").await; + multi_user.only_to_self += 1; + check_statistics(&get_statistics(alice).await?, &one_one, &multi_user); + + let empty_unencrypted = create_group_ex(alice, None, "Email thread").await?; + alice.send_text(empty_unencrypted, "foo").await; + multi_user.only_to_self += 1; + check_statistics(&get_statistics(alice).await?, &one_one, &multi_user); + + let group = alice + .create_group_with_members(ProtectionStatus::Unprotected, "Pizza 2", &[bob]) + .await; + alice.send_text(group, "foo").await; + multi_user.to_verified += 1; + check_statistics(&get_statistics(alice).await?, &one_one, &multi_user); + + let empty_broadcast = create_broadcast(alice, "Channel".to_string()).await?; + alice.send_text(empty_broadcast, "foo").await; + multi_user.only_to_self += 1; + check_statistics(&get_statistics(alice).await?, &one_one, &multi_user); + + // Incoming messages are not counted: + let rcvd = tcm.send_recv(bob, alice, "bar").await; + check_statistics(&get_statistics(alice).await?, &one_one, &multi_user); + + // Reactions are not counted: + crate::reaction::send_reaction(alice, rcvd.id, "👍") + .await + .unwrap(); + check_statistics(&get_statistics(alice).await?, &one_one, &multi_user); + + tcm.section("Test that after actually sending statistics, the message numbers are reset."); + let before_sending = get_statistics(alice).await.unwrap(); + + let stats = send_and_read_statistics(alice).await; + // The statistics are supposed not to have changed yet + assert_eq!(before_sending, stats); + + // Shift by 8 days so that the next statistics-sending is due: + SystemTime::shift(Duration::from_secs(8 * 24 * 3600)); + + let stats = send_and_read_statistics(alice).await; + assert_ne!(before_sending, stats); + + one_one = MessageStats { + to_verified: 0, + unverified_encrypted: 0, + unencrypted: 0, + only_to_self: 0, + }; + multi_user = MessageStats { + to_verified: 0, + unverified_encrypted: 0, + unencrypted: 0, + only_to_self: 0, + }; + check_statistics(&stats, &one_one, &multi_user); + + tcm.section( + "Test that after sending a message again, the message statistics start to fill again.", + ); + SystemTime::shift(Duration::from_secs(8 * 24 * 3600)); + tcm.send_recv(alice, bob, "Hi").await; + one_one.to_verified += 1; + check_statistics( + &send_and_read_statistics(alice).await, + &one_one, + &multi_user, + ); + + Ok(()) +} + +async fn send_and_read_statistics(context: &TestContext) -> String { + let chat_id = maybe_send_statistics(&context).await.unwrap().unwrap(); + let msg = context.get_last_msg_in(chat_id).await; + assert_eq!(msg.get_filename().unwrap(), "statistics.txt"); + + let stats = tokio::fs::read(msg.get_file(&context).unwrap()) + .await + .unwrap(); + String::from_utf8(stats).unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_statistics_securejoin_source_stats() -> Result<()> { + async fn check_statistics(context: &TestContext, expected: &SecurejoinSourceStats) { + let statistics = get_statistics(context).await.unwrap(); + let actual: serde_json::Value = serde_json::from_str(&statistics).unwrap(); + let actual = &actual["securejoin_source_stats"]; + + let expected = serde_json::to_string_pretty(&expected).unwrap(); + let expected: serde_json::Value = serde_json::from_str(&expected).unwrap(); + + assert_eq!(actual, &expected); + } + + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + alice.set_config_bool(Config::SendStatistics, true).await?; + + let mut expected = SecurejoinSourceStats { + unknown: 0, + external_link: 0, + internal_link: 0, + clipboard: 0, + image_loaded: 0, + scan: 0, + }; + + check_statistics(alice, &expected).await; + + let qr = get_securejoin_qr(bob, None).await?; + + join_securejoin(alice, &qr).await?; + expected.unknown += 1; + check_statistics(alice, &expected).await; + + join_securejoin(alice, &qr).await?; + expected.unknown += 1; + check_statistics(alice, &expected).await; + + join_securejoin_with_source(alice, &qr, Some(SecurejoinSource::Clipboard as u32), None).await?; + expected.clipboard += 1; + check_statistics(alice, &expected).await; + + join_securejoin_with_source( + alice, + &qr, + Some(SecurejoinSource::ExternalLink as u32), + None, + ) + .await?; + expected.external_link += 1; + check_statistics(alice, &expected).await; + + join_securejoin_with_source( + alice, + &qr, + Some(SecurejoinSource::InternalLink as u32), + None, + ) + .await?; + expected.internal_link += 1; + check_statistics(alice, &expected).await; + + join_securejoin_with_source(alice, &qr, Some(SecurejoinSource::ImageLoaded as u32), None) + .await?; + expected.image_loaded += 1; + check_statistics(alice, &expected).await; + + join_securejoin_with_source(alice, &qr, Some(SecurejoinSource::Scan as u32), None).await?; + expected.scan += 1; + check_statistics(alice, &expected).await; + + join_securejoin_with_source(alice, &qr, Some(SecurejoinSource::Clipboard as u32), None).await?; + expected.clipboard += 1; + check_statistics(alice, &expected).await; + + join_securejoin_with_source(alice, &qr, Some(SecurejoinSource::Clipboard as u32), None).await?; + expected.clipboard += 1; + check_statistics(alice, &expected).await; + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_statistics_securejoin_uipath_stats() -> Result<()> { + async fn check_statistics(context: &TestContext, expected: &SecurejoinUIPathStats) { + let stats = get_statistics(context).await.unwrap(); + let actual: serde_json::Value = serde_json::from_str(&stats).unwrap(); + let actual = &actual["securejoin_uipath_stats"]; + + let expected = serde_json::to_string_pretty(&expected).unwrap(); + let expected: serde_json::Value = serde_json::from_str(&expected).unwrap(); + + assert_eq!(actual, &expected); + } + + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + alice.set_config_bool(Config::SendStatistics, true).await?; + + let mut expected = SecurejoinUIPathStats { + other: 0, + qr_icon: 0, + new_contact: 0, + }; + + check_statistics(alice, &expected).await; + + let qr = get_securejoin_qr(bob, None).await?; + + join_securejoin(alice, &qr).await?; + expected.other += 1; + check_statistics(alice, &expected).await; + + join_securejoin(alice, &qr).await?; + expected.other += 1; + check_statistics(alice, &expected).await; + + join_securejoin_with_source( + alice, + &qr, + Some(0), + Some(SecurejoinUIPath::NewContact as u32), + ) + .await?; + expected.new_contact += 1; + check_statistics(alice, &expected).await; + + join_securejoin_with_source( + alice, + &qr, + Some(0), + Some(SecurejoinUIPath::NewContact as u32), + ) + .await?; + expected.new_contact += 1; + check_statistics(alice, &expected).await; + + join_securejoin_with_source(alice, &qr, Some(0), Some(SecurejoinUIPath::QrIcon as u32)).await?; + expected.qr_icon += 1; + check_statistics(alice, &expected).await; + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_statistics_securejoin_invites() -> Result<()> { + async fn check_statistics(context: &TestContext, expected: &[JoinedInvite]) { + let stats = get_statistics(context).await.unwrap(); + let actual: serde_json::Value = serde_json::from_str(&stats).unwrap(); + let actual = &actual["securejoin_invites_stats"]; + + let expected = serde_json::to_string_pretty(&expected).unwrap(); + let expected: serde_json::Value = serde_json::from_str(&expected).unwrap(); + + assert_eq!(actual, &expected); + } + + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let charlie = &tcm.charlie().await; + alice.set_config_bool(Config::SendStatistics, true).await?; + + let mut expected = vec![]; + + check_statistics(alice, &expected).await; + + let qr = get_securejoin_qr(bob, None).await?; + + // The UI will call `check_qr()` first, which must not make the statistics wrong: + check_qr(alice, &qr).await?; + tcm.exec_securejoin_qr(alice, bob, &qr).await; + expected.push(JoinedInvite { + contact_created: true, + already_verified: false, + typ: "contact".to_string(), + }); + check_statistics(alice, &expected).await; + + check_qr(alice, &qr).await?; + tcm.exec_securejoin_qr(alice, bob, &qr).await; + expected.push(JoinedInvite { + contact_created: false, + already_verified: true, + typ: "contact".to_string(), + }); + check_statistics(alice, &expected).await; + + let group_id = create_group_chat(bob, ProtectionStatus::Unprotected, "Group chat").await?; + let qr = get_securejoin_qr(bob, Some(group_id)).await?; + + check_qr(alice, &qr).await?; + tcm.exec_securejoin_qr(alice, bob, &qr).await; + expected.push(JoinedInvite { + contact_created: false, + already_verified: true, + typ: "group".to_string(), + }); + check_statistics(alice, &expected).await; + + // A contact with Charlie exists already: + alice.add_or_lookup_contact(charlie).await; + let group_id = + create_group_chat(charlie, ProtectionStatus::Unprotected, "Group chat 2").await?; + let qr = get_securejoin_qr(charlie, Some(group_id)).await?; + + check_qr(alice, &qr).await?; + tcm.exec_securejoin_qr(alice, bob, &qr).await; + expected.push(JoinedInvite { + contact_created: false, + already_verified: false, + typ: "group".to_string(), + }); + check_statistics(alice, &expected).await; + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_statistics_is_chatmail() -> Result<()> { + let alice = &TestContext::new_alice().await; + alice.set_config_bool(Config::SendStatistics, true).await?; + + let r = get_statistics(alice).await?; + let r: serde_json::Value = serde_json::from_str(&r)?; + assert_eq!(r.get("is_chatmail").unwrap().as_bool().unwrap(), false); + + alice.set_config_bool(Config::IsChatmail, true).await?; + + let r = get_statistics(alice).await?; + let r: serde_json::Value = serde_json::from_str(&r)?; + assert_eq!(r.get("is_chatmail").unwrap().as_bool().unwrap(), true); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_statistics_key_creation_timestamp() -> Result<()> { + // Alice uses a pregenerated key. It was created at this timestamp: + const ALICE_KEY_CREATION_TIME: u128 = 1582855645; + + let alice = &TestContext::new_alice().await; + alice.set_config_bool(Config::SendStatistics, true).await?; + + let r = get_statistics(alice).await?; + let r: serde_json::Value = serde_json::from_str(&r)?; + let key_created = r.get("key_created").unwrap().as_array().unwrap(); + assert_eq!( + key_created, + &vec![Value::Number( + Number::from_u128(ALICE_KEY_CREATION_TIME).unwrap() + )] + ); + + Ok(()) +} From a1c8beb4dafb2351eb9315e1261e5699aa80be8c Mon Sep 17 00:00:00 2001 From: Hocuri Date: Thu, 14 Aug 2025 20:24:51 +0200 Subject: [PATCH 02/21] small renames --- src/config.rs | 6 ++--- src/lib.rs | 2 +- src/securejoin.rs | 4 ++++ src/sql/migrations.rs | 6 ++--- src/statistics.rs | 38 +++++++++++++++--------------- src/statistics/statistics_tests.rs | 18 +++++++------- 6 files changed, 39 insertions(+), 35 deletions(-) diff --git a/src/config.rs b/src/config.rs index f92bf14ad5..a2056975aa 100644 --- a/src/config.rs +++ b/src/config.rs @@ -14,7 +14,6 @@ use tokio::fs; use crate::blob::BlobObject; use crate::configure::EnteredLoginParam; -use crate::constants; use crate::context::Context; use crate::events::EventType; use crate::log::{LogExt, info}; @@ -23,6 +22,7 @@ use crate::mimefactory::RECOMMENDED_FILE_SIZE; use crate::provider::{Provider, get_provider_by_id}; use crate::sync::{self, Sync::*, SyncData}; use crate::tools::get_abs_path; +use crate::{constants, statistics}; /// The available configuration keys. #[derive( @@ -846,8 +846,8 @@ impl Context { } Config::SendStatistics => { self.sql.set_raw_config(key.as_ref(), value).await?; - crate::statistics::set_last_excluded_msg_id(self).await?; - crate::statistics::set_last_old_contact_id(self).await?; + statistics::set_last_excluded_msg_id(self).await?; + statistics::set_last_old_contact_id(self).await?; } _ => { self.sql.set_raw_config(key.as_ref(), value).await?; diff --git a/src/lib.rs b/src/lib.rs index b3c26484fc..61879d65e5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -98,7 +98,7 @@ pub mod html; pub mod net; pub mod plaintext; mod push; -pub mod statistics; +pub(crate) mod statistics; pub mod summary; mod debug_logging; diff --git a/src/securejoin.rs b/src/securejoin.rs index 4b2bcc9db9..85aacf0ec0 100644 --- a/src/securejoin.rs +++ b/src/securejoin.rs @@ -156,6 +156,10 @@ pub async fn join_securejoin(context: &Context, qr: &str) -> Result { /// for more details. /// /// The function returns immediately and the handshake will run in background. +/// +/// **source** and **uipath** are for statistics-sending, +/// if the user enabled it in the settings; +/// if you don't have statistics-sending implemented, just pass `None` here. pub async fn join_securejoin_with_source( context: &Context, qr: &str, diff --git a/src/sql/migrations.rs b/src/sql/migrations.rs index 39b134c0b7..045918681a 100644 --- a/src/sql/migrations.rs +++ b/src/sql/migrations.rs @@ -1264,15 +1264,15 @@ CREATE INDEX gossip_timestamp_index ON gossip_timestamp (chat_id, fingerprint); inc_and_check(&mut migration_version, 134)?; if dbversion < migration_version { sql.execute_migration( - "CREATE TABLE stats_securejoin_sources( + "CREATE TABLE statistics_securejoin_sources( source INTEGER PRIMARY KEY, count INTEGER NOT NULL DEFAULT 0 ) STRICT; - CREATE TABLE stats_securejoin_uipaths( + CREATE TABLE statistics_securejoin_uipaths( uipath INTEGER PRIMARY KEY, count INTEGER NOT NULL DEFAULT 0 ) STRICT; - CREATE TABLE stats_securejoin_invites( + CREATE TABLE statistics_securejoin_invites( contact_created INTEGER NOT NULL, already_verified INTEGER NOT NULL, type TEXT NOT NULL diff --git a/src/statistics.rs b/src/statistics.rs index 5b5593ef2a..afef19ed2f 100644 --- a/src/statistics.rs +++ b/src/statistics.rs @@ -33,9 +33,9 @@ struct Statistics { contact_stats: Vec, message_stats_one_one: MessageStats, message_stats_multi_user: MessageStats, - securejoin_source_stats: SecurejoinSourceStats, - securejoin_uipath_stats: SecurejoinUIPathStats, - securejoin_invites_stats: Vec, + securejoin_sources: SecurejoinSources, + securejoin_uipaths: SecurejoinUIPaths, + securejoin_invites: Vec, } #[derive(Serialize, PartialEq)] @@ -84,7 +84,7 @@ enum SecurejoinSource { } #[derive(Serialize)] -struct SecurejoinSourceStats { +struct SecurejoinSources { unknown: u32, external_link: u32, internal_link: u32, @@ -101,7 +101,7 @@ enum SecurejoinUIPath { } #[derive(Serialize)] -struct SecurejoinUIPathStats { +struct SecurejoinUIPaths { other: u32, qr_icon: u32, new_contact: u32, @@ -155,7 +155,7 @@ See TODO[blog post] for more information." Some("text/plain"), )?; - crate::chat::send_msg(context, chat_id, &mut msg) + chat::send_msg(context, chat_id, &mut msg) .await .context("Failed to send statistics message") .log_err(context) @@ -240,9 +240,9 @@ async fn get_statistics(context: &Context) -> Result { contact_stats: get_contact_stats(context, last_old_contact).await?, message_stats_one_one: get_message_stats(context, last_excluded_msg, true).await?, message_stats_multi_user: get_message_stats(context, last_excluded_msg, false).await?, - securejoin_source_stats: get_securejoin_source_stats(context).await?, - securejoin_uipath_stats: get_securejoin_uipath_stats(context).await?, - securejoin_invites_stats: get_securejoin_invite_stats(context).await?, + securejoin_sources: get_securejoin_source_stats(context).await?, + securejoin_uipaths: get_securejoin_uipath_stats(context).await?, + securejoin_invites: get_securejoin_invite_stats(context).await?, }; Ok(serde_json::to_string_pretty(&statistics)?) @@ -556,7 +556,7 @@ pub(crate) async fn count_securejoin_source( context .sql .execute( - "INSERT INTO stats_securejoin_sources VALUES (?, 1) + "INSERT INTO statistics_securejoin_sources VALUES (?, 1) ON CONFLICT (source) DO UPDATE SET count=count+1;", (source,), ) @@ -569,7 +569,7 @@ pub(crate) async fn count_securejoin_source( context .sql .execute( - "INSERT INTO stats_securejoin_uipaths VALUES (?, 1) + "INSERT INTO statistics_securejoin_uipaths VALUES (?, 1) ON CONFLICT (uipath) DO UPDATE SET count=count+1;", (uipath,), ) @@ -577,11 +577,11 @@ pub(crate) async fn count_securejoin_source( Ok(()) } -async fn get_securejoin_source_stats(context: &Context) -> Result { +async fn get_securejoin_source_stats(context: &Context) -> Result { let map = context .sql .query_map( - "SELECT source, count FROM stats_securejoin_sources", + "SELECT source, count FROM statistics_securejoin_sources", (), |row| { let source: SecurejoinSource = row.get(0)?; @@ -592,7 +592,7 @@ async fn get_securejoin_source_stats(context: &Context) -> Result Result Result { +async fn get_securejoin_uipath_stats(context: &Context) -> Result { let map = context .sql .query_map( - "SELECT uipath, count FROM stats_securejoin_uipaths", + "SELECT uipath, count FROM statistics_securejoin_uipaths", (), |row| { let uipath: SecurejoinUIPath = row.get(0)?; @@ -619,7 +619,7 @@ async fn get_securejoin_uipath_stats(context: &Context) -> Result Result = context .sql .query_map( - "SELECT contact_created, already_verified, type FROM stats_securejoin_invites", + "SELECT contact_created, already_verified, type FROM statistics_securejoin_invites", (), |row| { let contact_created: bool = row.get(0)?; diff --git a/src/statistics/statistics_tests.rs b/src/statistics/statistics_tests.rs index d35af2d7ec..5868f67001 100644 --- a/src/statistics/statistics_tests.rs +++ b/src/statistics/statistics_tests.rs @@ -252,11 +252,11 @@ async fn send_and_read_statistics(context: &TestContext) -> String { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_statistics_securejoin_source_stats() -> Result<()> { - async fn check_statistics(context: &TestContext, expected: &SecurejoinSourceStats) { +async fn test_statistics_securejoin_sources() -> Result<()> { + async fn check_statistics(context: &TestContext, expected: &SecurejoinSources) { let statistics = get_statistics(context).await.unwrap(); let actual: serde_json::Value = serde_json::from_str(&statistics).unwrap(); - let actual = &actual["securejoin_source_stats"]; + let actual = &actual["securejoin_sources"]; let expected = serde_json::to_string_pretty(&expected).unwrap(); let expected: serde_json::Value = serde_json::from_str(&expected).unwrap(); @@ -269,7 +269,7 @@ async fn test_statistics_securejoin_source_stats() -> Result<()> { let bob = &tcm.bob().await; alice.set_config_bool(Config::SendStatistics, true).await?; - let mut expected = SecurejoinSourceStats { + let mut expected = SecurejoinSources { unknown: 0, external_link: 0, internal_link: 0, @@ -335,11 +335,11 @@ async fn test_statistics_securejoin_source_stats() -> Result<()> { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_statistics_securejoin_uipath_stats() -> Result<()> { - async fn check_statistics(context: &TestContext, expected: &SecurejoinUIPathStats) { +async fn test_statistics_securejoin_uipaths() -> Result<()> { + async fn check_statistics(context: &TestContext, expected: &SecurejoinUIPaths) { let stats = get_statistics(context).await.unwrap(); let actual: serde_json::Value = serde_json::from_str(&stats).unwrap(); - let actual = &actual["securejoin_uipath_stats"]; + let actual = &actual["securejoin_uipaths"]; let expected = serde_json::to_string_pretty(&expected).unwrap(); let expected: serde_json::Value = serde_json::from_str(&expected).unwrap(); @@ -352,7 +352,7 @@ async fn test_statistics_securejoin_uipath_stats() -> Result<()> { let bob = &tcm.bob().await; alice.set_config_bool(Config::SendStatistics, true).await?; - let mut expected = SecurejoinUIPathStats { + let mut expected = SecurejoinUIPaths { other: 0, qr_icon: 0, new_contact: 0, @@ -402,7 +402,7 @@ async fn test_statistics_securejoin_invites() -> Result<()> { async fn check_statistics(context: &TestContext, expected: &[JoinedInvite]) { let stats = get_statistics(context).await.unwrap(); let actual: serde_json::Value = serde_json::from_str(&stats).unwrap(); - let actual = &actual["securejoin_invites_stats"]; + let actual = &actual["securejoin_invites"]; let expected = serde_json::to_string_pretty(&expected).unwrap(); let expected: serde_json::Value = serde_json::from_str(&expected).unwrap(); From dd3ef4b2d43aea3e5d114ec90075a619cfd421f4 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Thu, 14 Aug 2025 20:35:59 +0200 Subject: [PATCH 03/21] fix: Don't reset last_old_contact_id if the user disables&reenables the setting --- src/statistics.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/statistics.rs b/src/statistics.rs index afef19ed2f..86e6688cdf 100644 --- a/src/statistics.rs +++ b/src/statistics.rs @@ -185,6 +185,17 @@ pub(crate) async fn set_last_excluded_msg_id(context: &Context) -> Result<()> { } pub(crate) async fn set_last_old_contact_id(context: &Context) -> Result<()> { + let config_exists = context + .sql + .get_raw_config(Config::StatsLastOldContactId.as_ref()) + .await? + .is_some(); + if config_exists { + // The user had statistics-sending enabled already in the past, + // keep the 'last old contact id' as-is + return Ok(()); + } + let last_contact_id: u64 = context .sql .query_get_value("SELECT MAX(id) FROM contacts", ()) From a152d8eb1f4fe900c86cb00ad20f0f96c66efb28 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Fri, 15 Aug 2025 09:22:47 +0200 Subject: [PATCH 04/21] Skip serializing bot, direct_chat, and new if they are false --- src/statistics.rs | 17 +++++++++++++++-- src/statistics/statistics_tests.rs | 5 +---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/statistics.rs b/src/statistics.rs index 86e6688cdf..7671774d1d 100644 --- a/src/statistics.rs +++ b/src/statistics.rs @@ -53,17 +53,30 @@ struct ContactStat { id: ContactId, verified: VerifiedStatus, + + // If one of the boolean properties is false, + // we leave them away. + // This way, the Json file becomes a lot smaller. + #[serde(skip_serializing_if = "is_false")] bot: bool, + + #[serde(skip_serializing_if = "is_false")] direct_chat: bool, + last_seen: u64, #[serde(skip_serializing_if = "Option::is_none")] transitive_chain: Option, /// Whether the contact was established after stats-sending was enabled + #[serde(skip_serializing_if = "is_false")] new: bool, } +fn is_false(b: &bool) -> bool { + !b +} + #[derive(Serialize)] struct MessageStats { to_verified: u32, @@ -379,8 +392,8 @@ async fn get_contact_stats(context: &Context, last_old_contact: u32) -> Result Result<()> { let contact_stats = r2.get("contact_stats").unwrap().as_array().unwrap(); assert_eq!(contact_stats.len(), 1); let contact_info = &contact_stats[0]; - assert_eq!( - contact_info.get("bot").unwrap(), - &serde_json::Value::Bool(false) - ); + assert!(contact_info.get("bot").is_none()); assert_eq!( contact_info.get("direct_chat").unwrap(), &serde_json::Value::Bool(true) From 3ba68a642e3e58ad76fd89096e0238d51bd81a1c Mon Sep 17 00:00:00 2001 From: Hocuri Date: Fri, 15 Aug 2025 09:51:20 +0200 Subject: [PATCH 05/21] Move a struct --- src/statistics.rs | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/src/statistics.rs b/src/statistics.rs index 7671774d1d..496b3044db 100644 --- a/src/statistics.rs +++ b/src/statistics.rs @@ -120,6 +120,23 @@ struct SecurejoinUIPaths { new_contact: u32, } +/// Some information on an invite-joining event +/// (i.e. a qr scan or a clicked link). +#[derive(Serialize)] +struct JoinedInvite { + /// Whether the contact was newly created right now. + /// If this is false, then a contact existed already before. + contact_created: bool, + /// If a contact already existed, + /// this tells us whether the contact was verified already. + already_verified: bool, + /// The type of the invite: + /// "contact" for 1:1 invites that setup a verified contact, + /// "group" for invites that invite to a group + /// and also perform the contact verification 'along the way'. + typ: String, +} + /// Sends a message with statistics about the usage of Delta Chat, /// if the last time such a message was sent /// was more than a week ago. @@ -513,8 +530,7 @@ async fn get_message_stats( let unverified_encrypted = t.query_row( &format!( - // (param GLOB '*\nc=1*' OR param GLOB 'c=1*')` - // matches all messages that are end-to-end encrypted + // (param GLOB '*\nc=1*' OR param GLOB 'c=1*') matches all messages that are end-to-end encrypted "SELECT COUNT(*) FROM msgs WHERE chat_id NOT IN temp.verified_chats AND chat_id NOT IN temp.empty_chats AND (param GLOB '*\nc=1*' OR param GLOB 'c=1*') @@ -686,23 +702,6 @@ pub(crate) async fn count_securejoin_invite(context: &Context, invite: &QrInvite Ok(()) } -/// Some information on an invite-joining event -/// (i.e. a qr scan or a clicked link). -#[derive(Serialize)] -struct JoinedInvite { - /// Whether the contact was newly created right now. - /// If this is false, then a contact existed already before. - contact_created: bool, - /// If a contact already existed, - /// this tells us whether the contact was verified already. - already_verified: bool, - /// The type of the invite: - /// "contact" for 1:1 invites that setup a verified contact, - /// "group" for invites that invite to a group - /// and also perform the contact verification 'along the way'. - typ: String, -} - async fn get_securejoin_invite_stats(context: &Context) -> Result> { let qr_scans: Vec = context .sql From 4f973be6e673a080b1ec624d9e712841a5741971 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Fri, 15 Aug 2025 13:23:59 +0200 Subject: [PATCH 06/21] fix: Looks like yerpc doesn't allow me to have optional parameters --- deltachat-jsonrpc/src/api.rs | 18 +++++++++++------- src/securejoin.rs | 6 +++--- src/statistics.rs | 2 +- src/statistics/statistics_tests.rs | 26 +++++++++++++++----------- 4 files changed, 30 insertions(+), 22 deletions(-) diff --git a/deltachat-jsonrpc/src/api.rs b/deltachat-jsonrpc/src/api.rs index 01322f51e7..22b4f33634 100644 --- a/deltachat-jsonrpc/src/api.rs +++ b/deltachat-jsonrpc/src/api.rs @@ -871,15 +871,19 @@ impl CommandApi { /// /// **qr**: The text of the scanned QR code. Typically, the same string as given /// to `check_qr()`. - /// - /// **source** and **uipath** are for statistics-sending, - /// if the user enabled it in the settings; - /// if you don't have statistics-sending implemented, just pass `None` here. - /// /// **returns**: The chat ID of the joined chat, the UI may redirect to the this chat. /// A returned chat ID does not guarantee that the chat is protected or the belonging contact is verified. /// - async fn secure_join( + async fn secure_join(&self, account_id: u32, qr: String) -> Result { + let ctx = self.get_context(account_id).await?; + let chat_id = securejoin::join_securejoin(&ctx, &qr).await?; + Ok(chat_id.to_u32()) + } + + /// Like `secure_join()`, but allows to pass a source and a UI-path. + /// You only need this if your UI has an option to send statistics + /// to Delta Chat's developers. + async fn secure_join_with_ux_info( &self, account_id: u32, qr: String, @@ -887,7 +891,7 @@ impl CommandApi { uipath: Option, ) -> Result { let ctx = self.get_context(account_id).await?; - let chat_id = securejoin::join_securejoin_with_source(&ctx, &qr, source, uipath).await?; + let chat_id = securejoin::join_securejoin_with_ux_info(&ctx, &qr, source, uipath).await?; Ok(chat_id.to_u32()) } diff --git a/src/securejoin.rs b/src/securejoin.rs index 85aacf0ec0..11622c664b 100644 --- a/src/securejoin.rs +++ b/src/securejoin.rs @@ -147,7 +147,7 @@ async fn get_self_fingerprint(context: &Context) -> Result { /// /// The function returns immediately and the handshake will run in background. pub async fn join_securejoin(context: &Context, qr: &str) -> Result { - join_securejoin_with_source(context, qr, None, None).await + join_securejoin_with_ux_info(context, qr, None, None).await } /// Take a scanned QR-code and do the setup-contact/join-group/invite handshake. @@ -160,7 +160,7 @@ pub async fn join_securejoin(context: &Context, qr: &str) -> Result { /// **source** and **uipath** are for statistics-sending, /// if the user enabled it in the settings; /// if you don't have statistics-sending implemented, just pass `None` here. -pub async fn join_securejoin_with_source( +pub async fn join_securejoin_with_ux_info( context: &Context, qr: &str, source: Option, @@ -173,7 +173,7 @@ pub async fn join_securejoin_with_source( err })?; - statistics::count_securejoin_source(context, source, uipath) + statistics::count_securejoin_ux_info(context, source, uipath) .await .log_err(context) .ok(); diff --git a/src/statistics.rs b/src/statistics.rs index 496b3044db..caba3666c2 100644 --- a/src/statistics.rs +++ b/src/statistics.rs @@ -579,7 +579,7 @@ async fn get_message_stats( Ok(message_stats) } -pub(crate) async fn count_securejoin_source( +pub(crate) async fn count_securejoin_ux_info( context: &Context, source: Option, uipath: Option, diff --git a/src/statistics/statistics_tests.rs b/src/statistics/statistics_tests.rs index b6e4999eef..3cc36444b1 100644 --- a/src/statistics/statistics_tests.rs +++ b/src/statistics/statistics_tests.rs @@ -4,7 +4,7 @@ use super::*; use crate::chat::{Chat, create_broadcast, create_group_chat, create_group_ex}; use crate::mimeparser::SystemMessage; use crate::qr::check_qr; -use crate::securejoin::{get_securejoin_qr, join_securejoin, join_securejoin_with_source}; +use crate::securejoin::{get_securejoin_qr, join_securejoin, join_securejoin_with_ux_info}; use crate::test_utils::{TestContext, TestContextManager, get_chat_msg}; use crate::tools::SystemTime; use pretty_assertions::assert_eq; @@ -287,11 +287,12 @@ async fn test_statistics_securejoin_sources() -> Result<()> { expected.unknown += 1; check_statistics(alice, &expected).await; - join_securejoin_with_source(alice, &qr, Some(SecurejoinSource::Clipboard as u32), None).await?; + join_securejoin_with_ux_info(alice, &qr, Some(SecurejoinSource::Clipboard as u32), None) + .await?; expected.clipboard += 1; check_statistics(alice, &expected).await; - join_securejoin_with_source( + join_securejoin_with_ux_info( alice, &qr, Some(SecurejoinSource::ExternalLink as u32), @@ -301,7 +302,7 @@ async fn test_statistics_securejoin_sources() -> Result<()> { expected.external_link += 1; check_statistics(alice, &expected).await; - join_securejoin_with_source( + join_securejoin_with_ux_info( alice, &qr, Some(SecurejoinSource::InternalLink as u32), @@ -311,20 +312,22 @@ async fn test_statistics_securejoin_sources() -> Result<()> { expected.internal_link += 1; check_statistics(alice, &expected).await; - join_securejoin_with_source(alice, &qr, Some(SecurejoinSource::ImageLoaded as u32), None) + join_securejoin_with_ux_info(alice, &qr, Some(SecurejoinSource::ImageLoaded as u32), None) .await?; expected.image_loaded += 1; check_statistics(alice, &expected).await; - join_securejoin_with_source(alice, &qr, Some(SecurejoinSource::Scan as u32), None).await?; + join_securejoin_with_ux_info(alice, &qr, Some(SecurejoinSource::Scan as u32), None).await?; expected.scan += 1; check_statistics(alice, &expected).await; - join_securejoin_with_source(alice, &qr, Some(SecurejoinSource::Clipboard as u32), None).await?; + join_securejoin_with_ux_info(alice, &qr, Some(SecurejoinSource::Clipboard as u32), None) + .await?; expected.clipboard += 1; check_statistics(alice, &expected).await; - join_securejoin_with_source(alice, &qr, Some(SecurejoinSource::Clipboard as u32), None).await?; + join_securejoin_with_ux_info(alice, &qr, Some(SecurejoinSource::Clipboard as u32), None) + .await?; expected.clipboard += 1; check_statistics(alice, &expected).await; @@ -367,7 +370,7 @@ async fn test_statistics_securejoin_uipaths() -> Result<()> { expected.other += 1; check_statistics(alice, &expected).await; - join_securejoin_with_source( + join_securejoin_with_ux_info( alice, &qr, Some(0), @@ -377,7 +380,7 @@ async fn test_statistics_securejoin_uipaths() -> Result<()> { expected.new_contact += 1; check_statistics(alice, &expected).await; - join_securejoin_with_source( + join_securejoin_with_ux_info( alice, &qr, Some(0), @@ -387,7 +390,8 @@ async fn test_statistics_securejoin_uipaths() -> Result<()> { expected.new_contact += 1; check_statistics(alice, &expected).await; - join_securejoin_with_source(alice, &qr, Some(0), Some(SecurejoinUIPath::QrIcon as u32)).await?; + join_securejoin_with_ux_info(alice, &qr, Some(0), Some(SecurejoinUIPath::QrIcon as u32)) + .await?; expected.qr_icon += 1; check_statistics(alice, &expected).await; From 23b8fda6a8463bd90be91ebe5a1c6b9474d6b1cd Mon Sep 17 00:00:00 2001 From: Hocuri Date: Fri, 15 Aug 2025 13:42:43 +0200 Subject: [PATCH 07/21] fix: Don't send statistics from a platform other than Android --- src/context.rs | 6 ++---- src/statistics.rs | 21 ++++++++++++++++++--- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/context.rs b/src/context.rs index 29f7aa82ab..4753e5716f 100644 --- a/src/context.rs +++ b/src/context.rs @@ -14,7 +14,6 @@ use ratelimit::Ratelimit; use tokio::sync::{Mutex, Notify, RwLock}; use crate::chat::{ChatId, get_chat_cnt}; -use crate::chatlist_events; use crate::config::Config; use crate::constants::{self, DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT, DC_VERSION_STR}; use crate::contact::{Contact, ContactId}; @@ -34,6 +33,7 @@ use crate::sql::Sql; use crate::stock_str::StockStrings; use crate::timesmearing::SmearedTimestamp; use crate::tools::{self, duration_to_str, time, time_elapsed}; +use crate::{chatlist_events, statistics}; /// Builder for the [`Context`]. /// @@ -1084,9 +1084,7 @@ impl Context { ); res.insert( "send_statistics", - self.get_config_bool(Config::SendStatistics) - .await? - .to_string(), + statistics::should_send_statistics(self).await?.to_string(), ); res.insert( "last_statistics_sent", diff --git a/src/statistics.rs b/src/statistics.rs index caba3666c2..5c24bbcaf5 100644 --- a/src/statistics.rs +++ b/src/statistics.rs @@ -144,7 +144,7 @@ struct JoinedInvite { /// On the other end, a bot will receive the message and make it available /// to Delta Chat's developers. pub async fn maybe_send_statistics(context: &Context) -> Result> { - if context.get_config_bool(Config::SendStatistics).await? { + if should_send_statistics(context).await? { let last_sending_time = context.get_config_i64(Config::LastStatisticsSent).await?; let next_sending_time = last_sending_time.saturating_add(30); // TODO increase to 1 day or 1 week if next_sending_time <= time() { @@ -154,6 +154,21 @@ pub async fn maybe_send_statistics(context: &Context) -> Result> Ok(None) } +pub(crate) async fn should_send_statistics(_context: &Context) -> Result { + #[cfg(any(target_os = "android", test))] + { + _context.get_config_bool(Config::SendStatistics).await + } + + // If the user enables statistics-sending on Android, + // and then transfers the account to e.g. Desktop, + // we should not send any statistics: + #[cfg(not(any(target_os = "android", test)))] + { + Ok(false) + } +} + async fn send_statistics(context: &Context) -> Result { info!(context, "Sending statistics."); @@ -584,7 +599,7 @@ pub(crate) async fn count_securejoin_ux_info( source: Option, uipath: Option, ) -> Result<()> { - if !context.get_config_bool(Config::SendStatistics).await? { + if !should_send_statistics(context).await? { return Ok(()); } @@ -669,7 +684,7 @@ async fn get_securejoin_uipath_stats(context: &Context) -> Result Result<()> { - if !context.get_config_bool(Config::SendStatistics).await? { + if !should_send_statistics(context).await? { return Ok(()); } From e283df8346029ac9a1a814ddad83eae9d4a9ca37 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Fri, 15 Aug 2025 13:57:05 +0200 Subject: [PATCH 08/21] small comment fix --- deltachat-jsonrpc/src/api.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/deltachat-jsonrpc/src/api.rs b/deltachat-jsonrpc/src/api.rs index 22b4f33634..9d878e9086 100644 --- a/deltachat-jsonrpc/src/api.rs +++ b/deltachat-jsonrpc/src/api.rs @@ -871,6 +871,7 @@ impl CommandApi { /// /// **qr**: The text of the scanned QR code. Typically, the same string as given /// to `check_qr()`. + /// /// **returns**: The chat ID of the joined chat, the UI may redirect to the this chat. /// A returned chat ID does not guarantee that the chat is protected or the belonging contact is verified. /// From 8e3f2e5fd11fa763f1e496add05940b03318d586 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Fri, 15 Aug 2025 14:11:04 +0200 Subject: [PATCH 09/21] cargo fmt --- deltachat-jsonrpc/src/api/types/chat.rs | 4 ++-- deltachat-jsonrpc/src/api/types/chat_list.rs | 2 +- deltachat-jsonrpc/src/api/types/http.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deltachat-jsonrpc/src/api/types/chat.rs b/deltachat-jsonrpc/src/api/types/chat.rs index 26e7b4293d..96388c27be 100644 --- a/deltachat-jsonrpc/src/api/types/chat.rs +++ b/deltachat-jsonrpc/src/api/types/chat.rs @@ -1,7 +1,7 @@ use std::time::{Duration, SystemTime}; -use anyhow::{Context as _, Result, bail}; -use deltachat::chat::{self, ChatVisibility, get_chat_contacts, get_past_chat_contacts}; +use anyhow::{bail, Context as _, Result}; +use deltachat::chat::{self, get_chat_contacts, get_past_chat_contacts, ChatVisibility}; use deltachat::chat::{Chat, ChatId}; use deltachat::constants::Chattype; use deltachat::contact::{Contact, ContactId}; diff --git a/deltachat-jsonrpc/src/api/types/chat_list.rs b/deltachat-jsonrpc/src/api/types/chat_list.rs index 1a9e851086..b5d31a7913 100644 --- a/deltachat-jsonrpc/src/api/types/chat_list.rs +++ b/deltachat-jsonrpc/src/api/types/chat_list.rs @@ -4,7 +4,7 @@ use deltachat::chatlist::get_last_message_for_chat; use deltachat::constants::*; use deltachat::contact::{Contact, ContactId}; use deltachat::{ - chat::{ChatVisibility, get_chat_contacts}, + chat::{get_chat_contacts, ChatVisibility}, chatlist::Chatlist, }; use num_traits::cast::ToPrimitive; diff --git a/deltachat-jsonrpc/src/api/types/http.rs b/deltachat-jsonrpc/src/api/types/http.rs index d370ba8f7c..9121a677ec 100644 --- a/deltachat-jsonrpc/src/api/types/http.rs +++ b/deltachat-jsonrpc/src/api/types/http.rs @@ -16,7 +16,7 @@ pub struct HttpResponse { impl From for HttpResponse { fn from(response: CoreHttpResponse) -> Self { - use base64::{Engine as _, engine::general_purpose}; + use base64::{engine::general_purpose, Engine as _}; let blob = general_purpose::STANDARD_NO_PAD.encode(response.blob); let mimetype = response.mimetype; let encoding = response.encoding; From 7986b1163fd60ceaaf11cee3512ee4921222ac05 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Fri, 15 Aug 2025 14:18:41 +0200 Subject: [PATCH 10/21] clippy --- src/statistics.rs | 7 ++++--- src/statistics/statistics_tests.rs | 20 ++++++++++---------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/statistics.rs b/src/statistics.rs index 5c24bbcaf5..fbc22558d0 100644 --- a/src/statistics.rs +++ b/src/statistics.rs @@ -154,10 +154,11 @@ pub async fn maybe_send_statistics(context: &Context) -> Result> Ok(None) } -pub(crate) async fn should_send_statistics(_context: &Context) -> Result { +#[allow(clippy::unused_async, unused)] +pub(crate) async fn should_send_statistics(context: &Context) -> Result { #[cfg(any(target_os = "android", test))] { - _context.get_config_bool(Config::SendStatistics).await + context.get_config_bool(Config::SendStatistics).await } // If the user enables statistics-sending on Android, @@ -438,7 +439,7 @@ async fn get_contact_stats(context: &Context, last_old_contact: u32) -> Result Result<()> { alice.set_config_bool(Config::SendStatistics, true).await?; - let chat_id = maybe_send_statistics(&alice).await?.unwrap(); - let msg = get_chat_msg(&alice, chat_id, 0, 2).await; + let chat_id = maybe_send_statistics(alice).await?.unwrap(); + let msg = get_chat_msg(alice, chat_id, 0, 2).await; assert_eq!(msg.get_info_type(), SystemMessage::ChatProtectionEnabled); - let chat = Chat::load_from_db(&alice, chat_id).await?; + let chat = Chat::load_from_db(alice, chat_id).await?; assert!(chat.is_protected()); - let msg = get_chat_msg(&alice, chat_id, 1, 2).await; + let msg = get_chat_msg(alice, chat_id, 1, 2).await; assert_eq!(msg.get_filename().unwrap(), "statistics.txt"); - let stats = tokio::fs::read(msg.get_file(&alice).unwrap()).await?; + let stats = tokio::fs::read(msg.get_file(alice).unwrap()).await?; let stats = std::str::from_utf8(&stats)?; - println!("\nEmpty account:\n{}\n", stats); + println!("\nEmpty account:\n{stats}\n"); assert!(stats.contains(r#""contact_stats": []"#)); - let r: serde_json::Value = serde_json::from_str(&stats)?; + let r: serde_json::Value = serde_json::from_str(stats)?; assert_eq!( r.get("contact_stats").unwrap(), &serde_json::Value::Array(vec![]) @@ -96,7 +96,7 @@ async fn test_message_stats() -> Result<()> { expected_one_one: &MessageStats, expected_multi_user: &MessageStats, ) { - let actual: serde_json::Value = serde_json::from_str(&stats).unwrap(); + let actual: serde_json::Value = serde_json::from_str(stats).unwrap(); for (expected, key) in [ (expected_one_one, "message_stats_one_one"), @@ -238,11 +238,11 @@ async fn test_message_stats() -> Result<()> { } async fn send_and_read_statistics(context: &TestContext) -> String { - let chat_id = maybe_send_statistics(&context).await.unwrap().unwrap(); + let chat_id = maybe_send_statistics(context).await.unwrap().unwrap(); let msg = context.get_last_msg_in(chat_id).await; assert_eq!(msg.get_filename().unwrap(), "statistics.txt"); - let stats = tokio::fs::read(msg.get_file(&context).unwrap()) + let stats = tokio::fs::read(msg.get_file(context).unwrap()) .await .unwrap(); String::from_utf8(stats).unwrap() From 228eda85a92d35b57ca793cc98c439c5be0a0d9e Mon Sep 17 00:00:00 2001 From: Hocuri Date: Fri, 15 Aug 2025 14:24:52 +0200 Subject: [PATCH 11/21] Increase sending interval to 1 week --- src/statistics.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/statistics.rs b/src/statistics.rs index fbc22558d0..51b781be9a 100644 --- a/src/statistics.rs +++ b/src/statistics.rs @@ -23,6 +23,7 @@ use crate::tools::{create_id, time}; pub(crate) const STATISTICS_BOT_EMAIL: &str = "self_reporting@testrun.org"; const STATISTICS_BOT_VCARD: &str = include_str!("../assets/statistics-bot.vcf"); +const SENDING_INTERVAL_SECONDS: i64 = 3600 * 24 * 7; // 1 week #[derive(Serialize)] struct Statistics { @@ -146,7 +147,7 @@ struct JoinedInvite { pub async fn maybe_send_statistics(context: &Context) -> Result> { if should_send_statistics(context).await? { let last_sending_time = context.get_config_i64(Config::LastStatisticsSent).await?; - let next_sending_time = last_sending_time.saturating_add(30); // TODO increase to 1 day or 1 week + let next_sending_time = last_sending_time.saturating_add(SENDING_INTERVAL_SECONDS); if next_sending_time <= time() { return Ok(Some(send_statistics(context).await?)); } From e4783419933d7dcb12db85d7ac89a1b54fe5b10d Mon Sep 17 00:00:00 2001 From: Hocuri Date: Sat, 23 Aug 2025 22:09:25 +0200 Subject: [PATCH 12/21] Document SecurejoinSource --- deltachat-jsonrpc/src/api.rs | 33 +++++++++++++++++++++++++++++++++ src/statistics.rs | 10 ++++++++++ 2 files changed, 43 insertions(+) diff --git a/deltachat-jsonrpc/src/api.rs b/deltachat-jsonrpc/src/api.rs index 9d878e9086..803132969b 100644 --- a/deltachat-jsonrpc/src/api.rs +++ b/deltachat-jsonrpc/src/api.rs @@ -884,6 +884,39 @@ impl CommandApi { /// Like `secure_join()`, but allows to pass a source and a UI-path. /// You only need this if your UI has an option to send statistics /// to Delta Chat's developers. + /// + /// **source**: The source where the QR code came from. One of: + /// ```rust + /// enum SecurejoinSource { + /// /// Because of some problem, it is unknown where the QR code came from. + /// Unknown = 0, + /// /// The user opened a link somewhere outside Delta Chat + /// ExternalLink = 1, + /// /// The user clicked on a link in a message inside Delta Chat + /// InternalLink = 2, + /// /// The user clicked "Paste from Clipboard" in the QR scan activity + /// Clipboard = 3, + /// /// The user clicked "Load QR code as image" in the QR scan activity + /// ImageLoaded = 4, + /// /// The user scanned a QR code + /// Scan = 5, + /// } + /// ``` + /// + /// **uipath**: Which UI path did the user use to arrive at the QR code screen. + /// If the SecurejoinSource was ExternalLink or InternalLink, + /// you can just pass 0 here, because the QR code screen wasn't even opened. + /// ```rust + /// enum SecurejoinUIPath { + /// /// The UI path is unknown, or the user didn't open the QR code screen at all. + /// Unknown = 0, + /// /// The user directly clicked on the QR icon in the main screen + /// QrIcon = 1, + /// /// The user first clicked on the `+` button in the main screen, + /// /// and then on "New Contact" + /// NewContact = 2, + /// } + /// ``` async fn secure_join_with_ux_info( &self, account_id: u32, diff --git a/src/statistics.rs b/src/statistics.rs index 51b781be9a..48880ec449 100644 --- a/src/statistics.rs +++ b/src/statistics.rs @@ -89,11 +89,17 @@ struct MessageStats { #[repr(u32)] #[derive(Debug, Clone, Copy, FromPrimitive, FromSql, PartialEq, Eq, PartialOrd, Ord)] enum SecurejoinSource { + /// Because of some problem, it is unknown where the QR code came from. Unknown = 0, + /// The user opened a link somewhere outside Delta Chat ExternalLink = 1, + /// The user clicked on a link in a message inside Delta Chat InternalLink = 2, + /// The user clicked "Paste from Clipboard" in the QR scan activity Clipboard = 3, + /// The user clicked "Load QR code as image" in the QR scan activity ImageLoaded = 4, + /// The user scanned a QR code Scan = 5, } @@ -109,8 +115,12 @@ struct SecurejoinSources { #[derive(Debug, Clone, Copy, FromPrimitive, FromSql, PartialEq, Eq, PartialOrd, Ord)] enum SecurejoinUIPath { + /// The UI path is unknown, or the user didn't open the QR code screen at all. Unknown = 0, + /// The user directly clicked on the QR icon in the main screen QrIcon = 1, + /// The user first clicked on the `+` button in the main screen, + /// and then on "New Contact" NewContact = 2, } From 5fac395826dd63756e75a67b86de6d3fdc42ee91 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Sat, 23 Aug 2025 22:25:36 +0200 Subject: [PATCH 13/21] refactor: Prefix all relevant configs with `stats` --- src/config.rs | 12 ++++++------ src/context.rs | 12 +++++------- src/context/context_tests.rs | 2 +- src/statistics.rs | 30 +++++++++++++++--------------- src/statistics/statistics_tests.rs | 16 ++++++++-------- 5 files changed, 35 insertions(+), 37 deletions(-) diff --git a/src/config.rs b/src/config.rs index a2056975aa..cdf193d8a9 100644 --- a/src/config.rs +++ b/src/config.rs @@ -433,20 +433,20 @@ pub enum Config { /// Send statistics to Delta Chat's developers. /// Can be exposed to the user as a setting. - SendStatistics, + StatsSending, /// Last time statistics were sent to Delta Chat's developers - LastStatisticsSent, + StatsLastSent, /// This key is sent to the statistics bot so that the bot can recognize the user /// without storing the email address - StatisticsId, + StatsId, /// The last message id that was already included in the previously sent statistics, /// or that already existed before the user opted in. /// Only messages with an id larger than this /// will be counted in the next statistics. - StatsLastExcludedMsgId, + StatsLastCountedMsgId, /// The last contact id that already existed when statistics-sending was enabled. /// All newer contacts get the `"new": true` attribute. @@ -844,9 +844,9 @@ impl Context { .await?; } } - Config::SendStatistics => { + Config::StatsSending => { self.sql.set_raw_config(key.as_ref(), value).await?; - statistics::set_last_excluded_msg_id(self).await?; + statistics::set_last_counted_msg_id(self).await?; statistics::set_last_old_contact_id(self).await?; } _ => { diff --git a/src/context.rs b/src/context.rs index 4753e5716f..8b634355a6 100644 --- a/src/context.rs +++ b/src/context.rs @@ -1077,18 +1077,16 @@ impl Context { .unwrap_or_default(), ); res.insert( - "statistics_id", - self.get_config_bool(Config::StatisticsId) - .await? - .to_string(), + "stats_id", + self.get_config_bool(Config::StatsId).await?.to_string(), ); res.insert( - "send_statistics", + "stats_sending", statistics::should_send_statistics(self).await?.to_string(), ); res.insert( - "last_statistics_sent", - self.get_config_i64(Config::LastStatisticsSent) + "stats_last_sent", + self.get_config_i64(Config::StatsLastSent) .await? .to_string(), ); diff --git a/src/context/context_tests.rs b/src/context/context_tests.rs index 2613e29d85..af103f8e3a 100644 --- a/src/context/context_tests.rs +++ b/src/context/context_tests.rs @@ -295,7 +295,7 @@ async fn test_get_info_completeness() { "webxdc_integration", "device_token", "encrypted_device_token", - "stats_last_excluded_msg_id", + "stats_last_counted_msg_id", "stats_last_old_contact_id", ]; let t = TestContext::new().await; diff --git a/src/statistics.rs b/src/statistics.rs index 48880ec449..c0166dfed5 100644 --- a/src/statistics.rs +++ b/src/statistics.rs @@ -156,7 +156,7 @@ struct JoinedInvite { /// to Delta Chat's developers. pub async fn maybe_send_statistics(context: &Context) -> Result> { if should_send_statistics(context).await? { - let last_sending_time = context.get_config_i64(Config::LastStatisticsSent).await?; + let last_sending_time = context.get_config_i64(Config::StatsLastSent).await?; let next_sending_time = last_sending_time.saturating_add(SENDING_INTERVAL_SECONDS); if next_sending_time <= time() { return Ok(Some(send_statistics(context).await?)); @@ -169,7 +169,7 @@ pub async fn maybe_send_statistics(context: &Context) -> Result> pub(crate) async fn should_send_statistics(context: &Context) -> Result { #[cfg(any(target_os = "android", test))] { - context.get_config_bool(Config::SendStatistics).await + context.get_config_bool(Config::StatsSending).await } // If the user enables statistics-sending on Android, @@ -187,7 +187,7 @@ async fn send_statistics(context: &Context) -> Result { // Setting this config at the beginning avoids endless loops when things do not // work out for whatever reason. context - .set_config_internal(Config::LastStatisticsSent, Some(&time().to_string())) + .set_config_internal(Config::StatsLastSent, Some(&time().to_string())) .await .log_err(context) .ok(); @@ -218,12 +218,12 @@ See TODO[blog post] for more information." .log_err(context) .ok(); - set_last_excluded_msg_id(context).await?; + set_last_counted_msg_id(context).await?; Ok(chat_id) } -pub(crate) async fn set_last_excluded_msg_id(context: &Context) -> Result<()> { +pub(crate) async fn set_last_counted_msg_id(context: &Context) -> Result<()> { let last_msgid: u64 = context .sql .query_get_value("SELECT MAX(id) FROM msgs", ()) @@ -233,7 +233,7 @@ pub(crate) async fn set_last_excluded_msg_id(context: &Context) -> Result<()> { context .sql .set_raw_config( - Config::StatsLastExcludedMsgId.as_ref(), + Config::StatsLastCountedMsgId.as_ref(), Some(&last_msgid.to_string()), ) .await?; @@ -273,8 +273,8 @@ pub(crate) async fn set_last_old_contact_id(context: &Context) -> Result<()> { async fn get_statistics(context: &Context) -> Result { // The ID of the last msg that was already counted in the previously sent statistics. // Only newer messages will be counted in the current statistics. - let last_excluded_msg = context - .get_config_u32(Config::StatsLastExcludedMsgId) + let last_counted_msg = context + .get_config_u32(Config::StatsLastCountedMsgId) .await?; // The Id of the last contact that already existed when the user enabled the setting. @@ -289,12 +289,12 @@ async fn get_statistics(context: &Context) -> Result { .map(|k| k.created_at().timestamp()) .collect(); - let statistics_id = match context.get_config(Config::StatisticsId).await? { + let statistics_id = match context.get_config(Config::StatsId).await? { Some(id) => id, None => { let id = create_id(); context - .set_config_internal(Config::StatisticsId, Some(&id)) + .set_config_internal(Config::StatsId, Some(&id)) .await?; id } @@ -306,8 +306,8 @@ async fn get_statistics(context: &Context) -> Result { statistics_id, is_chatmail: context.is_chatmail().await?, contact_stats: get_contact_stats(context, last_old_contact).await?, - message_stats_one_one: get_message_stats(context, last_excluded_msg, true).await?, - message_stats_multi_user: get_message_stats(context, last_excluded_msg, false).await?, + message_stats_one_one: get_message_stats(context, last_counted_msg, true).await?, + message_stats_multi_user: get_message_stats(context, last_counted_msg, false).await?, securejoin_sources: get_securejoin_source_stats(context).await?, securejoin_uipaths: get_securejoin_uipath_stats(context).await?, securejoin_invites: get_securejoin_invite_stats(context).await?, @@ -453,11 +453,11 @@ async fn get_contact_stats(context: &Context, last_old_contact: u32) -> Result Result { ensure!( - last_excluded_msg >= 9, + last_counted_msg >= 9, "Last_msgid < 9 would mean including 'special' messages in the statistics" ); @@ -543,7 +543,7 @@ async fn get_message_stats( } else { general_requirements += " AND chat_id NOT IN temp.one_one_chats"; } - let params = (ContactId::SELF, statistics_bot_chat_id, last_excluded_msg); + let params = (ContactId::SELF, statistics_bot_chat_id, last_counted_msg); let to_verified = t.query_row( &format!( diff --git a/src/statistics/statistics_tests.rs b/src/statistics/statistics_tests.rs index c812087afa..1fb4dfa70d 100644 --- a/src/statistics/statistics_tests.rs +++ b/src/statistics/statistics_tests.rs @@ -14,7 +14,7 @@ use serde_json::{Number, Value}; async fn test_maybe_send_statistics() -> Result<()> { let alice = &TestContext::new_alice().await; - alice.set_config_bool(Config::SendStatistics, true).await?; + alice.set_config_bool(Config::StatsSending, true).await?; let chat_id = maybe_send_statistics(alice).await?.unwrap(); let msg = get_chat_msg(alice, chat_id, 0, 2).await; @@ -48,7 +48,7 @@ async fn test_statistics_one_contact() -> Result<()> { let mut tcm = TestContextManager::new(); let alice = &tcm.alice().await; let bob = &tcm.bob().await; - alice.set_config_bool(Config::SendStatistics, true).await?; + alice.set_config_bool(Config::StatsSending, true).await?; let stats = get_statistics(alice).await?; let r: serde_json::Value = serde_json::from_str(&stats)?; @@ -114,7 +114,7 @@ async fn test_message_stats() -> Result<()> { let mut tcm = TestContextManager::new(); let alice = &tcm.alice().await; let bob = &tcm.bob().await; - alice.set_config_bool(Config::SendStatistics, true).await?; + alice.set_config_bool(Config::StatsSending, true).await?; let email_chat = alice.create_email_chat(bob).await; let encrypted_chat = alice.create_chat(bob).await; @@ -264,7 +264,7 @@ async fn test_statistics_securejoin_sources() -> Result<()> { let mut tcm = TestContextManager::new(); let alice = &tcm.alice().await; let bob = &tcm.bob().await; - alice.set_config_bool(Config::SendStatistics, true).await?; + alice.set_config_bool(Config::StatsSending, true).await?; let mut expected = SecurejoinSources { unknown: 0, @@ -350,7 +350,7 @@ async fn test_statistics_securejoin_uipaths() -> Result<()> { let mut tcm = TestContextManager::new(); let alice = &tcm.alice().await; let bob = &tcm.bob().await; - alice.set_config_bool(Config::SendStatistics, true).await?; + alice.set_config_bool(Config::StatsSending, true).await?; let mut expected = SecurejoinUIPaths { other: 0, @@ -415,7 +415,7 @@ async fn test_statistics_securejoin_invites() -> Result<()> { let alice = &tcm.alice().await; let bob = &tcm.bob().await; let charlie = &tcm.charlie().await; - alice.set_config_bool(Config::SendStatistics, true).await?; + alice.set_config_bool(Config::StatsSending, true).await?; let mut expected = vec![]; @@ -475,7 +475,7 @@ async fn test_statistics_securejoin_invites() -> Result<()> { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_statistics_is_chatmail() -> Result<()> { let alice = &TestContext::new_alice().await; - alice.set_config_bool(Config::SendStatistics, true).await?; + alice.set_config_bool(Config::StatsSending, true).await?; let r = get_statistics(alice).await?; let r: serde_json::Value = serde_json::from_str(&r)?; @@ -496,7 +496,7 @@ async fn test_statistics_key_creation_timestamp() -> Result<()> { const ALICE_KEY_CREATION_TIME: u128 = 1582855645; let alice = &TestContext::new_alice().await; - alice.set_config_bool(Config::SendStatistics, true).await?; + alice.set_config_bool(Config::StatsSending, true).await?; let r = get_statistics(alice).await?; let r: serde_json::Value = serde_json::from_str(&r)?; From a3a02e9fbb44a988db58cdf5835445a04dd1b699 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Sat, 23 Aug 2025 22:28:16 +0200 Subject: [PATCH 14/21] some more of iequidoo's suggestions --- src/config.rs | 1 - src/statistics.rs | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/config.rs b/src/config.rs index cdf193d8a9..8d62e29077 100644 --- a/src/config.rs +++ b/src/config.rs @@ -449,7 +449,6 @@ pub enum Config { StatsLastCountedMsgId, /// The last contact id that already existed when statistics-sending was enabled. - /// All newer contacts get the `"new": true` attribute. StatsLastOldContactId, /// MsgId of webxdc map integration. diff --git a/src/statistics.rs b/src/statistics.rs index c0166dfed5..ab5c3da651 100644 --- a/src/statistics.rs +++ b/src/statistics.rs @@ -28,7 +28,7 @@ const SENDING_INTERVAL_SECONDS: i64 = 3600 * 24 * 7; // 1 week #[derive(Serialize)] struct Statistics { core_version: String, - key_created: Vec, + key_create_timestamps: Vec, statistics_id: String, is_chatmail: bool, contact_stats: Vec, @@ -283,7 +283,7 @@ async fn get_statistics(context: &Context) -> Result { .get_config_u32(Config::StatsLastOldContactId) .await?; - let key_created: Vec = load_self_public_keyring(context) + let key_create_timestamps: Vec = load_self_public_keyring(context) .await? .iter() .map(|k| k.created_at().timestamp()) @@ -302,7 +302,7 @@ async fn get_statistics(context: &Context) -> Result { let statistics = Statistics { core_version: get_version_str().to_string(), - key_created, + key_create_timestamps, statistics_id, is_chatmail: context.is_chatmail().await?, contact_stats: get_contact_stats(context, last_old_contact).await?, From ebf0a2db8e6dc7d45949884c6180ac78cca4584e Mon Sep 17 00:00:00 2001 From: Hocuri Date: Mon, 25 Aug 2025 15:56:01 +0200 Subject: [PATCH 15/21] Update src/statistics.rs Co-authored-by: iequidoo <117991069+iequidoo@users.noreply.github.com> --- src/statistics.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/statistics.rs b/src/statistics.rs index ab5c3da651..38503974d2 100644 --- a/src/statistics.rs +++ b/src/statistics.rs @@ -198,7 +198,7 @@ async fn send_statistics(context: &Context) -> Result { msg.set_text( "The attachment contains anonymous usage statistics, \ because you enabled this in the settings. \ -This helps us improve the security of Delta Chat. \ +This helps us improve Delta Chat. \ See TODO[blog post] for more information." .to_string(), ); From 7d6aebff766137a90591bf847a865141b63392fd Mon Sep 17 00:00:00 2001 From: Hocuri Date: Sat, 23 Aug 2025 22:28:16 +0200 Subject: [PATCH 16/21] some more of iequidoo's suggestions --- src/statistics/statistics_tests.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/statistics/statistics_tests.rs b/src/statistics/statistics_tests.rs index 1fb4dfa70d..16de79d94d 100644 --- a/src/statistics/statistics_tests.rs +++ b/src/statistics/statistics_tests.rs @@ -60,8 +60,8 @@ async fn test_statistics_one_contact() -> Result<()> { let r2: serde_json::Value = serde_json::from_str(&stats)?; assert_eq!( - r.get("key_created").unwrap(), - r2.get("key_created").unwrap() + r.get("key_create_timestamps").unwrap(), + r2.get("key_create_timestamps").unwrap() ); assert_eq!( r.get("statistics_id").unwrap(), @@ -500,9 +500,9 @@ async fn test_statistics_key_creation_timestamp() -> Result<()> { let r = get_statistics(alice).await?; let r: serde_json::Value = serde_json::from_str(&r)?; - let key_created = r.get("key_created").unwrap().as_array().unwrap(); + let key_create_timestamps = r.get("key_create_timestamps").unwrap().as_array().unwrap(); assert_eq!( - key_created, + key_create_timestamps, &vec![Value::Number( Number::from_u128(ALICE_KEY_CREATION_TIME).unwrap() )] From c21b8d319220561ae5bc60a8b5aaf59200d746de Mon Sep 17 00:00:00 2001 From: Hocuri Date: Mon, 25 Aug 2025 16:36:06 +0200 Subject: [PATCH 17/21] iequidoo's review: Make sure statistics are sent again after rewound system time --- deltachat-time/src/lib.rs | 5 +++++ src/statistics.rs | 20 ++++++++++++-------- src/statistics/statistics_tests.rs | 22 ++++++++++++++++++++++ 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/deltachat-time/src/lib.rs b/deltachat-time/src/lib.rs index c8d7d6f6c2..7e92a3f61a 100644 --- a/deltachat-time/src/lib.rs +++ b/deltachat-time/src/lib.rs @@ -20,6 +20,11 @@ impl SystemTimeTools { pub fn shift(duration: Duration) { *SYSTEM_TIME_SHIFT.write().unwrap() += duration; } + + /// Simulates the system clock being rewound by `duration`. + pub fn shift_backwards(duration: Duration) { + *SYSTEM_TIME_SHIFT.write().unwrap() -= duration; + } } #[cfg(test)] diff --git a/src/statistics.rs b/src/statistics.rs index 38503974d2..7070d2369e 100644 --- a/src/statistics.rs +++ b/src/statistics.rs @@ -159,7 +159,19 @@ pub async fn maybe_send_statistics(context: &Context) -> Result> let last_sending_time = context.get_config_i64(Config::StatsLastSent).await?; let next_sending_time = last_sending_time.saturating_add(SENDING_INTERVAL_SECONDS); if next_sending_time <= time() { + // Setting this config at the beginning avoids endless loops when things do not + // work out for whatever reason. + context + .set_config_internal(Config::StatsLastSent, Some(&time().to_string())) + .await?; + return Ok(Some(send_statistics(context).await?)); + } else if time() < last_sending_time { + // The clock was rewound. + // Reset the config, so that the statistics will be sent normally in a week. + context + .set_config_internal(Config::StatsLastSent, Some(&time().to_string())) + .await?; } } Ok(None) @@ -184,14 +196,6 @@ pub(crate) async fn should_send_statistics(context: &Context) -> Result { async fn send_statistics(context: &Context) -> Result { info!(context, "Sending statistics."); - // Setting this config at the beginning avoids endless loops when things do not - // work out for whatever reason. - context - .set_config_internal(Config::StatsLastSent, Some(&time().to_string())) - .await - .log_err(context) - .ok(); - let chat_id = get_statistics_bot(context).await?; let mut msg = Message::new(Viewtype::File); diff --git a/src/statistics/statistics_tests.rs b/src/statistics/statistics_tests.rs index 16de79d94d..393678e39f 100644 --- a/src/statistics/statistics_tests.rs +++ b/src/statistics/statistics_tests.rs @@ -43,6 +43,28 @@ async fn test_maybe_send_statistics() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_rewound_time() -> Result<()> { + let alice = &TestContext::new_alice().await; + alice.set_config_bool(Config::StatsSending, true).await?; + + const EIGHT_DAYS: Duration = Duration::from_secs(3600 * 24 * 14); + SystemTime::shift(EIGHT_DAYS); + + maybe_send_statistics(alice).await?.unwrap(); + + // The system's time is rewound + SystemTime::shift_backwards(EIGHT_DAYS); + + assert!(maybe_send_statistics(alice).await?.is_none()); + + // After eight days pass again, statistics are sent again + SystemTime::shift(EIGHT_DAYS); + maybe_send_statistics(alice).await?.unwrap(); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_statistics_one_contact() -> Result<()> { let mut tcm = TestContextManager::new(); From 7ef56d6ba0d9eafd37355d2ca47418af827692d2 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Mon, 25 Aug 2025 17:23:11 +0200 Subject: [PATCH 18/21] iequidoo's review: Rename get_statistics_bot() -> get_statistics_chat_id() --- src/statistics.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/statistics.rs b/src/statistics.rs index 7070d2369e..efd2a52775 100644 --- a/src/statistics.rs +++ b/src/statistics.rs @@ -196,7 +196,7 @@ pub(crate) async fn should_send_statistics(context: &Context) -> Result { async fn send_statistics(context: &Context) -> Result { info!(context, "Sending statistics."); - let chat_id = get_statistics_bot(context).await?; + let chat_id = get_statistics_chat_id(context).await?; let mut msg = Message::new(Viewtype::File); msg.set_text( @@ -320,7 +320,7 @@ async fn get_statistics(context: &Context) -> Result { Ok(serde_json::to_string_pretty(&statistics)?) } -async fn get_statistics_bot(context: &Context) -> Result { +async fn get_statistics_chat_id(context: &Context) -> Result { let contact_id: ContactId = *import_vcard(context, STATISTICS_BOT_VCARD) .await? .first() @@ -465,7 +465,7 @@ async fn get_message_stats( "Last_msgid < 9 would mean including 'special' messages in the statistics" ); - let statistics_bot_chat_id = get_statistics_bot(context).await?; + let statistics_bot_chat_id = get_statistics_chat_id(context).await?; let trans_fn = |t: &mut rusqlite::Transaction| { t.pragma_update(None, "query_only", "0")?; From b77c815f1a731adc92c055ba51b24887c219b4b2 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Wed, 27 Aug 2025 14:39:46 +0200 Subject: [PATCH 19/21] iequidoo's second review --- deltachat-time/src/lib.rs | 2 +- src/sql/migrations.rs | 2 +- src/statistics.rs | 18 +++++++++--------- src/statistics/statistics_tests.rs | 10 +++++----- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/deltachat-time/src/lib.rs b/deltachat-time/src/lib.rs index 7e92a3f61a..b3b8b2a219 100644 --- a/deltachat-time/src/lib.rs +++ b/deltachat-time/src/lib.rs @@ -22,7 +22,7 @@ impl SystemTimeTools { } /// Simulates the system clock being rewound by `duration`. - pub fn shift_backwards(duration: Duration) { + pub fn shift_back(duration: Duration) { *SYSTEM_TIME_SHIFT.write().unwrap() -= duration; } } diff --git a/src/sql/migrations.rs b/src/sql/migrations.rs index 045918681a..0144b4bab5 100644 --- a/src/sql/migrations.rs +++ b/src/sql/migrations.rs @@ -1273,7 +1273,7 @@ CREATE INDEX gossip_timestamp_index ON gossip_timestamp (chat_id, fingerprint); count INTEGER NOT NULL DEFAULT 0 ) STRICT; CREATE TABLE statistics_securejoin_invites( - contact_created INTEGER NOT NULL, + already_existed INTEGER NOT NULL, already_verified INTEGER NOT NULL, type TEXT NOT NULL ) STRICT;", diff --git a/src/statistics.rs b/src/statistics.rs index efd2a52775..cdc3e2317b 100644 --- a/src/statistics.rs +++ b/src/statistics.rs @@ -135,9 +135,9 @@ struct SecurejoinUIPaths { /// (i.e. a qr scan or a clicked link). #[derive(Serialize)] struct JoinedInvite { - /// Whether the contact was newly created right now. - /// If this is false, then a contact existed already before. - contact_created: bool, + /// Whether the contact already existed before. + /// If this is false, then a contact was newly created. + already_existed: bool, /// If a contact already existed, /// this tells us whether the contact was verified already. already_verified: bool, @@ -711,7 +711,7 @@ pub(crate) async fn count_securejoin_invite(context: &Context, invite: &QrInvite // but it was not visible in the contacts list in the UI // e.g. because it's a past contact of a group we're in), // then its origin is UnhandledSecurejoinQrScan. - let contact_created = contact.origin == Origin::UnhandledSecurejoinQrScan; + let already_existed = contact.origin > Origin::UnhandledSecurejoinQrScan; // Check whether the contact was verified already before the QR scan. let already_verified = contact.is_verified(context).await?; @@ -724,9 +724,9 @@ pub(crate) async fn count_securejoin_invite(context: &Context, invite: &QrInvite context .sql .execute( - "INSERT INTO statistics_securejoin_invites (contact_created, already_verified, type) + "INSERT INTO statistics_securejoin_invites (already_existed, already_verified, type) VALUES (?, ?, ?)", - (contact_created, already_verified, typ), + (already_existed, already_verified, typ), ) .await?; @@ -737,15 +737,15 @@ async fn get_securejoin_invite_stats(context: &Context) -> Result = context .sql .query_map( - "SELECT contact_created, already_verified, type FROM statistics_securejoin_invites", + "SELECT already_existed, already_verified, type FROM statistics_securejoin_invites", (), |row| { - let contact_created: bool = row.get(0)?; + let already_existed: bool = row.get(0)?; let already_verified: bool = row.get(1)?; let typ: String = row.get(2)?; Ok(JoinedInvite { - contact_created, + already_existed, already_verified, typ, }) diff --git a/src/statistics/statistics_tests.rs b/src/statistics/statistics_tests.rs index 393678e39f..b8808c4928 100644 --- a/src/statistics/statistics_tests.rs +++ b/src/statistics/statistics_tests.rs @@ -54,7 +54,7 @@ async fn test_rewound_time() -> Result<()> { maybe_send_statistics(alice).await?.unwrap(); // The system's time is rewound - SystemTime::shift_backwards(EIGHT_DAYS); + SystemTime::shift_back(EIGHT_DAYS); assert!(maybe_send_statistics(alice).await?.is_none()); @@ -449,7 +449,7 @@ async fn test_statistics_securejoin_invites() -> Result<()> { check_qr(alice, &qr).await?; tcm.exec_securejoin_qr(alice, bob, &qr).await; expected.push(JoinedInvite { - contact_created: true, + already_existed: false, already_verified: false, typ: "contact".to_string(), }); @@ -458,7 +458,7 @@ async fn test_statistics_securejoin_invites() -> Result<()> { check_qr(alice, &qr).await?; tcm.exec_securejoin_qr(alice, bob, &qr).await; expected.push(JoinedInvite { - contact_created: false, + already_existed: true, already_verified: true, typ: "contact".to_string(), }); @@ -470,7 +470,7 @@ async fn test_statistics_securejoin_invites() -> Result<()> { check_qr(alice, &qr).await?; tcm.exec_securejoin_qr(alice, bob, &qr).await; expected.push(JoinedInvite { - contact_created: false, + already_existed: true, already_verified: true, typ: "group".to_string(), }); @@ -485,7 +485,7 @@ async fn test_statistics_securejoin_invites() -> Result<()> { check_qr(alice, &qr).await?; tcm.exec_securejoin_qr(alice, bob, &qr).await; expected.push(JoinedInvite { - contact_created: false, + already_existed: true, already_verified: false, typ: "group".to_string(), }); From 3295ed488c3a148b903e20e1377565947b02488e Mon Sep 17 00:00:00 2001 From: Hocuri Date: Wed, 27 Aug 2025 15:07:38 +0200 Subject: [PATCH 20/21] feat: Set the StatsLastSent setting after sending statistics --- src/statistics.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/statistics.rs b/src/statistics.rs index cdc3e2317b..707edc65e3 100644 --- a/src/statistics.rs +++ b/src/statistics.rs @@ -159,13 +159,20 @@ pub async fn maybe_send_statistics(context: &Context) -> Result> let last_sending_time = context.get_config_i64(Config::StatsLastSent).await?; let next_sending_time = last_sending_time.saturating_add(SENDING_INTERVAL_SECONDS); if next_sending_time <= time() { - // Setting this config at the beginning avoids endless loops when things do not - // work out for whatever reason. + // If something goes wrong, try again in 1 minute. + // This prevents infinite loops in the (unlikely) case of an error: + let one_minute_later = last_sending_time.saturating_add(60).to_string(); + context + .set_config_internal(Config::StatsLastSent, Some(&one_minute_later)) + .await?; + + let chat_id = Some(send_statistics(context).await?); + context .set_config_internal(Config::StatsLastSent, Some(&time().to_string())) .await?; - return Ok(Some(send_statistics(context).await?)); + return Ok(chat_id); } else if time() < last_sending_time { // The clock was rewound. // Reset the config, so that the statistics will be sent normally in a week. From 6c592d98bdaa0f83d8fa045bdd63a42d595b4b49 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Wed, 27 Aug 2025 16:34:14 +0200 Subject: [PATCH 21/21] fix: StatsId is not a bool --- src/context.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/context.rs b/src/context.rs index 8b634355a6..96bc572c53 100644 --- a/src/context.rs +++ b/src/context.rs @@ -1078,7 +1078,9 @@ impl Context { ); res.insert( "stats_id", - self.get_config_bool(Config::StatsId).await?.to_string(), + self.get_config(Config::StatsId) + .await? + .unwrap_or_else(|| "".to_string()), ); res.insert( "stats_sending",