Opt-in weekly sending of statistics

deltachat-jsonrpc/src/api.rs

@@ -383,11 +383,6 @@ impl CommandApi {
         Ok(BlobObject::create_and_deduplicate(&ctx, file, file)?.to_abs_path())
     }
 
-    async fn draft_self_report(&self, account_id: u32) -> Result<u32> {
-        let ctx = self.get_context(account_id).await?;
-        Ok(ctx.draft_self_report().await?.to_u32())
-    }
-
     /// Sets the given configuration key.
     async fn set_config(&self, account_id: u32, key: String, value: Option<String>) -> Result<()> {
         let ctx = self.get_context(account_id).await?;
@@ -886,6 +881,21 @@ impl CommandApi {
         Ok(chat_id.to_u32())
     }
 
+    /// Like `secure_join()`, but allows to pass a source and a UI-path.
+    /// You only need this if your UI has an option to send statistics
+    /// to Delta Chat's developers.
+    async fn secure_join_with_ux_info(
+        &self,
+        account_id: u32,
+        qr: String,
+        source: Option<u32>,
+        uipath: Option<u32>,
+    ) -> Result<u32> {
+        let ctx = self.get_context(account_id).await?;
+        let chat_id = securejoin::join_securejoin_with_ux_info(&ctx, &qr, source, uipath).await?;
+        Ok(chat_id.to_u32())
+    }
+
     async fn leave_group(&self, account_id: u32, chat_id: u32) -> Result<()> {
         let ctx = self.get_context(account_id).await?;
         remove_contact_from_chat(&ctx, ChatId::new(chat_id), ContactId::SELF).await

src/config.rs

@@ -14,7 +14,6 @@ use tokio::fs;
 
 use crate::blob::BlobObject;
 use crate::configure::EnteredLoginParam;
-use crate::constants;
 use crate::context::Context;
 use crate::events::EventType;
 use crate::log::{LogExt, info};
@@ -23,6 +22,7 @@ use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::provider::{Provider, get_provider_by_id};
 use crate::sync::{self, Sync::*, SyncData};
 use crate::tools::get_abs_path;
+use crate::{constants, statistics};
 
 /// The available configuration keys.
 #[derive(
@@ -431,9 +431,26 @@ pub enum Config {
     /// used for signatures, encryption to self and included in `Autocrypt` header.
     KeyId,
 
-    /// This key is sent to the self_reporting bot so that the bot can recognize the user
+    /// Send statistics to Delta Chat's developers.
+    /// Can be exposed to the user as a setting.
+    SendStatistics,
+
+    /// Last time statistics were sent to Delta Chat's developers
+    LastStatisticsSent,
+
+    /// This key is sent to the statistics bot so that the bot can recognize the user
     /// without storing the email address
-    SelfReportingId,
+    StatisticsId,
+
+    /// The last message id that was already included in the previously sent statistics,
+    /// or that already existed before the user opted in.
+    /// Only messages with an id larger than this
+    /// will be counted in the next statistics.
+    StatsLastExcludedMsgId,
+
+    /// The last contact id that already existed when statistics-sending was enabled.
+    /// All newer contacts get the `"new": true` attribute.
+    StatsLastOldContactId,
 
     /// MsgId of webxdc map integration.
     WebxdcIntegration,
@@ -827,6 +844,11 @@ impl Context {
                     .await?;
                 }
             }
+            Config::SendStatistics => {
+                self.sql.set_raw_config(key.as_ref(), value).await?;
+                statistics::set_last_excluded_msg_id(self).await?;
+                statistics::set_last_old_contact_id(self).await?;
+            }
             _ => {
                 self.sql.set_raw_config(key.as_ref(), value).await?;
             }

src/context.rs

@@ -10,35 +10,30 @@ use std::time::Duration;
 
 use anyhow::{Context as _, Result, bail, ensure};
 use async_channel::{self as channel, Receiver, Sender};
-use pgp::types::PublicKeyTrait;
 use ratelimit::Ratelimit;
 use tokio::sync::{Mutex, Notify, RwLock};
 
-use crate::chat::{ChatId, ProtectionStatus, get_chat_cnt};
-use crate::chatlist_events;
+use crate::chat::{ChatId, get_chat_cnt};
 use crate::config::Config;
-use crate::constants::{
-    self, DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT, DC_CHAT_ID_TRASH, DC_VERSION_STR,
-};
-use crate::contact::{Contact, ContactId, import_vcard, mark_contact_id_as_verified};
+use crate::constants::{self, DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT, DC_VERSION_STR};
+use crate::contact::{Contact, ContactId};
 use crate::debug_logging::DebugLogging;
-use crate::download::DownloadState;
 use crate::events::{Event, EventEmitter, EventType, Events};
 use crate::imap::{FolderMeaning, Imap, ServerMetadata};
-use crate::key::{load_self_secret_key, self_fingerprint};
+use crate::key::self_fingerprint;
 use crate::log::{info, warn};
 use crate::logged_debug_assert;
 use crate::login_param::{ConfiguredLoginParam, EnteredLoginParam};
-use crate::message::{self, Message, MessageState, MsgId};
-use crate::param::{Param, Params};
+use crate::message::{self, MessageState, MsgId};
 use crate::peer_channels::Iroh;
 use crate::push::PushSubscriber;
 use crate::quota::QuotaInfo;
 use crate::scheduler::{SchedulerState, convert_folder_meaning};
 use crate::sql::Sql;
 use crate::stock_str::StockStrings;
 use crate::timesmearing::SmearedTimestamp;
-use crate::tools::{self, create_id, duration_to_str, time, time_elapsed};
+use crate::tools::{self, duration_to_str, time, time_elapsed};
+use crate::{chatlist_events, statistics};
 
 /// Builder for the [`Context`].
 ///
@@ -1076,154 +1071,29 @@ impl Context {
                 .await?
                 .unwrap_or_default(),
         );
+        res.insert(
+            "statistics_id",
+            self.get_config_bool(Config::StatisticsId)
+                .await?
+                .to_string(),
+        );
+        res.insert(
+            "send_statistics",
+            statistics::should_send_statistics(self).await?.to_string(),
+        );
+        res.insert(
+            "last_statistics_sent",
+            self.get_config_i64(Config::LastStatisticsSent)
+                .await?
+                .to_string(),
+        );
 
         let elapsed = time_elapsed(&self.creation_time);
         res.insert("uptime", duration_to_str(elapsed));
 
         Ok(res)
     }
 
-    async fn get_self_report(&self) -> Result<String> {
-        #[derive(Default)]
-        struct ChatNumbers {
-            protected: u32,
-            opportunistic_dc: u32,
-            opportunistic_mua: u32,
-            unencrypted_dc: u32,
-            unencrypted_mua: u32,
-        }
-
-        let mut res = String::new();
-        res += &format!("core_version {}\n", get_version_str());
-
-        let num_msgs: u32 = self
-            .sql
-            .query_get_value(
-                "SELECT COUNT(*) FROM msgs WHERE hidden=0 AND chat_id!=?",
-                (DC_CHAT_ID_TRASH,),
-            )
-            .await?
-            .unwrap_or_default();
-        res += &format!("num_msgs {num_msgs}\n");
-
-        let num_chats: u32 = self
-            .sql
-            .query_get_value("SELECT COUNT(*) FROM chats WHERE id>9 AND blocked!=1", ())
-            .await?
-            .unwrap_or_default();
-        res += &format!("num_chats {num_chats}\n");
-
-        let db_size = tokio::fs::metadata(&self.sql.dbfile).await?.len();
-        res += &format!("db_size_bytes {db_size}\n");
-
-        let secret_key = &load_self_secret_key(self).await?.primary_key;
-        let key_created = secret_key.public_key().created_at().timestamp();
-        res += &format!("key_created {key_created}\n");
-
-        // how many of the chats active in the last months are:
-        // - protected
-        // - opportunistic-encrypted and the contact uses Delta Chat
-        // - opportunistic-encrypted and the contact uses a classical MUA
-        // - unencrypted and the contact uses Delta Chat
-        // - unencrypted and the contact uses a classical MUA
-        let three_months_ago = time().saturating_sub(3600 * 24 * 30 * 3);
-        let chats = self
-            .sql
-            .query_map(
-                "SELECT c.protected, m.param, m.msgrmsg
-                    FROM chats c
-                    JOIN msgs m
-                        ON c.id=m.chat_id
-                        AND m.id=(
-                                SELECT id
-                                FROM msgs
-                                WHERE chat_id=c.id
-                                AND hidden=0
-                                AND download_state=?
-                                AND to_id!=?
-                                ORDER BY timestamp DESC, id DESC LIMIT 1)
-                    WHERE c.id>9
-                    AND (c.blocked=0 OR c.blocked=2)
-                    AND IFNULL(m.timestamp,c.created_timestamp) > ?
-                    GROUP BY c.id",
-                (DownloadState::Done, ContactId::INFO, three_months_ago),
-                |row| {
-                    let protected: ProtectionStatus = row.get(0)?;
-                    let message_param: Params =
-                        row.get::<_, String>(1)?.parse().unwrap_or_default();
-                    let is_dc_message: bool = row.get(2)?;
-                    Ok((protected, message_param, is_dc_message))
-                },
-                |rows| {
-                    let mut chats = ChatNumbers::default();
-                    for row in rows {
-                        let (protected, message_param, is_dc_message) = row?;
-                        let encrypted = message_param
-                            .get_bool(Param::GuaranteeE2ee)
-                            .unwrap_or(false);
-
-                        if protected == ProtectionStatus::Protected {
-                            chats.protected += 1;
-                        } else if encrypted {
-                            if is_dc_message {
-                                chats.opportunistic_dc += 1;
-                            } else {
-                                chats.opportunistic_mua += 1;
-                            }
-                        } else if is_dc_message {
-                            chats.unencrypted_dc += 1;
-                        } else {
-                            chats.unencrypted_mua += 1;
-                        }
-                    }
-                    Ok(chats)
-                },
-            )
-            .await?;
-        res += &format!("chats_protected {}\n", chats.protected);
-        res += &format!("chats_opportunistic_dc {}\n", chats.opportunistic_dc);
-        res += &format!("chats_opportunistic_mua {}\n", chats.opportunistic_mua);
-        res += &format!("chats_unencrypted_dc {}\n", chats.unencrypted_dc);
-        res += &format!("chats_unencrypted_mua {}\n", chats.unencrypted_mua);
-
-        let self_reporting_id = match self.get_config(Config::SelfReportingId).await? {
-            Some(id) => id,
-            None => {
-                let id = create_id();
-                self.set_config(Config::SelfReportingId, Some(&id)).await?;
-                id
-            }
-        };
-        res += &format!("self_reporting_id {self_reporting_id}");
-
-        Ok(res)
-    }
-
-    /// Drafts a message with statistics about the usage of Delta Chat.
-    /// The user can inspect the message if they want, and then hit "Send".
-    ///
-    /// On the other end, a bot will receive the message and make it available
-    /// to Delta Chat's developers.
-    pub async fn draft_self_report(&self) -> Result<ChatId> {
-        const SELF_REPORTING_BOT_VCARD: &str = include_str!("../assets/self-reporting-bot.vcf");
-        let contact_id: ContactId = *import_vcard(self, SELF_REPORTING_BOT_VCARD)
-            .await?
-            .first()
-            .context("Self reporting bot vCard does not contain a contact")?;
-        mark_contact_id_as_verified(self, contact_id, ContactId::SELF).await?;
-
-        let chat_id = ChatId::create_for_contact(self, contact_id).await?;
-        chat_id
-            .set_protection(self, ProtectionStatus::Protected, time(), Some(contact_id))
-            .await?;
-
-        let mut msg = Message::new_text(self.get_self_report().await?);
-
-        chat_id.set_draft(self, Some(&mut msg)).await?;
-
-        Ok(chat_id)
-    }
-
     /// Get a list of fresh, unmuted messages in unblocked chats.
     ///
     /// The list starts with the most recent message

src/context/context_tests.rs

@@ -6,9 +6,9 @@ use super::*;
 use crate::chat::{Chat, MuteDuration, get_chat_contacts, get_chat_msgs, send_msg, set_muted};
 use crate::chatlist::Chatlist;
 use crate::constants::Chattype;
-use crate::mimeparser::SystemMessage;
+use crate::message::Message;
 use crate::receive_imf::receive_imf;
-use crate::test_utils::{E2EE_INFO_MSGS, TestContext, get_chat_msg};
+use crate::test_utils::{E2EE_INFO_MSGS, TestContext};
 use crate::tools::{SystemTime, create_outgoing_rfc724_mid};
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -276,7 +276,6 @@ async fn test_get_info_completeness() {
         "mail_port",
         "mail_security",
         "notify_about_wrong_pw",
-        "self_reporting_id",
         "selfstatus",
         "send_server",
         "send_user",
@@ -296,6 +295,8 @@ async fn test_get_info_completeness() {
         "webxdc_integration",
         "device_token",
         "encrypted_device_token",
+        "stats_last_excluded_msg_id",
+        "stats_last_old_contact_id",
     ];
     let t = TestContext::new().await;
     let info = t.get_info().await.unwrap();
@@ -598,26 +599,6 @@ async fn test_get_next_msgs() -> Result<()> {
     Ok(())
 }
 
-#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-async fn test_draft_self_report() -> Result<()> {
-    let alice = TestContext::new_alice().await;
-
-    let chat_id = alice.draft_self_report().await?;
-    let msg = get_chat_msg(&alice, chat_id, 0, 1).await;
-    assert_eq!(msg.get_info_type(), SystemMessage::ChatProtectionEnabled);
-
-    let chat = Chat::load_from_db(&alice, chat_id).await?;
-    assert!(chat.is_protected());
-
-    let mut draft = chat_id.get_draft(&alice).await?.unwrap();
-    assert!(draft.text.starts_with("core_version"));
-
-    // Test that sending into the protected chat works:
-    let _sent = alice.send_msg(chat_id, &mut draft).await;
-
-    Ok(())
-}
-
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_cache_is_cleared_when_io_is_started() -> Result<()> {
     let alice = TestContext::new_alice().await;

src/lib.rs

@@ -98,6 +98,7 @@ pub mod html;
 pub mod net;
 pub mod plaintext;
 mod push;
+pub(crate) mod statistics;
 pub mod summary;
 
 mod debug_logging;

src/receive_imf.rs

@@ -42,6 +42,7 @@ use crate::reaction::{Reaction, set_msg_reaction};
 use crate::rusqlite::OptionalExtension;
 use crate::securejoin::{self, handle_securejoin_handshake, observe_securejoin_on_other_device};
 use crate::simplify;
+use crate::statistics::STATISTICS_BOT_EMAIL;
 use crate::stock_str;
 use crate::sync::Sync::*;
 use crate::tools::{self, buf_compress, remove_subject_prefix};
@@ -1744,7 +1745,11 @@ async fn add_parts(
 
     let state = if !mime_parser.incoming {
         MessageState::OutDelivered
-    } else if seen || is_mdn || chat_id_blocked == Blocked::Yes || group_changes.silent
+    } else if seen
+        || is_mdn
+        || chat_id_blocked == Blocked::Yes
+        || group_changes.silent
+        || mime_parser.from.addr == STATISTICS_BOT_EMAIL
     // No check for `hidden` because only reactions are such and they should be `InFresh`.
     {
         MessageState::InSeen