enh: collector_mail_attach Decrypt GPG attachments (#2623)

Author: e3rd

.github/workflows/unittests.yml

@@ -72,7 +72,8 @@ jobs:
     - name: Run basic testsuite
       if: ${{ matrix.type == 'basic' }}
       run: pytest --cov intelmq/ --cov-report=xml --cov-branch intelmq/
-
+    - name: Assure permissions
+      run: chmod -R u=rwX,go= intelmq/tests/bots/collectors/mail/gpg_ring/
     - name: Run full testsuite
       if: ${{ matrix.type == 'full' }}
       run: pytest --cov intelmq/ --cov-report=xml --cov-branch intelmq/ contrib/
@@ -81,3 +82,4 @@ jobs:
         INTELMQ_TEST_DATABASES: 1
         INTELMQ_TEST_EXOTIC: 1
         INTELMQ_TEST_INSTALLATION: 1
+        GPG_RING_PATH: intelmq/tests/bots/collectors/mail/gpg_ring/

.reuse/dep5

@@ -33,3 +33,15 @@ License:   AGPL-3.0-or-later
 Files:     docs/_static/n6/data-flow.png docs/_static/n6/n6-schemat2.png
 Copyright: CERT.pl <[email protected]>
 License:   AGPL-3.0-only
+
+Files:  intelmq/tests/bots/collectors/mail/gpg_attachment.eml
+Copyright: 2025 Edvard Rejthar, CSIRT.cz <[email protected]>
+License: AGPL-3.0-or-later
+
+Files: intelmq/tests/bots/collectors/mail/gpg_wrong_attachment.eml
+Copyright: 2025 Edvard Rejthar, CSIRT.cz <[email protected]>
+License: AGPL-3.0-or-later
+
+Files: intelmq/tests/bots/collectors/mail/gpg_ring/**
+Copyright: 2025 Edvard Rejthar, CSIRT.cz <[email protected]>
+License: AGPL-3.0-or-later

CHANGELOG.md

@@ -29,6 +29,7 @@ Please refer to the [NEWS](NEWS.md) for a list of changes which have an affect o
 
 ### Bots
 #### Collectors
+- `intelmq.bots.collectors.mail.collector_mail_attach`: Decrypt GPG attachments (PR#2623 by Edvard Rejthar).
 - `intelmq.bots.collectors.shodan.collector_alert`: Added a new collector to query the Shodan Alert API (PR#2618 by Sebastian Wagner and Malawi CERT).
 - Remove `intelmq.bots.collectors.blueliv` as it uses an unmaintained library, does not work any more and breaks other CI tests (fixes #2593, PR#2632 by Sebastian Wagner).
 

docs/user/bots.md

@@ -486,6 +486,18 @@ The fields can be used by parsers to identify the feed and are not automatically
 
 (optional, boolean) Whether to extract compress files from the attachment. Defaults to true.
 
+**`decrypt_openpgp`**
+
+(optional, boolean) Whether to decrypt the attachment with GPG. Defaults to false.
+
+**`openpgp_passphrase`**
+
+(optional, string) The OpenPGP private key passhrase.
+
+**`gpg_home`**
+
+(optional, string) Change the GPG home directory.
+
 **`sent_from`**
 
 (optional, string) Only process messages sent from this address. Defaults to null (any sender).

intelmq/bots/collectors/mail/REQUIREMENTS.txt

@@ -2,3 +2,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
 imbox>=0.8.5
+python-gnupg>=0.5

intelmq/bots/collectors/mail/collector_mail_attach.py

@@ -11,14 +11,16 @@
 Uses the common mail iteration method from the lib file.
 """
 import re
+
 from intelmq.lib.utils import unzip
-from intelmq.lib.exceptions import InvalidArgument
+from intelmq.lib.exceptions import InvalidArgument, MissingDependencyError
 
 from ._lib import MailCollectorBot
 
 
 class MailAttachCollectorBot(MailCollectorBot):
     """Monitor IMAP mailboxes and retrieve mail attachments"""
+
     attach_regex: str = "csv.zip"
     extract_files: bool = True
     folder: str = "INBOX"
@@ -28,11 +30,25 @@ class MailAttachCollectorBot(MailCollectorBot):
     mail_user: str = "<user>"
     rate_limit: int = 60
     subject_regex: str = "<subject>"
+    decrypt_openpgp: bool = False
+    """ Decrypt the attachment with OpenPGP """
+
+    openpgp_passphrase: str = ""
+    """ The OpenPGP private key passhrase """
+
+    gpg_home: str = ""
+    """ Change the GPG home directory """
 
     def init(self):
         super().init()
         if self.attach_regex is None:
             raise InvalidArgument('attach_regex', expected='string')
+        if self.decrypt_openpgp:
+            try:
+                from gnupg import GPG
+            except ImportError:
+                raise MissingDependencyError("python-gnupg", ">=0.5")
+            self._gpg = GPG(gnupghome=self.gpg_home)
 
     def process_message(self, uid, message):
         seen = False
@@ -63,6 +79,15 @@ def process_message(self, uid, message):
                     raw_reports = ((attach_filename, attach['content'].read()), )
 
                 for file_name, raw_report in raw_reports:
+                    if self.decrypt_openpgp:
+                        gpg = self._gpg.decrypt(
+                            raw_report, passphrase=self.openpgp_passphrase
+                        )
+                        if gpg.ok:
+                            raw_report = gpg.data
+                        else:
+                            self.logger.error('Could not decrypt attachment %s: %s.', file_name, gpg.status)
+                            continue
                     report = self.new_report()
                     report.add("raw", raw_report)
                     if file_name:

intelmq/tests/bots/collectors/mail/gpg_attachment.eml

@@ -0,0 +1,40 @@
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="===============0256050913907025376=="
+Subject: foobar zip
+From: Sebastian Wagner <[email protected]>
+To: [email protected]
+Message-ID: <[email protected]>
+Date: Tue, 3 Sep 2019 16:57:40 +0200
+Content-Language: en-US
+
+--===============0256050913907025376==
+Content-Type: text/html; charset="utf-8"
+Content-Transfer-Encoding: 7bit
+
+<html>
+  <head>
+
+    <meta http-equiv="content-type" content="text/html; charset=UTF-8">
+  </head>
+  <body text="#000000" bgcolor="#FFFFFF">
+    Please look at the attachment<br>
+  </body>
+</html>
+
+--===============0256050913907025376==
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+Content-Disposition: attachment; filename="foobar.txt.gpg"
+MIME-Version: 1.0
+
+hQGMA9ig68HPFWOpAQv/fsRXK1mdmGw83CdbIOZeUlFVx2KKEONWVYIu/CHPxqxREYUWTva+XxYE
+LirKR9xK9lG19H6BSSZD3xXMahAShRzgs1dHH4UMxGqdpiXo5DbdVkTS7kg5+hG7QX20x+ExwsRd
+r76/gzY+Lmsv4hydxMHyG/w0OBsV/0mLWGAAVzer9Y8xkUP6jB16XzoSQGkfCP+7DpK224135JgU
+E6XSD2p+WKzALxXiET4IfinJnG8sSTHlkCQZgBp8lpDPdP2BbHo0KmvAofxaRqkvMfpVk7kUyy2W
+8tENKSEPu0sKEX8HZhT8Sw3X2pn8ggbAadHRjOzOO58MUvWJvwURqa1LIITA4pmyhw4svp1UzODb
+Q2Cd+MyWikjD4CebJ7P/JpnoDokux9qxWKnuuvmM4mqr6bvL+Sztud4V2Z60idokbzuEaJR6ZztT
+SBMatdh/OHcUnCaYCJiEOlrgk0hYcZ12XR5C8OBAI4St9F5eGjMB0l1tZz7CeQ6MrAoPY2pZ54XO
+0k0BA3PpG+K/khPOns8rW2mlYKgWfJqRdc91EasA4+iFoM+iFwyDQGBKbAdCu0BeIf9j7t5s5LOG
+iPWI5m2AIWf+aFNqKotGFAWXeSbMWw==
+
+--===============0256050913907025376==--

intelmq/tests/bots/collectors/mail/gpg_ring/crls.d/DIR.txt

@@ -0,0 +1 @@
+v:1:

intelmq/tests/bots/collectors/mail/gpg_ring/openpgp-revocs.d/3C8124A8245618D286CF871E94CE2905DB00CDB7.rev

@@ -0,0 +1,35 @@
+This is a revocation certificate for the OpenPGP key:
+
+pub   rsa3072 2019-12-11 [S]
+      3C8124A8245618D286CF871E94CE2905DB00CDB7
+uid          Example identity 2 <[email protected]>
+
+A revocation certificate is a kind of "kill switch" to publicly
+declare that a key shall not anymore be used.  It is not possible
+to retract such a revocation certificate once it has been published.
+
+Use it to revoke this key in case of a compromise or loss of
+the secret key.  However, if the secret key is still accessible,
+it is better to generate a new revocation certificate and give
+a reason for the revocation.  For details see the description of
+of the gpg command "--generate-revocation" in the GnuPG manual.
+
+To avoid an accidental use of this file, a colon has been inserted
+before the 5 dashes below.  Remove this colon with a text editor
+before importing and publishing this revocation certificate.
+
+:-----BEGIN PGP PUBLIC KEY BLOCK-----
+Comment: This is a revocation certificate
+
+iQG2BCABCgAgFiEEPIEkqCRWGNKGz4celM4pBdsAzbcFAl3xFwwCHQAACgkQlM4p
+BdsAzbeofgv+JUQ4Qb40Z0p2ML+jciJjNiRdKYOUyhY1UKECwq6QlbW/MSkQMn4g
+esJfevFwGHKxaI4bw9ywFMtR/UB91YDY4D+lURm4qMuc8pFYEBSMhro0x8ToWz1E
+vupwAqTF484ybBrujg2UaOox9BbyhIbsiAH3ttB6wThCbXdhkxp/w6qUaWo+n5Ra
+5xWrtFkHJx+WIE4mzxqvnuN3RtA9tkr3L8oavw8Vy0j44lhkVE4Mrl/XDGOnryvv
+3WYuiHjOdTP5plb/p4veTvCfUZOcrmhTiwU8gQWMQfy3Nr1NmIhNwoXRTwB41ogt
+j3/mijdKVlRkzzmungIt0G3NJM8sAI9TiZ6KyaH4g3fVHU0ddfUwd5sCV3Q+UHfJ
+iUZeO1qHA8PA0NZwquRE2rnGdc63nHE/7AD9SjpvqIoBcvfoZCniCBm70pSmJvSP
+Csd2/TvaI8PQ28vAVY2LKqCX6JX1MvkiJOZbDGl+VBhCavQCQ8lR0d2b+wriDa3y
+2rhCXk0+ozc+
+=92kE
+-----END PGP PUBLIC KEY BLOCK-----

intelmq/tests/bots/collectors/mail/gpg_ring/openpgp-revocs.d/F14F2E8097E0CCDE93C4E871F4A4F26779FA03BB.rev

@@ -0,0 +1,35 @@
+This is a revocation certificate for the OpenPGP key:
+
+pub   rsa3072 2019-12-11 [S]
+      F14F2E8097E0CCDE93C4E871F4A4F26779FA03BB
+uid          Example identity <[email protected]>
+
+A revocation certificate is a kind of "kill switch" to publicly
+declare that a key shall not anymore be used.  It is not possible
+to retract such a revocation certificate once it has been published.
+
+Use it to revoke this key in case of a compromise or loss of
+the secret key.  However, if the secret key is still accessible,
+it is better to generate a new revocation certificate and give
+a reason for the revocation.  For details see the description of
+of the gpg command "--generate-revocation" in the GnuPG manual.
+
+To avoid an accidental use of this file, a colon has been inserted
+before the 5 dashes below.  Remove this colon with a text editor
+before importing and publishing this revocation certificate.
+
+:-----BEGIN PGP PUBLIC KEY BLOCK-----
+Comment: This is a revocation certificate
+
+iQG2BCABCgAgFiEE8U8ugJfgzN6TxOhx9KTyZ3n6A7sFAl3xFtwCHQAACgkQ9KTy
+Z3n6A7vCwwv/U1mKrClA3lxfI+lTAecMeYwlYtYcAveDjJkMs3AtAUQyqd6B7I5F
+RXQL50xjiB+S+yv3WoXd4eQbT9Z2g1OnMJfSHzENM0K+yosUJS+TpN3SU7YxJ9GR
+4J3eDxo0IyB0mL1QFQXcsWSu92yAyQpwJnscg6S0hvxiq8ij+OyDPNctDtpxcArr
+orh9+rGfLBNABemtKgVgzmedjyB7fdPYjgCi/xJL8eYUxlRUgwuwCS7A3DBaSdwU
+bd4dGvGTvzSmKp8OrbW1j7yf4Vq91qVcSqAdv1y0EpKwnnefAPJV8TsFr4wg0GpI
+ocKDGMiQMPekxfeDPrZcII4jjYXlk6WRzXPTQySZcHBN6o/TKBjdanZv5iV4JktU
+jIvru6M7JAM0PJPFj+AwY6hLV3p741qCYYEUlFs9Yjq7j4vhoHqgdVbmZSVcditn
+KLcM6F7jEx0odcbL157/xLHHqsCPVoZrr9uZVuN8J6uIYw0GTXFRlpMhDEpVg+Jd
+mB4WSiNPkql0
+=rwt1
+-----END PGP PUBLIC KEY BLOCK-----