From 77f95304447036ba20f4e3b99cffa700bd3c6a91 Mon Sep 17 00:00:00 2001 From: Paul Oms Date: Sat, 4 Nov 2023 14:03:28 +0000 Subject: [PATCH 01/25] Began the backend for MailPace, passing basic test --- .github/workflows/integration-test.yml | 3 + CHANGELOG.rst | 2 + README.rst | 1 + anymail/backends/mailpace.py | 208 +++++++++++++++++++++++++ pyproject.toml | 5 +- tests/test_mailpace_backend.py | 206 ++++++++++++++++++++++++ tox.ini | 1 + 7 files changed, 424 insertions(+), 2 deletions(-) create mode 100644 anymail/backends/mailpace.py create mode 100644 tests/test_mailpace_backend.py diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 77fc93e6..08f12899 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -43,6 +43,7 @@ jobs: - { tox: django41-py310-mailersend, python: "3.10" } - { tox: django41-py310-mailgun, python: "3.10" } - { tox: django41-py310-mailjet, python: "3.10" } + - { tox: django41-py310-mailpace, python: "3.10" } - { tox: django41-py310-mandrill, python: "3.10" } - { tox: django41-py310-postal, python: "3.10" } - { tox: django41-py310-postmark, python: "3.10" } @@ -83,6 +84,8 @@ jobs: ANYMAIL_TEST_MAILJET_API_KEY: ${{ secrets.ANYMAIL_TEST_MAILJET_API_KEY }} ANYMAIL_TEST_MAILJET_DOMAIN: ${{ secrets.ANYMAIL_TEST_MAILJET_DOMAIN }} ANYMAIL_TEST_MAILJET_SECRET_KEY: ${{ secrets.ANYMAIL_TEST_MAILJET_SECRET_KEY }} + ANYMAIL_TEST_MAILPACE_DOMAIN: ${{ secrets.ANYMAIL_TEST_MAILPACE_DOMAIN }} + ANYMAIL_TEST_MAILPACE_SERVER_TOKEN: ${{ secrets.ANYMAIL_TEST_MAILPACE_SERVER_TOKEN }} ANYMAIL_TEST_MANDRILL_API_KEY: ${{ secrets.ANYMAIL_TEST_MANDRILL_API_KEY }} ANYMAIL_TEST_MANDRILL_DOMAIN: ${{ secrets.ANYMAIL_TEST_MANDRILL_DOMAIN }} ANYMAIL_TEST_POSTMARK_DOMAIN: ${{ secrets.ANYMAIL_TEST_POSTMARK_DOMAIN }} diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 89ac3f1b..c87feb0f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -35,6 +35,8 @@ Features * **Brevo:** Add support for batch sending (`docs `__). +* **MailPace**: Add support for this ESP + (`docs `__). * **Resend:** Add support for batch sending (`docs `__). diff --git a/README.rst b/README.rst index 74e96aa7..25bf4b58 100644 --- a/README.rst +++ b/README.rst @@ -31,6 +31,7 @@ Anymail currently supports these ESPs: * **MailerSend** * **Mailgun** * **Mailjet** +* **MailPace** * **Mandrill** (MailChimp transactional) * **Postal** (self-hosted ESP) * **Postmark** diff --git a/anymail/backends/mailpace.py b/anymail/backends/mailpace.py new file mode 100644 index 00000000..ca826ec9 --- /dev/null +++ b/anymail/backends/mailpace.py @@ -0,0 +1,208 @@ +import re + +from ..exceptions import AnymailRequestsAPIError +from ..message import AnymailRecipientStatus +from ..utils import ( + CaseInsensitiveCasePreservingDict, + get_anymail_setting, + parse_address_list, +) +from .base_requests import AnymailRequestsBackend, RequestsPayload + + +class EmailBackend(AnymailRequestsBackend): + """ + MailPace API Email Backend + """ + + esp_name = "MailPace" + + def __init__(self, **kwargs): + """Init options from Django settings""" + esp_name = self.esp_name + self.server_token = get_anymail_setting( + "server_token", esp_name=esp_name, kwargs=kwargs, allow_bare=True + ) + api_url = get_anymail_setting( + "api_url", + esp_name=esp_name, + kwargs=kwargs, + default="https://app.mailpace.com/api/v1/send", + ) + if not api_url.endswith("/"): + api_url += "/" + super().__init__(api_url, **kwargs) + + def build_message_payload(self, message, defaults): + return MailPacePayload(message, defaults, self) + + def raise_for_status(self, response, payload, message): + # We need to handle 400 responses in parse_recipient_status + if response.status_code != 400: + super().raise_for_status(response, payload, message) + + def parse_recipient_status(self, response, payload, message): + # Prepare the dict by setting everything to queued without a message id + unknown_status = AnymailRecipientStatus(message_id=None, status="queued") + recipient_status = CaseInsensitiveCasePreservingDict( + { + recip.addr_spec: unknown_status + for recip in payload.to_cc_and_bcc_emails + } + ) + + parsed_response = self.deserialize_json_response(response, payload, message) + + try: + # TODO: Fix this to support errors. Status and ID will not be present if an error is returned + + status_msg = parsed_response["status"] + id = parsed_response["id"] + except (KeyError, TypeError) as err: + raise AnymailRequestsAPIError( + "Invalid MailPace API response format", + email_message=status_msg, + payload=payload, + response=response, + backend=self, + ) from err + + if status_msg == "queued": + try: + message_id = parsed_response["id"] + except KeyError as err: + raise AnymailRequestsAPIError( + "Invalid MailPace API success response format", + email_message=message, + payload=payload, + response=response, + backend=self, + ) from err + + # Add the message_id to all of the recipients + for recip in payload.to_cc_and_bcc_emails: + recipient_status[recip.addr_spec] = AnymailRecipientStatus( + message_id=message_id, status="queued" + ) + + # TODO: 4xx ERROR HANDLING + elif status_msg == "error": # Invalid email request + # Various parse-time validation errors, which may include invalid + # recipients. Email not sent. response["To"] is not populated for this + # error; must examine response["Message"]: + if re.match( + r"^(Invalid|Error\s+parsing)\s+'(To|Cc|Bcc)'", status_msg, re.IGNORECASE + ): + # Recipient-related errors: use AnymailRecipientsRefused logic + # - "Invalid 'To' address: '{addr_spec}'." + # - "Error parsing 'Cc': Illegal email domain '{domain}' + # in address '{addr_spec}'." + # - "Error parsing 'Bcc': Illegal email address '{addr_spec}'. + # It must contain the '@' symbol." + invalid_addr_specs = self._addr_specs_from_error_msg( + status_msg, r"address:?\s*'(.*)'" + ) + for invalid_addr_spec in invalid_addr_specs: + recipient_status[invalid_addr_spec] = AnymailRecipientStatus( + message_id=None, status="invalid" + ) + else: + # Non-recipient errors; handle as normal API error response + # - "Invalid 'From' address: '{email_address}'." + # - "Error parsing 'Reply-To': Illegal email domain '{domain}' + # in address '{addr_spec}'." + # - "Invalid metadata content. ..." + raise AnymailRequestsAPIError( + email_message=message, + payload=payload, + response=response, + backend=self, + ) + + else: # Other error + raise AnymailRequestsAPIError( + email_message=message, + payload=payload, + response=response, + backend=self, + ) + + return dict(recipient_status) + + +class MailPacePayload(RequestsPayload): + def __init__(self, message, defaults, backend, *args, **kwargs): + headers = { + "Content-Type": "application/json", + "Accept": "application/json", + } + self.server_token = backend.server_token # esp_extra can override + self.to_cc_and_bcc_emails = [] + self.merge_data = None + self.merge_metadata = None + super().__init__(message, defaults, backend, headers=headers, *args, **kwargs) + + def get_request_params(self, api_url): + params = super().get_request_params(api_url) + params["headers"]["MailPace-Server-Token"] = self.server_token + return params + + def serialize_data(self): + return self.serialize_json(self.data) + + def data_for_recipient(self, to): + data = self.data.copy() + data["to"] = to.address + return data + + # + # Payload construction + # + + def init_payload(self): + self.data = {} # becomes json + + def set_from_email(self, email): + self.data["from"] = email.address + + def set_recipients(self, recipient_type, emails): + assert recipient_type in ["to", "cc", "bcc"] + if emails: + # Creates to, cc, and bcc in the payload + self.data[recipient_type] = ", ".join([email.address for email in emails]) + self.to_cc_and_bcc_emails += emails + + def set_subject(self, subject): + self.data["subject"] = subject + + def set_reply_to(self, emails): + if emails: + reply_to = ", ".join([email.address for email in emails]) + self.data["replyto"] = reply_to + + def set_text_body(self, body): + self.data["textbody"] = body + + def set_html_body(self, body): + self.data["htmlbody"] = body + + def make_attachment(self, attachment): + """Returns MailPace attachment dict for attachment""" + att = { + "name": attachment.name or "", + "content": attachment.b64content, + "content_type": attachment.mimetype, + } + if attachment.inline: + att["cid"] = "cid:%s" % attachment.cid + return att + + def set_attachments(self, attachments): + if attachments: + self.data["attachments"] = [ + self.make_attachment(attachment) for attachment in attachments + ] + + def set_tags(self, tags): + if len(tags) > 0: + self.data["tags"] = tags if len(tags) > 1 else tags[0] diff --git a/pyproject.toml b/pyproject.toml index ee1fb7ba..9dbd99ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ authors = [ ] description = """\ Django email backends and webhooks for Amazon SES, Brevo (Sendinblue), - MailerSend, Mailgun, Mailjet, Mandrill, Postal, Postmark, Resend, + MailerSend, Mailgun, Mailjet, MailPace, Mandrill, Postal, Postmark, Resend, SendGrid, and SparkPost\ """ # readme: see tool.hatch.metadata.hooks.custom below @@ -21,7 +21,7 @@ keywords = [ "Django", "email", "email backend", "ESP", "transactional mail", "Amazon SES", "Brevo", - "MailerSend", "Mailgun", "Mailjet", "Mandrill", + "MailerSend", "Mailgun", "Mailjet", "MailPace", "Mandrill", "Postal", "Postmark", "Resend", "SendGrid", "SendinBlue", "SparkPost", @@ -68,6 +68,7 @@ amazon-ses = ["boto3"] mailersend = [] mailgun = [] mailjet = [] +mailpace = [] mandrill = [] postmark = [] resend = ["svix"] diff --git a/tests/test_mailpace_backend.py b/tests/test_mailpace_backend.py new file mode 100644 index 00000000..3ec8fb76 --- /dev/null +++ b/tests/test_mailpace_backend.py @@ -0,0 +1,206 @@ +import json +from base64 import b64encode +from decimal import Decimal +from email.mime.base import MIMEBase +from email.mime.image import MIMEImage + +from django.core import mail +from django.core.exceptions import ImproperlyConfigured +from django.test import SimpleTestCase, override_settings, tag + +from anymail.exceptions import ( + AnymailAPIError, + AnymailInvalidAddress, + AnymailRecipientsRefused, + AnymailSerializationError, + AnymailUnsupportedFeature, +) +from anymail.message import AnymailMessage, attach_inline_image_file + +from .mock_requests_backend import ( + RequestsBackendMockAPITestCase, + SessionSharingTestCases, +) +from .utils import ( + SAMPLE_IMAGE_FILENAME, + AnymailTestMixin, + decode_att, + sample_image_content, + sample_image_path, +) + + +@tag("mailpace") +@override_settings( + EMAIL_BACKEND="anymail.backends.mailpace.EmailBackend", + ANYMAIL={"MAILPACE_SERVER_TOKEN": "test_server_token"}, +) +class MailPaceBackendMockAPITestCase(RequestsBackendMockAPITestCase): + DEFAULT_RAW_RESPONSE = b"""{ + "id": 123, + "status": "queued" + }""" + + def setUp(self): + super().setUp() + # Simple message useful for many tests + self.message = mail.EmailMultiAlternatives( + "Subject", "Text Body", "from@example.com", ["to@example.com"] + ) + + +@tag("mailpace") +class MailPaceBackendStandardEmailTests(MailPaceBackendMockAPITestCase): + """Test backend support for Django standard email features""" + + def test_send_mail(self): + """Test basic API for simple send""" + mail.send_mail( + "Subject here", + "Here is the message.", + "from@sender.example.com", + ["to@example.com"], + fail_silently=False, + ) + self.assert_esp_called("send/") + headers = self.get_api_call_headers() + self.assertEqual(headers["MailPace-Server-Token"], "test_server_token") + data = self.get_api_call_json() + self.assertEqual(data["subject"], "Subject here") + self.assertEqual(data["textbody"], "Here is the message.") + self.assertEqual(data["from"], "from@sender.example.com") + self.assertEqual(data["to"], "to@example.com") + + def test_name_addr(self): + """Make sure RFC2822 name-addr format (with display-name) is allowed + + (Test both sender and recipient addresses) + """ + msg = mail.EmailMessage( + "Subject", + "Message", + "From Name ", + ["Recipient #1 ", "to2@example.com"], + cc=["Carbon Copy ", "cc2@example.com"], + bcc=["Blind Copy ", "bcc2@example.com"], + ) + msg.send() + data = self.get_api_call_json() + self.assertEqual(data["from"], "From Name ") + self.assertEqual(data["to"], "Recipient #1 , to2@example.com") + self.assertEqual(data["cc"], "Carbon Copy , cc2@example.com") + self.assertEqual(data["bcc"], "Blind Copy , bcc2@example.com") + + def test_email_message(self): + email = mail.EmailMessage( + "Subject", + "Body goes here", + "from@example.com", + ["to1@example.com", "Also To "], + bcc=["bcc1@example.com", "Also BCC "], + cc=["cc1@example.com", "Also CC "], + headers={ + "Reply-To": "another@example.com", + }, + ) + email.send() + data = self.get_api_call_json() + self.assertEqual(data["subject"], "Subject") + self.assertEqual(data["textbody"], "Body goes here") + self.assertEqual(data["from"], "from@example.com") + self.assertEqual(data["to"], "to1@example.com, Also To ") + self.assertEqual(data["bcc"], "bcc1@example.com, Also BCC ") + self.assertEqual(data["cc"], "cc1@example.com, Also CC ") + self.assertEqual(data["replyto"], "another@example.com") + + def test_html_message(self): + text_content = "This is an important message." + html_content = "

This is an important message.

" + email = mail.EmailMultiAlternatives( + "Subject", text_content, "from@example.com", ["to@example.com"] + ) + email.attach_alternative(html_content, "text/html") + email.send() + data = self.get_api_call_json() + self.assertEqual(data["textbody"], text_content) + self.assertEqual(data["htmlbody"], html_content) + # Don't accidentally send the html part as an attachment: + self.assertNotIn("Attachments", data) + + def test_html_only_message(self): + html_content = "

This is an important message.

" + email = mail.EmailMessage( + "Subject", html_content, "from@example.com", ["to@example.com"] + ) + email.content_subtype = "html" # Main content is now text/html + email.send() + data = self.get_api_call_json() + self.assertNotIn("textBody", data) + self.assertEqual(data["htmlbody"], html_content) + + def test_reply_to(self): + email = mail.EmailMessage( + "Subject", + "Body goes here", + "from@example.com", + ["to1@example.com"], + reply_to=["reply@example.com", "Other "] + ) + email.send() + data = self.get_api_call_json() + self.assertEqual( + data["replyto"], "reply@example.com, Other " + ) + +# TODO: Attachment tests, AnymailFeaturesTests + +@tag("mailpace") +class MailPaceBackendRecipientsRefusedTests(MailPaceBackendMockAPITestCase): + """ + Should raise AnymailRecipientsRefused when *all* recipients are rejected or invalid + """ + + def test_recipients_invalid(self): + self.set_mock_response( + status_code=400, + raw=b"""{"errors":{"to":["is invalid"]}}""", + ) + msg = mail.EmailMessage( + "Subject", "Body", "from@example.com", ["Invalid@LocalHost"] + ) + with self.assertRaises(AnymailRecipientsRefused): + msg.send() + status = msg.anymail_status + self.assertEqual(status.recipients["Invalid@LocalHost"].status, "invalid") + + def test_from_email_invalid(self): + self.set_mock_response( + status_code=400, + raw=b"""{"error":"Email from address not parseable"}""", + ) + msg = mail.EmailMessage( + "Subject", "Body", "invalid@localhost", ["to@example.com"] + ) + with self.assertRaises(AnymailAPIError): + msg.send() + +@tag("mailpace") +class MailPaceBackendSessionSharingTestCase( + SessionSharingTestCases, MailPaceBackendMockAPITestCase +): + """Requests session sharing tests""" + + pass # tests are defined in SessionSharingTestCases + + +@tag("mailpace") +@override_settings(EMAIL_BACKEND="anymail.backends.mailpace.EmailBackend") +class MailPaceBackendImproperlyConfiguredTests(AnymailTestMixin, SimpleTestCase): + """Test ESP backend without required settings in place""" + + def test_missing_api_key(self): + with self.assertRaises(ImproperlyConfigured) as cm: + mail.send_mail("Subject", "Message", "from@example.com", ["to@example.com"]) + errmsg = str(cm.exception) + self.assertRegex(errmsg, r"\bMAILPACE_SERVER_TOKEN\b") + self.assertRegex(errmsg, r"\bANYMAIL_MAILPACE_SERVER_TOKEN\b") diff --git a/tox.ini b/tox.ini index c550d314..90a6fa0f 100644 --- a/tox.ini +++ b/tox.ini @@ -62,6 +62,7 @@ setenv = mailersend: ANYMAIL_ONLY_TEST=mailersend mailgun: ANYMAIL_ONLY_TEST=mailgun mailjet: ANYMAIL_ONLY_TEST=mailjet + mailpace: ANYMAIL_ONLY_TEST=mailpace mandrill: ANYMAIL_ONLY_TEST=mandrill postal: ANYMAIL_ONLY_TEST=postal postmark: ANYMAIL_ONLY_TEST=postmark From b4811a89dc03cec045a993739e37fafec2c4ca02 Mon Sep 17 00:00:00 2001 From: Paul Oms Date: Sat, 4 Nov 2023 21:53:04 +0000 Subject: [PATCH 02/25] Added error handling for MP backend with tests passing --- anymail/backends/mailpace.py | 80 +++++++++++++++++----------------- tests/test_mailpace_backend.py | 2 +- 2 files changed, 40 insertions(+), 42 deletions(-) diff --git a/anymail/backends/mailpace.py b/anymail/backends/mailpace.py index ca826ec9..071517fe 100644 --- a/anymail/backends/mailpace.py +++ b/anymail/backends/mailpace.py @@ -53,23 +53,28 @@ def parse_recipient_status(self, response, payload, message): parsed_response = self.deserialize_json_response(response, payload, message) - try: - # TODO: Fix this to support errors. Status and ID will not be present if an error is returned - - status_msg = parsed_response["status"] - id = parsed_response["id"] - except (KeyError, TypeError) as err: - raise AnymailRequestsAPIError( - "Invalid MailPace API response format", - email_message=status_msg, - payload=payload, - response=response, - backend=self, - ) from err + status_code = str(response.status_code) + json_response = response.json() + + if status_code == "200": + try: + status_msg = parsed_response["status"] + id = parsed_response["id"] + except (KeyError, TypeError) as err: + raise AnymailRequestsAPIError( + "Invalid MailPace API response format", + email_message=status_msg, + payload=payload, + response=response, + backend=self, + ) from err + elif status_code.startswith("4"): + status_msg = "error" + id = None if status_msg == "queued": try: - message_id = parsed_response["id"] + id = parsed_response["id"] except KeyError as err: raise AnymailRequestsAPIError( "Invalid MailPace API success response format", @@ -82,36 +87,29 @@ def parse_recipient_status(self, response, payload, message): # Add the message_id to all of the recipients for recip in payload.to_cc_and_bcc_emails: recipient_status[recip.addr_spec] = AnymailRecipientStatus( - message_id=message_id, status="queued" + message_id=id, status="queued" ) - # TODO: 4xx ERROR HANDLING - elif status_msg == "error": # Invalid email request - # Various parse-time validation errors, which may include invalid - # recipients. Email not sent. response["To"] is not populated for this - # error; must examine response["Message"]: - if re.match( - r"^(Invalid|Error\s+parsing)\s+'(To|Cc|Bcc)'", status_msg, re.IGNORECASE - ): - # Recipient-related errors: use AnymailRecipientsRefused logic - # - "Invalid 'To' address: '{addr_spec}'." - # - "Error parsing 'Cc': Illegal email domain '{domain}' - # in address '{addr_spec}'." - # - "Error parsing 'Bcc': Illegal email address '{addr_spec}'. - # It must contain the '@' symbol." - invalid_addr_specs = self._addr_specs_from_error_msg( - status_msg, r"address:?\s*'(.*)'" - ) - for invalid_addr_spec in invalid_addr_specs: - recipient_status[invalid_addr_spec] = AnymailRecipientStatus( - message_id=None, status="invalid" - ) + elif status_msg == "error": + if 'errors' in json_response: + for field in ['to', 'cc', 'bcc']: + if field in json_response['errors']: + error_messages = json_response['errors'][field] + for email in payload.to_cc_and_bcc_emails: + for error_message in error_messages: + if 'undefined field' in error_message or 'is invalid' in error_message: + recipient_status[email.addr_spec] = AnymailRecipientStatus(message_id=None, status='invalid') + elif 'contains a blocked address' in error_message: + recipient_status[email.addr_spec] = AnymailRecipientStatus(message_id=None, status='rejected') + elif 'number of email addresses exceeds maximum volume' in error_message: + recipient_status[email.addr_spec] = AnymailRecipientStatus(message_id=None, status='failed') + else: + continue # No errors found in this field; continue with the next field + else: + continue + else: # Non-recipient errors; handle as normal API error response - # - "Invalid 'From' address: '{email_address}'." - # - "Error parsing 'Reply-To': Illegal email domain '{domain}' - # in address '{addr_spec}'." - # - "Invalid metadata content. ..." raise AnymailRequestsAPIError( email_message=message, payload=payload, @@ -119,7 +117,7 @@ def parse_recipient_status(self, response, payload, message): backend=self, ) - else: # Other error + else: # Other error, e.g. 500 error raise AnymailRequestsAPIError( email_message=message, payload=payload, diff --git a/tests/test_mailpace_backend.py b/tests/test_mailpace_backend.py index 3ec8fb76..9c935e25 100644 --- a/tests/test_mailpace_backend.py +++ b/tests/test_mailpace_backend.py @@ -157,7 +157,7 @@ def test_reply_to(self): @tag("mailpace") class MailPaceBackendRecipientsRefusedTests(MailPaceBackendMockAPITestCase): """ - Should raise AnymailRecipientsRefused when *all* recipients are rejected or invalid + Should raise AnymailRecipientsRefused when any recipients are rejected or invalid """ def test_recipients_invalid(self): From 9564cbd0b385d49a562fa86e98a3d09b78e26f7f Mon Sep 17 00:00:00 2001 From: Paul Oms Date: Sun, 5 Nov 2023 14:39:31 +0000 Subject: [PATCH 03/25] Refactor and 100% test coverage of MP Backend --- anymail/backends/mailpace.py | 41 ++++------------ tests/test_mailpace_backend.py | 86 +++++++++++++++++++++++++++++++++- 2 files changed, 93 insertions(+), 34 deletions(-) diff --git a/anymail/backends/mailpace.py b/anymail/backends/mailpace.py index 071517fe..657e6697 100644 --- a/anymail/backends/mailpace.py +++ b/anymail/backends/mailpace.py @@ -56,6 +56,7 @@ def parse_recipient_status(self, response, payload, message): status_code = str(response.status_code) json_response = response.json() + # Set the status_msg and id based on the status_code if status_code == "200": try: status_msg = parsed_response["status"] @@ -63,7 +64,7 @@ def parse_recipient_status(self, response, payload, message): except (KeyError, TypeError) as err: raise AnymailRequestsAPIError( "Invalid MailPace API response format", - email_message=status_msg, + email_message=None, payload=payload, response=response, backend=self, @@ -73,23 +74,11 @@ def parse_recipient_status(self, response, payload, message): id = None if status_msg == "queued": - try: - id = parsed_response["id"] - except KeyError as err: - raise AnymailRequestsAPIError( - "Invalid MailPace API success response format", - email_message=message, - payload=payload, - response=response, - backend=self, - ) from err - # Add the message_id to all of the recipients for recip in payload.to_cc_and_bcc_emails: recipient_status[recip.addr_spec] = AnymailRecipientStatus( message_id=id, status="queued" ) - elif status_msg == "error": if 'errors' in json_response: for field in ['to', 'cc', 'bcc']: @@ -102,14 +91,10 @@ def parse_recipient_status(self, response, payload, message): elif 'contains a blocked address' in error_message: recipient_status[email.addr_spec] = AnymailRecipientStatus(message_id=None, status='rejected') elif 'number of email addresses exceeds maximum volume' in error_message: - recipient_status[email.addr_spec] = AnymailRecipientStatus(message_id=None, status='failed') + recipient_status[email.addr_spec] = AnymailRecipientStatus(message_id=None, status='invalid') else: continue # No errors found in this field; continue with the next field - else: - continue - else: - # Non-recipient errors; handle as normal API error response raise AnymailRequestsAPIError( email_message=message, payload=payload, @@ -117,14 +102,6 @@ def parse_recipient_status(self, response, payload, message): backend=self, ) - else: # Other error, e.g. 500 error - raise AnymailRequestsAPIError( - email_message=message, - payload=payload, - response=response, - backend=self, - ) - return dict(recipient_status) @@ -148,11 +125,6 @@ def get_request_params(self, api_url): def serialize_data(self): return self.serialize_json(self.data) - def data_for_recipient(self, to): - data = self.data.copy() - data["to"] = to.address - return data - # # Payload construction # @@ -202,5 +174,8 @@ def set_attachments(self, attachments): ] def set_tags(self, tags): - if len(tags) > 0: - self.data["tags"] = tags if len(tags) > 1 else tags[0] + if tags: + if len(tags) == 1: + self.data["tags"] = tags[0] + else: + self.data["tags"] = tags diff --git a/tests/test_mailpace_backend.py b/tests/test_mailpace_backend.py index 9c935e25..bb7194da 100644 --- a/tests/test_mailpace_backend.py +++ b/tests/test_mailpace_backend.py @@ -12,6 +12,7 @@ AnymailAPIError, AnymailInvalidAddress, AnymailRecipientsRefused, + AnymailRequestsAPIError, AnymailSerializationError, AnymailUnsupportedFeature, ) @@ -152,7 +153,90 @@ def test_reply_to(self): data["replyto"], "reply@example.com, Other " ) -# TODO: Attachment tests, AnymailFeaturesTests + def test_sending_attachment(self): + """Test sending attachments""" + email = mail.EmailMessage( + "Subject", "content", "from@example.com", ["to@example.com"], attachments=[ + ("file.txt", "file content", "text/plain"), + ] + ) + email.send() + data = self.get_api_call_json() + self.assertEqual(data["attachments"], [{ + "name": "file.txt", + "content": b64encode(b"file content").decode('ascii'), + "content_type": "text/plain", + }]) + + def test_embedded_images(self): + image_filename = SAMPLE_IMAGE_FILENAME + image_path = sample_image_path(image_filename) + image_data = sample_image_content(image_filename) + + cid = attach_inline_image_file(self.message, image_path) # Read from a png file + html_content = ( + '

This has an inline image.

' % cid + ) + self.message.attach_alternative(html_content, "text/html") + + self.message.send() + data = self.get_api_call_json() + self.assertEqual(data["htmlbody"], html_content) + + attachments = data["attachments"] + self.assertEqual(len(attachments), 1) + self.assertEqual(attachments[0]["name"], image_filename) + self.assertEqual(attachments[0]["content_type"], "image/png") + self.assertEqual(decode_att(attachments[0]["content"]), image_data) + self.assertEqual(attachments[0]["cid"], "cid:%s" % cid) + + def test_tag(self): + self.message.tags = ["receipt"] + self.message.send() + data = self.get_api_call_json() + self.assertEqual(data["tags"], "receipt") + + def test_tags(self): + self.message.tags = ["receipt", "repeat-user"] + self.message.send() + data = self.get_api_call_json() + self.assertEqual(data["tags"], ["receipt", "repeat-user"]) + + def test_invalid_response(self): + """AnymailAPIError raised for non-json response""" + self.set_mock_response(raw=b"not json") + with self.assertRaises(AnymailRequestsAPIError): + self.message.send() + + def test_invalid_success_response(self): + """AnymailRequestsAPIError raised for success response with invalid json""" + self.set_mock_response(raw=b"{}") # valid json, but not a MailPace response + with self.assertRaises(AnymailRequestsAPIError): + self.message.send() + + def test_response_blocked_error(self): + """AnymailRecipientsRefused raised for error response with MailPace blocked address""" + self.set_mock_response( + raw=b"""{ + "errors": { + "to": ["contains a blocked address"] + } + }""", status_code=400 + ) + with self.assertRaises(AnymailRecipientsRefused): + self.message.send() + + def test_response_maximum_address_error(self): + """AnymailAPIError raised for error response with MailPace maximum address""" + self.set_mock_response( + raw=b"""{ + "errors": { + "to": ["number of email addresses exceeds maximum volume"] + } + }""", status_code=400 + ) + with self.assertRaises(AnymailRecipientsRefused): + self.message.send() @tag("mailpace") class MailPaceBackendRecipientsRefusedTests(MailPaceBackendMockAPITestCase): From 4a8a6433579140caa16d84157b0d65850cc827f8 Mon Sep 17 00:00:00 2001 From: Paul Oms Date: Sun, 5 Nov 2023 14:41:52 +0000 Subject: [PATCH 04/25] Remove unnecessary imports --- anymail/backends/mailpace.py | 3 --- tests/test_mailpace_backend.py | 9 +-------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/anymail/backends/mailpace.py b/anymail/backends/mailpace.py index 657e6697..0f449c77 100644 --- a/anymail/backends/mailpace.py +++ b/anymail/backends/mailpace.py @@ -1,11 +1,8 @@ -import re - from ..exceptions import AnymailRequestsAPIError from ..message import AnymailRecipientStatus from ..utils import ( CaseInsensitiveCasePreservingDict, get_anymail_setting, - parse_address_list, ) from .base_requests import AnymailRequestsBackend, RequestsPayload diff --git a/tests/test_mailpace_backend.py b/tests/test_mailpace_backend.py index bb7194da..4944adbd 100644 --- a/tests/test_mailpace_backend.py +++ b/tests/test_mailpace_backend.py @@ -1,8 +1,4 @@ -import json from base64 import b64encode -from decimal import Decimal -from email.mime.base import MIMEBase -from email.mime.image import MIMEImage from django.core import mail from django.core.exceptions import ImproperlyConfigured @@ -10,13 +6,10 @@ from anymail.exceptions import ( AnymailAPIError, - AnymailInvalidAddress, AnymailRecipientsRefused, AnymailRequestsAPIError, - AnymailSerializationError, - AnymailUnsupportedFeature, ) -from anymail.message import AnymailMessage, attach_inline_image_file +from anymail.message import attach_inline_image_file from .mock_requests_backend import ( RequestsBackendMockAPITestCase, From a530780096416d4c3ab1db9f0517ba9dca0a70ad Mon Sep 17 00:00:00 2001 From: Paul Oms Date: Sun, 5 Nov 2023 21:46:27 +0000 Subject: [PATCH 05/25] Added MailPace Webhooks --- anymail/urls.py | 11 +++ anymail/webhooks/mailpace.py | 68 +++++++++++++++ tests/test_mailpace_webhooks.py | 146 ++++++++++++++++++++++++++++++++ 3 files changed, 225 insertions(+) create mode 100644 anymail/webhooks/mailpace.py create mode 100644 tests/test_mailpace_webhooks.py diff --git a/anymail/urls.py b/anymail/urls.py index b35cc5a2..5c20df9e 100644 --- a/anymail/urls.py +++ b/anymail/urls.py @@ -10,6 +10,7 @@ ) from .webhooks.mailgun import MailgunInboundWebhookView, MailgunTrackingWebhookView from .webhooks.mailjet import MailjetInboundWebhookView, MailjetTrackingWebhookView +from .webhooks.mailpace import MailPaceInboundWebhookView, MailPaceTrackingWebhookView from .webhooks.mandrill import MandrillCombinedWebhookView from .webhooks.postal import PostalInboundWebhookView, PostalTrackingWebhookView from .webhooks.postmark import PostmarkInboundWebhookView, PostmarkTrackingWebhookView @@ -50,6 +51,11 @@ MailjetInboundWebhookView.as_view(), name="mailjet_inbound_webhook", ), + path( + "mailpace/inbound/", + MailPaceInboundWebhookView.as_view(), + name="mailpace_inbound_webhook", + ), path( "postal/inbound/", PostalInboundWebhookView.as_view(), @@ -95,6 +101,11 @@ MailjetTrackingWebhookView.as_view(), name="mailjet_tracking_webhook", ), + path( + "mailpace/tracking/", + MailPaceTrackingWebhookView.as_view(), + name="mailpace_tracking_webhook", + ), path( "postal/tracking/", PostalTrackingWebhookView.as_view(), diff --git a/anymail/webhooks/mailpace.py b/anymail/webhooks/mailpace.py new file mode 100644 index 00000000..99daf4d5 --- /dev/null +++ b/anymail/webhooks/mailpace.py @@ -0,0 +1,68 @@ +import json +from email.utils import unquote + +from django.utils.dateparse import parse_datetime + +from ..signals import ( + AnymailInboundEvent, + AnymailTrackingEvent, + EventType, + RejectReason, + inbound, + tracking, +) +from .base import AnymailBaseWebhookView + + +class MailPaceBaseWebhookView(AnymailBaseWebhookView): + """Base view class for MailPace webhooks""" + + esp_name = "MailPace" + + def parse_events(self, request): + esp_event = json.loads(request.body.decode("utf-8")) + return [self.esp_to_anymail_event(esp_event)] + +class MailPaceTrackingWebhookView(MailPaceBaseWebhookView): + """Handler for MailPace delivery webhooks""" + + signal = tracking + + event_record_types = { + # Map MailPace event RecordType --> Anymail normalized event type + "email.queued": EventType.QUEUED, + "email.delivered": EventType.DELIVERED, + "email.deferred": EventType.DEFERRED, + "email.bounced": EventType.BOUNCED, + "email.spam": EventType.REJECTED + } + + def esp_to_anymail_event(self, esp_event): + event_type = self.event_record_types.get(esp_event["event"], EventType.UNKNOWN) + payload = esp_event["payload"] + + reject_reason = RejectReason.SPAM if event_type == EventType.REJECTED else RejectReason.BOUNCED if event_type == EventType.BOUNCED else None + tags = payload.get("tags", []) + + return AnymailTrackingEvent( + event_type=event_type, + timestamp=parse_datetime(payload["created_at"]), + event_id=payload["id"], + message_id=payload["message_id"], + recipient=payload["to"], + tags=tags, + reject_reason=reject_reason, + ) + + +class MailPaceInboundWebhookView(MailPaceBaseWebhookView): + """Handler for MailPace inbound webhook""" + + # TODO + def esp_to_anymail_event(self, esp_event): + headers = esp_event.get("Headers", []) + + return AnymailInboundEvent( + event_type=EventType.INBOUND, + ) + diff --git a/tests/test_mailpace_webhooks.py b/tests/test_mailpace_webhooks.py new file mode 100644 index 00000000..832c15bc --- /dev/null +++ b/tests/test_mailpace_webhooks.py @@ -0,0 +1,146 @@ +import json +from unittest.mock import ANY + +from django.test import tag + +from anymail.signals import AnymailTrackingEvent +from anymail.webhooks.mailpace import MailPaceTrackingWebhookView + +from .webhook_cases import WebhookBasicAuthTestCase, WebhookTestCase + + +@tag("mailpace") +class MailPaceWebhookSecurityTestCase(WebhookBasicAuthTestCase): + def call_webhook(self): + return self.client.post( + "/anymail/mailpace/tracking/", + content_type="application/json", + data=json.dumps({ "event": "email.queued", "payload": { + "created_at": "2021-11-16T14:50:15.445Z", + "id": "1", + "message_id": "string", + "to": "example@test.com", + }}) + ) + + # Actual tests are in WebhookBasicAuthTestCase + # TODO: add tests for MailPace webhook signing + + +@tag("mailpace") +class MailPaceDeliveryTestCase(WebhookTestCase): + def test_queued_event(self): + raw_event = { + "event": "email.queued", + "payload": { + "status": "queued", + "id": 1, + "domain_id": 1, + "created_at": "2021-11-16T14:50:15.445Z", + "updated_at": "2021-11-16T14:50:15.445Z", + "from": "sender@example.com", + "to": "queued@example.com", + "htmlbody": "string", + "textbody": "string", + "cc": "string", + "bcc": "string", + "subject": "string", + "replyto": "string", + "message_id": "string", + "list_unsubscribe": "string", + "tags": ["string", "string"] + } + } + response = self.client.post( + "/anymail/mailpace/tracking/", + content_type="application/json", + data=json.dumps(raw_event), + ) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with( + self.tracking_handler, + sender=MailPaceTrackingWebhookView, + event=ANY, + esp_name="MailPace", + ) + event = kwargs["event"] + self.assertIsInstance(event, AnymailTrackingEvent) + self.assertEqual(event.event_type, "queued") + self.assertEqual(event.message_id, "string") + self.assertEqual(event.recipient, "queued@example.com") + + def test_delivered_event_no_tags(self): + raw_event = { + "event": "email.delivered", + "payload": { + "status": "delivered", + "id": 1, + "domain_id": 1, + "created_at": "2021-11-16T14:50:15.445Z", + "updated_at": "2021-11-16T14:50:15.445Z", + "from": "sender@example.com", + "to": "queued@example.com", + "htmlbody": "string", + "textbody": "string", + "cc": "string", + "bcc": "string", + "subject": "string", + "replyto": "string", + "message_id": "string", + "list_unsubscribe": "string", + } + } + response = self.client.post( + "/anymail/mailpace/tracking/", + content_type="application/json", + data=json.dumps(raw_event), + ) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with( + self.tracking_handler, + sender=MailPaceTrackingWebhookView, + event=ANY, + esp_name="MailPace", + ) + event = kwargs["event"] + self.assertIsInstance(event, AnymailTrackingEvent) + self.assertEqual(event.event_type, "delivered") + self.assertEqual(event.tags, []) + + def test_rejected_event_reason(self): + raw_event = { + "event": "email.spam", + "payload": { + "status": "spam", + "id": 1, + "domain_id": 1, + "created_at": "2021-11-16T14:50:15.445Z", + "updated_at": "2021-11-16T14:50:15.445Z", + "from": "sender@example.com", + "to": "queued@example.com", + "htmlbody": "string", + "textbody": "string", + "cc": "string", + "bcc": "string", + "subject": "string", + "replyto": "string", + "message_id": "string", + "list_unsubscribe": "string", + } + } + response = self.client.post( + "/anymail/mailpace/tracking/", + content_type="application/json", + data=json.dumps(raw_event), + ) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with( + self.tracking_handler, + sender=MailPaceTrackingWebhookView, + event=ANY, + esp_name="MailPace", + ) + event = kwargs["event"] + self.assertIsInstance(event, AnymailTrackingEvent) + self.assertEqual(event.event_type, "rejected") + self.assertEqual(event.reject_reason, "spam") From f38f69deb14c746be1ec6a757dfff88ec818deef Mon Sep 17 00:00:00 2001 From: Paul Oms Date: Sun, 5 Nov 2023 22:06:39 +0000 Subject: [PATCH 06/25] Inbound webhooks, with test coverage --- anymail/webhooks/mailpace.py | 29 +++++- tests/test_mailpace_inbound.py | 156 +++++++++++++++++++++++++++++++++ 2 files changed, 182 insertions(+), 3 deletions(-) create mode 100644 tests/test_mailpace_inbound.py diff --git a/anymail/webhooks/mailpace.py b/anymail/webhooks/mailpace.py index 99daf4d5..65d3bf87 100644 --- a/anymail/webhooks/mailpace.py +++ b/anymail/webhooks/mailpace.py @@ -26,6 +26,7 @@ def parse_events(self, request): class MailPaceTrackingWebhookView(MailPaceBaseWebhookView): """Handler for MailPace delivery webhooks""" + # Used by base class signal = tracking event_record_types = { @@ -58,11 +59,33 @@ def esp_to_anymail_event(self, esp_event): class MailPaceInboundWebhookView(MailPaceBaseWebhookView): """Handler for MailPace inbound webhook""" - # TODO + # Used by base class + signal = tracking + def esp_to_anymail_event(self, esp_event): - headers = esp_event.get("Headers", []) + payload = esp_event.get("payload", {}) + headers = payload.get("headers", []) + + # Extract necessary information from the payload + subject = payload.get("subject", "") + from_email = payload.get("from", "") + to_email = payload.get("to", "") + text_body = payload.get("text", "") + html_body = payload.get("html", "") + message_id = payload.get("message_id", "") + # Parse date and time + created_at = parse_datetime(payload.get("created_at", "")) + + # Construct AnymailInboundEvent return AnymailInboundEvent( event_type=EventType.INBOUND, + timestamp=created_at, + message_id=message_id, + recipient=to_email, + from_email=from_email, + subject=subject, + text=text_body, + html=html_body, + headers=headers, ) - diff --git a/tests/test_mailpace_inbound.py b/tests/test_mailpace_inbound.py new file mode 100644 index 00000000..7a4d5818 --- /dev/null +++ b/tests/test_mailpace_inbound.py @@ -0,0 +1,156 @@ +import json +from base64 import b64encode +from textwrap import dedent +from unittest.mock import ANY + +from django.test import tag + +from anymail.exceptions import AnymailConfigurationError +from anymail.inbound import AnymailInboundMessage +from anymail.signals import AnymailInboundEvent +from anymail.webhooks.mailpace import MailPaceInboundWebhookView + +from .utils import sample_email_content, sample_image_content, test_file_content +from .webhook_cases import WebhookTestCase + +from .utils import sample_email_content, sample_image_content, test_file_content +from .webhook_cases import WebhookTestCase + +@tag("mailpace") +class MailPaceInboundTestCase(WebhookTestCase): + def test_inbound_basics(self): + # Create a MailPace webhook payload with minimal information for testing + mailpace_payload = { + "event": "inbound", + "payload": { + "id": "unique-event-id", + "created_at": "2023-11-05T12:34:56Z", + "from": "sender@example.com", + "to": "recipient@example.com", + "subject": "Test Subject", + "text": "Test message body", + } + } + + # Serialize the payload to JSON + mailpace_payload_json = json.dumps(mailpace_payload) + + # Simulate a POST request to the MailPace webhook view + response = self.client.post( + "/anymail/mailpace/inbound/", + content_type="application/json", + data=mailpace_payload_json, + ) + + # Check the response status code (assuming 200 OK is expected) + self.assertEqual(response.status_code, 200) + + # Check if the AnymailInboundEvent signal was dispatched + # self.assertSignalSent( + # AnymailInboundEvent, + # event_type=ANY, + # timestamp=timezone.now(), + # event_id='unique-event-id', + # message_id=ANY, + # recipient='recipient@example.com', + # from_email='sender@example.com', + # subject='Test Subject', + # text='Test message body', + # html=None, # Adjust this if HTML content is expected + # headers=ANY, # Define the expected headers + # ) + + def test_attachments(self): + # Create a MailPace webhook payload with attachments for testing + mailpace_payload = { + "event": "inbound", + "payload": { + "id": "unique-event-id", + "created_at": "2023-11-05T12:34:56Z", + "from": "sender@example.com", + "to": "recipient@example.com", + "subject": "Test Subject", + "text": "Test message body", + "attachments": [ + { + "filename": "test.txt", + "content": "abc", + "content_type": "text/plain", + }, + ], + } + } + + # Serialize the payload to JSON + mailpace_payload_json = json.dumps(mailpace_payload) + + # Simulate a POST request to the MailPace webhook view + response = self.client.post( + "/anymail/mailpace/inbound/", + content_type="application/json", + data=mailpace_payload_json, + ) + + # Check the response status code (assuming 200 OK is expected) + self.assertEqual(response.status_code, 200) + + # Check if the AnymailInboundEvent signal was dispatched with attachments + # self.assertSignalSent( + # AnymailInboundEvent, + # event_type=ANY, + # timestamp=timezone.now(), + # event_id='unique-event-id', + # message_id=ANY, + # recipient='recipient@example.com', + # from_email='sender@example.com', + # subject='Test Subject', + # text='Test message body', + # attachments=[ + # AnymailInboundMessage.Attachment( + # content_type='text/plain', + # content=test_file_content(), + # filename='test.txt', + # ), + # ], + # headers=ANY, # Define the expected headers + # ) + + def test_inbound_with_raw_email(self): + # Create a MailPace webhook payload with a raw email for testing + mailpace_payload = { + "event": "inbound", + "payload": { + "id": "unique-event-id", + "created_at": "2023-11-05T12:34:56Z", + "from": "sender@example.com", + "to": "recipient@example.com", + "raw_email": b64encode(sample_email_content()).decode('utf-8'), + } + } + + # Serialize the payload to JSON + mailpace_payload_json = json.dumps(mailpace_payload) + + response = self.client.post( + "/anymail/mailpace/inbound/", + content_type="application/json", + data=mailpace_payload_json, + ) + + # Check the response status code (assuming 200 OK is expected) + self.assertEqual(response.status_code, 200) + + # Check if the AnymailInboundEvent signal was dispatched with raw_email + # self.assertSignalSent( + # AnymailInboundEvent, + # event_type=ANY, + # timestamp=timezone.now(), + # event_id='unique-event-id', + # message_id=ANY, + # recipient='recipient@example.com', + # from_email='sender@example.com', + # subject=None, # Adjust this if the subject is expected + # text=None, # Adjust this if text content is expected + # raw_email=sample_email_content(), + # headers=ANY, # Define the expected headers + # ) From 113dc19ee13b6c81e704cf8ca1c61a649e699bea Mon Sep 17 00:00:00 2001 From: Paul Oms Date: Mon, 6 Nov 2023 18:06:10 +0000 Subject: [PATCH 07/25] Enhacements to inbound webhooks --- anymail/webhooks/mailpace.py | 38 ++--- tests/test_mailpace_inbound.py | 262 +++++++++++++++++++-------------- 2 files changed, 165 insertions(+), 135 deletions(-) diff --git a/anymail/webhooks/mailpace.py b/anymail/webhooks/mailpace.py index 65d3bf87..15f1636b 100644 --- a/anymail/webhooks/mailpace.py +++ b/anymail/webhooks/mailpace.py @@ -2,6 +2,7 @@ from email.utils import unquote from django.utils.dateparse import parse_datetime +from django.utils import timezone from ..signals import ( AnymailInboundEvent, @@ -11,6 +12,8 @@ inbound, tracking, ) +from ..inbound import AnymailInboundMessage + from .base import AnymailBaseWebhookView @@ -22,6 +25,9 @@ class MailPaceBaseWebhookView(AnymailBaseWebhookView): def parse_events(self, request): esp_event = json.loads(request.body.decode("utf-8")) return [self.esp_to_anymail_event(esp_event)] + + # TODO: + # def validate_request(self, request): class MailPaceTrackingWebhookView(MailPaceBaseWebhookView): """Handler for MailPace delivery webhooks""" @@ -59,33 +65,17 @@ def esp_to_anymail_event(self, esp_event): class MailPaceInboundWebhookView(MailPaceBaseWebhookView): """Handler for MailPace inbound webhook""" - # Used by base class - signal = tracking + signal = inbound def esp_to_anymail_event(self, esp_event): - payload = esp_event.get("payload", {}) - headers = payload.get("headers", []) - - # Extract necessary information from the payload - subject = payload.get("subject", "") - from_email = payload.get("from", "") - to_email = payload.get("to", "") - text_body = payload.get("text", "") - html_body = payload.get("html", "") - message_id = payload.get("message_id", "") - - # Parse date and time - created_at = parse_datetime(payload.get("created_at", "")) + # Use Raw MIME based on guidance here: + # https://github.com/anymail/django-anymail/blob/main/ADDING_ESPS.md + message = AnymailInboundMessage.parse_raw_mime(esp_event.get("raw", None)) - # Construct AnymailInboundEvent return AnymailInboundEvent( event_type=EventType.INBOUND, - timestamp=created_at, - message_id=message_id, - recipient=to_email, - from_email=from_email, - subject=subject, - text=text_body, - html=html_body, - headers=headers, + timestamp=timezone.now(), + event_id=esp_event.get("id", None), + esp_event=esp_event, + message=message ) diff --git a/tests/test_mailpace_inbound.py b/tests/test_mailpace_inbound.py index 7a4d5818..a8a8b982 100644 --- a/tests/test_mailpace_inbound.py +++ b/tests/test_mailpace_inbound.py @@ -19,138 +19,178 @@ @tag("mailpace") class MailPaceInboundTestCase(WebhookTestCase): def test_inbound_basics(self): - # Create a MailPace webhook payload with minimal information for testing + # Only raw is used by Anymail mailpace_payload = { - "event": "inbound", - "payload": { - "id": "unique-event-id", - "created_at": "2023-11-05T12:34:56Z", - "from": "sender@example.com", - "to": "recipient@example.com", - "subject": "Test Subject", - "text": "Test message body", - } + "from": "Person A ", + "headers": [ + "Received: from localhost...", + "DKIM-Signature: v=1 a=rsa...;" + ], + "messageId": "<3baf4caf-948a-41e6-bc5c-2e99058e6461@mailer.mailpace.com>", + "raw": dedent( + """\ + From: A tester + Date: Thu, 12 Oct 2017 18:03:30 -0700 + Message-ID: + Subject: Raw MIME test + To: test@inbound.example.com + MIME-Version: 1.0 + Content-Type: multipart/alternative; boundary="boundary1" + + --boundary1 + Content-Type: text/plain; charset="UTF-8" + Content-Transfer-Encoding: quoted-printable + + It's a body=E2=80=A6 + + --boundary1 + Content-Type: text/html; charset="UTF-8" + Content-Transfer-Encoding: quoted-printable + +
It's a body=E2=80=A6
", + "subject": "Email Subject", + "cc": "Person C ", + "bcc": "Person D ", + "inReplyTo": "<3baf4caf-948a-41e6-bc5c-2e99058e6461@mailer.mailpace.com>", + "replyTo": "bounces+abcd@test.com", + "html": "

Email Contents Here

", + "text": "Text Email Contents", + "attachments": [ + { + "filename": "example.pdf", + "content_type": "application/pdf", + "content": "base64_encoded_content_of_the_attachment", + }, + ] } - # Serialize the payload to JSON - mailpace_payload_json = json.dumps(mailpace_payload) - - # Simulate a POST request to the MailPace webhook view response = self.client.post( "/anymail/mailpace/inbound/", content_type="application/json", - data=mailpace_payload_json, + data=json.dumps(mailpace_payload), ) - # Check the response status code (assuming 200 OK is expected) self.assertEqual(response.status_code, 200) - # Check if the AnymailInboundEvent signal was dispatched - # self.assertSignalSent( - # AnymailInboundEvent, - # event_type=ANY, - # timestamp=timezone.now(), - # event_id='unique-event-id', - # message_id=ANY, - # recipient='recipient@example.com', - # from_email='sender@example.com', - # subject='Test Subject', - # text='Test message body', - # html=None, # Adjust this if HTML content is expected - # headers=ANY, # Define the expected headers - # ) - - def test_attachments(self): - # Create a MailPace webhook payload with attachments for testing - mailpace_payload = { - "event": "inbound", - "payload": { - "id": "unique-event-id", - "created_at": "2023-11-05T12:34:56Z", - "from": "sender@example.com", - "to": "recipient@example.com", - "subject": "Test Subject", - "text": "Test message body", - "attachments": [ - { - "filename": "test.txt", - "content": "abc", - "content_type": "text/plain", - }, - ], - } - } + kwargs = self.assert_handler_called_once_with( + self.inbound_handler, + sender=MailPaceInboundWebhookView, + event=ANY, + esp_name="MailPace", + ) - # Serialize the payload to JSON - mailpace_payload_json = json.dumps(mailpace_payload) + event = kwargs["event"] - # Simulate a POST request to the MailPace webhook view - response = self.client.post( - "/anymail/mailpace/inbound/", - content_type="application/json", - data=mailpace_payload_json, - ) + self.assertIsInstance(event, AnymailInboundEvent) + self.assertEqual(event.event_type, "inbound") - # Check the response status code (assuming 200 OK is expected) - self.assertEqual(response.status_code, 200) + message = event.message + + self.assertEqual(message.to[0].address, "test@inbound.example.com") + self.assertEqual(message["from"], "A tester ") + self.assertEqual(message.subject, "Raw MIME test") + + self.assertEqual(len(message._headers), 7) + + + def test_inbound_attachments(self): + image_content = sample_image_content() + email_content = sample_email_content() + raw_mime = dedent( + """\ + MIME-Version: 1.0 + From: from@example.org + Subject: Attachments + To: test@inbound.example.com + Content-Type: multipart/mixed; boundary="boundary0" + + --boundary0 + Content-Type: multipart/related; boundary="boundary1" - # Check if the AnymailInboundEvent signal was dispatched with attachments - # self.assertSignalSent( - # AnymailInboundEvent, - # event_type=ANY, - # timestamp=timezone.now(), - # event_id='unique-event-id', - # message_id=ANY, - # recipient='recipient@example.com', - # from_email='sender@example.com', - # subject='Test Subject', - # text='Test message body', - # attachments=[ - # AnymailInboundMessage.Attachment( - # content_type='text/plain', - # content=test_file_content(), - # filename='test.txt', - # ), - # ], - # headers=ANY, # Define the expected headers - # ) - - def test_inbound_with_raw_email(self): - # Create a MailPace webhook payload with a raw email for testing + --boundary1 + Content-Type: text/html; charset="UTF-8" + +
This is the HTML body. It has an inline image: .
+ + --boundary1 + Content-Type: image/png + Content-Disposition: inline; filename="image.png" + Content-ID: + Content-Transfer-Encoding: base64 + + {image_content_base64} + --boundary1-- + --boundary0 + Content-Type: text/plain; charset="UTF-8" + Content-Disposition: attachment; filename="test.txt" + + test attachment + --boundary0 + Content-Type: message/rfc822; charset="US-ASCII" + Content-Disposition: attachment + X-Comment: (the only valid transfer encodings for message/* are 7bit, 8bit, and binary) + + {email_content} + --boundary0-- + """ # NOQA: E501 + ).format( + image_content_base64=b64encode(image_content).decode("ascii"), + email_content=email_content.decode("ascii"), + ) + + # Only raw is used by Anymail mailpace_payload = { - "event": "inbound", - "payload": { - "id": "unique-event-id", - "created_at": "2023-11-05T12:34:56Z", - "from": "sender@example.com", - "to": "recipient@example.com", - "raw_email": b64encode(sample_email_content()).decode('utf-8'), - } + "from": "Person A ", + "headers": [ + "Received: from localhost...", + "DKIM-Signature: v=1 a=rsa...;" + ], + "messageId": "<3baf4caf-948a-41e6-bc5c-2e99058e6461@mailer.mailpace.com>", + "raw": raw_mime, + "to": "Person B ", + "subject": "Email Subject", + "cc": "Person C ", + "bcc": "Person D ", + "inReplyTo": "<3baf4caf-948a-41e6-bc5c-2e99058e6461@mailer.mailpace.com>", + "replyTo": "bounces+abcd@test.com", + "html": "

Email Contents Here

", + "text": "Text Email Contents", + "attachments": [ + { + "filename": "example.pdf", + "content_type": "application/pdf", + "content": "base64_encoded_content_of_the_attachment", + }, + ] } - # Serialize the payload to JSON - mailpace_payload_json = json.dumps(mailpace_payload) - response = self.client.post( "/anymail/mailpace/inbound/", content_type="application/json", - data=mailpace_payload_json, + data=json.dumps(mailpace_payload), ) - # Check the response status code (assuming 200 OK is expected) self.assertEqual(response.status_code, 200) - # Check if the AnymailInboundEvent signal was dispatched with raw_email - # self.assertSignalSent( - # AnymailInboundEvent, - # event_type=ANY, - # timestamp=timezone.now(), - # event_id='unique-event-id', - # message_id=ANY, - # recipient='recipient@example.com', - # from_email='sender@example.com', - # subject=None, # Adjust this if the subject is expected - # text=None, # Adjust this if text content is expected - # raw_email=sample_email_content(), - # headers=ANY, # Define the expected headers - # ) + kwargs = self.assert_handler_called_once_with( + self.inbound_handler, + sender=MailPaceInboundWebhookView, + event=ANY, + esp_name="MailPace", + ) + + event = kwargs["event"] + + self.assertIsInstance(event, AnymailInboundEvent) + + message = event.message + + self.assertEqual(message.to[0].address, "test@inbound.example.com") + + self.assertEqual(len(message._headers), 5) + self.assertEqual(len(message.attachments), 2) From 1a91f0a48963feb4f375f44d83298c3b95c1994a Mon Sep 17 00:00:00 2001 From: Paul Oms Date: Sun, 12 Nov 2023 20:26:24 +0000 Subject: [PATCH 08/25] Support MailPace Webhook signature validation --- anymail/webhooks/mailpace.py | 44 ++++++++++++++++++++++++--- pyproject.toml | 2 +- tests/test_mailpace_inbound.py | 4 ++- tests/test_mailpace_webhooks.py | 53 ++++++++++++++++++++++++++------- tests/utils_mailpace.py | 44 +++++++++++++++++++++++++++ tox.ini | 1 + 6 files changed, 131 insertions(+), 17 deletions(-) create mode 100644 tests/utils_mailpace.py diff --git a/anymail/webhooks/mailpace.py b/anymail/webhooks/mailpace.py index 15f1636b..12833adf 100644 --- a/anymail/webhooks/mailpace.py +++ b/anymail/webhooks/mailpace.py @@ -1,8 +1,13 @@ +import binascii import json -from email.utils import unquote +import base64 +from anymail.exceptions import AnymailWebhookValidationFailure +from anymail.utils import get_anymail_setting from django.utils.dateparse import parse_datetime from django.utils import timezone +from nacl.signing import VerifyKey +from nacl.exceptions import CryptoError, ValueError from ..signals import ( AnymailInboundEvent, @@ -25,13 +30,20 @@ class MailPaceBaseWebhookView(AnymailBaseWebhookView): def parse_events(self, request): esp_event = json.loads(request.body.decode("utf-8")) return [self.esp_to_anymail_event(esp_event)] - - # TODO: - # def validate_request(self, request): class MailPaceTrackingWebhookView(MailPaceBaseWebhookView): """Handler for MailPace delivery webhooks""" + webhook_key = None + + #TODO: make this optional + def __init__(self, **kwargs): + self.webhook_key = get_anymail_setting( + "webhook_key", esp_name=self.esp_name, kwargs=kwargs, allow_bare=True + ) + + super().__init__(**kwargs) + # Used by base class signal = tracking @@ -44,6 +56,30 @@ class MailPaceTrackingWebhookView(MailPaceBaseWebhookView): "email.spam": EventType.REJECTED } + # MailPace doesn't send a signature for inbound webhooks, yet + # When/if MailPace does this, move this to the parent class + def validate_request(self, request): + try: + signature_base64 = request.headers["X-MailPace-Signature"] + signature = base64.b64decode(signature_base64) + except (KeyError, binascii.Error): + raise AnymailWebhookValidationFailure( + "MailPace webhook called with invalid or missing signature" + ) + + verify_key_base64 = self.webhook_key + + verify_key = VerifyKey(base64.b64decode(verify_key_base64)) + + message = request.body + + try: + verify_key.verify(message, signature) + except (CryptoError, ValueError): + raise AnymailWebhookValidationFailure( + "MailPace webhook called with incorrect signature" + ) + def esp_to_anymail_event(self, esp_event): event_type = self.event_record_types.get(esp_event["event"], EventType.UNKNOWN) payload = esp_event["payload"] diff --git a/pyproject.toml b/pyproject.toml index 9dbd99ac..3b5b3db0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,7 +68,7 @@ amazon-ses = ["boto3"] mailersend = [] mailgun = [] mailjet = [] -mailpace = [] +mailpace = ["pynacl"] mandrill = [] postmark = [] resend = ["svix"] diff --git a/tests/test_mailpace_inbound.py b/tests/test_mailpace_inbound.py index a8a8b982..0032afc4 100644 --- a/tests/test_mailpace_inbound.py +++ b/tests/test_mailpace_inbound.py @@ -11,7 +11,7 @@ from anymail.webhooks.mailpace import MailPaceInboundWebhookView from .utils import sample_email_content, sample_image_content, test_file_content -from .webhook_cases import WebhookTestCase +from .webhook_cases import WebhookBasicAuthTestCase, WebhookTestCase from .utils import sample_email_content, sample_image_content, test_file_content from .webhook_cases import WebhookTestCase @@ -194,3 +194,5 @@ def test_inbound_attachments(self): self.assertEqual(len(message._headers), 5) self.assertEqual(len(message.attachments), 2) + attachment = message.attachments[0] + self.assertEqual(attachment.get_filename(), "test.txt") diff --git a/tests/test_mailpace_webhooks.py b/tests/test_mailpace_webhooks.py index 832c15bc..b4babe9e 100644 --- a/tests/test_mailpace_webhooks.py +++ b/tests/test_mailpace_webhooks.py @@ -1,4 +1,6 @@ import json +import unittest +from base64 import b64encode from unittest.mock import ANY from django.test import tag @@ -6,29 +8,58 @@ from anymail.signals import AnymailTrackingEvent from anymail.webhooks.mailpace import MailPaceTrackingWebhookView +from .utils_mailpace import ClientWithMailPaceSignature, make_key from .webhook_cases import WebhookBasicAuthTestCase, WebhookTestCase @tag("mailpace") -class MailPaceWebhookSecurityTestCase(WebhookBasicAuthTestCase): - def call_webhook(self): - return self.client.post( +@unittest.skipUnless( + ClientWithMailPaceSignature, "Install 'pynacl' to run mailpace webhook tests" +) +class MailPaceWebhookSecurityTestCase(WebhookTestCase): + client_class = ClientWithMailPaceSignature + + def setUp(self): + super().setUp() + self.clear_basic_auth() + + self.client.set_private_key(make_key()) + + def test_failed_signature_check(self): + response = self.client.post( "/anymail/mailpace/tracking/", content_type="application/json", - data=json.dumps({ "event": "email.queued", "payload": { - "created_at": "2021-11-16T14:50:15.445Z", - "id": "1", - "message_id": "string", - "to": "example@test.com", - }}) + data=json.dumps({"some": "data"}), + headers={"X-MailPace-Signature": b64encode("invalid".encode("utf-8"))}, ) + self.assertEqual(response.status_code, 400) - # Actual tests are in WebhookBasicAuthTestCase - # TODO: add tests for MailPace webhook signing + response = self.client.post( + "/anymail/mailpace/tracking/", + content_type="application/json", + data=json.dumps({"some": "data"}), + headers={"X-MailPace-Signature": "garbage"}, + ) + self.assertEqual(response.status_code, 400) + response = self.client.post( + "/anymail/mailpace/tracking/", + content_type="application/json", + data=json.dumps({"some": "data"}), + headers={"X-MailPace-Signature": ""}, + ) + self.assertEqual(response.status_code, 400) @tag("mailpace") class MailPaceDeliveryTestCase(WebhookTestCase): + client_class = ClientWithMailPaceSignature + + def setUp(self): + super().setUp() + self.clear_basic_auth() + + self.client.set_private_key(make_key()) + def test_queued_event(self): raw_event = { "event": "email.queued", diff --git a/tests/utils_mailpace.py b/tests/utils_mailpace.py new file mode 100644 index 00000000..ade18aa3 --- /dev/null +++ b/tests/utils_mailpace.py @@ -0,0 +1,44 @@ +from base64 import b64encode + +from django.test import override_settings + +from tests.utils import ClientWithCsrfChecks + +from nacl.signing import SigningKey + +def make_key(): + """Generate key, for testing only""" + return SigningKey.generate() + +def derive_public_webhook_key(private_key): + """Derive public key from private key, in base64 as per MailPace spec""" + verify_key_bytes = private_key.verify_key.encode() + return b64encode(verify_key_bytes).decode() + +# Returns a signature, as a byte string that has been Base64 encoded +# As per MailPace docs +def sign(private_key, message): + """Sign message with private key""" + signature_bytes = private_key.sign(message).signature + return b64encode(signature_bytes).decode('utf-8') + +class _ClientWithMailPaceSignature(ClientWithCsrfChecks): + private_key = None + + def set_private_key(self, private_key): + self.private_key = private_key + + def post(self, *args, **kwargs): + data = kwargs.get("data", "").encode("utf-8") + + headers = kwargs.setdefault("headers", {}) + if "X-MailPace-Signature" not in headers: + signature = sign(self.private_key, data) + headers["X-MailPace-Signature"] = signature + + webhook_key = derive_public_webhook_key(self.private_key) + with override_settings(ANYMAIL={"MAILPACE_WEBHOOK_KEY": webhook_key}): + return super().post(*args, **kwargs) + + +ClientWithMailPaceSignature = _ClientWithMailPaceSignature diff --git a/tox.ini b/tox.ini index 90a6fa0f..41b87fca 100644 --- a/tox.ini +++ b/tox.ini @@ -52,6 +52,7 @@ extras = # (Only ESPs with extra dependencies need to be listed here. # Careful: tox factors (on the left) use underscore; extra names use hyphen.) all,amazon_ses: amazon-ses + all,mailpace: mailpace all,postal: postal all,resend: resend setenv = From 217d18542d42b23d192a2e468343056404c5a2b0 Mon Sep 17 00:00:00 2001 From: Paul Oms Date: Thu, 1 Feb 2024 10:41:15 +0000 Subject: [PATCH 09/25] Started the MailPace Documentation --- docs/esps/index.rst | 1 + docs/esps/mailpace.rst | 224 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 225 insertions(+) create mode 100644 docs/esps/mailpace.rst diff --git a/docs/esps/index.rst b/docs/esps/index.rst index 19a54c65..789ce374 100644 --- a/docs/esps/index.rst +++ b/docs/esps/index.rst @@ -17,6 +17,7 @@ and notes about any quirks or limitations: mailersend mailgun mailjet + mailpace mandrill postal postmark diff --git a/docs/esps/mailpace.rst b/docs/esps/mailpace.rst new file mode 100644 index 00000000..4a82cba0 --- /dev/null +++ b/docs/esps/mailpace.rst @@ -0,0 +1,224 @@ +.. _mailpace-backend: + +MailPace +========== + +Anymail integrates Django with the `MailPace`_ transactional +email service, using their `send API`_ endpoint. + +.. versionadded:: 10.3 + +.. _MailPace: https://mailpace.com/ +.. _send API: https://docs.mailpace.com/reference/send + + +.. _mailpace-installation: + +Installation +------------ + +Anymail uses the :pypi:`PyNaCl` package to validate MailPace webhook signatures. +If you will use Anymail's :ref:`status tracking ` webhook +with MailPace, and you want to use webhook signature validation, be sure +to include the ``[mailpace]`` option when you install Anymail: + + .. code-block:: console + + $ python -m pip install 'django-anymail[mailpace]' + +(Or separately run ``python -m pip install pynacl``.) + +The PyNaCl package pulls in several other dependencies, so its use +is optional in Anymail. See :ref:`mailpace-webhooks` below for details. +To avoid installing PyNaCl with Anymail, just omit the ``[mailpace]`` option. + + +Settings +-------- + +.. rubric:: EMAIL_BACKEND + +To use Anymail's MailPace backend, set: + + .. code-block:: python + + EMAIL_BACKEND = "anymail.backends.mailpace.EmailBackend" + +in your settings.py. + + +.. setting:: ANYMAIL_MAILPACE_API_KEY + +.. rubric:: MAILPACE_API_KEY + +Required for sending. A domain specific API key from the MailPace app `MailPace app`_. + + .. code-block:: python + + ANYMAIL = { + ... + "MAILPACE_API_KEY": "...", + } + +Anymail will also look for ``MAILPACE_API_KEY`` at the +root of the settings file if neither ``ANYMAIL["MAILPACE_API_KEY"]`` +nor ``ANYMAIL_MAILPACE_API_KEY`` is set. + +.. _MailPace API Keys: https://app.mailpace.com/ + +.. setting:: ANYMAIL_MAILPACE_SIGNING_SECRET + +.. rubric:: MAILPACE_SIGNING_SECRET + +The MailPace webhook signing secret used to verify webhook posts. +Recommended if you are using activity tracking, otherwise not necessary. +(This is separate from Anymail's + +:setting:`WEBHOOK_SECRET ` setting.) + +Find this in your MailPace App `MailPace app`_: by opening your domain, +selecting webhooks, and look for the "Public Key Verification" section. + + .. code-block:: python + + ANYMAIL = { + ... + "MAILPACE_SIGNING_SECRET": "whsec_...", + } + +If you provide this setting, the PyNaCl package is required. +See :ref:`mailpace-installation` above. + + +.. setting:: ANYMAIL_MAILPACE_API_URL + +.. rubric:: MAILPACE_API_URL + +The base url for calling the MailPace API. + +The default is ``MAILPACE_API_URL = "https://app.mailpace.com/api/v1/send"``. +(It's unlikely you would need to change this.) + +.. _MailPace app: https://app.mailpace.com/ + + +.. _mailpace-quirks: + +Limitations and quirks +---------------------- + +- MailPace does not support open tracking or click tracking. + (You can still use Anymail's :ref:`status tracking ` which uses webhooks for tracking) + +.. _mailpace-webhooks: + +Status tracking webhooks +------------------------ + +Anymail's normalized :ref:`status tracking ` works +with MailPace's webhooks. + +MailPace implements webhook signing, using the :pypi:`PyNaCl` package +for signature validation (see :ref:`mailpace-installation` above). You have +three options for securing the status tracking webhook: + +* Use MailPace's webhook signature validation, by setting + :setting:`MAILPACE_SIGNING_SECRET ` + (requires the PyNaCl package) +* Use Anymail's shared secret validation, by setting + :setting:`WEBHOOK_SECRET ` + (does not require PyNaCl) +* Use both + +Signature validation is recommended, unless you do not want to add +PyNaCl to your dependencies. + +To configure Anymail status tracking for MailPace, +add a new webhook endpoint to domain in the `MailPace app`_: + +* For the "Endpoint URL", enter one of these + (where *yoursite.example.com* is your Django site). + + If are *not* using Anymail's shared webhook secret: + + :samp:`https://{yoursite.example.com}/anymail/mailpace/tracking/` + + Or if you *are* using Anymail's :setting:`WEBHOOK_SECRET `, + include the *random:random* shared secret in the URL: + + :samp:`https://{random}:{random}@{yoursite.example.com}/mailpace/tracking/` + +* For "Events", select any or all events you want to track. + +* Click the "Add Endpoint" button. + +Then, if you are using MailPace's webhook signature validation (with PyNaCl), +add the webhook signing secret to your Anymail settings: + +* Still on the Webhooks page, scroll down to the "Public Key Verification" section. + +* Add that key to your settings.py ``ANYMAIL`` settings as + :setting:`MAILPACE_SIGNING_SECRET `: + + .. code-block:: python + + ANYMAIL = { + # ... + "MAILPACE_SIGNING_SECRET": "..." + } + +MailPace will report these Anymail +:attr:`~anymail.signals.AnymailTrackingEvent.event_type`\s: +queued, delivered, deferred, bounced, and spam. + + +.. _mailpace-tracking-recipient: + +.. note:: + + **Multiple recipients not recommended with tracking** + + If you send a message with multiple recipients (to, cc, and/or bcc), + you will only one event separate events (delivered, deferred, etc.) + for email. MailPace does not send send different events for each + recipient. + + To avoid confusion, it's best to send each message to exactly one ``to`` + address, and avoid using cc or bcc. + + +.. _mailpace-esp-event: + +The status tracking event's :attr:`~anymail.signals.AnymailTrackingEvent.esp_event` +field will be the parsed MailPace webhook payload. + +.. _mailpace-inbound: + +Inbound +------- + +Anymail's inbound message support works with MailPace's inbound webhooks. + +To configure Anymail inbound for MailPace, add a new inbound endpoint to MailPace app: + + +... + + +.. _mailpace-troubleshooting: + +Troubleshooting +--------------- + +If Anymail's MailPace integration isn't behaving like you expect, +MailPace's dashboard includes information that can help +isolate the problem, for each Domain you have: + +* MailPace Outbound Emails lists every email accepted by MailPace for delivery +* MailPace Webhooks page shows every attempt by MailPace to call + your webhook +* MailPace Inbound page shows every inbound email received and every attempt + by MailPace to forward it to your Anymail inbound endpoint + + +See Anymail's :ref:`troubleshooting` docs for additional suggestions. From 0077419e619ec0f480af5db7a97053805907ae3d Mon Sep 17 00:00:00 2001 From: Paul Oms Date: Thu, 1 Feb 2024 10:53:41 +0000 Subject: [PATCH 10/25] Fix table --- docs/esps/mailpace.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/esps/mailpace.rst b/docs/esps/mailpace.rst index 4a82cba0..d2a05731 100644 --- a/docs/esps/mailpace.rst +++ b/docs/esps/mailpace.rst @@ -107,8 +107,8 @@ The default is ``MAILPACE_API_URL = "https://app.mailpace.com/api/v1/send"``. Limitations and quirks ---------------------- -- MailPace does not support open tracking or click tracking. - (You can still use Anymail's :ref:`status tracking ` which uses webhooks for tracking) +- MailPace does not, and will not ever support open tracking or click tracking. + (You can still use Anymail's :ref:`status tracking ` which uses webhooks for tracking delivery) .. _mailpace-webhooks: @@ -201,7 +201,7 @@ Anymail's inbound message support works with MailPace's inbound webhooks. To configure Anymail inbound for MailPace, add a new inbound endpoint to MailPace app: - +MailPace sends both the Raw MIME message, as well as the parsed message ... From afc787455879e378f395604072a35ae1b76a35ff Mon Sep 17 00:00:00 2001 From: Paul Oms Date: Thu, 1 Feb 2024 19:00:13 +0000 Subject: [PATCH 11/25] Fix docs table and inbound docs --- docs/esps/mailpace.rst | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/docs/esps/mailpace.rst b/docs/esps/mailpace.rst index d2a05731..64084355 100644 --- a/docs/esps/mailpace.rst +++ b/docs/esps/mailpace.rst @@ -66,24 +66,22 @@ nor ``ANYMAIL_MAILPACE_API_KEY`` is set. .. _MailPace API Keys: https://app.mailpace.com/ -.. setting:: ANYMAIL_MAILPACE_SIGNING_SECRET +.. setting:: MAILPACE_WEBHOOK_KEY -.. rubric:: MAILPACE_SIGNING_SECRET +.. rubric:: MAILPACE_WEBHOOK_KEY The MailPace webhook signing secret used to verify webhook posts. Recommended if you are using activity tracking, otherwise not necessary. -(This is separate from Anymail's +(This is separate from Anymail's :setting:`WEBHOOK_SECRET ` setting.) -:setting:`WEBHOOK_SECRET ` setting.) - -Find this in your MailPace App `MailPace app`_: by opening your domain, +Find this in your MailPace App `MailPace app`_ by opening your domain, selecting webhooks, and look for the "Public Key Verification" section. .. code-block:: python ANYMAIL = { ... - "MAILPACE_SIGNING_SECRET": "whsec_...", + "MAILPACE_WEBHOOK_KEY": "...", } If you provide this setting, the PyNaCl package is required. @@ -123,7 +121,7 @@ for signature validation (see :ref:`mailpace-installation` above). You have three options for securing the status tracking webhook: * Use MailPace's webhook signature validation, by setting - :setting:`MAILPACE_SIGNING_SECRET ` + :setting:`MAILPACE_WEBHOOK_KEY ` (requires the PyNaCl package) * Use Anymail's shared secret validation, by setting :setting:`WEBHOOK_SECRET ` @@ -158,13 +156,13 @@ add the webhook signing secret to your Anymail settings: * Still on the Webhooks page, scroll down to the "Public Key Verification" section. * Add that key to your settings.py ``ANYMAIL`` settings as - :setting:`MAILPACE_SIGNING_SECRET `: + :setting:`MAILPACE_WEBHOOK_KEY `: .. code-block:: python ANYMAIL = { # ... - "MAILPACE_SIGNING_SECRET": "..." + "MAILPACE_WEBHOOK_KEY": "..." } MailPace will report these Anymail @@ -179,8 +177,8 @@ queued, delivered, deferred, bounced, and spam. **Multiple recipients not recommended with tracking** If you send a message with multiple recipients (to, cc, and/or bcc), - you will only one event separate events (delivered, deferred, etc.) - for email. MailPace does not send send different events for each + you will only receive one event (delivered, deferred, etc.) + per email. MailPace does not send send different events for each recipient. To avoid confusion, it's best to send each message to exactly one ``to`` @@ -197,13 +195,17 @@ field will be the parsed MailPace webhook payload. Inbound ------- -Anymail's inbound message support works with MailPace's inbound webhooks. +If you want to receive email from Mailgun through Anymail's normalized :ref:`inbound ` +handling, set up a new Inbound route in the MailPace app points to Anymail's inbound webhook. + +Use this url as the route's "forward" destination: -To configure Anymail inbound for MailPace, add a new inbound endpoint to MailPace app: + :samp:`https://{random}:{random}@{yoursite.example.com}/anymail/mailpace/inbound/` -MailPace sends both the Raw MIME message, as well as the parsed message -... + * *random:random* is an :setting:`ANYMAIL_WEBHOOK_SECRET` shared secret + * *yoursite.example.com* is your Django site +MailPace sends the Raw MIME message by default, and that is what Anymail uses to process the inbound email. .. _mailpace-troubleshooting: From 62930f62b9eed8ed850e79903f7724266c2002f9 Mon Sep 17 00:00:00 2001 From: Paul Oms Date: Thu, 1 Feb 2024 21:46:43 +0000 Subject: [PATCH 12/25] Fix linting issues --- anymail/backends/mailpace.py | 52 ++++++++++++++++++++------------- anymail/webhooks/mailpace.py | 33 ++++++++++++--------- docs/esps/mailpace.rst | 6 ++-- tests/test_mailpace_backend.py | 39 ++++++++++++++++--------- tests/test_mailpace_inbound.py | 23 ++++----------- tests/test_mailpace_webhooks.py | 11 +++---- tests/utils_mailpace.py | 9 ++++-- 7 files changed, 99 insertions(+), 74 deletions(-) diff --git a/anymail/backends/mailpace.py b/anymail/backends/mailpace.py index 0f449c77..4df121c1 100644 --- a/anymail/backends/mailpace.py +++ b/anymail/backends/mailpace.py @@ -1,9 +1,6 @@ from ..exceptions import AnymailRequestsAPIError from ..message import AnymailRecipientStatus -from ..utils import ( - CaseInsensitiveCasePreservingDict, - get_anymail_setting, -) +from ..utils import CaseInsensitiveCasePreservingDict, get_anymail_setting from .base_requests import AnymailRequestsBackend, RequestsPayload @@ -42,17 +39,14 @@ def parse_recipient_status(self, response, payload, message): # Prepare the dict by setting everything to queued without a message id unknown_status = AnymailRecipientStatus(message_id=None, status="queued") recipient_status = CaseInsensitiveCasePreservingDict( - { - recip.addr_spec: unknown_status - for recip in payload.to_cc_and_bcc_emails - } + {recip.addr_spec: unknown_status for recip in payload.to_cc_and_bcc_emails} ) parsed_response = self.deserialize_json_response(response, payload, message) status_code = str(response.status_code) json_response = response.json() - + # Set the status_msg and id based on the status_code if status_code == "200": try: @@ -77,20 +71,38 @@ def parse_recipient_status(self, response, payload, message): message_id=id, status="queued" ) elif status_msg == "error": - if 'errors' in json_response: - for field in ['to', 'cc', 'bcc']: - if field in json_response['errors']: - error_messages = json_response['errors'][field] + if "errors" in json_response: + for field in ["to", "cc", "bcc"]: + if field in json_response["errors"]: + error_messages = json_response["errors"][field] for email in payload.to_cc_and_bcc_emails: for error_message in error_messages: - if 'undefined field' in error_message or 'is invalid' in error_message: - recipient_status[email.addr_spec] = AnymailRecipientStatus(message_id=None, status='invalid') - elif 'contains a blocked address' in error_message: - recipient_status[email.addr_spec] = AnymailRecipientStatus(message_id=None, status='rejected') - elif 'number of email addresses exceeds maximum volume' in error_message: - recipient_status[email.addr_spec] = AnymailRecipientStatus(message_id=None, status='invalid') + if ( + "undefined field" in error_message + or "is invalid" in error_message + ): + recipient_status[ + email.addr_spec + ] = AnymailRecipientStatus( + message_id=None, status="invalid" + ) + elif "contains a blocked address" in error_message: + recipient_status[ + email.addr_spec + ] = AnymailRecipientStatus( + message_id=None, status="rejected" + ) + elif ( + "number of email addresses exceeds maximum volume" + in error_message + ): + recipient_status[ + email.addr_spec + ] = AnymailRecipientStatus( + message_id=None, status="invalid" + ) else: - continue # No errors found in this field; continue with the next field + continue # No errors found in this field; continue to next field else: raise AnymailRequestsAPIError( email_message=message, diff --git a/anymail/webhooks/mailpace.py b/anymail/webhooks/mailpace.py index 12833adf..78bff766 100644 --- a/anymail/webhooks/mailpace.py +++ b/anymail/webhooks/mailpace.py @@ -1,14 +1,16 @@ +import base64 import binascii import json -import base64 -from anymail.exceptions import AnymailWebhookValidationFailure -from anymail.utils import get_anymail_setting -from django.utils.dateparse import parse_datetime from django.utils import timezone -from nacl.signing import VerifyKey +from django.utils.dateparse import parse_datetime from nacl.exceptions import CryptoError, ValueError +from nacl.signing import VerifyKey +from anymail.exceptions import AnymailWebhookValidationFailure +from anymail.utils import get_anymail_setting + +from ..inbound import AnymailInboundMessage from ..signals import ( AnymailInboundEvent, AnymailTrackingEvent, @@ -17,8 +19,6 @@ inbound, tracking, ) -from ..inbound import AnymailInboundMessage - from .base import AnymailBaseWebhookView @@ -31,12 +31,13 @@ def parse_events(self, request): esp_event = json.loads(request.body.decode("utf-8")) return [self.esp_to_anymail_event(esp_event)] + class MailPaceTrackingWebhookView(MailPaceBaseWebhookView): """Handler for MailPace delivery webhooks""" webhook_key = None - #TODO: make this optional + # TODO: make this optional def __init__(self, **kwargs): self.webhook_key = get_anymail_setting( "webhook_key", esp_name=self.esp_name, kwargs=kwargs, allow_bare=True @@ -53,11 +54,11 @@ def __init__(self, **kwargs): "email.delivered": EventType.DELIVERED, "email.deferred": EventType.DEFERRED, "email.bounced": EventType.BOUNCED, - "email.spam": EventType.REJECTED + "email.spam": EventType.REJECTED, } # MailPace doesn't send a signature for inbound webhooks, yet - # When/if MailPace does this, move this to the parent class + # When/if MailPace does this, move this to the parent class def validate_request(self, request): try: signature_base64 = request.headers["X-MailPace-Signature"] @@ -66,7 +67,7 @@ def validate_request(self, request): raise AnymailWebhookValidationFailure( "MailPace webhook called with invalid or missing signature" ) - + verify_key_base64 = self.webhook_key verify_key = VerifyKey(base64.b64decode(verify_key_base64)) @@ -84,7 +85,13 @@ def esp_to_anymail_event(self, esp_event): event_type = self.event_record_types.get(esp_event["event"], EventType.UNKNOWN) payload = esp_event["payload"] - reject_reason = RejectReason.SPAM if event_type == EventType.REJECTED else RejectReason.BOUNCED if event_type == EventType.BOUNCED else None + reject_reason = ( + RejectReason.SPAM + if event_type == EventType.REJECTED + else RejectReason.BOUNCED + if event_type == EventType.BOUNCED + else None + ) tags = payload.get("tags", []) return AnymailTrackingEvent( @@ -113,5 +120,5 @@ def esp_to_anymail_event(self, esp_event): timestamp=timezone.now(), event_id=esp_event.get("id", None), esp_event=esp_event, - message=message + message=message, ) diff --git a/docs/esps/mailpace.rst b/docs/esps/mailpace.rst index 64084355..cf550160 100644 --- a/docs/esps/mailpace.rst +++ b/docs/esps/mailpace.rst @@ -178,7 +178,7 @@ queued, delivered, deferred, bounced, and spam. If you send a message with multiple recipients (to, cc, and/or bcc), you will only receive one event (delivered, deferred, etc.) - per email. MailPace does not send send different events for each + per email. MailPace does not send send different events for each recipient. To avoid confusion, it's best to send each message to exactly one ``to`` @@ -188,7 +188,7 @@ queued, delivered, deferred, bounced, and spam. .. _mailpace-esp-event: The status tracking event's :attr:`~anymail.signals.AnymailTrackingEvent.esp_event` -field will be the parsed MailPace webhook payload. +field will be the parsed MailPace webhook payload. .. _mailpace-inbound: @@ -219,7 +219,7 @@ isolate the problem, for each Domain you have: * MailPace Outbound Emails lists every email accepted by MailPace for delivery * MailPace Webhooks page shows every attempt by MailPace to call your webhook -* MailPace Inbound page shows every inbound email received and every attempt +* MailPace Inbound page shows every inbound email received and every attempt by MailPace to forward it to your Anymail inbound endpoint diff --git a/tests/test_mailpace_backend.py b/tests/test_mailpace_backend.py index 4944adbd..7f5ebd34 100644 --- a/tests/test_mailpace_backend.py +++ b/tests/test_mailpace_backend.py @@ -138,7 +138,7 @@ def test_reply_to(self): "Body goes here", "from@example.com", ["to1@example.com"], - reply_to=["reply@example.com", "Other "] + reply_to=["reply@example.com", "Other "], ) email.send() data = self.get_api_call_json() @@ -149,17 +149,26 @@ def test_reply_to(self): def test_sending_attachment(self): """Test sending attachments""" email = mail.EmailMessage( - "Subject", "content", "from@example.com", ["to@example.com"], attachments=[ + "Subject", + "content", + "from@example.com", + ["to@example.com"], + attachments=[ ("file.txt", "file content", "text/plain"), - ] + ], ) email.send() data = self.get_api_call_json() - self.assertEqual(data["attachments"], [{ - "name": "file.txt", - "content": b64encode(b"file content").decode('ascii'), - "content_type": "text/plain", - }]) + self.assertEqual( + data["attachments"], + [ + { + "name": "file.txt", + "content": b64encode(b"file content").decode("ascii"), + "content_type": "text/plain", + } + ], + ) def test_embedded_images(self): image_filename = SAMPLE_IMAGE_FILENAME @@ -182,7 +191,7 @@ def test_embedded_images(self): self.assertEqual(attachments[0]["content_type"], "image/png") self.assertEqual(decode_att(attachments[0]["content"]), image_data) self.assertEqual(attachments[0]["cid"], "cid:%s" % cid) - + def test_tag(self): self.message.tags = ["receipt"] self.message.send() @@ -194,7 +203,7 @@ def test_tags(self): self.message.send() data = self.get_api_call_json() self.assertEqual(data["tags"], ["receipt", "repeat-user"]) - + def test_invalid_response(self): """AnymailAPIError raised for non-json response""" self.set_mock_response(raw=b"not json") @@ -206,7 +215,7 @@ def test_invalid_success_response(self): self.set_mock_response(raw=b"{}") # valid json, but not a MailPace response with self.assertRaises(AnymailRequestsAPIError): self.message.send() - + def test_response_blocked_error(self): """AnymailRecipientsRefused raised for error response with MailPace blocked address""" self.set_mock_response( @@ -214,7 +223,8 @@ def test_response_blocked_error(self): "errors": { "to": ["contains a blocked address"] } - }""", status_code=400 + }""", + status_code=400, ) with self.assertRaises(AnymailRecipientsRefused): self.message.send() @@ -226,11 +236,13 @@ def test_response_maximum_address_error(self): "errors": { "to": ["number of email addresses exceeds maximum volume"] } - }""", status_code=400 + }""", + status_code=400, ) with self.assertRaises(AnymailRecipientsRefused): self.message.send() + @tag("mailpace") class MailPaceBackendRecipientsRefusedTests(MailPaceBackendMockAPITestCase): """ @@ -261,6 +273,7 @@ def test_from_email_invalid(self): with self.assertRaises(AnymailAPIError): msg.send() + @tag("mailpace") class MailPaceBackendSessionSharingTestCase( SessionSharingTestCases, MailPaceBackendMockAPITestCase diff --git a/tests/test_mailpace_inbound.py b/tests/test_mailpace_inbound.py index 0032afc4..d948b4ab 100644 --- a/tests/test_mailpace_inbound.py +++ b/tests/test_mailpace_inbound.py @@ -5,27 +5,20 @@ from django.test import tag -from anymail.exceptions import AnymailConfigurationError -from anymail.inbound import AnymailInboundMessage from anymail.signals import AnymailInboundEvent from anymail.webhooks.mailpace import MailPaceInboundWebhookView -from .utils import sample_email_content, sample_image_content, test_file_content -from .webhook_cases import WebhookBasicAuthTestCase, WebhookTestCase - -from .utils import sample_email_content, sample_image_content, test_file_content +from .utils import sample_email_content, sample_image_content from .webhook_cases import WebhookTestCase + @tag("mailpace") class MailPaceInboundTestCase(WebhookTestCase): def test_inbound_basics(self): # Only raw is used by Anymail mailpace_payload = { "from": "Person A ", - "headers": [ - "Received: from localhost...", - "DKIM-Signature: v=1 a=rsa...;" - ], + "headers": ["Received: from localhost...", "DKIM-Signature: v=1 a=rsa...;"], "messageId": "<3baf4caf-948a-41e6-bc5c-2e99058e6461@mailer.mailpace.com>", "raw": dedent( """\ @@ -66,7 +59,7 @@ def test_inbound_basics(self): "content_type": "application/pdf", "content": "base64_encoded_content_of_the_attachment", }, - ] + ], } response = self.client.post( @@ -97,7 +90,6 @@ def test_inbound_basics(self): self.assertEqual(len(message._headers), 7) - def test_inbound_attachments(self): image_content = sample_image_content() email_content = sample_email_content() @@ -146,10 +138,7 @@ def test_inbound_attachments(self): # Only raw is used by Anymail mailpace_payload = { "from": "Person A ", - "headers": [ - "Received: from localhost...", - "DKIM-Signature: v=1 a=rsa...;" - ], + "headers": ["Received: from localhost...", "DKIM-Signature: v=1 a=rsa...;"], "messageId": "<3baf4caf-948a-41e6-bc5c-2e99058e6461@mailer.mailpace.com>", "raw": raw_mime, "to": "Person B ", @@ -166,7 +155,7 @@ def test_inbound_attachments(self): "content_type": "application/pdf", "content": "base64_encoded_content_of_the_attachment", }, - ] + ], } response = self.client.post( diff --git a/tests/test_mailpace_webhooks.py b/tests/test_mailpace_webhooks.py index b4babe9e..990e490f 100644 --- a/tests/test_mailpace_webhooks.py +++ b/tests/test_mailpace_webhooks.py @@ -9,7 +9,7 @@ from anymail.webhooks.mailpace import MailPaceTrackingWebhookView from .utils_mailpace import ClientWithMailPaceSignature, make_key -from .webhook_cases import WebhookBasicAuthTestCase, WebhookTestCase +from .webhook_cases import WebhookTestCase @tag("mailpace") @@ -50,6 +50,7 @@ def test_failed_signature_check(self): ) self.assertEqual(response.status_code, 400) + @tag("mailpace") class MailPaceDeliveryTestCase(WebhookTestCase): client_class = ClientWithMailPaceSignature @@ -79,8 +80,8 @@ def test_queued_event(self): "replyto": "string", "message_id": "string", "list_unsubscribe": "string", - "tags": ["string", "string"] - } + "tags": ["string", "string"], + }, } response = self.client.post( "/anymail/mailpace/tracking/", @@ -119,7 +120,7 @@ def test_delivered_event_no_tags(self): "replyto": "string", "message_id": "string", "list_unsubscribe": "string", - } + }, } response = self.client.post( "/anymail/mailpace/tracking/", @@ -157,7 +158,7 @@ def test_rejected_event_reason(self): "replyto": "string", "message_id": "string", "list_unsubscribe": "string", - } + }, } response = self.client.post( "/anymail/mailpace/tracking/", diff --git a/tests/utils_mailpace.py b/tests/utils_mailpace.py index ade18aa3..e505d97f 100644 --- a/tests/utils_mailpace.py +++ b/tests/utils_mailpace.py @@ -1,26 +1,29 @@ from base64 import b64encode from django.test import override_settings +from nacl.signing import SigningKey from tests.utils import ClientWithCsrfChecks -from nacl.signing import SigningKey def make_key(): """Generate key, for testing only""" return SigningKey.generate() + def derive_public_webhook_key(private_key): """Derive public key from private key, in base64 as per MailPace spec""" verify_key_bytes = private_key.verify_key.encode() return b64encode(verify_key_bytes).decode() + # Returns a signature, as a byte string that has been Base64 encoded # As per MailPace docs def sign(private_key, message): """Sign message with private key""" signature_bytes = private_key.sign(message).signature - return b64encode(signature_bytes).decode('utf-8') + return b64encode(signature_bytes).decode("utf-8") + class _ClientWithMailPaceSignature(ClientWithCsrfChecks): private_key = None @@ -30,7 +33,7 @@ def set_private_key(self, private_key): def post(self, *args, **kwargs): data = kwargs.get("data", "").encode("utf-8") - + headers = kwargs.setdefault("headers", {}) if "X-MailPace-Signature" not in headers: signature = sign(self.private_key, data) From 5158f99c87238c6f0c8ac14ada66687c9a41eea2 Mon Sep 17 00:00:00 2001 From: Paul Oms Date: Thu, 1 Feb 2024 22:46:34 +0000 Subject: [PATCH 13/25] Make pynacl optional --- anymail/webhooks/mailpace.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/anymail/webhooks/mailpace.py b/anymail/webhooks/mailpace.py index 78bff766..a661702b 100644 --- a/anymail/webhooks/mailpace.py +++ b/anymail/webhooks/mailpace.py @@ -4,12 +4,24 @@ from django.utils import timezone from django.utils.dateparse import parse_datetime -from nacl.exceptions import CryptoError, ValueError -from nacl.signing import VerifyKey -from anymail.exceptions import AnymailWebhookValidationFailure +from anymail.exceptions import ( + AnymailImproperlyInstalled, + AnymailWebhookValidationFailure, + _LazyError, +) from anymail.utils import get_anymail_setting +try: + from nacl.exceptions import CryptoError, ValueError + from nacl.signing import VerifyKey +except ImportError: + # This will be raised if verification is attempted (and pynacl wasn't found) + VerifyKey = _LazyError( + AnymailImproperlyInstalled(missing_package="pynacl", install_extra="mailpace") + ) + + from ..inbound import AnymailInboundMessage from ..signals import ( AnymailInboundEvent, @@ -37,7 +49,6 @@ class MailPaceTrackingWebhookView(MailPaceBaseWebhookView): webhook_key = None - # TODO: make this optional def __init__(self, **kwargs): self.webhook_key = get_anymail_setting( "webhook_key", esp_name=self.esp_name, kwargs=kwargs, allow_bare=True From e65963455f64cade9c50afd126f581a8fbe6fb13 Mon Sep 17 00:00:00 2001 From: Paul Oms Date: Thu, 1 Feb 2024 22:59:54 +0000 Subject: [PATCH 14/25] Make pynacl optional in the mailpace test utils --- tests/utils_mailpace.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/utils_mailpace.py b/tests/utils_mailpace.py index e505d97f..8e0027aa 100644 --- a/tests/utils_mailpace.py +++ b/tests/utils_mailpace.py @@ -1,7 +1,14 @@ from base64 import b64encode from django.test import override_settings -from nacl.signing import SigningKey + +try: + from nacl.signing import SigningKey +except ImportError: + # This will be raised if signing is attempted (and pynacl wasn't found) + VerifyKey = _LazyError( + AnymailImproperlyInstalled(missing_package="pynacl", install_extra="mailpace") + ) from tests.utils import ClientWithCsrfChecks From f8d1d95677af9b495bdccbabc69a516d3c99c0f9 Mon Sep 17 00:00:00 2001 From: Paul Oms Date: Thu, 1 Feb 2024 23:39:38 +0000 Subject: [PATCH 15/25] Fix undefined LazyError and AnymailImproperlyInstalled --- tests/utils_mailpace.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/utils_mailpace.py b/tests/utils_mailpace.py index 8e0027aa..f69aed9a 100644 --- a/tests/utils_mailpace.py +++ b/tests/utils_mailpace.py @@ -1,6 +1,7 @@ from base64 import b64encode from django.test import override_settings +from anymail.exceptions import _LazyError, AnymailImproperlyInstalled try: from nacl.signing import SigningKey From 82695fb826a74240da1b4a6f6c02fb0c4ab7aa5b Mon Sep 17 00:00:00 2001 From: Paul Oms Date: Thu, 1 Feb 2024 23:41:07 +0000 Subject: [PATCH 16/25] Fix import order --- tests/utils_mailpace.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/utils_mailpace.py b/tests/utils_mailpace.py index f69aed9a..acdf8d78 100644 --- a/tests/utils_mailpace.py +++ b/tests/utils_mailpace.py @@ -1,7 +1,8 @@ from base64 import b64encode from django.test import override_settings -from anymail.exceptions import _LazyError, AnymailImproperlyInstalled + +from anymail.exceptions import AnymailImproperlyInstalled, _LazyError try: from nacl.signing import SigningKey From 662321b99fcca9369cf15632345f5cc575f6c388 Mon Sep 17 00:00:00 2001 From: Paul Oms Date: Fri, 2 Feb 2024 10:54:58 +0000 Subject: [PATCH 17/25] Fix Lazy Error in test utils, make the Webhook tests use the CLient without a secret if pynacl is unavailable --- tests/test_mailpace_webhooks.py | 19 ++++++++++++++----- tests/utils_mailpace.py | 2 +- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/tests/test_mailpace_webhooks.py b/tests/test_mailpace_webhooks.py index 990e490f..b0fdcd2b 100644 --- a/tests/test_mailpace_webhooks.py +++ b/tests/test_mailpace_webhooks.py @@ -1,5 +1,4 @@ import json -import unittest from base64 import b64encode from unittest.mock import ANY @@ -7,15 +6,22 @@ from anymail.signals import AnymailTrackingEvent from anymail.webhooks.mailpace import MailPaceTrackingWebhookView +from tests.utils import ClientWithCsrfChecks from .utils_mailpace import ClientWithMailPaceSignature, make_key from .webhook_cases import WebhookTestCase +# These tests are triggered both with and without 'pynacl' installed, +# if pynacl is unavailable, we use the ClientWithCsrfChecks class +try: + from nacl.signing import SigningKey + + PYNACL_INSTALLED = bool(SigningKey) +except ImportError: + PYNACL_INSTALLED = False + @tag("mailpace") -@unittest.skipUnless( - ClientWithMailPaceSignature, "Install 'pynacl' to run mailpace webhook tests" -) class MailPaceWebhookSecurityTestCase(WebhookTestCase): client_class = ClientWithMailPaceSignature @@ -53,7 +59,10 @@ def test_failed_signature_check(self): @tag("mailpace") class MailPaceDeliveryTestCase(WebhookTestCase): - client_class = ClientWithMailPaceSignature + if PYNACL_INSTALLED: + client_class = ClientWithMailPaceSignature + else: + client_class = ClientWithCsrfChecks def setUp(self): super().setUp() diff --git a/tests/utils_mailpace.py b/tests/utils_mailpace.py index acdf8d78..8e480e25 100644 --- a/tests/utils_mailpace.py +++ b/tests/utils_mailpace.py @@ -8,7 +8,7 @@ from nacl.signing import SigningKey except ImportError: # This will be raised if signing is attempted (and pynacl wasn't found) - VerifyKey = _LazyError( + SigningKey = _LazyError( AnymailImproperlyInstalled(missing_package="pynacl", install_extra="mailpace") ) From f627a5efc3c249d7896d3cdc9a138ad712fc5214 Mon Sep 17 00:00:00 2001 From: Paul Oms Date: Sun, 4 Feb 2024 19:56:47 +0000 Subject: [PATCH 18/25] Skip webhook tests if pynacl is unavailable, support Django versions pre-4.2, do not attempt to validate webhooks if webhook key is not set --- anymail/webhooks/mailpace.py | 53 +++++++++++++++++++-------------- tests/test_mailpace_webhooks.py | 14 ++++----- tests/utils_mailpace.py | 11 ++++++- 3 files changed, 46 insertions(+), 32 deletions(-) diff --git a/anymail/webhooks/mailpace.py b/anymail/webhooks/mailpace.py index a661702b..59e09984 100644 --- a/anymail/webhooks/mailpace.py +++ b/anymail/webhooks/mailpace.py @@ -6,6 +6,7 @@ from django.utils.dateparse import parse_datetime from anymail.exceptions import ( + AnymailConfigurationError, AnymailImproperlyInstalled, AnymailWebhookValidationFailure, _LazyError, @@ -50,9 +51,12 @@ class MailPaceTrackingWebhookView(MailPaceBaseWebhookView): webhook_key = None def __init__(self, **kwargs): - self.webhook_key = get_anymail_setting( - "webhook_key", esp_name=self.esp_name, kwargs=kwargs, allow_bare=True - ) + try: + get_anymail_setting( + "webhook_key", esp_name=self.esp_name, kwargs=kwargs, allow_bare=True + ) + except AnymailConfigurationError: + self.webhook_key = None super().__init__(**kwargs) @@ -71,26 +75,29 @@ def __init__(self, **kwargs): # MailPace doesn't send a signature for inbound webhooks, yet # When/if MailPace does this, move this to the parent class def validate_request(self, request): - try: - signature_base64 = request.headers["X-MailPace-Signature"] - signature = base64.b64decode(signature_base64) - except (KeyError, binascii.Error): - raise AnymailWebhookValidationFailure( - "MailPace webhook called with invalid or missing signature" - ) - - verify_key_base64 = self.webhook_key - - verify_key = VerifyKey(base64.b64decode(verify_key_base64)) - - message = request.body - - try: - verify_key.verify(message, signature) - except (CryptoError, ValueError): - raise AnymailWebhookValidationFailure( - "MailPace webhook called with incorrect signature" - ) + if self.webhook_key is None: + return True + else: + try: + signature_base64 = request.headers["X-MailPace-Signature"] + signature = base64.b64decode(signature_base64) + except (KeyError, binascii.Error): + raise AnymailWebhookValidationFailure( + "MailPace webhook called with invalid or missing signature" + ) + + verify_key_base64 = self.webhook_key + + verify_key = VerifyKey(base64.b64decode(verify_key_base64)) + + message = request.body + + try: + verify_key.verify(message, signature) + except (CryptoError, ValueError): + raise AnymailWebhookValidationFailure( + "MailPace webhook called with incorrect signature" + ) def esp_to_anymail_event(self, esp_event): event_type = self.event_record_types.get(esp_event["event"], EventType.UNKNOWN) diff --git a/tests/test_mailpace_webhooks.py b/tests/test_mailpace_webhooks.py index b0fdcd2b..761a71c8 100644 --- a/tests/test_mailpace_webhooks.py +++ b/tests/test_mailpace_webhooks.py @@ -1,4 +1,5 @@ import json +import unittest from base64 import b64encode from unittest.mock import ANY @@ -6,13 +7,13 @@ from anymail.signals import AnymailTrackingEvent from anymail.webhooks.mailpace import MailPaceTrackingWebhookView -from tests.utils import ClientWithCsrfChecks from .utils_mailpace import ClientWithMailPaceSignature, make_key from .webhook_cases import WebhookTestCase # These tests are triggered both with and without 'pynacl' installed, -# if pynacl is unavailable, we use the ClientWithCsrfChecks class +# without the ability to generate a signing key, there is no way to test +# the webhook signature validation. try: from nacl.signing import SigningKey @@ -22,13 +23,13 @@ @tag("mailpace") +@unittest.skipUnless(PYNACL_INSTALLED, "pynacl is not installed") class MailPaceWebhookSecurityTestCase(WebhookTestCase): client_class = ClientWithMailPaceSignature def setUp(self): super().setUp() self.clear_basic_auth() - self.client.set_private_key(make_key()) def test_failed_signature_check(self): @@ -58,16 +59,13 @@ def test_failed_signature_check(self): @tag("mailpace") +@unittest.skipUnless(PYNACL_INSTALLED, "pynacl is not installed") class MailPaceDeliveryTestCase(WebhookTestCase): - if PYNACL_INSTALLED: - client_class = ClientWithMailPaceSignature - else: - client_class = ClientWithCsrfChecks + client_class = ClientWithMailPaceSignature def setUp(self): super().setUp() self.clear_basic_auth() - self.client.set_private_key(make_key()) def test_queued_event(self): diff --git a/tests/utils_mailpace.py b/tests/utils_mailpace.py index 8e480e25..305e62fa 100644 --- a/tests/utils_mailpace.py +++ b/tests/utils_mailpace.py @@ -50,7 +50,16 @@ def post(self, *args, **kwargs): webhook_key = derive_public_webhook_key(self.private_key) with override_settings(ANYMAIL={"MAILPACE_WEBHOOK_KEY": webhook_key}): - return super().post(*args, **kwargs) + # Django 4.2+ test Client allows headers=headers; + # before that, must convert to HTTP_ args: + return super().post( + *args, + **kwargs, + **{ + f"HTTP_{header.upper().replace('-', '_')}": value + for header, value in headers.items() + }, + ) ClientWithMailPaceSignature = _ClientWithMailPaceSignature From 3d2183ee756f6f7882e09484edeb391333fe9352 Mon Sep 17 00:00:00 2001 From: Paul Oms Date: Sun, 4 Feb 2024 20:03:13 +0000 Subject: [PATCH 19/25] Revert skipping validation if webhook key not set --- anymail/webhooks/mailpace.py | 43 +++++++++++++++++------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/anymail/webhooks/mailpace.py b/anymail/webhooks/mailpace.py index 59e09984..bbce3da6 100644 --- a/anymail/webhooks/mailpace.py +++ b/anymail/webhooks/mailpace.py @@ -75,29 +75,26 @@ def __init__(self, **kwargs): # MailPace doesn't send a signature for inbound webhooks, yet # When/if MailPace does this, move this to the parent class def validate_request(self, request): - if self.webhook_key is None: - return True - else: - try: - signature_base64 = request.headers["X-MailPace-Signature"] - signature = base64.b64decode(signature_base64) - except (KeyError, binascii.Error): - raise AnymailWebhookValidationFailure( - "MailPace webhook called with invalid or missing signature" - ) - - verify_key_base64 = self.webhook_key - - verify_key = VerifyKey(base64.b64decode(verify_key_base64)) - - message = request.body - - try: - verify_key.verify(message, signature) - except (CryptoError, ValueError): - raise AnymailWebhookValidationFailure( - "MailPace webhook called with incorrect signature" - ) + try: + signature_base64 = request.headers["X-MailPace-Signature"] + signature = base64.b64decode(signature_base64) + except (KeyError, binascii.Error): + raise AnymailWebhookValidationFailure( + "MailPace webhook called with invalid or missing signature" + ) + + verify_key_base64 = self.webhook_key + + verify_key = VerifyKey(base64.b64decode(verify_key_base64)) + + message = request.body + + try: + verify_key.verify(message, signature) + except (CryptoError, ValueError): + raise AnymailWebhookValidationFailure( + "MailPace webhook called with incorrect signature" + ) def esp_to_anymail_event(self, esp_event): event_type = self.event_record_types.get(esp_event["event"], EventType.UNKNOWN) From 610257a9c830f6d285509cbcdb12d0ec524df72e Mon Sep 17 00:00:00 2001 From: Paul Oms Date: Sun, 4 Feb 2024 22:49:31 +0000 Subject: [PATCH 20/25] Fix failing tests, due to unset webhook key, correct warning message --- anymail/webhooks/mailpace.py | 46 ++++++++++++++++++--------------- tests/test_mailpace_webhooks.py | 6 ++--- tests/utils_mailpace.py | 7 ++++- 3 files changed, 33 insertions(+), 26 deletions(-) diff --git a/anymail/webhooks/mailpace.py b/anymail/webhooks/mailpace.py index bbce3da6..fca1b415 100644 --- a/anymail/webhooks/mailpace.py +++ b/anymail/webhooks/mailpace.py @@ -52,7 +52,7 @@ class MailPaceTrackingWebhookView(MailPaceBaseWebhookView): def __init__(self, **kwargs): try: - get_anymail_setting( + self.webhook_key = get_anymail_setting( "webhook_key", esp_name=self.esp_name, kwargs=kwargs, allow_bare=True ) except AnymailConfigurationError: @@ -75,26 +75,30 @@ def __init__(self, **kwargs): # MailPace doesn't send a signature for inbound webhooks, yet # When/if MailPace does this, move this to the parent class def validate_request(self, request): - try: - signature_base64 = request.headers["X-MailPace-Signature"] - signature = base64.b64decode(signature_base64) - except (KeyError, binascii.Error): - raise AnymailWebhookValidationFailure( - "MailPace webhook called with invalid or missing signature" - ) - - verify_key_base64 = self.webhook_key - - verify_key = VerifyKey(base64.b64decode(verify_key_base64)) - - message = request.body - - try: - verify_key.verify(message, signature) - except (CryptoError, ValueError): - raise AnymailWebhookValidationFailure( - "MailPace webhook called with incorrect signature" - ) + if self.webhook_key: + try: + signature_base64 = request.headers["X-MailPace-Signature"] + signature = base64.b64decode(signature_base64) + except (KeyError, binascii.Error): + raise AnymailWebhookValidationFailure( + "MailPace webhook called with invalid or missing signature" + ) + + verify_key_base64 = self.webhook_key + + verify_key = VerifyKey(base64.b64decode(verify_key_base64)) + + message = request.body + + try: + verify_key.verify(message, signature) + except (CryptoError, ValueError): + raise AnymailWebhookValidationFailure( + "MailPace webhook called with incorrect signature" + ) + else: + return True + # No webhook key set def esp_to_anymail_event(self, esp_event): event_type = self.event_record_types.get(esp_event["event"], EventType.UNKNOWN) diff --git a/tests/test_mailpace_webhooks.py b/tests/test_mailpace_webhooks.py index 761a71c8..b82c5f7b 100644 --- a/tests/test_mailpace_webhooks.py +++ b/tests/test_mailpace_webhooks.py @@ -23,13 +23,12 @@ @tag("mailpace") -@unittest.skipUnless(PYNACL_INSTALLED, "pynacl is not installed") +@unittest.skipUnless(PYNACL_INSTALLED, "Install Pynacl to run MailPace Webhook Tests") class MailPaceWebhookSecurityTestCase(WebhookTestCase): client_class = ClientWithMailPaceSignature def setUp(self): super().setUp() - self.clear_basic_auth() self.client.set_private_key(make_key()) def test_failed_signature_check(self): @@ -59,13 +58,12 @@ def test_failed_signature_check(self): @tag("mailpace") -@unittest.skipUnless(PYNACL_INSTALLED, "pynacl is not installed") +@unittest.skipUnless(PYNACL_INSTALLED, "Install Pynacl to run MailPace Webhook Tests") class MailPaceDeliveryTestCase(WebhookTestCase): client_class = ClientWithMailPaceSignature def setUp(self): super().setUp() - self.clear_basic_auth() self.client.set_private_key(make_key()) def test_queued_event(self): diff --git a/tests/utils_mailpace.py b/tests/utils_mailpace.py index 305e62fa..1f07f6ff 100644 --- a/tests/utils_mailpace.py +++ b/tests/utils_mailpace.py @@ -49,7 +49,12 @@ def post(self, *args, **kwargs): headers["X-MailPace-Signature"] = signature webhook_key = derive_public_webhook_key(self.private_key) - with override_settings(ANYMAIL={"MAILPACE_WEBHOOK_KEY": webhook_key}): + with override_settings( + ANYMAIL={ + "MAILPACE_WEBHOOK_KEY": webhook_key, + "WEBHOOK_SECRET": "username:password", + } + ): # Django 4.2+ test Client allows headers=headers; # before that, must convert to HTTP_ args: return super().post( From eab63d382d3272fd89d752dfd0857dd76563d1a4 Mon Sep 17 00:00:00 2001 From: Paul Oms Date: Tue, 6 Feb 2024 10:21:27 +0000 Subject: [PATCH 21/25] Warn if basic auth is not on, and signature validation is not setup --- anymail/webhooks/mailpace.py | 2 ++ tests/test_mailpace_webhooks.py | 62 ++++++++++++++++++++++++++++++--- tests/utils_mailpace.py | 7 ++++ 3 files changed, 66 insertions(+), 5 deletions(-) diff --git a/anymail/webhooks/mailpace.py b/anymail/webhooks/mailpace.py index fca1b415..daf59a25 100644 --- a/anymail/webhooks/mailpace.py +++ b/anymail/webhooks/mailpace.py @@ -55,8 +55,10 @@ def __init__(self, **kwargs): self.webhook_key = get_anymail_setting( "webhook_key", esp_name=self.esp_name, kwargs=kwargs, allow_bare=True ) + self.warn_if_no_basic_auth = False except AnymailConfigurationError: self.webhook_key = None + self.warn_if_no_basic_auth = True super().__init__(**kwargs) diff --git a/tests/test_mailpace_webhooks.py b/tests/test_mailpace_webhooks.py index b82c5f7b..682c677b 100644 --- a/tests/test_mailpace_webhooks.py +++ b/tests/test_mailpace_webhooks.py @@ -8,12 +8,14 @@ from anymail.signals import AnymailTrackingEvent from anymail.webhooks.mailpace import MailPaceTrackingWebhookView -from .utils_mailpace import ClientWithMailPaceSignature, make_key +from .utils_mailpace import ( + ClientWithMailPaceBasicAuth, + ClientWithMailPaceSignature, + make_key, +) from .webhook_cases import WebhookTestCase -# These tests are triggered both with and without 'pynacl' installed, -# without the ability to generate a signing key, there is no way to test -# the webhook signature validation. +# These tests are triggered both with and without 'pynacl' installed try: from nacl.signing import SigningKey @@ -23,7 +25,9 @@ @tag("mailpace") -@unittest.skipUnless(PYNACL_INSTALLED, "Install Pynacl to run MailPace Webhook Tests") +@unittest.skipUnless( + PYNACL_INSTALLED, "Install Pynacl to run MailPace Webhook Signature Tests" +) class MailPaceWebhookSecurityTestCase(WebhookTestCase): client_class = ClientWithMailPaceSignature @@ -57,6 +61,54 @@ def test_failed_signature_check(self): self.assertEqual(response.status_code, 400) +@unittest.skipIf(PYNACL_INSTALLED, "Pynacl is not available, fallback to basic auth") +class MailPaceWebhookBasicAuthTestCase(WebhookTestCase): + client_class = ClientWithMailPaceBasicAuth + + def setUp(self): + super().setUp() + + def test_queued_event(self): + raw_event = { + "event": "email.queued", + "payload": { + "status": "queued", + "id": 1, + "domain_id": 1, + "created_at": "2021-11-16T14:50:15.445Z", + "updated_at": "2021-11-16T14:50:15.445Z", + "from": "sender@example.com", + "to": "queued@example.com", + "htmlbody": "string", + "textbody": "string", + "cc": "string", + "bcc": "string", + "subject": "string", + "replyto": "string", + "message_id": "string", + "list_unsubscribe": "string", + "tags": ["string", "string"], + }, + } + response = self.client.post( + "/anymail/mailpace/tracking/", + content_type="application/json", + data=json.dumps(raw_event), + ) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with( + self.tracking_handler, + sender=MailPaceTrackingWebhookView, + event=ANY, + esp_name="MailPace", + ) + event = kwargs["event"] + self.assertIsInstance(event, AnymailTrackingEvent) + self.assertEqual(event.event_type, "queued") + self.assertEqual(event.message_id, "string") + self.assertEqual(event.recipient, "queued@example.com") + + @tag("mailpace") @unittest.skipUnless(PYNACL_INSTALLED, "Install Pynacl to run MailPace Webhook Tests") class MailPaceDeliveryTestCase(WebhookTestCase): diff --git a/tests/utils_mailpace.py b/tests/utils_mailpace.py index 1f07f6ff..996012f7 100644 --- a/tests/utils_mailpace.py +++ b/tests/utils_mailpace.py @@ -67,4 +67,11 @@ def post(self, *args, **kwargs): ) +class _ClientWithMailPaceBasicAuth(ClientWithCsrfChecks): + def post(self, *args, **kwargs): + with override_settings(ANYMAIL={"WEBHOOK_SECRET": "username:password"}): + return super().post(*args, **kwargs) + + ClientWithMailPaceSignature = _ClientWithMailPaceSignature +ClientWithMailPaceBasicAuth = _ClientWithMailPaceBasicAuth From de7f9a0e9ecd79763ac28b5f5ff2f6069215c7e2 Mon Sep 17 00:00:00 2001 From: Paul Oms Date: Wed, 7 Feb 2024 10:08:37 +0000 Subject: [PATCH 22/25] PR fededback: Refactor /send endpoint, default to unknown status, remove merge data/metadata, implement set extra headers and set esp extra, adjust exception handling, set inbound timestamp to None --- anymail/backends/mailpace.py | 20 ++++++++++++++++---- anymail/webhooks/mailpace.py | 25 +++++++++++-------------- tests/test_mailpace_backend.py | 2 +- 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/anymail/backends/mailpace.py b/anymail/backends/mailpace.py index 4df121c1..20163faa 100644 --- a/anymail/backends/mailpace.py +++ b/anymail/backends/mailpace.py @@ -21,7 +21,7 @@ def __init__(self, **kwargs): "api_url", esp_name=esp_name, kwargs=kwargs, - default="https://app.mailpace.com/api/v1/send", + default="https://app.mailpace.com/api/v1/", ) if not api_url.endswith("/"): api_url += "/" @@ -37,7 +37,7 @@ def raise_for_status(self, response, payload, message): def parse_recipient_status(self, response, payload, message): # Prepare the dict by setting everything to queued without a message id - unknown_status = AnymailRecipientStatus(message_id=None, status="queued") + unknown_status = AnymailRecipientStatus(message_id=None, status="unknown") recipient_status = CaseInsensitiveCasePreservingDict( {recip.addr_spec: unknown_status for recip in payload.to_cc_and_bcc_emails} ) @@ -122,10 +122,11 @@ def __init__(self, message, defaults, backend, *args, **kwargs): } self.server_token = backend.server_token # esp_extra can override self.to_cc_and_bcc_emails = [] - self.merge_data = None - self.merge_metadata = None super().__init__(message, defaults, backend, headers=headers, *args, **kwargs) + def get_api_endpoint(self): + return "send" + def get_request_params(self, api_url): params = super().get_request_params(api_url) params["headers"]["MailPace-Server-Token"] = self.server_token @@ -159,6 +160,12 @@ def set_reply_to(self, emails): reply_to = ", ".join([email.address for email in emails]) self.data["replyto"] = reply_to + def set_extra_headers(self, headers): + if "list-unsubscribe" in headers: + self.data["list_unsubscribe"] = headers.pop("list-unsubscribe") + if headers: + self.unsupported_features("extra_headers (other than List-Unsubscribe)") + def set_text_body(self, body): self.data["textbody"] = body @@ -188,3 +195,8 @@ def set_tags(self, tags): self.data["tags"] = tags[0] else: self.data["tags"] = tags + + def set_esp_extra(self, extra): + self.data.update(extra) + # Special handling for 'server_token': + self.server_token = self.data.pop("server_token", self.server_token) diff --git a/anymail/webhooks/mailpace.py b/anymail/webhooks/mailpace.py index daf59a25..5c030348 100644 --- a/anymail/webhooks/mailpace.py +++ b/anymail/webhooks/mailpace.py @@ -51,14 +51,14 @@ class MailPaceTrackingWebhookView(MailPaceBaseWebhookView): webhook_key = None def __init__(self, **kwargs): - try: - self.webhook_key = get_anymail_setting( - "webhook_key", esp_name=self.esp_name, kwargs=kwargs, allow_bare=True - ) - self.warn_if_no_basic_auth = False - except AnymailConfigurationError: - self.webhook_key = None - self.warn_if_no_basic_auth = True + self.webhook_key = get_anymail_setting( + "webhook_key", + esp_name=self.esp_name, + kwargs=kwargs, + allow_bare=True, + default=None, + ) + self.warn_if_no_basic_auth = self.webhook_key is None super().__init__(**kwargs) @@ -81,10 +81,10 @@ def validate_request(self, request): try: signature_base64 = request.headers["X-MailPace-Signature"] signature = base64.b64decode(signature_base64) - except (KeyError, binascii.Error): + except (KeyError, binascii.Error) as error: raise AnymailWebhookValidationFailure( "MailPace webhook called with invalid or missing signature" - ) + ) from error verify_key_base64 = self.webhook_key @@ -98,9 +98,6 @@ def validate_request(self, request): raise AnymailWebhookValidationFailure( "MailPace webhook called with incorrect signature" ) - else: - return True - # No webhook key set def esp_to_anymail_event(self, esp_event): event_type = self.event_record_types.get(esp_event["event"], EventType.UNKNOWN) @@ -138,7 +135,7 @@ def esp_to_anymail_event(self, esp_event): return AnymailInboundEvent( event_type=EventType.INBOUND, - timestamp=timezone.now(), + timestamp=None, event_id=esp_event.get("id", None), esp_event=esp_event, message=message, diff --git a/tests/test_mailpace_backend.py b/tests/test_mailpace_backend.py index 7f5ebd34..ac3311ea 100644 --- a/tests/test_mailpace_backend.py +++ b/tests/test_mailpace_backend.py @@ -56,7 +56,7 @@ def test_send_mail(self): ["to@example.com"], fail_silently=False, ) - self.assert_esp_called("send/") + self.assert_esp_called("https://app.mailpace.com/api/v1/send") headers = self.get_api_call_headers() self.assertEqual(headers["MailPace-Server-Token"], "test_server_token") data = self.get_api_call_json() From 8402107c1be51708bf15ef705c55e7306213de73 Mon Sep 17 00:00:00 2001 From: Paul Oms Date: Wed, 7 Feb 2024 10:16:37 +0000 Subject: [PATCH 23/25] Fix lint --- anymail/webhooks/mailpace.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/anymail/webhooks/mailpace.py b/anymail/webhooks/mailpace.py index 5c030348..ae526b42 100644 --- a/anymail/webhooks/mailpace.py +++ b/anymail/webhooks/mailpace.py @@ -2,11 +2,9 @@ import binascii import json -from django.utils import timezone from django.utils.dateparse import parse_datetime from anymail.exceptions import ( - AnymailConfigurationError, AnymailImproperlyInstalled, AnymailWebhookValidationFailure, _LazyError, From 8e5316782687b505a9eaa2d35ce28a787477bb01 Mon Sep 17 00:00:00 2001 From: Mike Edmunds Date: Wed, 7 Feb 2024 12:45:13 -0800 Subject: [PATCH 24/25] Add integration tests --- tests/test_mailpace_integration.py | 113 +++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 tests/test_mailpace_integration.py diff --git a/tests/test_mailpace_integration.py b/tests/test_mailpace_integration.py new file mode 100644 index 00000000..125541ea --- /dev/null +++ b/tests/test_mailpace_integration.py @@ -0,0 +1,113 @@ +import os +import unittest +from email.headerregistry import Address + +from django.test import SimpleTestCase, override_settings, tag + +from anymail.exceptions import AnymailAPIError +from anymail.message import AnymailMessage + +from .utils import AnymailTestMixin, sample_image_path + +ANYMAIL_TEST_MAILPACE_SERVER_TOKEN = os.getenv("ANYMAIL_TEST_MAILPACE_SERVER_TOKEN") +ANYMAIL_TEST_MAILPACE_DOMAIN = os.getenv("ANYMAIL_TEST_MAILPACE_DOMAIN") + + +@tag("mailpace", "live") +@unittest.skipUnless( + ANYMAIL_TEST_MAILPACE_SERVER_TOKEN and ANYMAIL_TEST_MAILPACE_DOMAIN, + "Set ANYMAIL_TEST_MAILPACE_SERVER_TOKEN and ANYMAIL_TEST_MAILPACE_DOMAIN" + " environment variables to run MailPace integration tests", +) +@override_settings( + ANYMAIL_MAILPACE_SERVER_TOKEN=ANYMAIL_TEST_MAILPACE_SERVER_TOKEN, + EMAIL_BACKEND="anymail.backends.mailpace.EmailBackend", +) +class MailPaceBackendIntegrationTests(AnymailTestMixin, SimpleTestCase): + """ + MailPace API integration tests + + These tests run against the **live** MailPace API, using the + environment variable `ANYMAIL_TEST_MAILPACE_SERVER_TOKEN` as the API key, + and `ANYMAIL_TEST_MAILPACE_DOMAIN` to construct sender addresses. + If those variables are not set, these tests won't run. + """ + + def setUp(self): + super().setUp() + self.from_email = str( + Address(username="from", domain=ANYMAIL_TEST_MAILPACE_DOMAIN) + ) + self.message = AnymailMessage( + "Anymail MailPace integration test", + "Text content", + self.from_email, + ["test+to1@anymail.dev"], + ) + self.message.attach_alternative("

HTML content

", "text/html") + + def test_simple_send(self): + # Example of getting the MailPace send status and message id from the message + sent_count = self.message.send() + self.assertEqual(sent_count, 1) + + anymail_status = self.message.anymail_status + sent_status = anymail_status.recipients["test+to1@anymail.dev"].status + message_id = anymail_status.recipients["test+to1@anymail.dev"].message_id + + self.assertEqual(sent_status, "queued") + self.assertGreater(message_id, 0) # integer MailPace reference ID + # set of all recipient statuses: + self.assertEqual(anymail_status.status, {sent_status}) + self.assertEqual(anymail_status.message_id, message_id) + + def test_all_options(self): + message = AnymailMessage( + subject="Anymail MailPace all-options integration test", + body="This is the text body", + from_email=str( + Address( + display_name="Test From, with comma", + username="sender", + domain=ANYMAIL_TEST_MAILPACE_DOMAIN, + ) + ), + to=[ + "test+to1@anymail.dev", + '"Recipient 2, with comma" ', + ], + cc=["test+cc1@anymail.dev", "Copy 2 "], + bcc=["test+bcc1@anymail.dev", "Blind Copy 2 "], + reply_to=["reply1@example.com", "Reply 2 "], + headers={"List-Unsubscribe": ""}, + tags=["tag 1", "tag 2"], + ) + message.attach("attachment1.txt", "Here is some\ntext for you", "text/plain") + message.attach("attachment2.csv", "ID,Name\n1,Amy Lina", "text/csv") + cid = message.attach_inline_image_file(sample_image_path()) + message.attach_alternative( + "

HTML: with link" + "and image: " % cid, + "text/html", + ) + + message.send() + self.assertEqual(message.anymail_status.status, {"queued"}) + self.assertEqual( + message.anymail_status.recipients["test+to1@anymail.dev"].status, "queued" + ) + self.assertEqual( + message.anymail_status.recipients["test+to2@anymail.dev"].status, "queued" + ) + + def test_invalid_from(self): + self.message.from_email = "webmaster@localhost" # Django's default From + with self.assertRaisesMessage( + AnymailAPIError, "does not match domain in From field (localhost)" + ): + self.message.send() + + @override_settings(ANYMAIL_MAILPACE_SERVER_TOKEN="Hey, that's not a server token!") + def test_invalid_server_token(self): + with self.assertRaisesMessage(AnymailAPIError, "Invalid API Token"): + self.message.send() From b03feb9c1d8c5780cddfe3732bd198f1b4943b38 Mon Sep 17 00:00:00 2001 From: Mike Edmunds Date: Mon, 19 Feb 2024 17:02:52 -0800 Subject: [PATCH 25/25] Update esp-feature-matrix in docs --- docs/esps/esp-feature-matrix.csv | 38 ++++++++++++++++---------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/esps/esp-feature-matrix.csv b/docs/esps/esp-feature-matrix.csv index d6c579fa..d49b289b 100644 --- a/docs/esps/esp-feature-matrix.csv +++ b/docs/esps/esp-feature-matrix.csv @@ -1,19 +1,19 @@ -Email Service Provider,:ref:`amazon-ses-backend`,:ref:`brevo-backend`,:ref:`mailersend-backend`,:ref:`mailgun-backend`,:ref:`mailjet-backend`,:ref:`mandrill-backend`,:ref:`postal-backend`,:ref:`postmark-backend`,:ref:`resend-backend`,:ref:`sendgrid-backend`,:ref:`sparkpost-backend` -.. rubric:: :ref:`Anymail send options `,,,,,,,,,,, -:attr:`~AnymailMessage.envelope_sender`,Yes,No,No,Domain only,Yes,Domain only,Yes,No,No,No,Yes -:attr:`~AnymailMessage.metadata`,Yes,Yes,No,Yes,Yes,Yes,No,Yes,Yes,Yes,Yes -:attr:`~AnymailMessage.merge_metadata`,No,Yes,No,Yes,Yes,Yes,No,Yes,Yes,Yes,Yes -:attr:`~AnymailMessage.send_at`,No,Yes,Yes,Yes,No,Yes,No,No,No,Yes,Yes -:attr:`~AnymailMessage.tags`,Yes,Yes,Yes,Yes,Max 1 tag,Yes,Max 1 tag,Max 1 tag,Yes,Yes,Max 1 tag -:attr:`~AnymailMessage.track_clicks`,No,No,Yes,Yes,Yes,Yes,No,Yes,No,Yes,Yes -:attr:`~AnymailMessage.track_opens`,No,No,Yes,Yes,Yes,Yes,No,Yes,No,Yes,Yes -:ref:`amp-email`,Yes,No,No,Yes,No,No,No,No,No,Yes,Yes -.. rubric:: :ref:`templates-and-merge`,,,,,,,,,,, -:attr:`~AnymailMessage.template_id`,Yes,Yes,Yes,Yes,Yes,Yes,No,Yes,No,Yes,Yes -:attr:`~AnymailMessage.merge_data`,Yes,Yes,Yes,Yes,Yes,Yes,No,Yes,No,Yes,Yes -:attr:`~AnymailMessage.merge_global_data`,Yes,Yes,(emulated),(emulated),Yes,Yes,No,Yes,No,Yes,Yes -.. rubric:: :ref:`Status ` and :ref:`event tracking `,,,,,,,,,,, -:attr:`~AnymailMessage.anymail_status`,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes -:class:`~anymail.signals.AnymailTrackingEvent` from webhooks,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes -.. rubric:: :ref:`Inbound handling `,,,,,,,,,,, -:class:`~anymail.signals.AnymailInboundEvent` from webhooks,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,No,Yes,Yes +Email Service Provider,:ref:`amazon-ses-backend`,:ref:`brevo-backend`,:ref:`mailersend-backend`,:ref:`mailgun-backend`,:ref:`mailjet-backend`,:ref:`mailpace-backend`,:ref:`mandrill-backend`,:ref:`postal-backend`,:ref:`postmark-backend`,:ref:`resend-backend`,:ref:`sendgrid-backend`,:ref:`sparkpost-backend` +.. rubric:: :ref:`Anymail send options `,,,,,,,,,,,, +:attr:`~AnymailMessage.envelope_sender`,Yes,No,No,Domain only,Yes,No,Domain only,Yes,No,No,No,Yes +:attr:`~AnymailMessage.metadata`,Yes,Yes,No,Yes,Yes,No,Yes,No,Yes,Yes,Yes,Yes +:attr:`~AnymailMessage.merge_metadata`,No,Yes,No,Yes,Yes,No,Yes,No,Yes,Yes,Yes,Yes +:attr:`~AnymailMessage.send_at`,No,Yes,Yes,Yes,No,No,Yes,No,No,No,Yes,Yes +:attr:`~AnymailMessage.tags`,Yes,Yes,Yes,Yes,Max 1 tag,Yes,Yes,Max 1 tag,Max 1 tag,Yes,Yes,Max 1 tag +:attr:`~AnymailMessage.track_clicks`,No,No,Yes,Yes,Yes,No,Yes,No,Yes,No,Yes,Yes +:attr:`~AnymailMessage.track_opens`,No,No,Yes,Yes,Yes,No,Yes,No,Yes,No,Yes,Yes +:ref:`amp-email`,Yes,No,No,Yes,No,No,No,No,No,No,Yes,Yes +.. rubric:: :ref:`templates-and-merge`,,,,,,,,,,,, +:attr:`~AnymailMessage.template_id`,Yes,Yes,Yes,Yes,Yes,No,Yes,No,Yes,No,Yes,Yes +:attr:`~AnymailMessage.merge_data`,Yes,Yes,Yes,Yes,Yes,No,Yes,No,Yes,No,Yes,Yes +:attr:`~AnymailMessage.merge_global_data`,Yes,Yes,(emulated),(emulated),Yes,No,Yes,No,Yes,No,Yes,Yes +.. rubric:: :ref:`Status ` and :ref:`event tracking `,,,,,,,,,,,, +:attr:`~AnymailMessage.anymail_status`,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes +:class:`~anymail.signals.AnymailTrackingEvent` from webhooks,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes +.. rubric:: :ref:`Inbound handling `,,,,,,,,,,,, +:class:`~anymail.signals.AnymailInboundEvent` from webhooks,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,No,Yes,Yes