From 36786e7339603daec8096d4ed436e0d1ceb5bcd3 Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Thu, 21 Mar 2024 17:12:05 -0400 Subject: [PATCH 01/68] initial version send/receive o365 mail --- django_mail_admin/backends.py | 45 ++++++++ django_mail_admin/models/configurations.py | 16 ++- django_mail_admin/o365_util.py | 123 +++++++++++++++++++++ django_mail_admin/transports/__init__.py | 1 + django_mail_admin/transports/o365.py | 42 +++++++ example/example/settings.py | 10 +- 6 files changed, 235 insertions(+), 2 deletions(-) create mode 100644 django_mail_admin/o365_util.py create mode 100644 django_mail_admin/transports/o365.py diff --git a/django_mail_admin/backends.py b/django_mail_admin/backends.py index 5bd77f2..4682e5c 100644 --- a/django_mail_admin/backends.py +++ b/django_mail_admin/backends.py @@ -1,6 +1,8 @@ import logging import threading +from urllib import parse + from django.core.mail.backends.base import BaseEmailBackend from django.core.mail.backends.smtp import EmailBackend @@ -8,6 +10,8 @@ from .models import Outbox, create_attachments from .utils import PRIORITY +from django_mail_admin.o365_util import O365Connection + logger = logging.getLogger(__name__) @@ -66,3 +70,44 @@ def send_messages(self, email_messages): logger.exception('Email queue failed') return len(email_messages) + +class O365Backend(BaseEmailBackend): + """ + Backend to handle sending emails via o365 connection + """ + scheme = O365Connection.scheme #"office365" + def __init__(self, fail_silently: bool=False, **kwargs) -> None: + super().__init__(fail_silently, **kwargs) + self.conn: O365Connection = None + self.from_email: str = None + self.fail_silently: bool = fail_silently + self._lock = threading.RLock() + + def open(self): + configuration: Outbox = None + + configurations = Outbox.objects.filter(active=True) + if len(configurations) > 1 or len(configurations) == 0: + raise ValueError('Got %(l)s active configurations, expected 1' % {'l': len(configurations)}) + else: + configuration = configurations.first() + + self.from_email = configuration.email_host_user + + self.connection = None + parseresult=parse.urlparse(configuration.email_host) + if not self.scheme == parseresult.scheme.lower(): + raise ValueError(f'Invalid EMAIL_HOST scheme, expected "{self.scheme}", got "{parseresult.scheme}"') + query_dict=dict(parse.parse_qsl(parseresult.query)) + client_id_key=query_dict.get( "client_id_key", "" ) + client_secret_key=query_dict.get("client_secret_key", "") + + self.conn = O365Connection( + client_id_key=client_id_key, + client_secret_key=client_secret_key + ) + + def send_messages(self, email_messages) -> int: + if not self.conn or not self.from_email: + raise Exception(f"Backend not yet ready to send_messages") + return self.conn.send_messages(self.from_email, email_messages, fail_silently=self.fail_silently) \ No newline at end of file diff --git a/django_mail_admin/models/configurations.py b/django_mail_admin/models/configurations.py index 42ae6ea..f587da3 100644 --- a/django_mail_admin/models/configurations.py +++ b/django_mail_admin/models/configurations.py @@ -21,7 +21,7 @@ from django_mail_admin.signals import message_received from django_mail_admin.transports import Pop3Transport, ImapTransport, \ MaildirTransport, MboxTransport, BabylTransport, MHTransport, \ - MMDFTransport, GmailImapTransport + MMDFTransport, GmailImapTransport, O365Transport logger = logging.getLogger(__name__) @@ -93,6 +93,11 @@ class Mailbox(models.Model): "
" "Be sure to urlencode your username and password should they " "contain illegal characters (like @, :, etc)." + "
" + "For MS-365 email accounts use: 
office365:username@example.com:/?"
+            "client_id=&client_secret=&
" + "supports only on-behalf-of-a-user; thus requires user's auth & consent" + "
" )), blank=True, null=True, @@ -110,6 +115,8 @@ class Mailbox(models.Model): "setting is unnecessary.
" "If you send e-mail without setting this, your 'From' header will'" "be set to match the setting `DEFAULT_FROM_EMAIL`." + "
" + "Required for Office 365 mailbox" )), blank=True, null=True, @@ -234,6 +241,13 @@ def get_connection(self): archive=self.archive ) conn.connect(self.username, self.password) + elif self.type == O365Transport.scheme: #'office365' + conn = O365Transport(last_polled=self.last_polling) + conn.connect( + client_id_key=self._query_string.get( "client_id_key", [""])[0], + client_secret_key=self._query_string.get("client_secret_key", [""])[0], + owner_email=self.from_email, + ) elif self.type == 'pop3': conn = Pop3Transport( self.location, diff --git a/django_mail_admin/o365_util.py b/django_mail_admin/o365_util.py new file mode 100644 index 0000000..175e897 --- /dev/null +++ b/django_mail_admin/o365_util.py @@ -0,0 +1,123 @@ +""" +Helper utility classes/ functions for O365 support +""" +import logging + +from base64 import b64encode +from datetime import datetime +from django.conf import settings + +from O365 import MSGraphProtocol +from O365 import Account, FileSystemTokenBackend + +logger = logging.getLogger(__name__) + +class O365Connection: + scheme = "office365" + o365_protocol = "MSGraphProtocol" + + def __init__(self, + client_id_key:str, + client_secret_key:str, + protocol=o365_protocol) -> None: + self.account = None + try: + client_id, client_secret = self._get_auth_info(client_id_key, client_secret_key) + self._connect(client_id, client_secret, protocol) + except (TypeError, ValueError) as e: + logger.warning("O365Connection: Couldn't authenticate %s" % e) + + def _get_auth_info(self, client_id_key:str, client_secret_key:str): + if not settings.O365_ADMIN_SETTINGS: + raise Exception("O365_ADMIN_SETTINGS configuration missing.") + + client_id = settings.O365_ADMIN_SETTINGS.get(client_id_key,"") + if not client_id: + raise Exception(f"O365_ADMIN_SETTINGS.{client_id_key} not set! ") + + client_secret = settings.O365_ADMIN_SETTINGS.get(client_secret_key,"") + if not client_secret: + raise Exception(f"O365_ADMIN_SETTINGS.{client_secret_key} not set!") + + return client_id, client_secret + + def _connect(self, client_id, client_secret, protocol) -> None: + # connect_id/ and secret should have been already setup + # for offline & message_all scopes, for on behalf of user access. + protocol_selected= MSGraphProtocol(api_version='beta') if protocol == self.o365_protocol else None + if not protocol_selected: + raise Exception(f"Unsupported protocol {protocol}") + + token_path=settings.O365_ADMIN_SETTINGS.get("O365_AUTH_BACKEND_TOKEN_DIR") + if not token_path: + token_path = "." + logger.warning(f"Using default path '{token_path}'; highly recommended to set explicit path in O365_ADMIN_SETTINGS.O365_AUTH_BACKEND_TOKEN_DIR") + + token_filename=settings.O365_ADMIN_SETTINGS.get("O365_AUTH_BACKEND_TOKEN_FILE") + if not token_filename: + token_filename = "o365_token.txt" + logger.warning(f"Using default token filename {token_filename}; highly recommended to set explicit path in O365_ADMIN_SETTINGS.O365_AUTH_BACKEND_TOKEN_DIR") + + token_backend = FileSystemTokenBackend( + token_path=token_path, + token_filename=token_filename + ) + + self.account = Account(credentials=(client_id, client_secret), protocol=protocol_selected, scopes=['offline_access', 'message_all'], token_backend=token_backend) + + if not self.account.is_authenticated: + logger.info("Hold on .... going for authentication!") + self.account.authenticate() + + def _prepare_attachment_for_dispatch(self, attachment): + """bridge from Django EmailMessage Attachment to O365 Attachment""" + content=attachment[1] + b64content=b64encode( content if isinstance(content, bytes) else bytes(content, 'utf-8') ).decode('utf-8') + return { "name": f"{attachment[0]}" + , "content": b64content + , "on_disk": False + } + + def send_messages(self, + from_email:str, + email_messages, + fail_silently:bool = False) -> int: + if not from_email: + raise ValueError("from_email can not be empty!") + + sent_messages = 0 + mailbox = self.account.mailbox(from_email) + for msg in email_messages: + try: + m = mailbox.new_message() + m.to.add(msg.to) + m.cc.add(msg.cc) + m.bcc.add(msg.bcc) + m.subject = msg.subject + m.body = msg.body + m.save_message() + m.attachments.add( [self._prepare_attachment_for_dispatch(attachment) for attachment in msg.attachments] ) + for attachment in m.attachments: + #workaround: avoid NoneType compare w/ int, O365 exception + attachment.size = len(attachment.content) + m.save_draft() + m.send() + sent_messages += 1 + except Exception as e: + logger.error(f"Exception in sending message: error info: {e}") + if not fail_silently: + raise e + return sent_messages + + def get_messages(self, owner_email:str, last_polled: datetime, condition): + if not self.account.is_authenticated: + logger.error(f"get_messages unavailable; account not authenticated!") + return + mailbox = self.account.mailbox(owner_email) + mail_folder = mailbox.get_folder(folder_name="Inbox") + if last_polled: + #ISO 8601 format AND in UTC time. + #For example, midnight UTC on Jan 1, 2022 is 2022-01-01T00:00:00Z. + qstr=f"receivedDateTime gt {last_polled.replace(microsecond=0).isoformat()[:-6]}Z" + for mail in mail_folder.get_messages(query=qstr): + yield(mail.get_mime_content()) diff --git a/django_mail_admin/transports/__init__.py b/django_mail_admin/transports/__init__.py index fdc51d8..b518417 100644 --- a/django_mail_admin/transports/__init__.py +++ b/django_mail_admin/transports/__init__.py @@ -6,3 +6,4 @@ from django_mail_admin.transports.mh import MHTransport from django_mail_admin.transports.mmdf import MMDFTransport from django_mail_admin.transports.gmail import GmailImapTransport +from django_mail_admin.transports.o365 import O365Transport \ No newline at end of file diff --git a/django_mail_admin/transports/o365.py b/django_mail_admin/transports/o365.py new file mode 100644 index 0000000..07c46e1 --- /dev/null +++ b/django_mail_admin/transports/o365.py @@ -0,0 +1,42 @@ +""" +EmailTransport implementation for O365 +""" +import logging + +from datetime import datetime + +from django_mail_admin.o365_util import O365Connection + +from .base import EmailTransport + +logger = logging.getLogger(__name__) + +class O365Transport(EmailTransport): + scheme = O365Connection.scheme + + def __init__(self, last_polled: datetime = None) -> None: + super().__init__() + self.conn: O365Connection = None + self.owner_email: str = None + self.last_polled: datetime | None = last_polled + + def connect(self, + client_id_key: str, + client_secret_key: str, + owner_email: str, + o365_protocol="MSGraphProtocol" ) -> None: + try: + self.conn = O365Connection( client_id_key = client_id_key, + client_secret_key = client_secret_key, + protocol = o365_protocol + ) + self.owner_email = owner_email + except (TypeError, ValueError) as e: + logger.warning("Couldn't authenticate %s" % e) + + def get_message(self, condition): + if not self.conn: + logger.error(f"get_message unavailable; account not connected yet") + return + for mail in self.conn.get_messages(self.owner_email, self.last_polled, condition): + yield(self.get_email_from_bytes(mail)) diff --git a/example/example/settings.py b/example/example/settings.py index ea2da64..3b4029e 100644 --- a/example/example/settings.py +++ b/example/example/settings.py @@ -12,6 +12,7 @@ import os import sys +from decouple import config # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -121,4 +122,11 @@ INSTALLED_APPS += ( 'django_admin_row_actions', ) -EMAIL_BACKEND = 'django_mail_admin.backends.CustomEmailBackend' +EMAIL_BACKEND = 'django_mail_admin.backends.O365Backend' + +O365_ADMIN_SETTINGS = { + "O365_AUTH_BACKEND_TOKEN_DIR":config("O365_AUTH_BACKEND_TOKEN_DIR"), + "O365_AUTH_BACKEND_TOKEN_FILE":config("O365_AUTH_BACKEND_TOKEN_FILE"), + "O365_CLIENT_ID":config("O365_CLIENT_ID"), + "O365_CLIENT_SECRET":config("O365_CLIENT_SECRET") +} \ No newline at end of file From be791230562b151c8392e9438e9bbaa943bee4e3 Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Sat, 23 Mar 2024 14:06:36 -0400 Subject: [PATCH 02/68] linked in-reply-to message, support reply-to. --- django_mail_admin/o365_util.py | 37 ++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/django_mail_admin/o365_util.py b/django_mail_admin/o365_util.py index 175e897..89d86e7 100644 --- a/django_mail_admin/o365_util.py +++ b/django_mail_admin/o365_util.py @@ -6,9 +6,11 @@ from base64 import b64encode from datetime import datetime from django.conf import settings +from django.core.mail.message import EmailMessage from O365 import MSGraphProtocol -from O365 import Account, FileSystemTokenBackend +from O365 import Account, Message, FileSystemTokenBackend +from O365.mailbox import MailBox logger = logging.getLogger(__name__) @@ -69,7 +71,32 @@ def _connect(self, client_id, client_secret, protocol) -> None: logger.info("Hold on .... going for authentication!") self.account.authenticate() - def _prepare_attachment_for_dispatch(self, attachment): + def _get_message_by_id(self, mailbox:MailBox, message_id:str, folder_name="Inbox") -> Message|None: + """retrieve message by id from given mailbox""" + if not message_id: + return None + mail_folder = mailbox.get_folder(folder_name=folder_name) if mailbox else None + qstr = f"internetMessageId eq '{message_id}'" + emails = mail_folder.get_messages(query=qstr) if mail_folder else [] + for email in emails: + return email + return None + + def _get_reply_to_message(self, mailbox:MailBox, msg: EmailMessage) -> Message|None: + """retrieve message representing in-reply-to id in msg headers""" + if not msg or not msg.extra_headers or not mailbox: + return None + msg_id = msg.extra_headers.get('In-Reply-To', None) + reply_to_msg = self._get_message_by_id( mailbox, msg_id ) if msg_id else None + return reply_to_msg + + def _prepare_new_message(self, mailbox, msg:EmailMessage): + """create new or reply-to draft based on msg in-reply-to header""" + reply_to_message = self._get_reply_to_message(mailbox, msg) + new_draft_message = reply_to_message.reply() if reply_to_message else mailbox.new_message() + return new_draft_message + + def _prepare_attachment_for_dispatch(self, attachment) -> dict: """bridge from Django EmailMessage Attachment to O365 Attachment""" content=attachment[1] b64content=b64encode( content if isinstance(content, bytes) else bytes(content, 'utf-8') ).decode('utf-8') @@ -89,12 +116,14 @@ def send_messages(self, mailbox = self.account.mailbox(from_email) for msg in email_messages: try: - m = mailbox.new_message() + m = self._prepare_new_message(mailbox, msg) m.to.add(msg.to) m.cc.add(msg.cc) m.bcc.add(msg.bcc) - m.subject = msg.subject + m.reply_to.add(msg.reply_to) m.body = msg.body + if msg.subject: + m.subject = msg.subject m.save_message() m.attachments.add( [self._prepare_attachment_for_dispatch(attachment) for attachment in msg.attachments] ) for attachment in m.attachments: From d8d45e0d542151e1473aa39a2d9d023a3dcb43c2 Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Sat, 23 Mar 2024 14:41:19 -0400 Subject: [PATCH 03/68] tidy up with black --- .pre-commit-config.yaml | 13 +++ django_mail_admin/backends.py | 80 +++++++++----- django_mail_admin/o365_util.py | 129 ++++++++++++++--------- django_mail_admin/transports/__init__.py | 2 +- django_mail_admin/transports/o365.py | 28 +++-- example/example/settings.py | 94 ++++++++--------- 6 files changed, 208 insertions(+), 138 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..294ef2d --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,13 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + - repo: https://github.com/psf/black + rev: 22.10.0 + hooks: + - id: black diff --git a/django_mail_admin/backends.py b/django_mail_admin/backends.py index 4682e5c..a615b20 100644 --- a/django_mail_admin/backends.py +++ b/django_mail_admin/backends.py @@ -16,26 +16,45 @@ class CustomEmailBackend(EmailBackend): - def __init__(self, host=None, port=None, username=None, password=None, - use_tls=None, fail_silently=False, use_ssl=None, timeout=None, - ssl_keyfile=None, ssl_certfile=None, - **kwargs): + def __init__( + self, + host=None, + port=None, + username=None, + password=None, + use_tls=None, + fail_silently=False, + use_ssl=None, + timeout=None, + ssl_keyfile=None, + ssl_certfile=None, + **kwargs, + ): super(CustomEmailBackend, self).__init__(fail_silently=fail_silently) # TODO: implement choosing backend for a letter as a param configurations = Outbox.objects.filter(active=True) if len(configurations) > 1 or len(configurations) == 0: - raise ValueError('Got %(l)s active configurations, expected 1' % {'l': len(configurations)}) + raise ValueError( + "Got %(l)s active configurations, expected 1" + % {"l": len(configurations)} + ) else: configuration = configurations.first() self.host = host or configuration.email_host self.port = port or configuration.email_port self.username = configuration.email_host_user if username is None else username - self.password = configuration.email_host_password if password is None else password + self.password = ( + configuration.email_host_password if password is None else password + ) self.use_tls = configuration.email_use_tls if use_tls is None else use_tls self.use_ssl = configuration.email_use_ssl if use_ssl is None else use_ssl self.timeout = configuration.email_timeout if timeout is None else timeout - self.ssl_keyfile = configuration.email_ssl_keyfile if ssl_keyfile is None else ssl_keyfile - self.ssl_certfile = configuration.email_ssl_certfile if ssl_certfile is None else ssl_certfile + self.ssl_keyfile = ( + configuration.email_ssl_keyfile if ssl_keyfile is None else ssl_keyfile + ) + self.ssl_certfile = ( + configuration.email_ssl_certfile if ssl_certfile is None else ssl_certfile + ) self.connection = None self._lock = threading.RLock() @@ -54,9 +73,9 @@ def send_messages(self, email_messages): headers=msg.extra_headers, priority=PRIORITY.medium, ) - alternatives = getattr(msg, 'alternatives', []) + alternatives = getattr(msg, "alternatives", []) for content, mimetype in alternatives: - if mimetype == 'text/html': + if mimetype == "text/html": email.html_message = content email.save() @@ -67,16 +86,19 @@ def send_messages(self, email_messages): except Exception: if not self.fail_silently: raise - logger.exception('Email queue failed') + logger.exception("Email queue failed") return len(email_messages) + class O365Backend(BaseEmailBackend): """ Backend to handle sending emails via o365 connection """ - scheme = O365Connection.scheme #"office365" - def __init__(self, fail_silently: bool=False, **kwargs) -> None: + + scheme = O365Connection.scheme # "office365" + + def __init__(self, fail_silently: bool = False, **kwargs) -> None: super().__init__(fail_silently, **kwargs) self.conn: O365Connection = None self.from_email: str = None @@ -88,26 +110,32 @@ def open(self): configurations = Outbox.objects.filter(active=True) if len(configurations) > 1 or len(configurations) == 0: - raise ValueError('Got %(l)s active configurations, expected 1' % {'l': len(configurations)}) + raise ValueError( + "Got %(l)s active configurations, expected 1" + % {"l": len(configurations)} + ) else: configuration = configurations.first() - + self.from_email = configuration.email_host_user - + self.connection = None - parseresult=parse.urlparse(configuration.email_host) + parseresult = parse.urlparse(configuration.email_host) if not self.scheme == parseresult.scheme.lower(): - raise ValueError(f'Invalid EMAIL_HOST scheme, expected "{self.scheme}", got "{parseresult.scheme}"') - query_dict=dict(parse.parse_qsl(parseresult.query)) - client_id_key=query_dict.get( "client_id_key", "" ) - client_secret_key=query_dict.get("client_secret_key", "") - + raise ValueError( + f'Invalid EMAIL_HOST scheme, expected "{self.scheme}", got "{parseresult.scheme}"' + ) + query_dict = dict(parse.parse_qsl(parseresult.query)) + client_id_key = query_dict.get("client_id_key", "") + client_secret_key = query_dict.get("client_secret_key", "") + self.conn = O365Connection( - client_id_key=client_id_key, - client_secret_key=client_secret_key + client_id_key=client_id_key, client_secret_key=client_secret_key ) - + def send_messages(self, email_messages) -> int: if not self.conn or not self.from_email: raise Exception(f"Backend not yet ready to send_messages") - return self.conn.send_messages(self.from_email, email_messages, fail_silently=self.fail_silently) \ No newline at end of file + return self.conn.send_messages( + self.from_email, email_messages, fail_silently=self.fail_silently + ) diff --git a/django_mail_admin/o365_util.py b/django_mail_admin/o365_util.py index 89d86e7..ebb3cd0 100644 --- a/django_mail_admin/o365_util.py +++ b/django_mail_admin/o365_util.py @@ -14,64 +14,82 @@ logger = logging.getLogger(__name__) + class O365Connection: scheme = "office365" o365_protocol = "MSGraphProtocol" - def __init__(self, - client_id_key:str, - client_secret_key:str, - protocol=o365_protocol) -> None: + def __init__( + self, client_id_key: str, client_secret_key: str, protocol=o365_protocol + ) -> None: self.account = None try: - client_id, client_secret = self._get_auth_info(client_id_key, client_secret_key) + client_id, client_secret = self._get_auth_info( + client_id_key, client_secret_key + ) self._connect(client_id, client_secret, protocol) except (TypeError, ValueError) as e: logger.warning("O365Connection: Couldn't authenticate %s" % e) - def _get_auth_info(self, client_id_key:str, client_secret_key:str): + def _get_auth_info(self, client_id_key: str, client_secret_key: str): if not settings.O365_ADMIN_SETTINGS: raise Exception("O365_ADMIN_SETTINGS configuration missing.") - - client_id = settings.O365_ADMIN_SETTINGS.get(client_id_key,"") + + client_id = settings.O365_ADMIN_SETTINGS.get(client_id_key, "") if not client_id: raise Exception(f"O365_ADMIN_SETTINGS.{client_id_key} not set! ") - - client_secret = settings.O365_ADMIN_SETTINGS.get(client_secret_key,"") + + client_secret = settings.O365_ADMIN_SETTINGS.get(client_secret_key, "") if not client_secret: raise Exception(f"O365_ADMIN_SETTINGS.{client_secret_key} not set!") - + return client_id, client_secret def _connect(self, client_id, client_secret, protocol) -> None: - # connect_id/ and secret should have been already setup + # connect_id/ and secret should have been already setup # for offline & message_all scopes, for on behalf of user access. - protocol_selected= MSGraphProtocol(api_version='beta') if protocol == self.o365_protocol else None + protocol_selected = ( + MSGraphProtocol(api_version="beta") + if protocol == self.o365_protocol + else None + ) if not protocol_selected: raise Exception(f"Unsupported protocol {protocol}") - - token_path=settings.O365_ADMIN_SETTINGS.get("O365_AUTH_BACKEND_TOKEN_DIR") + + token_path = settings.O365_ADMIN_SETTINGS.get("O365_AUTH_BACKEND_TOKEN_DIR") if not token_path: token_path = "." - logger.warning(f"Using default path '{token_path}'; highly recommended to set explicit path in O365_ADMIN_SETTINGS.O365_AUTH_BACKEND_TOKEN_DIR") - - token_filename=settings.O365_ADMIN_SETTINGS.get("O365_AUTH_BACKEND_TOKEN_FILE") + logger.warning( + f"Using default path '{token_path}'; highly recommended to set explicit path in O365_ADMIN_SETTINGS.O365_AUTH_BACKEND_TOKEN_DIR" + ) + + token_filename = settings.O365_ADMIN_SETTINGS.get( + "O365_AUTH_BACKEND_TOKEN_FILE" + ) if not token_filename: token_filename = "o365_token.txt" - logger.warning(f"Using default token filename {token_filename}; highly recommended to set explicit path in O365_ADMIN_SETTINGS.O365_AUTH_BACKEND_TOKEN_DIR") - + logger.warning( + f"Using default token filename {token_filename}; highly recommended to set explicit path in O365_ADMIN_SETTINGS.O365_AUTH_BACKEND_TOKEN_DIR" + ) + token_backend = FileSystemTokenBackend( - token_path=token_path, - token_filename=token_filename + token_path=token_path, token_filename=token_filename ) - - self.account = Account(credentials=(client_id, client_secret), protocol=protocol_selected, scopes=['offline_access', 'message_all'], token_backend=token_backend) - + + self.account = Account( + credentials=(client_id, client_secret), + protocol=protocol_selected, + scopes=["offline_access", "message_all"], + token_backend=token_backend, + ) + if not self.account.is_authenticated: logger.info("Hold on .... going for authentication!") self.account.authenticate() - def _get_message_by_id(self, mailbox:MailBox, message_id:str, folder_name="Inbox") -> Message|None: + def _get_message_by_id( + self, mailbox: MailBox, message_id: str, folder_name="Inbox" + ) -> Message | None: """retrieve message by id from given mailbox""" if not message_id: return None @@ -82,33 +100,35 @@ def _get_message_by_id(self, mailbox:MailBox, message_id:str, folder_name="Inbox return email return None - def _get_reply_to_message(self, mailbox:MailBox, msg: EmailMessage) -> Message|None: + def _get_reply_to_message( + self, mailbox: MailBox, msg: EmailMessage + ) -> Message | None: """retrieve message representing in-reply-to id in msg headers""" if not msg or not msg.extra_headers or not mailbox: return None - msg_id = msg.extra_headers.get('In-Reply-To', None) - reply_to_msg = self._get_message_by_id( mailbox, msg_id ) if msg_id else None + msg_id = msg.extra_headers.get("In-Reply-To", None) + reply_to_msg = self._get_message_by_id(mailbox, msg_id) if msg_id else None return reply_to_msg - - def _prepare_new_message(self, mailbox, msg:EmailMessage): + + def _prepare_new_message(self, mailbox, msg: EmailMessage): """create new or reply-to draft based on msg in-reply-to header""" reply_to_message = self._get_reply_to_message(mailbox, msg) - new_draft_message = reply_to_message.reply() if reply_to_message else mailbox.new_message() + new_draft_message = ( + reply_to_message.reply() if reply_to_message else mailbox.new_message() + ) return new_draft_message - + def _prepare_attachment_for_dispatch(self, attachment) -> dict: """bridge from Django EmailMessage Attachment to O365 Attachment""" - content=attachment[1] - b64content=b64encode( content if isinstance(content, bytes) else bytes(content, 'utf-8') ).decode('utf-8') - return { "name": f"{attachment[0]}" - , "content": b64content - , "on_disk": False - } - - def send_messages(self, - from_email:str, - email_messages, - fail_silently:bool = False) -> int: + content = attachment[1] + b64content = b64encode( + content if isinstance(content, bytes) else bytes(content, "utf-8") + ).decode("utf-8") + return {"name": f"{attachment[0]}", "content": b64content, "on_disk": False} + + def send_messages( + self, from_email: str, email_messages, fail_silently: bool = False + ) -> int: if not from_email: raise ValueError("from_email can not be empty!") @@ -125,9 +145,14 @@ def send_messages(self, if msg.subject: m.subject = msg.subject m.save_message() - m.attachments.add( [self._prepare_attachment_for_dispatch(attachment) for attachment in msg.attachments] ) + m.attachments.add( + [ + self._prepare_attachment_for_dispatch(attachment) + for attachment in msg.attachments + ] + ) for attachment in m.attachments: - #workaround: avoid NoneType compare w/ int, O365 exception + # workaround: avoid NoneType compare w/ int, O365 exception attachment.size = len(attachment.content) m.save_draft() m.send() @@ -137,16 +162,16 @@ def send_messages(self, if not fail_silently: raise e return sent_messages - - def get_messages(self, owner_email:str, last_polled: datetime, condition): + + def get_messages(self, owner_email: str, last_polled: datetime, condition): if not self.account.is_authenticated: logger.error(f"get_messages unavailable; account not authenticated!") return mailbox = self.account.mailbox(owner_email) mail_folder = mailbox.get_folder(folder_name="Inbox") if last_polled: - #ISO 8601 format AND in UTC time. - #For example, midnight UTC on Jan 1, 2022 is 2022-01-01T00:00:00Z. - qstr=f"receivedDateTime gt {last_polled.replace(microsecond=0).isoformat()[:-6]}Z" + # ISO 8601 format AND in UTC time. + # For example, midnight UTC on Jan 1, 2022 is 2022-01-01T00:00:00Z. + qstr = f"receivedDateTime gt {last_polled.replace(microsecond=0).isoformat()[:-6]}Z" for mail in mail_folder.get_messages(query=qstr): - yield(mail.get_mime_content()) + yield (mail.get_mime_content()) diff --git a/django_mail_admin/transports/__init__.py b/django_mail_admin/transports/__init__.py index b518417..33c53b0 100644 --- a/django_mail_admin/transports/__init__.py +++ b/django_mail_admin/transports/__init__.py @@ -6,4 +6,4 @@ from django_mail_admin.transports.mh import MHTransport from django_mail_admin.transports.mmdf import MMDFTransport from django_mail_admin.transports.gmail import GmailImapTransport -from django_mail_admin.transports.o365 import O365Transport \ No newline at end of file +from django_mail_admin.transports.o365 import O365Transport diff --git a/django_mail_admin/transports/o365.py b/django_mail_admin/transports/o365.py index 07c46e1..fbf13e6 100644 --- a/django_mail_admin/transports/o365.py +++ b/django_mail_admin/transports/o365.py @@ -11,6 +11,7 @@ logger = logging.getLogger(__name__) + class O365Transport(EmailTransport): scheme = O365Connection.scheme @@ -20,16 +21,19 @@ def __init__(self, last_polled: datetime = None) -> None: self.owner_email: str = None self.last_polled: datetime | None = last_polled - def connect(self, - client_id_key: str, - client_secret_key: str, - owner_email: str, - o365_protocol="MSGraphProtocol" ) -> None: + def connect( + self, + client_id_key: str, + client_secret_key: str, + owner_email: str, + o365_protocol="MSGraphProtocol", + ) -> None: try: - self.conn = O365Connection( client_id_key = client_id_key, - client_secret_key = client_secret_key, - protocol = o365_protocol - ) + self.conn = O365Connection( + client_id_key=client_id_key, + client_secret_key=client_secret_key, + protocol=o365_protocol, + ) self.owner_email = owner_email except (TypeError, ValueError) as e: logger.warning("Couldn't authenticate %s" % e) @@ -38,5 +42,7 @@ def get_message(self, condition): if not self.conn: logger.error(f"get_message unavailable; account not connected yet") return - for mail in self.conn.get_messages(self.owner_email, self.last_polled, condition): - yield(self.get_email_from_bytes(mail)) + for mail in self.conn.get_messages( + self.owner_email, self.last_polled, condition + ): + yield (self.get_email_from_bytes(mail)) diff --git a/example/example/settings.py b/example/example/settings.py index 3b4029e..c8fcd52 100644 --- a/example/example/settings.py +++ b/example/example/settings.py @@ -25,61 +25,61 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = ['*'] +ALLOWED_HOSTS = ["*"] # Application definition INSTALLED_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - - 'django_mail_admin', - + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "django_mail_admin", # if your app has other dependencies that need to be added to the site # they should be added here ] MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", ] -ROOT_URLCONF = 'example.urls' +ROOT_URLCONF = "example.urls" TEMPLATES = [ { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [os.path.join(BASE_DIR, 'templates'), ], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [ + os.path.join(BASE_DIR, "templates"), + ], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", ], }, }, ] -WSGI_APPLICATION = 'example.wsgi.application' +WSGI_APPLICATION = "example.wsgi.application" # Database # https://docs.djangoproject.com/en/1.9/ref/settings/#databases DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": os.path.join(BASE_DIR, "db.sqlite3"), } } @@ -88,25 +88,25 @@ AUTH_PASSWORD_VALIDATORS = [ { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", }, ] # Internationalization # https://docs.djangoproject.com/en/1.9/topics/i18n/ -LANGUAGE_CODE = 'en-us' +LANGUAGE_CODE = "en-us" -TIME_ZONE = 'UTC' +TIME_ZONE = "UTC" USE_I18N = True @@ -117,16 +117,14 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.9/howto/static-files/ -STATIC_URL = '/static/' -MEDIA_URL = '/media/' -INSTALLED_APPS += ( - 'django_admin_row_actions', -) -EMAIL_BACKEND = 'django_mail_admin.backends.O365Backend' +STATIC_URL = "/static/" +MEDIA_URL = "/media/" +INSTALLED_APPS += ("django_admin_row_actions",) +EMAIL_BACKEND = "django_mail_admin.backends.O365Backend" O365_ADMIN_SETTINGS = { - "O365_AUTH_BACKEND_TOKEN_DIR":config("O365_AUTH_BACKEND_TOKEN_DIR"), - "O365_AUTH_BACKEND_TOKEN_FILE":config("O365_AUTH_BACKEND_TOKEN_FILE"), - "O365_CLIENT_ID":config("O365_CLIENT_ID"), - "O365_CLIENT_SECRET":config("O365_CLIENT_SECRET") -} \ No newline at end of file + "O365_AUTH_BACKEND_TOKEN_DIR": config("O365_AUTH_BACKEND_TOKEN_DIR"), + "O365_AUTH_BACKEND_TOKEN_FILE": config("O365_AUTH_BACKEND_TOKEN_FILE"), + "O365_CLIENT_ID": config("O365_CLIENT_ID"), + "O365_CLIENT_SECRET": config("O365_CLIENT_SECRET"), +} From 2ddb8b12a6283f3ae41c94b6dedbd8ff4125d2c4 Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Sat, 23 Mar 2024 14:53:29 -0400 Subject: [PATCH 04/68] black/flake8 fixes --- django_mail_admin/o365_util.py | 8 +++++--- django_mail_admin/transports/o365.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/django_mail_admin/o365_util.py b/django_mail_admin/o365_util.py index ebb3cd0..ea92402 100644 --- a/django_mail_admin/o365_util.py +++ b/django_mail_admin/o365_util.py @@ -59,8 +59,9 @@ def _connect(self, client_id, client_secret, protocol) -> None: token_path = settings.O365_ADMIN_SETTINGS.get("O365_AUTH_BACKEND_TOKEN_DIR") if not token_path: token_path = "." + logger.warning(f"Using default token path '{token_path}';") logger.warning( - f"Using default path '{token_path}'; highly recommended to set explicit path in O365_ADMIN_SETTINGS.O365_AUTH_BACKEND_TOKEN_DIR" + " - set explicit path in O365_ADMIN_SETTINGS. O365_AUTH_BACKEND_TOKEN_DIR" ) token_filename = settings.O365_ADMIN_SETTINGS.get( @@ -68,8 +69,9 @@ def _connect(self, client_id, client_secret, protocol) -> None: ) if not token_filename: token_filename = "o365_token.txt" + logger.warning(f"Using default token filename {token_filename};") logger.warning( - f"Using default token filename {token_filename}; highly recommended to set explicit path in O365_ADMIN_SETTINGS.O365_AUTH_BACKEND_TOKEN_DIR" + " - set explicit path in O365_ADMIN_SETTINGS.O365_AUTH_BACKEND_TOKEN_FILE" ) token_backend = FileSystemTokenBackend( @@ -165,7 +167,7 @@ def send_messages( def get_messages(self, owner_email: str, last_polled: datetime, condition): if not self.account.is_authenticated: - logger.error(f"get_messages unavailable; account not authenticated!") + logger.error("get_messages unavailable; account not authenticated!") return mailbox = self.account.mailbox(owner_email) mail_folder = mailbox.get_folder(folder_name="Inbox") diff --git a/django_mail_admin/transports/o365.py b/django_mail_admin/transports/o365.py index fbf13e6..7a712f2 100644 --- a/django_mail_admin/transports/o365.py +++ b/django_mail_admin/transports/o365.py @@ -40,7 +40,7 @@ def connect( def get_message(self, condition): if not self.conn: - logger.error(f"get_message unavailable; account not connected yet") + logger.error("get_message unavailable; account not connected yet") return for mail in self.conn.get_messages( self.owner_email, self.last_polled, condition From 2329137a353c2f6e696e5b9f648e8441ce00b1e0 Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Mon, 25 Mar 2024 14:49:36 -0400 Subject: [PATCH 05/68] added end-2-end test for O365 --- CONTRIBUTING.rst | 10 ++-- django_mail_admin/o365_util.py | 10 +++- tests/o365/__init__.py | 0 tests/o365/test_commands.py | 75 +++++++++++++++++++++++++++++ tests/settings.py | 86 +++++++++++++++++++++++----------- 5 files changed, 147 insertions(+), 34 deletions(-) create mode 100644 tests/o365/__init__.py create mode 100644 tests/o365/test_commands.py diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index c927724..fc36b6a 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -3,7 +3,7 @@ Contributing ============ Contributions are welcome, and they are greatly appreciated! Every -little bit helps, and credit will always be given. +little bit helps, and credit will always be given. You can contribute in many ways: @@ -36,7 +36,7 @@ is open to whoever wants to implement it. Write Documentation ~~~~~~~~~~~~~~~~~~~ -django mail admin could always use more documentation, whether as part of the +django mail admin could always use more documentation, whether as part of the official django mail admin docs, in docstrings, or even on the web in blog posts, articles, and such. @@ -78,10 +78,10 @@ Ready to contribute? Here's how to set up `django_mail_admin` for local developm tests, including testing other Python versions with tox:: $ flake8 django_mail_admin tests - $ python setup.py test + $ python manage.py test $ tox - To get flake8 and tox, just pip install them into your virtualenv. + To get flake8 and tox, just pip install them into your virtualenv. 6. Commit your changes and push your branch to GitHub:: @@ -100,7 +100,7 @@ Before you submit a pull request, check that it meets these guidelines: 2. If the pull request adds functionality, the docs should be updated. Put your new functionality into a function with a docstring, and add the feature to the list in README.rst. -3. The pull request should work for Python 2.6, 2.7, and 3.3, and for PyPy. Check +3. The pull request should work for Python 2.6, 2.7, and 3.3, and for PyPy. Check https://travis-ci.org/delneg/django_mail_admin/pull_requests and make sure that the tests pass for all supported Python versions. diff --git a/django_mail_admin/o365_util.py b/django_mail_admin/o365_util.py index ea92402..93abafc 100644 --- a/django_mail_admin/o365_util.py +++ b/django_mail_admin/o365_util.py @@ -59,20 +59,24 @@ def _connect(self, client_id, client_secret, protocol) -> None: token_path = settings.O365_ADMIN_SETTINGS.get("O365_AUTH_BACKEND_TOKEN_DIR") if not token_path: token_path = "." + """ logger.warning(f"Using default token path '{token_path}';") logger.warning( - " - set explicit path in O365_ADMIN_SETTINGS. O365_AUTH_BACKEND_TOKEN_DIR" + " - set explicit path in O365_ADMIN_SETTINGS.O365_AUTH_BACKEND_TOKEN_DIR" ) + """ token_filename = settings.O365_ADMIN_SETTINGS.get( "O365_AUTH_BACKEND_TOKEN_FILE" ) if not token_filename: token_filename = "o365_token.txt" + """ logger.warning(f"Using default token filename {token_filename};") logger.warning( " - set explicit path in O365_ADMIN_SETTINGS.O365_AUTH_BACKEND_TOKEN_FILE" ) + """ token_backend = FileSystemTokenBackend( token_path=token_path, token_filename=token_filename @@ -171,9 +175,11 @@ def get_messages(self, owner_email: str, last_polled: datetime, condition): return mailbox = self.account.mailbox(owner_email) mail_folder = mailbox.get_folder(folder_name="Inbox") + qstr = "" + order_by = f"receivedDateTime DESC" if last_polled: # ISO 8601 format AND in UTC time. # For example, midnight UTC on Jan 1, 2022 is 2022-01-01T00:00:00Z. qstr = f"receivedDateTime gt {last_polled.replace(microsecond=0).isoformat()[:-6]}Z" - for mail in mail_folder.get_messages(query=qstr): + for mail in mail_folder.get_messages(order_by=order_by, query=qstr): yield (mail.get_mime_content()) diff --git a/tests/o365/__init__.py b/tests/o365/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/o365/test_commands.py b/tests/o365/test_commands.py new file mode 100644 index 0000000..4c1664d --- /dev/null +++ b/tests/o365/test_commands.py @@ -0,0 +1,75 @@ +import time +import random + +from urllib import parse +from datetime import datetime + +from django.test import TestCase +from django.conf import settings +from django.core.management import call_command + +from django_mail_admin.models import Outbox, IncomingEmail, OutgoingEmail, STATUS +from django_mail_admin.models import Mailbox + + +class O365CommandTest(TestCase): + def print_incoming_emails(self): + for email in IncomingEmail.objects.all().order_by("processed"): + print(f"{email.message_id} | {email.subject} | {email.processed}") + + def test_o365(self): + user1_email = settings.O365_TEST_ACCOUNT + user1_email_str = parse.quote(user1_email) + o365_con = f"office365://{user1_email_str}@outlook.office365.com?client_id_key=O365_CLIENT_ID&client_secret_key=O365_CLIENT_SECRET&archive=Archived" + + inbox = Mailbox.objects.create( + name="O365CommandTest_Inbox", uri=o365_con, from_email=user1_email + ) + inbox.get_new_mail() + print(f"received {IncomingEmail.objects.count()} new emails") + # self.print_incoming_emails() + + outbox = Outbox.objects.create( + name="O365CommandTest_Outbox", + email_host=o365_con, + email_host_user=user1_email, + email_host_password="ase123hgfd", + active=True, + ) + + test_subject = f"UnitTest Subject Dated {datetime.now()}" + test_body = f"UnitTest Body Random {random.random()}" + """ + print(f"\ntest_subject: {test_subject}") + print(f"test_body: {test_body}\n") + """ + OutgoingEmail.objects.create( + from_email=user1_email, + to=[user1_email], + status=STATUS.queued, + subject=test_subject, + message=test_body, + backend_alias="o365", + ) + call_command("send_queued_mail", log_level=0) + time.sleep(5) + max_attempts = 3 + latest_email = None + while max_attempts: + max_attempts -= 1 + new_emails = inbox.get_new_mail() + if not new_emails: + print( + f"new mail not seen yet; checking again, attempts remaining {max_attempts}" + ) + time.sleep(5) + else: + latest_email = new_emails[0] + break + """ + print(f"\nlatest_email.subject: {latest_email.subject}") + print(f"latest_email.text: {latest_email.text}") + """ + assert latest_email + assert latest_email.subject == test_subject + assert latest_email.text == test_body diff --git a/tests/settings.py b/tests/settings.py index f3c3bc9..39fb59f 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -4,6 +4,8 @@ import django import os +from decouple import config + DEBUG = True USE_TZ = True @@ -16,45 +18,75 @@ DATABASES = { "default": { "ENGINE": "django.db.backends.sqlite3", - 'NAME': os.path.join(os.path.dirname(__file__), 'test.db'), - 'TEST_NAME': os.path.join(os.path.dirname(__file__), 'test.db'), + "NAME": os.path.join(os.path.dirname(__file__), "test.db"), + "TEST_NAME": os.path.join(os.path.dirname(__file__), "test.db"), } } CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - 'TIMEOUT': 36000, - 'KEY_PREFIX': 'django_mail_admin', + "default": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + "TIMEOUT": 36000, + "KEY_PREFIX": "django_mail_admin", + }, + "django_mail_admin": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + "TIMEOUT": 36000, + "KEY_PREFIX": "django_mail_admin", }, - 'django_mail_admin': { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - 'TIMEOUT': 36000, - 'KEY_PREFIX': 'django_mail_admin', - } } DJANGO_MAIL_ADMIN = { - 'BACKENDS': { - 'default': 'django.core.mail.backends.dummy.EmailBackend', - 'locmem': 'django.core.mail.backends.locmem.EmailBackend', - 'error': 'tests.test_backends.ErrorRaisingBackend', - 'smtp': 'django.core.mail.backends.smtp.EmailBackend', - 'connection_tester': 'django_mail_admin.tests.test_mail.ConnectionTestingBackend', - 'custom': 'django_mail_admin.backends.CustomEmailBackend' + "BACKENDS": { + "default": "django.core.mail.backends.dummy.EmailBackend", + "locmem": "django.core.mail.backends.locmem.EmailBackend", + "error": "tests.test_backends.ErrorRaisingBackend", + "smtp": "django.core.mail.backends.smtp.EmailBackend", + "connection_tester": "django_mail_admin.tests.test_mail.ConnectionTestingBackend", + "custom": "django_mail_admin.backends.CustomEmailBackend", + "o365": "django_mail_admin.backends.O365Backend", } } +O365_ADMIN_SETTINGS = { + "TOKEN_BACKEND": "FileSystemTokenBackend", + "O365_CLIENT_ID": config("O365_CLIENT_ID"), + "O365_CLIENT_SECRET": config("O365_CLIENT_SECRET"), +} +O365_WEBAPP_SETTINGS = { + "test_webapp1": { + # TBD certificate support + "auth": "token_secret", #'certificate' + "O365_CLIENT_ID": config("O365_CLIENT_ID"), + "O365_CLIENT_SECRET": config("O365_CLIENT_SECRET"), + }, +} + +O365_TOKEN_BACKENDS = { + "FileSystemTokenBackend": { + "O365_AUTH_BACKEND_TOKEN_DIR": config("O365_AUTH_BACKEND_TOKEN_DIR"), + "O365_AUTH_BACKEND_TOKEN_FILE": config("O365_AUTH_BACKEND_TOKEN_FILE"), + }, +} + +O365_MAILBOXES = { + "TestUser1": {"email": "test_user@test.com", "webapp": "test_webapp1"}, +} + +O365_TEST_ACCOUNT = config("O365_TEST_ACCOUNT") + TEMPLATES = [ { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [os.path.join(BASE_DIR, 'templates'), ], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [ + os.path.join(BASE_DIR, "templates"), + ], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", ], }, }, From 03006d0e08644f28e678455b8fd474e3692b2994 Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Mon, 25 Mar 2024 17:29:12 -0400 Subject: [PATCH 06/68] added support for AzureTokenBackend --- django_mail_admin/backends.py | 10 +- django_mail_admin/models/configurations.py | 326 +++++++++--------- .../{o365_util.py => o365_utils.py} | 162 +++++++-- django_mail_admin/transports/o365.py | 9 +- example/example/settings.py | 7 + tests/o365/test_commands.py | 6 +- tests/settings.py | 7 + 7 files changed, 326 insertions(+), 201 deletions(-) rename django_mail_admin/{o365_util.py => o365_utils.py} (58%) diff --git a/django_mail_admin/backends.py b/django_mail_admin/backends.py index a615b20..26cb9be 100644 --- a/django_mail_admin/backends.py +++ b/django_mail_admin/backends.py @@ -10,7 +10,7 @@ from .models import Outbox, create_attachments from .utils import PRIORITY -from django_mail_admin.o365_util import O365Connection +from django_mail_admin.o365_utils import O365Connection logger = logging.getLogger(__name__) @@ -130,12 +130,12 @@ def open(self): client_secret_key = query_dict.get("client_secret_key", "") self.conn = O365Connection( - client_id_key=client_id_key, client_secret_key=client_secret_key + from_email=self.from_email, + client_id_key=client_id_key, + client_secret_key=client_secret_key, ) def send_messages(self, email_messages) -> int: if not self.conn or not self.from_email: raise Exception(f"Backend not yet ready to send_messages") - return self.conn.send_messages( - self.from_email, email_messages, fail_silently=self.fail_silently - ) + return self.conn.send_messages(email_messages, fail_silently=self.fail_silently) diff --git a/django_mail_admin/models/configurations.py b/django_mail_admin/models/configurations.py index f587da3..e29f9d3 100644 --- a/django_mail_admin/models/configurations.py +++ b/django_mail_admin/models/configurations.py @@ -15,29 +15,49 @@ from django.utils.translation import gettext_lazy as _ from django_mail_admin import utils -from django_mail_admin.settings import get_allowed_mimetypes, strip_unallowed_mimetypes, \ - get_altered_message_header, get_text_stored_mimetypes, get_store_original_message, \ - get_compress_original_message, get_attachment_interpolation_header +from django_mail_admin.settings import ( + get_allowed_mimetypes, + strip_unallowed_mimetypes, + get_altered_message_header, + get_text_stored_mimetypes, + get_store_original_message, + get_compress_original_message, + get_attachment_interpolation_header, +) from django_mail_admin.signals import message_received -from django_mail_admin.transports import Pop3Transport, ImapTransport, \ - MaildirTransport, MboxTransport, BabylTransport, MHTransport, \ - MMDFTransport, GmailImapTransport, O365Transport +from django_mail_admin.transports import ( + Pop3Transport, + ImapTransport, + MaildirTransport, + MboxTransport, + BabylTransport, + MHTransport, + MMDFTransport, + GmailImapTransport, + O365Transport, +) logger = logging.getLogger(__name__) class Outbox(models.Model): - name = models.CharField(_('Name'), max_length=255) - email_use_tls = models.BooleanField('EMAIL_USE_TLS', default=True) - email_use_ssl = models.BooleanField('EMAIL_USE_SSL', default=False) - email_ssl_keyfile = models.CharField('EMAIL_SSL_KEYFILE', max_length=1024, null=True, blank=True) - email_ssl_certfile = models.CharField('EMAIL_SSL_CERTFILE', max_length=1024, null=True, blank=True) - email_host = models.CharField('EMAIL_HOST', max_length=1024) - email_host_user = models.CharField('EMAIL_HOST_USER', max_length=255) - email_host_password = models.CharField('EMAIL_HOST_PASSWORD', max_length=255) - email_port = models.PositiveSmallIntegerField('EMAIL_PORT', default=587) - email_timeout = models.PositiveSmallIntegerField('EMAIL_TIMEOUT', null=True, blank=True) - active = models.BooleanField(_('Active'), default=False) + name = models.CharField(_("Name"), max_length=255) + email_use_tls = models.BooleanField("EMAIL_USE_TLS", default=True) + email_use_ssl = models.BooleanField("EMAIL_USE_SSL", default=False) + email_ssl_keyfile = models.CharField( + "EMAIL_SSL_KEYFILE", max_length=1024, null=True, blank=True + ) + email_ssl_certfile = models.CharField( + "EMAIL_SSL_CERTFILE", max_length=1024, null=True, blank=True + ) + email_host = models.CharField("EMAIL_HOST", max_length=1024) + email_host_user = models.CharField("EMAIL_HOST_USER", max_length=255) + email_host_password = models.CharField("EMAIL_HOST_PASSWORD", max_length=255) + email_port = models.PositiveSmallIntegerField("EMAIL_PORT", default=587) + email_timeout = models.PositiveSmallIntegerField( + "EMAIL_TIMEOUT", null=True, blank=True + ) + active = models.BooleanField(_("Active"), default=False) def save(self, *args, **kwargs): # Only one item can be active at a time @@ -56,12 +76,17 @@ def save(self, *args, **kwargs): def clean(self): if self.email_use_ssl and self.email_use_tls: raise ValidationError( - _("EMAIL_USE_TLS/EMAIL_USE_SSL are mutually exclusive, so only set one of those settings to True.")) + _( + "EMAIL_USE_TLS/EMAIL_USE_SSL are mutually exclusive, so only set one of those settings to True." + ) + ) def __str__(self): - return '%(email_host_user)s@%(email_host)s:%(email_port)s' % {'email_host_user': self.email_host_user, - 'email_host': self.email_host, - 'email_port': self.email_port} + return "%(email_host_user)s@%(email_host)s:%(email_port)s" % { + "email_host_user": self.email_host_user, + "email_host": self.email_host, + "email_port": self.email_port, + } class Meta: verbose_name = _("Outbox") @@ -70,79 +95,93 @@ class Meta: class ActiveMailboxManager(models.Manager): def get_queryset(self): - return super(ActiveMailboxManager, self).get_queryset().filter( - active=True, + return ( + super(ActiveMailboxManager, self) + .get_queryset() + .filter( + active=True, + ) ) class Mailbox(models.Model): name = models.CharField( - _('Name'), + _("Name"), max_length=255, ) uri = models.CharField( - _('URI'), + _("URI"), max_length=255, - help_text=(_( - "Example: imap+ssl://myusername:mypassword@someserver
" - "
" - "Internet transports include 'imap' and 'pop3'; " - "common local file transports include 'maildir', 'mbox', " - "and less commonly 'babyl', 'mh', and 'mmdf'.
" - "
" - "Be sure to urlencode your username and password should they " - "contain illegal characters (like @, :, etc)." - "
" - "For MS-365 email accounts use: 
office365:username@example.com:/?"
-            "client_id=&client_secret=&
" - "supports only on-behalf-of-a-user; thus requires user's auth & consent" - "
" - )), + help_text=( + _( + "Example: imap+ssl://myusername:mypassword@someserver
" + "
" + "Internet transports include 'imap' and 'pop3'; " + "common local file transports include 'maildir', 'mbox', " + "and less commonly 'babyl', 'mh', and 'mmdf'.
" + "
" + "Be sure to urlencode your username and password should they " + "contain illegal characters (like @, :, etc)." + "
" + "For MS-365 email accounts use: 
office365:username@example.com:/?"
+                "client_id=&client_secret=&
" + "supports only on-behalf-of-a-user; thus requires user's auth & consent" + "
" + ) + ), blank=True, null=True, default=None, ) from_email = models.CharField( - _('From email'), + _("From email"), max_length=255, - help_text=(_( - "Example: MailBot <mailbot@yourdomain.com>
" - "'From' header to set for outgoing email.
" - "
" - "If you do not use this e-mail inbox for outgoing mail, this " - "setting is unnecessary.
" - "If you send e-mail without setting this, your 'From' header will'" - "be set to match the setting `DEFAULT_FROM_EMAIL`." - "
" - "Required for Office 365 mailbox" - )), + help_text=( + _( + "Example: MailBot <mailbot@yourdomain.com>
" + "'From' header to set for outgoing email.
" + "
" + "If you do not use this e-mail inbox for outgoing mail, this " + "setting is unnecessary.
" + "If you send e-mail without setting this, your 'From' header will'" + "be set to match the setting `DEFAULT_FROM_EMAIL`." + "
" + "Required for Office 365 mailbox" + ) + ), blank=True, null=True, default=None, ) active = models.BooleanField( - _('Active'), - help_text=(_( - "Check this e-mail inbox for new e-mail messages during polling " - "cycles. This checkbox does not have an effect upon whether " - "mail is collected here when this mailbox receives mail from a " - "pipe, and does not affect whether e-mail messages can be " - "dispatched from this mailbox. " - )), + _("Active"), + help_text=( + _( + "Check this e-mail inbox for new e-mail messages during polling " + "cycles. This checkbox does not have an effect upon whether " + "mail is collected here when this mailbox receives mail from a " + "pipe, and does not affect whether e-mail messages can be " + "dispatched from this mailbox. " + ) + ), blank=True, default=True, ) last_polling = models.DateTimeField( _("Last polling"), - help_text=(_("The time of last successful polling for messages." - "It is blank for new mailboxes and is not set for " - "mailboxes that only receive messages via a pipe.")), + help_text=( + _( + "The time of last successful polling for messages." + "It is blank for new mailboxes and is not set for " + "mailboxes that only receive messages via a pipe." + ) + ), blank=True, - null=True + null=True, ) objects = models.Manager() @@ -178,30 +217,30 @@ def password(self): @property def location(self): """Returns the location (domain and path) of messages.""" - return self._domain if self._domain else '' + self._protocol_info.path + return self._domain if self._domain else "" + self._protocol_info.path @property def type(self): """Returns the 'transport' name for this mailbox.""" scheme = self._protocol_info.scheme.lower() - if '+' in scheme: - return scheme.split('+')[0] + if "+" in scheme: + return scheme.split("+")[0] return scheme @property def use_ssl(self): """Returns whether or not this mailbox's connection uses SSL.""" - return '+ssl' in self._protocol_info.scheme.lower() + return "+ssl" in self._protocol_info.scheme.lower() @property def use_tls(self): """Returns whether or not this mailbox's connection uses STARTTLS.""" - return '+tls' in self._protocol_info.scheme.lower() + return "+tls" in self._protocol_info.scheme.lower() @property def archive(self): """Returns (if specified) the folder to archive messages to.""" - archive_folder = self._query_string.get('archive', None) + archive_folder = self._query_string.get("archive", None) if not archive_folder: return None return archive_folder[0] @@ -209,7 +248,7 @@ def archive(self): @property def folder(self): """Returns (if specified) the folder to fetch mail from.""" - folder = self._query_string.get('folder', None) + folder = self._query_string.get("folder", None) if not folder: return None return folder[0] @@ -223,47 +262,46 @@ def get_connection(self): """ if not self.uri: return None - elif self.type == 'imap': + elif self.type == "imap": conn = ImapTransport( self.location, port=self.port if self.port else None, ssl=self.use_ssl, tls=self.use_tls, archive=self.archive, - folder=self.folder + folder=self.folder, ) conn.connect(self.username, self.password) - elif self.type == 'gmail': + elif self.type == "gmail": conn = GmailImapTransport( self.location, port=self.port if self.port else None, ssl=True, - archive=self.archive + archive=self.archive, ) conn.connect(self.username, self.password) - elif self.type == O365Transport.scheme: #'office365' - conn = O365Transport(last_polled=self.last_polling) + elif self.type == O365Transport.scheme: #'office365' + conn = O365Transport( + owner_email=self.from_email, last_polled=self.last_polling + ) conn.connect( - client_id_key=self._query_string.get( "client_id_key", [""])[0], + client_id_key=self._query_string.get("client_id_key", [""])[0], client_secret_key=self._query_string.get("client_secret_key", [""])[0], - owner_email=self.from_email, ) - elif self.type == 'pop3': + elif self.type == "pop3": conn = Pop3Transport( - self.location, - port=self.port if self.port else None, - ssl=self.use_ssl + self.location, port=self.port if self.port else None, ssl=self.use_ssl ) conn.connect(self.username, self.password) - elif self.type == 'maildir': + elif self.type == "maildir": conn = MaildirTransport(self.location) - elif self.type == 'mbox': + elif self.type == "mbox": conn = MboxTransport(self.location) - elif self.type == 'babyl': + elif self.type == "babyl": conn = BabylTransport(self.location) - elif self.type == 'mh': + elif self.type == "mh": conn = MHTransport(self.location) - elif self.type == 'mmdf': + elif self.type == "mmdf": conn = MMDFTransport(self.location) return conn @@ -286,9 +324,7 @@ def _get_dehydrated_message(self, msg, record): for header, value in msg.items(): new[header] = value for part in msg.get_payload(): - new.attach( - self._get_dehydrated_message(part, record) - ) + new.attach(self._get_dehydrated_message(part, record)) elif ( strip_unallowed_mimetypes() and not msg.get_content_type() in get_allowed_mimetypes() @@ -297,18 +333,13 @@ def _get_dehydrated_message(self, msg, record): new[header] = value # Delete header, otherwise when attempting to deserialize the # payload, it will be expecting a body for this. - del new['Content-Transfer-Encoding'] - new[get_altered_message_header()] = ( - 'Stripped; Content type %s not allowed' % ( - msg.get_content_type() - ) - ) - new.set_payload('') - elif ( - ( - msg.get_content_type() not in get_text_stored_mimetypes() - ) or - ('attachment' in msg.get('Content-Disposition', '')) + del new["Content-Transfer-Encoding"] + new[ + get_altered_message_header() + ] = "Stripped; Content type %s not allowed" % (msg.get_content_type()) + new.set_payload("") + elif (msg.get_content_type() not in get_text_stored_mimetypes()) or ( + "attachment" in msg.get("Content-Disposition", "") ): filename = None raw_filename = msg.get_filename() @@ -319,17 +350,13 @@ def _get_dehydrated_message(self, msg, record): else: _, extension = os.path.splitext(filename) if not extension: - extension = '.bin' + extension = ".bin" attachment = IncomingAttachment() attachment.document.save( uuid.uuid4().hex + extension, - ContentFile( - BytesIO( - msg.get_payload(decode=True) - ).getvalue() - ) + ContentFile(BytesIO(msg.get_payload(decode=True)).getvalue()), ) attachment.message = record for key, value in msg.items(): @@ -337,14 +364,12 @@ def _get_dehydrated_message(self, msg, record): attachment.save() placeholder = EmailMessage() - placeholder[ - get_attachment_interpolation_header() - ] = str(attachment.pk) + placeholder[get_attachment_interpolation_header()] = str(attachment.pk) new = placeholder else: content_charset = msg.get_content_charset() if not content_charset: - content_charset = 'ascii' + content_charset = "ascii" try: # Make sure that the payload can be properly decoded in the # defined charset, if it can't, let's mash some things @@ -352,50 +377,36 @@ def _get_dehydrated_message(self, msg, record): msg.get_payload(decode=True).decode(content_charset) except LookupError: logger.warning( - "Unknown encoding %s; interpreting as ASCII!", - content_charset - ) - msg.set_payload( - msg.get_payload(decode=True).decode( - 'ascii', - 'ignore' - ) + "Unknown encoding %s; interpreting as ASCII!", content_charset ) + msg.set_payload(msg.get_payload(decode=True).decode("ascii", "ignore")) except ValueError: logger.warning( "Decoding error encountered; interpreting %s as ASCII!", - content_charset - ) - msg.set_payload( - msg.get_payload(decode=True).decode( - 'ascii', - 'ignore' - ) + content_charset, ) + msg.set_payload(msg.get_payload(decode=True).decode("ascii", "ignore")) new = msg return new def _process_message(self, message): from django_mail_admin.models import IncomingEmail, OutgoingEmail + msg = IncomingEmail() if get_store_original_message(): self._process_save_original_message(message, msg) msg.mailbox = self - if 'subject' in message: - msg.subject = ( - utils.convert_header_to_unicode(message['subject'])[0:255] - ) - if 'message-id' in message: - msg.message_id = message['message-id'][0:255].strip() - if 'from' in message: - msg.from_header = utils.convert_header_to_unicode(message['from']) - if 'to' in message: - msg.to_header = utils.convert_header_to_unicode(message['to']) - elif 'Delivered-To' in message: - msg.to_header = utils.convert_header_to_unicode( - message['Delivered-To'] - ) + if "subject" in message: + msg.subject = utils.convert_header_to_unicode(message["subject"])[0:255] + if "message-id" in message: + msg.message_id = message["message-id"][0:255].strip() + if "from" in message: + msg.from_header = utils.convert_header_to_unicode(message["from"]) + if "to" in message: + msg.to_header = utils.convert_header_to_unicode(message["to"]) + elif "Delivered-To" in message: + msg.to_header = utils.convert_header_to_unicode(message["Delivered-To"]) msg.save() message = self._get_dehydrated_message(message, msg) try: @@ -403,15 +414,20 @@ def _process_message(self, message): except KeyError as exc: # email.message.replace_header may raise 'KeyError' if the header # 'content-transfer-encoding' is missing - logger.warning("Failed to parse message: %s", exc, ) + logger.warning( + "Failed to parse message: %s", + exc, + ) return None msg.set_body(body) - if message['in-reply-to']: + if message["in-reply-to"]: try: - in_reply_to = message['in-reply-to'].strip() + in_reply_to = message["in-reply-to"].strip() # Hack to work with db-independent JSONField (which is interpreted as string in db) msg.in_reply_to = OutgoingEmail.objects.filter( - headers__contains='"Message-ID":"' + message['in-reply-to'].strip() + '"' + headers__contains='"Message-ID":"' + + message["in-reply-to"].strip() + + '"' )[0] except IndexError: pass @@ -422,18 +438,12 @@ def _process_save_original_message(self, message, msg): if get_compress_original_message(): with NamedTemporaryFile(suffix=".eml.gz") as fp_tmp: with gzip.GzipFile(fileobj=fp_tmp, mode="w") as fp: - fp.write(message.as_string().encode('utf-8')) - msg.eml.save( - "%s.eml.gz" % (uuid.uuid4(),), - File(fp_tmp), - save=False - ) + fp.write(message.as_string().encode("utf-8")) + msg.eml.save("%s.eml.gz" % (uuid.uuid4(),), File(fp_tmp), save=False) else: msg.eml.save( - '%s.eml' % uuid.uuid4(), - ContentFile(message.as_string()), - save=False + "%s.eml" % uuid.uuid4(), ContentFile(message.as_string()), save=False ) def get_new_mail(self, condition=None): @@ -447,12 +457,12 @@ def get_new_mail(self, condition=None): if msg is not None: new_mail.append(msg) self.last_polling = now() - self.save(update_fields=['last_polling']) + self.save(update_fields=["last_polling"]) return new_mail def __str__(self): return self.name class Meta: - verbose_name = _('Mailbox') - verbose_name_plural = _('Mailboxes') + verbose_name = _("Mailbox") + verbose_name_plural = _("Mailboxes") diff --git a/django_mail_admin/o365_util.py b/django_mail_admin/o365_utils.py similarity index 58% rename from django_mail_admin/o365_util.py rename to django_mail_admin/o365_utils.py index 93abafc..7162e33 100644 --- a/django_mail_admin/o365_util.py +++ b/django_mail_admin/o365_utils.py @@ -2,12 +2,15 @@ Helper utility classes/ functions for O365 support """ import logging +import hashlib from base64 import b64encode from datetime import datetime from django.conf import settings from django.core.mail.message import EmailMessage +from O365.utils import BaseTokenBackend + from O365 import MSGraphProtocol from O365 import Account, Message, FileSystemTokenBackend from O365.mailbox import MailBox @@ -20,8 +23,16 @@ class O365Connection: o365_protocol = "MSGraphProtocol" def __init__( - self, client_id_key: str, client_secret_key: str, protocol=o365_protocol + self, + from_email: str, + client_id_key: str, + client_secret_key: str, + protocol=o365_protocol, ) -> None: + self.from_email = from_email + if not self.from_email: + raise ValueError("from_email required.") + self.account = None try: client_id, client_secret = self._get_auth_info( @@ -45,6 +56,18 @@ def _get_auth_info(self, client_id_key: str, client_secret_key: str): return client_id, client_secret + def _decorate_token_name(self): + bname = settings.O365_ADMIN_SETTINGS.get( + "O365_AUTH_BACKEND_AZ_BLOB_NAME", "o365_token.txt" + ) + return "{}/{}".format( + hashlib.sha1( + settings.O365_ADMIN_SETTINGS.get("O365_CLIENT_ID").encode("utf-8") + + bname.encode("utf-8") + ).hexdigest(), + bname, + ) + def _connect(self, client_id, client_secret, protocol) -> None: # connect_id/ and secret should have been already setup # for offline & message_all scopes, for on behalf of user access. @@ -56,30 +79,14 @@ def _connect(self, client_id, client_secret, protocol) -> None: if not protocol_selected: raise Exception(f"Unsupported protocol {protocol}") - token_path = settings.O365_ADMIN_SETTINGS.get("O365_AUTH_BACKEND_TOKEN_DIR") - if not token_path: - token_path = "." - """ - logger.warning(f"Using default token path '{token_path}';") - logger.warning( - " - set explicit path in O365_ADMIN_SETTINGS.O365_AUTH_BACKEND_TOKEN_DIR" - ) - """ - - token_filename = settings.O365_ADMIN_SETTINGS.get( - "O365_AUTH_BACKEND_TOKEN_FILE" - ) - if not token_filename: - token_filename = "o365_token.txt" - """ - logger.warning(f"Using default token filename {token_filename};") - logger.warning( - " - set explicit path in O365_ADMIN_SETTINGS.O365_AUTH_BACKEND_TOKEN_FILE" - ) - """ - - token_backend = FileSystemTokenBackend( - token_path=token_path, token_filename=token_filename + token_backend = AzureTokenBackend( + connection_str=settings.O365_ADMIN_SETTINGS.get( + "O365_AUTH_BACKEND_AZ_CONNECTION_STR" + ), + container_name=settings.O365_ADMIN_SETTINGS.get( + "O365_AUTH_BACKEND_AZ_CONTAINER_PATH" + ), + blob_name=self._decorate_token_name(), ) self.account = Account( @@ -132,14 +139,9 @@ def _prepare_attachment_for_dispatch(self, attachment) -> dict: ).decode("utf-8") return {"name": f"{attachment[0]}", "content": b64content, "on_disk": False} - def send_messages( - self, from_email: str, email_messages, fail_silently: bool = False - ) -> int: - if not from_email: - raise ValueError("from_email can not be empty!") - + def send_messages(self, email_messages, fail_silently: bool = False) -> int: sent_messages = 0 - mailbox = self.account.mailbox(from_email) + mailbox = self.account.mailbox(self.from_email) for msg in email_messages: try: m = self._prepare_new_message(mailbox, msg) @@ -183,3 +185,99 @@ def get_messages(self, owner_email: str, last_polled: datetime, condition): qstr = f"receivedDateTime gt {last_polled.replace(microsecond=0).isoformat()[:-6]}Z" for mail in mail_folder.get_messages(order_by=order_by, query=qstr): yield (mail.get_mime_content()) + + +class AzureTokenBackend(BaseTokenBackend): + """An Azure Blob backend to store tokens""" + + def __init__(self, connection_str, container_name, blob_name): + """ + Init Backend + :param str connection_str: Connection str for the Blob storage account + :param str container_name: Container string for blob file. + :param str blob_name: Blob name + """ + self.container_name = container_name + self.blob_name = blob_name + try: + from azure.storage.blob import BlobClient + except ModuleNotFoundError as e: + raise Exception( + "Please install the azure-storage-blob package to use this token backend." + ) from e + super().__init__() + + self._client = BlobClient.from_connection_string( + connection_str, # type: ignore + container_name=container_name, # type: ignore + blob_name=blob_name, # type: ignore + ) + + def __repr__(self): + return "AzureTokenBackend('{}', '{}')".format( + self.connection_str, self.container_name + ) + + def load_token(self): + """ + Retrieves the token from the store + :return dict or None: The token if exists, None otherwise + """ + token = None + try: + downloader = self._client.download_blob(max_concurrency=1, encoding="UTF-8") + token_str = downloader.readall() + token = self.token_constructor(self.serializer.loads(token_str)) + except Exception as e: + logger.error( + "Token (blob_text: {}/{}) could not be retrieved from the backend: {}".format( + self.container_name, self.blob_name, e + ) + ) + + return token + + def save_token(self): + """ + Saves the token dict in the store + :return bool: Success / Failure + """ + if self.token is None: + raise ValueError('You have to set the "token" first.') + + try: + r = self._client.upload_blob(self.serializer.dumps(self.token)) + except Exception as e: + logger.error("Token secret could not be created: {}".format(e)) + return False + return True + + def delete_token(self): + """ + Deletes the token from the store + :return bool: Success / Failure + """ + try: + r = self._client.delete_blob() + except Exception as e: + logger.error("Token secret could not be deleted: {}".format(e)) + return False + else: + logger.warning( + "Deleted token secret {} ({}).".format( + self.container_name, self.blob_name + ) + ) + return True + + def check_token(self): + """ + Checks if the token exists + :return bool: True if it exists on the store + """ + try: + _ = self._client.exists() + except: + return False + else: + return True diff --git a/django_mail_admin/transports/o365.py b/django_mail_admin/transports/o365.py index 7a712f2..82ca48e 100644 --- a/django_mail_admin/transports/o365.py +++ b/django_mail_admin/transports/o365.py @@ -5,7 +5,7 @@ from datetime import datetime -from django_mail_admin.o365_util import O365Connection +from django_mail_admin.o365_utils import O365Connection from .base import EmailTransport @@ -15,26 +15,25 @@ class O365Transport(EmailTransport): scheme = O365Connection.scheme - def __init__(self, last_polled: datetime = None) -> None: + def __init__(self, owner_email: str, last_polled: datetime = None) -> None: super().__init__() self.conn: O365Connection = None - self.owner_email: str = None + self.owner_email: str = owner_email self.last_polled: datetime | None = last_polled def connect( self, client_id_key: str, client_secret_key: str, - owner_email: str, o365_protocol="MSGraphProtocol", ) -> None: try: self.conn = O365Connection( + from_email=self.owner_email, client_id_key=client_id_key, client_secret_key=client_secret_key, protocol=o365_protocol, ) - self.owner_email = owner_email except (TypeError, ValueError) as e: logger.warning("Couldn't authenticate %s" % e) diff --git a/example/example/settings.py b/example/example/settings.py index c8fcd52..5eca7b1 100644 --- a/example/example/settings.py +++ b/example/example/settings.py @@ -127,4 +127,11 @@ "O365_AUTH_BACKEND_TOKEN_FILE": config("O365_AUTH_BACKEND_TOKEN_FILE"), "O365_CLIENT_ID": config("O365_CLIENT_ID"), "O365_CLIENT_SECRET": config("O365_CLIENT_SECRET"), + "O365_AUTH_BACKEND_AZ_BLOB_NAME": config("O365_AUTH_BACKEND_AZ_BLOB_NAME"), + "O365_AUTH_BACKEND_AZ_CONNECTION_STR": config( + "O365_AUTH_BACKEND_AZ_CONNECTION_STR" + ), + "O365_AUTH_BACKEND_AZ_CONTAINER_PATH": config( + "O365_AUTH_BACKEND_AZ_CONTAINER_PATH" + ), } diff --git a/tests/o365/test_commands.py b/tests/o365/test_commands.py index 4c1664d..ef7dd81 100644 --- a/tests/o365/test_commands.py +++ b/tests/o365/test_commands.py @@ -10,6 +10,7 @@ from django_mail_admin.models import Outbox, IncomingEmail, OutgoingEmail, STATUS from django_mail_admin.models import Mailbox +from django_mail_admin.mail import send_queued class O365CommandTest(TestCase): @@ -51,7 +52,10 @@ def test_o365(self): message=test_body, backend_alias="o365", ) - call_command("send_queued_mail", log_level=0) + # call_command("send_queued_mail", log_level=0) + total_sent, total_failed = send_queued() + assert total_failed == 0 + assert total_sent == 1 time.sleep(5) max_attempts = 3 latest_email = None diff --git a/tests/settings.py b/tests/settings.py index 39fb59f..e012f20 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -51,6 +51,13 @@ "TOKEN_BACKEND": "FileSystemTokenBackend", "O365_CLIENT_ID": config("O365_CLIENT_ID"), "O365_CLIENT_SECRET": config("O365_CLIENT_SECRET"), + "O365_AUTH_BACKEND_AZ_BLOB_NAME": config("O365_AUTH_BACKEND_AZ_BLOB_NAME"), + "O365_AUTH_BACKEND_AZ_CONNECTION_STR": config( + "O365_AUTH_BACKEND_AZ_CONNECTION_STR" + ), + "O365_AUTH_BACKEND_AZ_CONTAINER_PATH": config( + "O365_AUTH_BACKEND_AZ_CONTAINER_PATH" + ), } O365_WEBAPP_SETTINGS = { "test_webapp1": { From b6dd47e4830289b76646b8f58f105a12b7cc61df Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Wed, 27 Mar 2024 13:53:39 -0400 Subject: [PATCH 07/68] redirection auth, azure token-backend, example app --- django_mail_admin/backends.py | 8 +- django_mail_admin/models/configurations.py | 11 +- django_mail_admin/o365_utils.py | 119 +++++++++++++++------ django_mail_admin/transports/o365.py | 4 +- example/example/settings.py | 31 +++++- example/example/urls.py | 17 ++- example/example/views.py | 69 ++++++++++++ example/templates/mailbox/auth.html | 19 ++++ example/templates/mailbox/base.html | 30 ++++++ example/templates/mailbox/index.html | 40 +++++++ requirements.txt | 3 + 11 files changed, 300 insertions(+), 51 deletions(-) create mode 100644 example/example/views.py create mode 100644 example/templates/mailbox/auth.html create mode 100644 example/templates/mailbox/base.html create mode 100644 example/templates/mailbox/index.html diff --git a/django_mail_admin/backends.py b/django_mail_admin/backends.py index 26cb9be..12c0d6e 100644 --- a/django_mail_admin/backends.py +++ b/django_mail_admin/backends.py @@ -96,8 +96,6 @@ class O365Backend(BaseEmailBackend): Backend to handle sending emails via o365 connection """ - scheme = O365Connection.scheme # "office365" - def __init__(self, fail_silently: bool = False, **kwargs) -> None: super().__init__(fail_silently, **kwargs) self.conn: O365Connection = None @@ -121,16 +119,18 @@ def open(self): self.connection = None parseresult = parse.urlparse(configuration.email_host) - if not self.scheme == parseresult.scheme.lower(): + if not O365Connection.SCHEME == parseresult.scheme.lower(): raise ValueError( - f'Invalid EMAIL_HOST scheme, expected "{self.scheme}", got "{parseresult.scheme}"' + f'Invalid EMAIL_HOST scheme, expected "{O365Connection.SCHEME}", got "{parseresult.scheme}"' ) query_dict = dict(parse.parse_qsl(parseresult.query)) + client_app_id = query_dict.get("client_app_id", "") client_id_key = query_dict.get("client_id_key", "") client_secret_key = query_dict.get("client_secret_key", "") self.conn = O365Connection( from_email=self.from_email, + client_app_id=client_app_id, client_id_key=client_id_key, client_secret_key=client_secret_key, ) diff --git a/django_mail_admin/models/configurations.py b/django_mail_admin/models/configurations.py index e29f9d3..3fad992 100644 --- a/django_mail_admin/models/configurations.py +++ b/django_mail_admin/models/configurations.py @@ -123,10 +123,10 @@ class Mailbox(models.Model): "
" "Be sure to urlencode your username and password should they " "contain illegal characters (like @, :, etc)." - "
" - "For MS-365 email accounts use: 
office365:username@example.com:/?"
-                "client_id=&client_secret=&
" - "supports only on-behalf-of-a-user; thus requires user's auth & consent" + "

" + "For Office 365 email accounts use: 'office365:username@example.com:/?" + "client_id_key=&client_secret_key=&client_app_id=', provide either of client_app_id or client_id_key/client_secret_key pair; client_app_id overrides when both are provided" + "
supports only on-behalf-of-a-user; thus requires user's auth & consent in a separate authentication flow via console/ or web-browser." "
" ) ), @@ -280,11 +280,12 @@ def get_connection(self): archive=self.archive, ) conn.connect(self.username, self.password) - elif self.type == O365Transport.scheme: #'office365' + elif self.type == O365Transport.SCHEME: #'office365' conn = O365Transport( owner_email=self.from_email, last_polled=self.last_polling ) conn.connect( + client_app_id=self._query_string.get("client_app_id", [""])[0], client_id_key=self._query_string.get("client_id_key", [""])[0], client_secret_key=self._query_string.get("client_secret_key", [""])[0], ) diff --git a/django_mail_admin/o365_utils.py b/django_mail_admin/o365_utils.py index 7162e33..00e5e22 100644 --- a/django_mail_admin/o365_utils.py +++ b/django_mail_admin/o365_utils.py @@ -18,44 +18,95 @@ logger = logging.getLogger(__name__) +class O365NotAuthenticated(Exception): + pass + + class O365Connection: - scheme = "office365" - o365_protocol = "MSGraphProtocol" + SCHEME = "office365" + O365_PROTOCOL = "MSGraphProtocol" + DEFAULT_CLIENT_ID_KEY = "O365_CLIENT_ID" + DEFAULT_CLIENT_SECRET_KEY = "O365_CLIENT_SECRET" def __init__( self, from_email: str, + client_app_id: str, client_id_key: str, client_secret_key: str, - protocol=o365_protocol, + protocol: str = O365_PROTOCOL, ) -> None: self.from_email = from_email if not self.from_email: raise ValueError("from_email required.") self.account = None + self.selected_settings = None try: client_id, client_secret = self._get_auth_info( - client_id_key, client_secret_key + client_app_id, + client_id_key if client_id_key else self.DEFAULT_CLIENT_ID_KEY, + client_secret_key + if client_secret_key + else self.DEFAULT_CLIENT_SECRET_KEY, ) - self._connect(client_id, client_secret, protocol) + self._connect(client_app_id, client_id, client_secret, protocol) except (TypeError, ValueError) as e: logger.warning("O365Connection: Couldn't authenticate %s" % e) - def _get_auth_info(self, client_id_key: str, client_secret_key: str): - if not settings.O365_ADMIN_SETTINGS: - raise Exception("O365_ADMIN_SETTINGS configuration missing.") + def _get_auth_info( + self, client_app_id: str, client_id_key: str, client_secret_key: str + ): + selected_settings = ( + settings.O365_CLIENT_APP_SETTINGS.get(client_app_id, {}) + if client_app_id + else settings.O365_ADMIN_SETTINGS + ) + if not selected_settings: + raise ValueError("Selected settings not yet set!") - client_id = settings.O365_ADMIN_SETTINGS.get(client_id_key, "") + client_id = selected_settings.get(client_id_key, "") if not client_id: - raise Exception(f"O365_ADMIN_SETTINGS.{client_id_key} not set! ") + raise ValueError(f"{client_app_id}.{client_id_key} not set! ") - client_secret = settings.O365_ADMIN_SETTINGS.get(client_secret_key, "") + client_secret = selected_settings.get(client_secret_key, "") if not client_secret: - raise Exception(f"O365_ADMIN_SETTINGS.{client_secret_key} not set!") + raise ValueError(f"{client_app_id}.{client_secret_key} not set!") return client_id, client_secret + def _get_token_backend( + self, client_app_id: str | None = None + ) -> BaseTokenBackend | None: + selected_settings = ( + settings.O365_CLIENT_APP_SETTINGS.get(client_app_id, {}) + if client_app_id + else settings.O365_ADMIN_SETTINGS + ) + if not selected_settings: + raise ValueError("Selected settings not yet set!") + + token_backend = selected_settings.get("TOKEN_BACKEND", "FileSystemTokenBackend") + backend_settings = settings.O365_TOKEN_BACKENDS.get(token_backend, {}) + + if "FileSystemTokenBackend" == token_backend: + return FileSystemTokenBackend( + token_path=backend_settings.get("O365_AUTH_BACKEND_TOKEN_DIR", "."), + token_filename=backend_settings.get("O365_AUTH_BACKEND_TOKEN_FILE"), + ) + + if "AZBlobStorageTokenBackend" == token_backend: + return AZBlobStorageTokenBackend( + connection_str=backend_settings.get( + "O365_AUTH_BACKEND_AZ_CONNECTION_STR" + ), + container_name=backend_settings.get( + "O365_AUTH_BACKEND_AZ_CONTAINER_PATH" + ), + blob_name=self._decorate_token_name(), + ) + return None + def _decorate_token_name(self): bname = settings.O365_ADMIN_SETTINGS.get( "O365_AUTH_BACKEND_AZ_BLOB_NAME", "o365_token.txt" @@ -63,31 +114,23 @@ def _decorate_token_name(self): return "{}/{}".format( hashlib.sha1( settings.O365_ADMIN_SETTINGS.get("O365_CLIENT_ID").encode("utf-8") - + bname.encode("utf-8") + + self.from_email.encode("utf-8") ).hexdigest(), bname, ) - def _connect(self, client_id, client_secret, protocol) -> None: + def _connect(self, client_app_id, client_id, client_secret, protocol) -> None: # connect_id/ and secret should have been already setup # for offline & message_all scopes, for on behalf of user access. protocol_selected = ( MSGraphProtocol(api_version="beta") - if protocol == self.o365_protocol + if protocol == self.O365_PROTOCOL else None ) if not protocol_selected: - raise Exception(f"Unsupported protocol {protocol}") - - token_backend = AzureTokenBackend( - connection_str=settings.O365_ADMIN_SETTINGS.get( - "O365_AUTH_BACKEND_AZ_CONNECTION_STR" - ), - container_name=settings.O365_ADMIN_SETTINGS.get( - "O365_AUTH_BACKEND_AZ_CONTAINER_PATH" - ), - blob_name=self._decorate_token_name(), - ) + raise ValueError(f"Unsupported protocol {protocol}") + + token_backend = self._get_token_backend(client_app_id=client_app_id) self.account = Account( credentials=(client_id, client_secret), @@ -96,10 +139,6 @@ def _connect(self, client_id, client_secret, protocol) -> None: token_backend=token_backend, ) - if not self.account.is_authenticated: - logger.info("Hold on .... going for authentication!") - self.account.authenticate() - def _get_message_by_id( self, mailbox: MailBox, message_id: str, folder_name="Inbox" ) -> Message | None: @@ -141,6 +180,12 @@ def _prepare_attachment_for_dispatch(self, attachment) -> dict: def send_messages(self, email_messages, fail_silently: bool = False) -> int: sent_messages = 0 + if not self.account.is_authenticated: + logger.error("send_messages unavailable; account not authenticated!") + raise O365NotAuthenticated( + "get_messages unavailable; account not yet authenticated!" + ) + mailbox = self.account.mailbox(self.from_email) for msg in email_messages: try: @@ -174,7 +219,9 @@ def send_messages(self, email_messages, fail_silently: bool = False) -> int: def get_messages(self, owner_email: str, last_polled: datetime, condition): if not self.account.is_authenticated: logger.error("get_messages unavailable; account not authenticated!") - return + raise O365NotAuthenticated( + "get_messages unavailable; account not yet authenticated!" + ) mailbox = self.account.mailbox(owner_email) mail_folder = mailbox.get_folder(folder_name="Inbox") qstr = "" @@ -187,8 +234,8 @@ def get_messages(self, owner_email: str, last_polled: datetime, condition): yield (mail.get_mime_content()) -class AzureTokenBackend(BaseTokenBackend): - """An Azure Blob backend to store tokens""" +class AZBlobStorageTokenBackend(BaseTokenBackend): + """An Azure Blob store backend for token management""" def __init__(self, connection_str, container_name, blob_name): """ @@ -202,7 +249,7 @@ def __init__(self, connection_str, container_name, blob_name): try: from azure.storage.blob import BlobClient except ModuleNotFoundError as e: - raise Exception( + raise ModuleNotFoundError( "Please install the azure-storage-blob package to use this token backend." ) from e super().__init__() @@ -246,7 +293,9 @@ def save_token(self): raise ValueError('You have to set the "token" first.') try: - r = self._client.upload_blob(self.serializer.dumps(self.token)) + r = self._client.upload_blob( + self.serializer.dumps(self.token), overwrite=True + ) except Exception as e: logger.error("Token secret could not be created: {}".format(e)) return False diff --git a/django_mail_admin/transports/o365.py b/django_mail_admin/transports/o365.py index 82ca48e..6bcbbc1 100644 --- a/django_mail_admin/transports/o365.py +++ b/django_mail_admin/transports/o365.py @@ -13,7 +13,7 @@ class O365Transport(EmailTransport): - scheme = O365Connection.scheme + SCHEME = O365Connection.SCHEME def __init__(self, owner_email: str, last_polled: datetime = None) -> None: super().__init__() @@ -23,6 +23,7 @@ def __init__(self, owner_email: str, last_polled: datetime = None) -> None: def connect( self, + client_app_id: str, client_id_key: str, client_secret_key: str, o365_protocol="MSGraphProtocol", @@ -30,6 +31,7 @@ def connect( try: self.conn = O365Connection( from_email=self.owner_email, + client_app_id=client_app_id, client_id_key=client_id_key, client_secret_key=client_secret_key, protocol=o365_protocol, diff --git a/example/example/settings.py b/example/example/settings.py index 5eca7b1..3931477 100644 --- a/example/example/settings.py +++ b/example/example/settings.py @@ -11,7 +11,6 @@ """ import os -import sys from decouple import config # Build paths inside the project like this: os.path.join(BASE_DIR, ...) @@ -39,6 +38,7 @@ "django_mail_admin", # if your app has other dependencies that need to be added to the site # they should be added here + "sslserver", ] MIDDLEWARE = [ @@ -123,8 +123,7 @@ EMAIL_BACKEND = "django_mail_admin.backends.O365Backend" O365_ADMIN_SETTINGS = { - "O365_AUTH_BACKEND_TOKEN_DIR": config("O365_AUTH_BACKEND_TOKEN_DIR"), - "O365_AUTH_BACKEND_TOKEN_FILE": config("O365_AUTH_BACKEND_TOKEN_FILE"), + "TOKEN_BACKEND": "AZBlobStorageTokenBackend", # "FileSystemTokenBackend" "O365_CLIENT_ID": config("O365_CLIENT_ID"), "O365_CLIENT_SECRET": config("O365_CLIENT_SECRET"), "O365_AUTH_BACKEND_AZ_BLOB_NAME": config("O365_AUTH_BACKEND_AZ_BLOB_NAME"), @@ -135,3 +134,29 @@ "O365_AUTH_BACKEND_AZ_CONTAINER_PATH" ), } + +O365_CLIENT_APP_SETTINGS = { + "example_client_app1": { + # TBD certificate support + "auth": "token_secret", #'certificate' + "TOKEN_BACKEND": "AZBlobStorageTokenBackend", + "O365_CLIENT_ID": config("O365_CLIENT_ID"), + "O365_CLIENT_SECRET": config("O365_CLIENT_SECRET"), + }, +} + +O365_TOKEN_BACKENDS = { + "FileSystemTokenBackend": { + "O365_AUTH_BACKEND_TOKEN_DIR": config("O365_AUTH_BACKEND_TOKEN_DIR"), + "O365_AUTH_BACKEND_TOKEN_FILE": config("O365_AUTH_BACKEND_TOKEN_FILE"), + }, + "AZBlobStorageTokenBackend": { + "O365_AUTH_BACKEND_AZ_BLOB_NAME": config("O365_AUTH_BACKEND_AZ_BLOB_NAME"), + "O365_AUTH_BACKEND_AZ_CONTAINER_PATH": config( + "O365_AUTH_BACKEND_AZ_CONTAINER_PATH" + ), + "O365_AUTH_BACKEND_AZ_CONNECTION_STR": config( + "O365_AUTH_BACKEND_AZ_CONNECTION_STR" + ), + }, +} diff --git a/example/example/urls.py b/example/example/urls.py index b1ee031..5950cfb 100644 --- a/example/example/urls.py +++ b/example/example/urls.py @@ -13,14 +13,25 @@ 1. Import the include() function: from django.conf.urls import url, include 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) """ -from django.conf.urls import url, include +from django.urls import path, include, re_path + +# from django.conf.urls import url from django.contrib import admin from django.conf import settings from django.conf.urls.static import static +from . import views + urlpatterns = [ - url(r'^admin/', admin.site.urls), - url(r'', include('django_mail_admin.urls', namespace='django_mail_admin')), + re_path(r"^admin/", admin.site.urls), + re_path(r"", include("django_mail_admin.urls", namespace="django_mail_admin")), + path(r"example/mailbox/", views.mailbox, name="mailbox_list"), + path( + r"example/mailbox//auth1", + views.mailbox_auth_step1, + name="mailbox_auth_step1", + ), + path(r"example/mailbox/auth2", views.mailbox_auth_step2, name="mailbox_auth_step2"), ] if settings.DEBUG: # static files (images, css, javascript, etc.) diff --git a/example/example/views.py b/example/example/views.py new file mode 100644 index 0000000..fdfbe1a --- /dev/null +++ b/example/example/views.py @@ -0,0 +1,69 @@ +""" +Example illustration of two-step auth flow for O365 +""" +from django.urls import reverse +from django.shortcuts import get_object_or_404, render, redirect + +from django_mail_admin.models import Mailbox + + +def mailbox(request): + # Display latest recent EDI documents + mailbox_list = Mailbox.objects.order_by("-id")[:10] + context = { + "mailbox_list": mailbox_list, + } + # Allow adding new one, by uploading a document + return render(request, "mailbox/index.html", context) + + +def mailbox_auth_step1(request, id): + # callback = absolute url to o365_auth_step_two_callback() page, https://domain.tld/steptwo + callback = request.build_absolute_uri(reverse("mailbox_auth_step2")) + mbx = get_object_or_404(Mailbox, pk=id) + + transport = mbx.get_connection() + conn = transport.conn + account = conn.account + """ + The scopes here need to exactly match the O365 web-app provisioning in MS-Entra/Identity. Do not use the "message_all" etc helpers that O365 supports in Account.authenticate flow. + """ + scopes = ["Mail.ReadWrite", "Mail.Send", "offline_access"] + + #'state' is a variable that is guaranteed to be sent with redirect. thus we store the mailbox object id to update the auth-infor on redirect of respective Mailbox a/c + url, state = account.con.get_authorization_url( + requested_scopes=scopes, redirect_uri=callback, state=id + ) + + return redirect(url) + + +def mailbox_auth_step2(request): + queryprms = request.GET + state = queryprms.get("state", None) + mbx_id = state + if not mbx_id: + return render( + request, + "mailbox/auth.html", + {"result": False, "message": "failed to get MBX ID!"}, + ) + mbx = get_object_or_404(Mailbox, pk=mbx_id) + + transport = mbx.get_connection() + conn = transport.conn + account = conn.account + + # rebuild the redirect_uri used in mailbox_auth_step1 + callback = request.build_absolute_uri(reverse("mailbox_auth_step2")) + + # get the request URL of the page which will include additional auth information + # Example request: /steptwo?code=abc123&state=xyz456 + requested_url = request.build_absolute_uri() + result = account.con.request_token( + requested_url, state=state, redirect_uri=callback + ) + + # if result is True, then authentication was succesful + # and the auth token is stored in the token backend + return render(request, "mailbox/auth.html", {"result": result}) diff --git a/example/templates/mailbox/auth.html b/example/templates/mailbox/auth.html new file mode 100644 index 0000000..17dc260 --- /dev/null +++ b/example/templates/mailbox/auth.html @@ -0,0 +1,19 @@ + +{% extends "mailbox/base.html" %} + +{% block title %}Example - Mailbox: Listings{% endblock %} + +{% block site_head %}Example - Mailboxes {% endblock %} + +{% block content %} + +

Mailbox Auth

+{% if result %} +Authorization {% if result == True %} completed successfully {% else %} attempt failed '{{ message }}'' {% endif %} for Mailbox {{ mbx.id }}, with URI {{ mbx.uri }} +{% else %} +

Authorization results not yet available for Mailbox {{ mbx.id }}.

+{% endif %} + +{% endblock %} diff --git a/example/templates/mailbox/base.html b/example/templates/mailbox/base.html new file mode 100644 index 0000000..6f9c111 --- /dev/null +++ b/example/templates/mailbox/base.html @@ -0,0 +1,30 @@ + + + + {% load static %} + + + {% block title %}Example.com{% endblock %} + + + + +

+ {% block site_head %}Example.com{% endblock %} +

+ + +
+ {% block content %}{% endblock %} +
+ + + diff --git a/example/templates/mailbox/index.html b/example/templates/mailbox/index.html new file mode 100644 index 0000000..44c9364 --- /dev/null +++ b/example/templates/mailbox/index.html @@ -0,0 +1,40 @@ + +{% extends "mailbox/base.html" %} + +{% block title %}Example - Mailbox: Listings{% endblock %} + +{% block site_head %}Example - Mailboxes {% endblock %} + +{% block content %} + +

Mailbox

+{% if mailbox_list %} + + + + + + + + + + + {% for mbx in mailbox_list %} + + + + + + + + + + {% endfor %} +
IDNameFrom EmailURIActiveLast_pollingAction
{{ mbx.id }}{{ mbx.name }}{{ mbx.from_email }}{{ mbx.uri }}{{ mbx.active }}{{ mbx.last_polling }} authorize
+{% else %} +

No Mailboxes are configured yet.

+{% endif %} + +{% endblock %} diff --git a/requirements.txt b/requirements.txt index 8664816..08760ac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,6 @@ jsonfield # Additional requirements go here social-auth-app-django +asgiref==3.8.0 +azure-core==1.30.1 +azure-storage-blob==12.19.1 From bd20574f339fb3125ff517adad0a5e3b3e989fb2 Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Thu, 28 Mar 2024 17:21:44 -0400 Subject: [PATCH 08/68] updated test, decoreate token method --- django_mail_admin/o365_utils.py | 674 ++++++++++++++++---------------- tests/o365/test_commands.py | 77 +++- tests/settings.py | 16 +- 3 files changed, 413 insertions(+), 354 deletions(-) diff --git a/django_mail_admin/o365_utils.py b/django_mail_admin/o365_utils.py index 00e5e22..0146000 100644 --- a/django_mail_admin/o365_utils.py +++ b/django_mail_admin/o365_utils.py @@ -1,332 +1,342 @@ -""" -Helper utility classes/ functions for O365 support -""" -import logging -import hashlib - -from base64 import b64encode -from datetime import datetime -from django.conf import settings -from django.core.mail.message import EmailMessage - -from O365.utils import BaseTokenBackend - -from O365 import MSGraphProtocol -from O365 import Account, Message, FileSystemTokenBackend -from O365.mailbox import MailBox - -logger = logging.getLogger(__name__) - - -class O365NotAuthenticated(Exception): - pass - - -class O365Connection: - SCHEME = "office365" - O365_PROTOCOL = "MSGraphProtocol" - DEFAULT_CLIENT_ID_KEY = "O365_CLIENT_ID" - DEFAULT_CLIENT_SECRET_KEY = "O365_CLIENT_SECRET" - - def __init__( - self, - from_email: str, - client_app_id: str, - client_id_key: str, - client_secret_key: str, - protocol: str = O365_PROTOCOL, - ) -> None: - self.from_email = from_email - if not self.from_email: - raise ValueError("from_email required.") - - self.account = None - self.selected_settings = None - try: - client_id, client_secret = self._get_auth_info( - client_app_id, - client_id_key if client_id_key else self.DEFAULT_CLIENT_ID_KEY, - client_secret_key - if client_secret_key - else self.DEFAULT_CLIENT_SECRET_KEY, - ) - self._connect(client_app_id, client_id, client_secret, protocol) - except (TypeError, ValueError) as e: - logger.warning("O365Connection: Couldn't authenticate %s" % e) - - def _get_auth_info( - self, client_app_id: str, client_id_key: str, client_secret_key: str - ): - selected_settings = ( - settings.O365_CLIENT_APP_SETTINGS.get(client_app_id, {}) - if client_app_id - else settings.O365_ADMIN_SETTINGS - ) - if not selected_settings: - raise ValueError("Selected settings not yet set!") - - client_id = selected_settings.get(client_id_key, "") - if not client_id: - raise ValueError(f"{client_app_id}.{client_id_key} not set! ") - - client_secret = selected_settings.get(client_secret_key, "") - if not client_secret: - raise ValueError(f"{client_app_id}.{client_secret_key} not set!") - - return client_id, client_secret - - def _get_token_backend( - self, client_app_id: str | None = None - ) -> BaseTokenBackend | None: - selected_settings = ( - settings.O365_CLIENT_APP_SETTINGS.get(client_app_id, {}) - if client_app_id - else settings.O365_ADMIN_SETTINGS - ) - if not selected_settings: - raise ValueError("Selected settings not yet set!") - - token_backend = selected_settings.get("TOKEN_BACKEND", "FileSystemTokenBackend") - backend_settings = settings.O365_TOKEN_BACKENDS.get(token_backend, {}) - - if "FileSystemTokenBackend" == token_backend: - return FileSystemTokenBackend( - token_path=backend_settings.get("O365_AUTH_BACKEND_TOKEN_DIR", "."), - token_filename=backend_settings.get("O365_AUTH_BACKEND_TOKEN_FILE"), - ) - - if "AZBlobStorageTokenBackend" == token_backend: - return AZBlobStorageTokenBackend( - connection_str=backend_settings.get( - "O365_AUTH_BACKEND_AZ_CONNECTION_STR" - ), - container_name=backend_settings.get( - "O365_AUTH_BACKEND_AZ_CONTAINER_PATH" - ), - blob_name=self._decorate_token_name(), - ) - return None - - def _decorate_token_name(self): - bname = settings.O365_ADMIN_SETTINGS.get( - "O365_AUTH_BACKEND_AZ_BLOB_NAME", "o365_token.txt" - ) - return "{}/{}".format( - hashlib.sha1( - settings.O365_ADMIN_SETTINGS.get("O365_CLIENT_ID").encode("utf-8") - + self.from_email.encode("utf-8") - ).hexdigest(), - bname, - ) - - def _connect(self, client_app_id, client_id, client_secret, protocol) -> None: - # connect_id/ and secret should have been already setup - # for offline & message_all scopes, for on behalf of user access. - protocol_selected = ( - MSGraphProtocol(api_version="beta") - if protocol == self.O365_PROTOCOL - else None - ) - if not protocol_selected: - raise ValueError(f"Unsupported protocol {protocol}") - - token_backend = self._get_token_backend(client_app_id=client_app_id) - - self.account = Account( - credentials=(client_id, client_secret), - protocol=protocol_selected, - scopes=["offline_access", "message_all"], - token_backend=token_backend, - ) - - def _get_message_by_id( - self, mailbox: MailBox, message_id: str, folder_name="Inbox" - ) -> Message | None: - """retrieve message by id from given mailbox""" - if not message_id: - return None - mail_folder = mailbox.get_folder(folder_name=folder_name) if mailbox else None - qstr = f"internetMessageId eq '{message_id}'" - emails = mail_folder.get_messages(query=qstr) if mail_folder else [] - for email in emails: - return email - return None - - def _get_reply_to_message( - self, mailbox: MailBox, msg: EmailMessage - ) -> Message | None: - """retrieve message representing in-reply-to id in msg headers""" - if not msg or not msg.extra_headers or not mailbox: - return None - msg_id = msg.extra_headers.get("In-Reply-To", None) - reply_to_msg = self._get_message_by_id(mailbox, msg_id) if msg_id else None - return reply_to_msg - - def _prepare_new_message(self, mailbox, msg: EmailMessage): - """create new or reply-to draft based on msg in-reply-to header""" - reply_to_message = self._get_reply_to_message(mailbox, msg) - new_draft_message = ( - reply_to_message.reply() if reply_to_message else mailbox.new_message() - ) - return new_draft_message - - def _prepare_attachment_for_dispatch(self, attachment) -> dict: - """bridge from Django EmailMessage Attachment to O365 Attachment""" - content = attachment[1] - b64content = b64encode( - content if isinstance(content, bytes) else bytes(content, "utf-8") - ).decode("utf-8") - return {"name": f"{attachment[0]}", "content": b64content, "on_disk": False} - - def send_messages(self, email_messages, fail_silently: bool = False) -> int: - sent_messages = 0 - if not self.account.is_authenticated: - logger.error("send_messages unavailable; account not authenticated!") - raise O365NotAuthenticated( - "get_messages unavailable; account not yet authenticated!" - ) - - mailbox = self.account.mailbox(self.from_email) - for msg in email_messages: - try: - m = self._prepare_new_message(mailbox, msg) - m.to.add(msg.to) - m.cc.add(msg.cc) - m.bcc.add(msg.bcc) - m.reply_to.add(msg.reply_to) - m.body = msg.body - if msg.subject: - m.subject = msg.subject - m.save_message() - m.attachments.add( - [ - self._prepare_attachment_for_dispatch(attachment) - for attachment in msg.attachments - ] - ) - for attachment in m.attachments: - # workaround: avoid NoneType compare w/ int, O365 exception - attachment.size = len(attachment.content) - m.save_draft() - m.send() - sent_messages += 1 - except Exception as e: - logger.error(f"Exception in sending message: error info: {e}") - if not fail_silently: - raise e - return sent_messages - - def get_messages(self, owner_email: str, last_polled: datetime, condition): - if not self.account.is_authenticated: - logger.error("get_messages unavailable; account not authenticated!") - raise O365NotAuthenticated( - "get_messages unavailable; account not yet authenticated!" - ) - mailbox = self.account.mailbox(owner_email) - mail_folder = mailbox.get_folder(folder_name="Inbox") - qstr = "" - order_by = f"receivedDateTime DESC" - if last_polled: - # ISO 8601 format AND in UTC time. - # For example, midnight UTC on Jan 1, 2022 is 2022-01-01T00:00:00Z. - qstr = f"receivedDateTime gt {last_polled.replace(microsecond=0).isoformat()[:-6]}Z" - for mail in mail_folder.get_messages(order_by=order_by, query=qstr): - yield (mail.get_mime_content()) - - -class AZBlobStorageTokenBackend(BaseTokenBackend): - """An Azure Blob store backend for token management""" - - def __init__(self, connection_str, container_name, blob_name): - """ - Init Backend - :param str connection_str: Connection str for the Blob storage account - :param str container_name: Container string for blob file. - :param str blob_name: Blob name - """ - self.container_name = container_name - self.blob_name = blob_name - try: - from azure.storage.blob import BlobClient - except ModuleNotFoundError as e: - raise ModuleNotFoundError( - "Please install the azure-storage-blob package to use this token backend." - ) from e - super().__init__() - - self._client = BlobClient.from_connection_string( - connection_str, # type: ignore - container_name=container_name, # type: ignore - blob_name=blob_name, # type: ignore - ) - - def __repr__(self): - return "AzureTokenBackend('{}', '{}')".format( - self.connection_str, self.container_name - ) - - def load_token(self): - """ - Retrieves the token from the store - :return dict or None: The token if exists, None otherwise - """ - token = None - try: - downloader = self._client.download_blob(max_concurrency=1, encoding="UTF-8") - token_str = downloader.readall() - token = self.token_constructor(self.serializer.loads(token_str)) - except Exception as e: - logger.error( - "Token (blob_text: {}/{}) could not be retrieved from the backend: {}".format( - self.container_name, self.blob_name, e - ) - ) - - return token - - def save_token(self): - """ - Saves the token dict in the store - :return bool: Success / Failure - """ - if self.token is None: - raise ValueError('You have to set the "token" first.') - - try: - r = self._client.upload_blob( - self.serializer.dumps(self.token), overwrite=True - ) - except Exception as e: - logger.error("Token secret could not be created: {}".format(e)) - return False - return True - - def delete_token(self): - """ - Deletes the token from the store - :return bool: Success / Failure - """ - try: - r = self._client.delete_blob() - except Exception as e: - logger.error("Token secret could not be deleted: {}".format(e)) - return False - else: - logger.warning( - "Deleted token secret {} ({}).".format( - self.container_name, self.blob_name - ) - ) - return True - - def check_token(self): - """ - Checks if the token exists - :return bool: True if it exists on the store - """ - try: - _ = self._client.exists() - except: - return False - else: - return True +""" +Helper utility classes/ functions for O365 support +""" +import logging +import hashlib + +from base64 import b64encode +from datetime import datetime +from django.conf import settings +from django.core.mail.message import EmailMessage + +from O365.utils import BaseTokenBackend + +from O365 import MSGraphProtocol +from O365 import Account, Message, FileSystemTokenBackend +from O365.mailbox import MailBox + +logger = logging.getLogger(__name__) + + +class O365NotAuthenticated(Exception): + pass + + +class O365Connection: + SCHEME = "office365" + O365_PROTOCOL = "MSGraphProtocol" + DEFAULT_CLIENT_ID_KEY = "O365_CLIENT_ID" + DEFAULT_CLIENT_SECRET_KEY = "O365_CLIENT_SECRET" + + def __init__( + self, + from_email: str, + client_app_id: str, + client_id_key: str, + client_secret_key: str, + protocol: str = O365_PROTOCOL, + ) -> None: + self.from_email = from_email + if not self.from_email: + raise ValueError("from_email required.") + + self.account = None + self.selected_settings = None + try: + client_id, client_secret = self._get_auth_info( + client_app_id, + client_id_key if client_id_key else self.DEFAULT_CLIENT_ID_KEY, + client_secret_key + if client_secret_key + else self.DEFAULT_CLIENT_SECRET_KEY, + ) + self._connect(client_app_id, client_id, client_secret, protocol) + except (TypeError, ValueError) as e: + logger.warning("O365Connection: Couldn't authenticate %s" % e) + + def _get_auth_info( + self, client_app_id: str, client_id_key: str, client_secret_key: str + ): + selected_settings = ( + settings.O365_CLIENT_APP_SETTINGS.get(client_app_id, {}) + if client_app_id + else settings.O365_ADMIN_SETTINGS + ) + if not selected_settings: + raise ValueError("Selected settings not yet set!") + + client_id = selected_settings.get(client_id_key, "") + if not client_id: + raise ValueError(f"{client_app_id}.{client_id_key} not set! ") + + client_secret = selected_settings.get(client_secret_key, "") + if not client_secret: + raise ValueError(f"{client_app_id}.{client_secret_key} not set!") + + return client_id, client_secret + + def _get_token_backend( + self, client_app_id: str | None = None, client_id: str | None = None + ) -> BaseTokenBackend | None: + selected_settings = ( + settings.O365_CLIENT_APP_SETTINGS.get(client_app_id, {}) + if client_app_id + else settings.O365_ADMIN_SETTINGS + ) + if not selected_settings: + raise ValueError("Selected settings not yet set!") + + token_backend = selected_settings.get("TOKEN_BACKEND", "FileSystemTokenBackend") + backend_settings = settings.O365_TOKEN_BACKENDS.get(token_backend, {}) + + if "FileSystemTokenBackend" == token_backend: + return FileSystemTokenBackend( + token_path=backend_settings.get("O365_AUTH_BACKEND_TOKEN_DIR", "."), + token_filename=self._decorate_token_name( + client_id, + token_name_pattern=backend_settings.get( + "O365_AUTH_BACKEND_TOKEN_FILE" + ), + ), + ) + + if "AZBlobStorageTokenBackend" == token_backend: + return AZBlobStorageTokenBackend( + connection_str=backend_settings.get( + "O365_AUTH_BACKEND_AZ_CONNECTION_STR" + ), + container_name=backend_settings.get( + "O365_AUTH_BACKEND_AZ_CONTAINER_PATH" + ), + blob_name=self._decorate_token_name( + client_id, + token_name_pattern=settings.O365_ADMIN_SETTINGS.get( + "O365_AUTH_BACKEND_AZ_BLOB_NAME" + ), + ), + ) + return None + + def _decorate_token_name(self, client_id: str, token_name_pattern: str | None): + if not token_name_pattern: + token_name_pattern = "o365_token.txt" + return "{}/{}".format( + hashlib.sha1( + client_id.encode("utf-8") + self.from_email.encode("utf-8") + ).hexdigest(), + token_name_pattern, + ) + + def _connect(self, client_app_id, client_id, client_secret, protocol) -> None: + # connect_id/ and secret should have been already setup + # for offline & message_all scopes, for on behalf of user access. + protocol_selected = ( + MSGraphProtocol(api_version="beta") + if protocol == self.O365_PROTOCOL + else None + ) + if not protocol_selected: + raise ValueError(f"Unsupported protocol {protocol}") + + token_backend = self._get_token_backend( + client_app_id=client_app_id, client_id=client_id + ) + + self.account = Account( + credentials=(client_id, client_secret), + protocol=protocol_selected, + scopes=["offline_access", "message_all"], + token_backend=token_backend, + ) + + def _get_message_by_id( + self, mailbox: MailBox, message_id: str, folder_name="Inbox" + ) -> Message | None: + """retrieve message by id from given mailbox""" + if not message_id: + return None + mail_folder = mailbox.get_folder(folder_name=folder_name) if mailbox else None + qstr = f"internetMessageId eq '{message_id}'" + emails = mail_folder.get_messages(query=qstr) if mail_folder else [] + for email in emails: + return email + return None + + def _get_reply_to_message( + self, mailbox: MailBox, msg: EmailMessage + ) -> Message | None: + """retrieve message representing in-reply-to id in msg headers""" + if not msg or not msg.extra_headers or not mailbox: + return None + msg_id = msg.extra_headers.get("In-Reply-To", None) + reply_to_msg = self._get_message_by_id(mailbox, msg_id) if msg_id else None + return reply_to_msg + + def _prepare_new_message(self, mailbox, msg: EmailMessage): + """create new or reply-to draft based on msg in-reply-to header""" + reply_to_message = self._get_reply_to_message(mailbox, msg) + new_draft_message = ( + reply_to_message.reply() if reply_to_message else mailbox.new_message() + ) + return new_draft_message + + def _prepare_attachment_for_dispatch(self, attachment) -> dict: + """bridge from Django EmailMessage Attachment to O365 Attachment""" + content = attachment[1] + b64content = b64encode( + content if isinstance(content, bytes) else bytes(content, "utf-8") + ).decode("utf-8") + return {"name": f"{attachment[0]}", "content": b64content, "on_disk": False} + + def send_messages(self, email_messages, fail_silently: bool = False) -> int: + sent_messages = 0 + if not self.account.is_authenticated: + logger.error("send_messages unavailable; account not authenticated!") + if not fail_silently: + raise O365NotAuthenticated( + "get_messages unavailable; account not yet authenticated!" + ) + mailbox = self.account.mailbox(self.from_email) + for msg in email_messages: + try: + m = self._prepare_new_message(mailbox, msg) + m.to.add(msg.to) + m.cc.add(msg.cc) + m.bcc.add(msg.bcc) + m.reply_to.add(msg.reply_to) + m.body = msg.body + if msg.subject: + m.subject = msg.subject + m.save_message() + m.attachments.add( + [ + self._prepare_attachment_for_dispatch(attachment) + for attachment in msg.attachments + ] + ) + for attachment in m.attachments: + # workaround: avoid NoneType compare w/ int, O365 exception + attachment.size = len(attachment.content) + m.save_draft() + m.send() + sent_messages += 1 + except Exception as e: + logger.error(f"Exception in sending message: error info: {e}") + if not fail_silently: + raise e + return sent_messages + + def get_messages(self, owner_email: str, last_polled: datetime, condition): + if not self.account.is_authenticated: + logger.error("get_messages unavailable; account not authenticated!") + raise O365NotAuthenticated( + "get_messages unavailable; account not yet authenticated!" + ) + mailbox = self.account.mailbox(owner_email) + mail_folder = mailbox.get_folder(folder_name="Inbox") + qstr = "" + order_by = f"receivedDateTime DESC" + if last_polled: + # ISO 8601 format AND in UTC time. + # For example, midnight UTC on Jan 1, 2022 is 2022-01-01T00:00:00Z. + qstr = f"receivedDateTime gt {last_polled.replace(microsecond=0).isoformat()[:-6]}Z" + for mail in mail_folder.get_messages(order_by=order_by, query=qstr): + yield (mail.get_mime_content()) + + +class AZBlobStorageTokenBackend(BaseTokenBackend): + """An Azure Blob store backend for token management""" + + def __init__(self, connection_str, container_name, blob_name): + """ + Init Backend + :param str connection_str: Connection str for the Blob storage account + :param str container_name: Container string for blob file. + :param str blob_name: Blob name + """ + self.container_name = container_name + self.blob_name = blob_name + try: + from azure.storage.blob import BlobClient + except ModuleNotFoundError as e: + raise ModuleNotFoundError( + "Please install the azure-storage-blob package to use this token backend." + ) from e + super().__init__() + + self._client = BlobClient.from_connection_string( + connection_str, # type: ignore + container_name=container_name, # type: ignore + blob_name=blob_name, # type: ignore + ) + + def __repr__(self): + return "AzureTokenBackend('{}', '{}')".format( + self.connection_str, self.container_name + ) + + def load_token(self): + """ + Retrieves the token from the store + :return dict or None: The token if exists, None otherwise + """ + token = None + try: + downloader = self._client.download_blob(max_concurrency=1, encoding="UTF-8") + token_str = downloader.readall() + token = self.token_constructor(self.serializer.loads(token_str)) + except Exception as e: + logger.error( + "Token (blob_text: {}/{}) could not be retrieved from the backend: {}".format( + self.container_name, self.blob_name, e + ) + ) + + return token + + def save_token(self): + """ + Saves the token dict in the store + :return bool: Success / Failure + """ + if self.token is None: + raise ValueError('You have to set the "token" first.') + + try: + r = self._client.upload_blob( + self.serializer.dumps(self.token), overwrite=True + ) + except Exception as e: + logger.error("Token secret could not be created: {}".format(e)) + return False + return True + + def delete_token(self): + """ + Deletes the token from the store + :return bool: Success / Failure + """ + try: + r = self._client.delete_blob() + except Exception as e: + logger.error("Token secret could not be deleted: {}".format(e)) + return False + else: + logger.warning( + "Deleted token secret {} ({}).".format( + self.container_name, self.blob_name + ) + ) + return True + + def check_token(self): + """ + Checks if the token exists + :return bool: True if it exists on the store + """ + try: + _ = self._client.exists() + except: + return False + else: + return True diff --git a/tests/o365/test_commands.py b/tests/o365/test_commands.py index ef7dd81..7561e83 100644 --- a/tests/o365/test_commands.py +++ b/tests/o365/test_commands.py @@ -11,29 +11,56 @@ from django_mail_admin.models import Outbox, IncomingEmail, OutgoingEmail, STATUS from django_mail_admin.models import Mailbox from django_mail_admin.mail import send_queued +from django_mail_admin.connections import connections + +from django_mail_admin.o365_utils import O365NotAuthenticated class O365CommandTest(TestCase): + O365_BACKEND_ALIAS = "o365" + def print_incoming_emails(self): for email in IncomingEmail.objects.all().order_by("processed"): print(f"{email.message_id} | {email.subject} | {email.processed}") + def send_email(self, outgoing_email) -> bool: + retry_send = False + send_successful = False + while True: + try: + outgoing_email.dispatch(commit=False) + except O365NotAuthenticated as uae: + connection = connections[self.O365_BACKEND_ALIAS] + print(f"warning: send_mail: {uae};") + if not retry_send: + retry_send = connection.conn.account.authenticate() + print( + f"authentication attempt: " + + ("successful " if retry_send else " failed!") + ) + else: + retry_send = False + except Exception as e: + print(f"error: send_mail: {e}") + else: + send_successful = True + retry_send = False + + if send_successful or not retry_send: + break + return send_successful + def test_o365(self): - user1_email = settings.O365_TEST_ACCOUNT - user1_email_str = parse.quote(user1_email) - o365_con = f"office365://{user1_email_str}@outlook.office365.com?client_id_key=O365_CLIENT_ID&client_secret_key=O365_CLIENT_SECRET&archive=Archived" + from_user_email = settings.O365_TEST_ADMIN.get("test_from_email") + to_user_email = settings.O365_TEST_ADMIN.get("test_to_email", from_user_email) - inbox = Mailbox.objects.create( - name="O365CommandTest_Inbox", uri=o365_con, from_email=user1_email - ) - inbox.get_new_mail() - print(f"received {IncomingEmail.objects.count()} new emails") - # self.print_incoming_emails() + from_user_email_str = parse.quote(from_user_email) + from_user_o365_con = f"office365://{from_user_email_str}@outlook.office365.com?client_id_key=O365_CLIENT_ID&client_secret_key=O365_CLIENT_SECRET&archive=Archived" - outbox = Outbox.objects.create( + Outbox.objects.create( name="O365CommandTest_Outbox", - email_host=o365_con, - email_host_user=user1_email, + email_host=from_user_o365_con, + email_host_user=from_user_email, email_host_password="ase123hgfd", active=True, ) @@ -44,19 +71,29 @@ def test_o365(self): print(f"\ntest_subject: {test_subject}") print(f"test_body: {test_body}\n") """ - OutgoingEmail.objects.create( - from_email=user1_email, - to=[user1_email], + outgoing_email = OutgoingEmail.objects.create( + from_email=from_user_email, + to=[to_user_email], status=STATUS.queued, subject=test_subject, message=test_body, - backend_alias="o365", + backend_alias=self.O365_BACKEND_ALIAS, ) - # call_command("send_queued_mail", log_level=0) - total_sent, total_failed = send_queued() - assert total_failed == 0 - assert total_sent == 1 + send_successful = self.send_email(outgoing_email) + + assert send_successful == True time.sleep(5) + + to_user_email_str = parse.quote(to_user_email) + to_user_o365_con = f"office365://{to_user_email_str}@outlook.office365.com?client_id_key=O365_CLIENT_ID&client_secret_key=O365_CLIENT_SECRET&archive=Archived" + + inbox = Mailbox.objects.create( + name="O365CommandTest_Inbox", uri=to_user_o365_con, from_email=to_user_email + ) + inbox.get_new_mail() + print(f"received {IncomingEmail.objects.count()} new emails") + # self.print_incoming_emails() + max_attempts = 3 latest_email = None while max_attempts: diff --git a/tests/settings.py b/tests/settings.py index e012f20..64d770a 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -48,7 +48,7 @@ } O365_ADMIN_SETTINGS = { - "TOKEN_BACKEND": "FileSystemTokenBackend", + "TOKEN_BACKEND": "AZBlobStorageTokenBackend", "O365_CLIENT_ID": config("O365_CLIENT_ID"), "O365_CLIENT_SECRET": config("O365_CLIENT_SECRET"), "O365_AUTH_BACKEND_AZ_BLOB_NAME": config("O365_AUTH_BACKEND_AZ_BLOB_NAME"), @@ -73,13 +73,25 @@ "O365_AUTH_BACKEND_TOKEN_DIR": config("O365_AUTH_BACKEND_TOKEN_DIR"), "O365_AUTH_BACKEND_TOKEN_FILE": config("O365_AUTH_BACKEND_TOKEN_FILE"), }, + "AZBlobStorageTokenBackend": { + "O365_AUTH_BACKEND_AZ_BLOB_NAME": config("O365_AUTH_BACKEND_AZ_BLOB_NAME"), + "O365_AUTH_BACKEND_AZ_CONTAINER_PATH": config( + "O365_AUTH_BACKEND_AZ_CONTAINER_PATH" + ), + "O365_AUTH_BACKEND_AZ_CONNECTION_STR": config( + "O365_AUTH_BACKEND_AZ_CONNECTION_STR" + ), + }, } O365_MAILBOXES = { "TestUser1": {"email": "test_user@test.com", "webapp": "test_webapp1"}, } -O365_TEST_ACCOUNT = config("O365_TEST_ACCOUNT") +O365_TEST_ADMIN = { + "test_from_email": config("O365_TEST_FROM_ACCOUNT"), + "test_to_email": config("O365_TEST_TO_ACCOUNT"), +} TEMPLATES = [ { From 6d0917884489415c226c18d320ca14f5a776b5a9 Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Fri, 29 Mar 2024 10:55:46 -0400 Subject: [PATCH 09/68] readme and test updates, fixes post tox run. --- README.rst | 2 + django_mail_admin/models/configurations.py | 8 +-- django_mail_admin/o365_utils.py | 11 ++-- requirements.txt | 1 + requirements_test.txt | 1 + tests/o365/test_commands.py | 41 +++++++------- tox.ini | 2 +- tt.txt | 63 ++++++++++++++++++++++ 8 files changed, 101 insertions(+), 28 deletions(-) create mode 100644 tt.txt diff --git a/README.rst b/README.rst index 4b0f16d..e692900 100644 --- a/README.rst +++ b/README.rst @@ -160,6 +160,8 @@ Optional requirements 1. `django_admin_row_actions` for some useful actions in the admin interface 2. `requests` & `social-auth-app-django` for Gmail +3. `python-o365` for O365 emails +4. `azure-storage-blob` Azure Blob-store token-backend for O365 auth-tokens FAQ diff --git a/django_mail_admin/models/configurations.py b/django_mail_admin/models/configurations.py index 3fad992..b299ab8 100644 --- a/django_mail_admin/models/configurations.py +++ b/django_mail_admin/models/configurations.py @@ -120,14 +120,14 @@ class Mailbox(models.Model): "Internet transports include 'imap' and 'pop3'; " "common local file transports include 'maildir', 'mbox', " "and less commonly 'babyl', 'mh', and 'mmdf'.
" - "
" - "Be sure to urlencode your username and password should they " - "contain illegal characters (like @, :, etc)." "

" "For Office 365 email accounts use: 'office365:username@example.com:/?" - "client_id_key=&client_secret_key=&client_app_id=', provide either of client_app_id or client_id_key/client_secret_key pair; client_app_id overrides when both are provided" + "client_id_key=&client_secret_key=&client_app_id='. Default values of client_id_key and client_secret_key are 'O365_CLIENT_ID', and 'O365_CLIENT_SECRET'. When all 3 are provided it will lookup client_id_key/secret_key values from client_app_id configuration." "
supports only on-behalf-of-a-user; thus requires user's auth & consent in a separate authentication flow via console/ or web-browser." "
" + "Be sure to urlencode your username and password should they " + "contain illegal characters (like @, :, etc)." + "
" ) ), blank=True, diff --git a/django_mail_admin/o365_utils.py b/django_mail_admin/o365_utils.py index 0146000..65869e8 100644 --- a/django_mail_admin/o365_utils.py +++ b/django_mail_admin/o365_utils.py @@ -3,6 +3,7 @@ """ import logging import hashlib +from typing import Optional from base64 import b64encode from datetime import datetime @@ -76,8 +77,8 @@ def _get_auth_info( return client_id, client_secret def _get_token_backend( - self, client_app_id: str | None = None, client_id: str | None = None - ) -> BaseTokenBackend | None: + self, client_app_id: str = None, client_id: str = None + ) -> Optional[object]: selected_settings = ( settings.O365_CLIENT_APP_SETTINGS.get(client_app_id, {}) if client_app_id @@ -117,7 +118,7 @@ def _get_token_backend( ) return None - def _decorate_token_name(self, client_id: str, token_name_pattern: str | None): + def _decorate_token_name(self, client_id: str, token_name_pattern: Optional[str]): if not token_name_pattern: token_name_pattern = "o365_token.txt" return "{}/{}".format( @@ -151,7 +152,7 @@ def _connect(self, client_app_id, client_id, client_secret, protocol) -> None: def _get_message_by_id( self, mailbox: MailBox, message_id: str, folder_name="Inbox" - ) -> Message | None: + ) -> Optional[Message]: """retrieve message by id from given mailbox""" if not message_id: return None @@ -164,7 +165,7 @@ def _get_message_by_id( def _get_reply_to_message( self, mailbox: MailBox, msg: EmailMessage - ) -> Message | None: + ) -> Optional[Message]: """retrieve message representing in-reply-to id in msg headers""" if not msg or not msg.extra_headers or not mailbox: return None diff --git a/requirements.txt b/requirements.txt index 08760ac..e5d59d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,6 @@ jsonfield # Additional requirements go here social-auth-app-django asgiref==3.8.0 +O365 azure-core==1.30.1 azure-storage-blob==12.19.1 diff --git a/requirements_test.txt b/requirements_test.txt index 45a130a..a428bb1 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -6,3 +6,4 @@ codecov>=2.0.0 mock==2.0.0 # Additional test requirements go here +python-decouple==3.8 diff --git a/tests/o365/test_commands.py b/tests/o365/test_commands.py index 7561e83..44a508a 100644 --- a/tests/o365/test_commands.py +++ b/tests/o365/test_commands.py @@ -51,6 +51,11 @@ def send_email(self, outgoing_email) -> bool: return send_successful def test_o365(self): + """ + An end-to-end test of sending one email from the mailbox of user1, and checking if it is received in the mailbox of configured user2, if no user2 is configured, sends it to user1. + + It is likely to through O365NotAuthenticated if the mailboxe authorizations are not already setup for user1 and user2. + """ from_user_email = settings.O365_TEST_ADMIN.get("test_from_email") to_user_email = settings.O365_TEST_ADMIN.get("test_to_email", from_user_email) @@ -79,10 +84,6 @@ def test_o365(self): message=test_body, backend_alias=self.O365_BACKEND_ALIAS, ) - send_successful = self.send_email(outgoing_email) - - assert send_successful == True - time.sleep(5) to_user_email_str = parse.quote(to_user_email) to_user_o365_con = f"office365://{to_user_email_str}@outlook.office365.com?client_id_key=O365_CLIENT_ID&client_secret_key=O365_CLIENT_SECRET&archive=Archived" @@ -90,27 +91,31 @@ def test_o365(self): inbox = Mailbox.objects.create( name="O365CommandTest_Inbox", uri=to_user_o365_con, from_email=to_user_email ) - inbox.get_new_mail() - print(f"received {IncomingEmail.objects.count()} new emails") - # self.print_incoming_emails() + # first poll for inbox emails of user2 + new_emails = inbox.get_new_mail() + print(f"received {len(new_emails)} new emails") + # time.sleep(10) + # send new email from user1 to user2 + send_successful = self.send_email(outgoing_email) + + assert send_successful == True + # sleep for sync + # time.sleep(10) + + # get inbox emails for user2 hoping to have received the recently sent email. max_attempts = 3 latest_email = None + # if the recent attempt did not succeed in receiving the new email, try again for max_attempts times to fetch new emails. while max_attempts: max_attempts -= 1 + time.sleep(2) new_emails = inbox.get_new_mail() - if not new_emails: - print( - f"new mail not seen yet; checking again, attempts remaining {max_attempts}" - ) - time.sleep(5) - else: - latest_email = new_emails[0] + print(f"received {len(new_emails)} new emails") + latest_email = new_emails[0] if new_emails else None + if latest_email: break - """ - print(f"\nlatest_email.subject: {latest_email.subject}") - print(f"latest_email.text: {latest_email.text}") - """ + print(f"\tNo new mails; retry attempts remaining {max_attempts}") assert latest_email assert latest_email.subject == test_subject assert latest_email.text == test_body diff --git a/tox.ini b/tox.ini index 1046e5c..5594631 100644 --- a/tox.ini +++ b/tox.ini @@ -24,7 +24,7 @@ deps = django22: Django>=2.2,<3.0 django30: Django>=3.0,<3.1 django40: Django>=4.0 -passenv=TRAVIS TRAVIS_JOB_ID TRAVIS_BRANCH +passenv=TRAVIS [travis:env] DJANGO = 1.8: django18 diff --git a/tt.txt b/tt.txt new file mode 100644 index 0000000..1d2ff11 --- /dev/null +++ b/tt.txt @@ -0,0 +1,63 @@ +asgiref==3.8.0 +azure-core==1.30.1 +azure-storage-blob==12.19.1 +beautifulsoup4==4.12.3 +black==24.3.0 +cachetools==5.3.3 +certifi==2024.2.2 +cffi==1.16.0 +cfgv==3.4.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +colorama==0.4.6 +cryptography==42.0.5 +defusedxml==0.8.0rc2 +distlib==0.3.8 +Django==5.0.3 +django-admin-row-actions==0.10.0 +django-mail-admin==0.3.2 +django-sslserver==0.22 +filelock==3.13.1 +flake8==7.0.0 +identify==2.5.35 +idna==3.6 +isodate==0.6.1 +jsonfield==3.1.0 +MarkupSafe==2.1.5 +mccabe==0.7.0 +mock==5.1.0 +mod_wsgi==5.0.0 +mypy-extensions==1.0.0 +nodeenv==1.8.0 +O365==2.0.34 +oauthlib==3.2.2 +packaging==24.0 +pathspec==0.12.1 +platformdirs==4.2.0 +pluggy==1.4.0 +pre-commit==3.6.2 +pycodestyle==2.11.1 +pycparser==2.21 +pyflakes==3.2.0 +PyJWT==2.8.0 +pyproject-api==1.6.1 +python-dateutil==2.9.0.post0 +python-decouple==3.8 +python3-openid==3.2.0 +pytz==2024.1 +PyYAML==6.0.1 +requests==2.31.0 +requests-oauthlib==1.4.0 +six==1.16.0 +social-auth-app-django==5.1.0 +social-auth-core==4.5.3 +soupsieve==2.5 +sqlparse==0.4.4 +stringcase==1.2.0 +tox==4.14.1 +typing_extensions==4.10.0 +tzdata==2024.1 +tzlocal==5.2 +urllib3==2.2.1 +virtualenv==20.25.1 From 1c6923a9096dac0cd224a54838e14cce6cae0465 Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Fri, 29 Mar 2024 13:30:51 -0400 Subject: [PATCH 10/68] remove unwanted file --- tt.txt | 63 ---------------------------------------------------------- 1 file changed, 63 deletions(-) delete mode 100644 tt.txt diff --git a/tt.txt b/tt.txt deleted file mode 100644 index 1d2ff11..0000000 --- a/tt.txt +++ /dev/null @@ -1,63 +0,0 @@ -asgiref==3.8.0 -azure-core==1.30.1 -azure-storage-blob==12.19.1 -beautifulsoup4==4.12.3 -black==24.3.0 -cachetools==5.3.3 -certifi==2024.2.2 -cffi==1.16.0 -cfgv==3.4.0 -chardet==5.2.0 -charset-normalizer==3.3.2 -click==8.1.7 -colorama==0.4.6 -cryptography==42.0.5 -defusedxml==0.8.0rc2 -distlib==0.3.8 -Django==5.0.3 -django-admin-row-actions==0.10.0 -django-mail-admin==0.3.2 -django-sslserver==0.22 -filelock==3.13.1 -flake8==7.0.0 -identify==2.5.35 -idna==3.6 -isodate==0.6.1 -jsonfield==3.1.0 -MarkupSafe==2.1.5 -mccabe==0.7.0 -mock==5.1.0 -mod_wsgi==5.0.0 -mypy-extensions==1.0.0 -nodeenv==1.8.0 -O365==2.0.34 -oauthlib==3.2.2 -packaging==24.0 -pathspec==0.12.1 -platformdirs==4.2.0 -pluggy==1.4.0 -pre-commit==3.6.2 -pycodestyle==2.11.1 -pycparser==2.21 -pyflakes==3.2.0 -PyJWT==2.8.0 -pyproject-api==1.6.1 -python-dateutil==2.9.0.post0 -python-decouple==3.8 -python3-openid==3.2.0 -pytz==2024.1 -PyYAML==6.0.1 -requests==2.31.0 -requests-oauthlib==1.4.0 -six==1.16.0 -social-auth-app-django==5.1.0 -social-auth-core==4.5.3 -soupsieve==2.5 -sqlparse==0.4.4 -stringcase==1.2.0 -tox==4.14.1 -typing_extensions==4.10.0 -tzdata==2024.1 -tzlocal==5.2 -urllib3==2.2.1 -virtualenv==20.25.1 From 7fc6f7ca7da079d564e93c9f86e5cd9da38db9a7 Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Mon, 1 Apr 2024 13:36:42 -0400 Subject: [PATCH 11/68] 1 fix in token_backend, example, n test cleanup. --- django_mail_admin/o365_utils.py | 8 +++++++- example/example/settings.py | 4 ++++ tests/o365/test_commands.py | 4 ++-- tests/settings.py | 19 ++++--------------- 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/django_mail_admin/o365_utils.py b/django_mail_admin/o365_utils.py index 65869e8..a0baec3 100644 --- a/django_mail_admin/o365_utils.py +++ b/django_mail_admin/o365_utils.py @@ -111,7 +111,7 @@ def _get_token_backend( ), blob_name=self._decorate_token_name( client_id, - token_name_pattern=settings.O365_ADMIN_SETTINGS.get( + token_name_pattern=backend_settings.get( "O365_AUTH_BACKEND_AZ_BLOB_NAME" ), ), @@ -255,6 +255,12 @@ def __init__(self, connection_str, container_name, blob_name): :param str container_name: Container string for blob file. :param str blob_name: Blob name """ + if not (connection_str and container_name and blob_name): + raise ValueError( + "At least one required inputs is empty! " + + f"connection_str:'{connection_str}', container_name:'{container_name}', blob_name:'{blob_name}'" + ) + self.container_name = container_name self.blob_name = blob_name try: diff --git a/example/example/settings.py b/example/example/settings.py index 3931477..48b066e 100644 --- a/example/example/settings.py +++ b/example/example/settings.py @@ -123,6 +123,10 @@ EMAIL_BACKEND = "django_mail_admin.backends.O365Backend" O365_ADMIN_SETTINGS = { + "TOKEN_BACKEND": "FileSystemTokenBackend", # "AZBlobStorageTokenBackend" +} + +EXAMPLE_O365_ADMIN_SETTINGS = { "TOKEN_BACKEND": "AZBlobStorageTokenBackend", # "FileSystemTokenBackend" "O365_CLIENT_ID": config("O365_CLIENT_ID"), "O365_CLIENT_SECRET": config("O365_CLIENT_SECRET"), diff --git a/tests/o365/test_commands.py b/tests/o365/test_commands.py index 44a508a..83a6b1c 100644 --- a/tests/o365/test_commands.py +++ b/tests/o365/test_commands.py @@ -60,7 +60,7 @@ def test_o365(self): to_user_email = settings.O365_TEST_ADMIN.get("test_to_email", from_user_email) from_user_email_str = parse.quote(from_user_email) - from_user_o365_con = f"office365://{from_user_email_str}@outlook.office365.com?client_id_key=O365_CLIENT_ID&client_secret_key=O365_CLIENT_SECRET&archive=Archived" + from_user_o365_con = f"office365://{from_user_email_str}@outlook.office365.com?client_app_id=test_webapp1" Outbox.objects.create( name="O365CommandTest_Outbox", @@ -86,7 +86,7 @@ def test_o365(self): ) to_user_email_str = parse.quote(to_user_email) - to_user_o365_con = f"office365://{to_user_email_str}@outlook.office365.com?client_id_key=O365_CLIENT_ID&client_secret_key=O365_CLIENT_SECRET&archive=Archived" + to_user_o365_con = f"office365://{to_user_email_str}@outlook.office365.com?client_app_id=test_webapp1" inbox = Mailbox.objects.create( name="O365CommandTest_Inbox", uri=to_user_o365_con, from_email=to_user_email diff --git a/tests/settings.py b/tests/settings.py index 64d770a..985b9d3 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -48,21 +48,14 @@ } O365_ADMIN_SETTINGS = { - "TOKEN_BACKEND": "AZBlobStorageTokenBackend", - "O365_CLIENT_ID": config("O365_CLIENT_ID"), - "O365_CLIENT_SECRET": config("O365_CLIENT_SECRET"), - "O365_AUTH_BACKEND_AZ_BLOB_NAME": config("O365_AUTH_BACKEND_AZ_BLOB_NAME"), - "O365_AUTH_BACKEND_AZ_CONNECTION_STR": config( - "O365_AUTH_BACKEND_AZ_CONNECTION_STR" - ), - "O365_AUTH_BACKEND_AZ_CONTAINER_PATH": config( - "O365_AUTH_BACKEND_AZ_CONTAINER_PATH" - ), + "TOKEN_BACKEND": "FileSystemTokenBackend", } -O365_WEBAPP_SETTINGS = { + +O365_CLIENT_APP_SETTINGS = { "test_webapp1": { # TBD certificate support "auth": "token_secret", #'certificate' + "TOKEN_BACKEND": "AZBlobStorageTokenBackend", "O365_CLIENT_ID": config("O365_CLIENT_ID"), "O365_CLIENT_SECRET": config("O365_CLIENT_SECRET"), }, @@ -84,10 +77,6 @@ }, } -O365_MAILBOXES = { - "TestUser1": {"email": "test_user@test.com", "webapp": "test_webapp1"}, -} - O365_TEST_ADMIN = { "test_from_email": config("O365_TEST_FROM_ACCOUNT"), "test_to_email": config("O365_TEST_TO_ACCOUNT"), From 79d53570b9faac09e162d7f2534081539b331df6 Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Mon, 1 Apr 2024 15:53:11 -0400 Subject: [PATCH 12/68] error handling with improved exception message --- django_mail_admin/o365_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django_mail_admin/o365_utils.py b/django_mail_admin/o365_utils.py index a0baec3..f2deae6 100644 --- a/django_mail_admin/o365_utils.py +++ b/django_mail_admin/o365_utils.py @@ -191,7 +191,7 @@ def _prepare_attachment_for_dispatch(self, attachment) -> dict: def send_messages(self, email_messages, fail_silently: bool = False) -> int: sent_messages = 0 - if not self.account.is_authenticated: + if not (self.account and self.account.is_authenticated): logger.error("send_messages unavailable; account not authenticated!") if not fail_silently: raise O365NotAuthenticated( @@ -228,7 +228,7 @@ def send_messages(self, email_messages, fail_silently: bool = False) -> int: return sent_messages def get_messages(self, owner_email: str, last_polled: datetime, condition): - if not self.account.is_authenticated: + if not (self.account and self.account.is_authenticated): logger.error("get_messages unavailable; account not authenticated!") raise O365NotAuthenticated( "get_messages unavailable; account not yet authenticated!" From 82d665188aa6fc751cc6299a6471b9ff340f071f Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Mon, 1 Apr 2024 16:14:20 -0400 Subject: [PATCH 13/68] improved error message --- django_mail_admin/o365_utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/django_mail_admin/o365_utils.py b/django_mail_admin/o365_utils.py index f2deae6..14d4fb1 100644 --- a/django_mail_admin/o365_utils.py +++ b/django_mail_admin/o365_utils.py @@ -64,7 +64,12 @@ def _get_auth_info( else settings.O365_ADMIN_SETTINGS ) if not selected_settings: - raise ValueError("Selected settings not yet set!") + if client_app_id: + raise ValueError( + f"Settings not found for client_app_id: '{client_app_id}'" + ) + else: + raise ValueError(f"O365_ADMIN_SETTINGS not defined!") client_id = selected_settings.get(client_id_key, "") if not client_id: From f21feb1e30785ea6406e67e03c50711f2026367b Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Sat, 6 Apr 2024 11:30:38 -0400 Subject: [PATCH 14/68] support sending html alternative when available --- django_mail_admin/o365_utils.py | 13 ++++- tests/o365/test_commands.py | 68 +++++++++++++++++++-------- tests/settings.py | 1 + tests/templates/html_email_draft.html | 19 ++++++++ 4 files changed, 80 insertions(+), 21 deletions(-) create mode 100644 tests/templates/html_email_draft.html diff --git a/django_mail_admin/o365_utils.py b/django_mail_admin/o365_utils.py index 14d4fb1..aac6158 100644 --- a/django_mail_admin/o365_utils.py +++ b/django_mail_admin/o365_utils.py @@ -8,7 +8,7 @@ from base64 import b64encode from datetime import datetime from django.conf import settings -from django.core.mail.message import EmailMessage +from django.core.mail.message import EmailMessage, EmailMultiAlternatives from O365.utils import BaseTokenBackend @@ -194,6 +194,14 @@ def _prepare_attachment_for_dispatch(self, attachment) -> dict: ).decode("utf-8") return {"name": f"{attachment[0]}", "content": b64content, "on_disk": False} + def _get_html_body(self, msg) -> Optional[str]: + if isinstance(msg, EmailMultiAlternatives): + for msg_alt in msg.alternatives: + alt_content, alt_content_type = msg_alt + if "text/html" == alt_content_type: + return alt_content + return None + def send_messages(self, email_messages, fail_silently: bool = False) -> int: sent_messages = 0 if not (self.account and self.account.is_authenticated): @@ -210,7 +218,8 @@ def send_messages(self, email_messages, fail_silently: bool = False) -> int: m.cc.add(msg.cc) m.bcc.add(msg.bcc) m.reply_to.add(msg.reply_to) - m.body = msg.body + html_body = self._get_html_body(msg) + m.body = msg.body if not html_body else html_body if msg.subject: m.subject = msg.subject m.save_message() diff --git a/tests/o365/test_commands.py b/tests/o365/test_commands.py index 83a6b1c..0e97f18 100644 --- a/tests/o365/test_commands.py +++ b/tests/o365/test_commands.py @@ -6,7 +6,7 @@ from django.test import TestCase from django.conf import settings -from django.core.management import call_command +from django.template.loader import render_to_string from django_mail_admin.models import Outbox, IncomingEmail, OutgoingEmail, STATUS from django_mail_admin.models import Mailbox @@ -23,6 +23,19 @@ def print_incoming_emails(self): for email in IncomingEmail.objects.all().order_by("processed"): print(f"{email.message_id} | {email.subject} | {email.processed}") + def _get_html_draft( + self, + text_body: str, + message_hash: str, + legal_disclaimer="This is a test legal disclaimer", + ): + context = { + "bodylines": text_body.splitlines(), + "message_hash": message_hash, + "legal_disclaimer": legal_disclaimer, + } + return render_to_string("html_email_draft.html", context) + def send_email(self, outgoing_email) -> bool: retry_send = False send_successful = False @@ -71,19 +84,27 @@ def test_o365(self): ) test_subject = f"UnitTest Subject Dated {datetime.now()}" - test_body = f"UnitTest Body Random {random.random()}" + message_hash = random.random() + # do not use '\n' sequence in test body, the returning email will collapse all new line sequences to a single space + text_body = f"UnitTest Body. Line1 Hi\nLine2 This is a test draft" + test_body = f"{text_body}\n{message_hash}" + test_html_body = self._get_html_draft(text_body, message_hash) """ print(f"\ntest_subject: {test_subject}") print(f"test_body: {test_body}\n") """ - outgoing_email = OutgoingEmail.objects.create( - from_email=from_user_email, - to=[to_user_email], - status=STATUS.queued, - subject=test_subject, - message=test_body, - backend_alias=self.O365_BACKEND_ALIAS, - ) + outgoing_emails = [] + for tst_body in [test_body, test_html_body]: + outgoing_emails.append( + OutgoingEmail.objects.create( + from_email=from_user_email, + to=[to_user_email], + status=STATUS.queued, + subject=test_subject, + message=tst_body, + backend_alias=self.O365_BACKEND_ALIAS, + ) + ) to_user_email_str = parse.quote(to_user_email) to_user_o365_con = f"office365://{to_user_email_str}@outlook.office365.com?client_app_id=test_webapp1" @@ -96,8 +117,9 @@ def test_o365(self): print(f"received {len(new_emails)} new emails") # time.sleep(10) - # send new email from user1 to user2 - send_successful = self.send_email(outgoing_email) + # send new email(s) from user1 to user2 + for outgoing_email in outgoing_emails: + send_successful = self.send_email(outgoing_email) assert send_successful == True # sleep for sync @@ -105,17 +127,25 @@ def test_o365(self): # get inbox emails for user2 hoping to have received the recently sent email. max_attempts = 3 - latest_email = None # if the recent attempt did not succeed in receiving the new email, try again for max_attempts times to fetch new emails. + latest_emails = [] while max_attempts: max_attempts -= 1 time.sleep(2) new_emails = inbox.get_new_mail() print(f"received {len(new_emails)} new emails") - latest_email = new_emails[0] if new_emails else None - if latest_email: + latest_emails.extend(new_emails) + if len(latest_emails) == 2: break - print(f"\tNo new mails; retry attempts remaining {max_attempts}") - assert latest_email - assert latest_email.subject == test_subject - assert latest_email.text == test_body + print( + f"\tContinue seeking new mails; retry attempts remaining {max_attempts}" + ) + + assert latest_emails + test_body_wo_newlines = test_body.replace("\n", " ") + for latest_email in latest_emails: + assert latest_email.subject == test_subject + body_matches = (latest_email.text == test_body_wo_newlines) or ( + str(message_hash) in latest_email.html + ) + assert body_matches diff --git a/tests/settings.py b/tests/settings.py index 985b9d3..0e8ece2 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -87,6 +87,7 @@ "BACKEND": "django.template.backends.django.DjangoTemplates", "DIRS": [ os.path.join(BASE_DIR, "templates"), + os.path.join(BASE_DIR, "tests/templates"), ], "APP_DIRS": True, "OPTIONS": { diff --git a/tests/templates/html_email_draft.html b/tests/templates/html_email_draft.html new file mode 100644 index 0000000..62d2acf --- /dev/null +++ b/tests/templates/html_email_draft.html @@ -0,0 +1,19 @@ + + + + + + + +

+ {% for bodyline in bodylines %} + {{bodyline}}
+ {% endfor %} +

+

Kindly retain the following when replying +
{{ message_hash}} +

+

---------------------------------------------------------

+

{{legal_disclaimer}}

+ + From e98af63cdf6d0c73dff535d794bc0e09de143d05 Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Sat, 6 Apr 2024 12:46:18 -0400 Subject: [PATCH 15/68] + 0003_alter_mailbox_from_email_alter_mailbox_uri --- ...er_mailbox_from_email_alter_mailbox_uri.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 django_mail_admin/migrations/0003_alter_mailbox_from_email_alter_mailbox_uri.py diff --git a/django_mail_admin/migrations/0003_alter_mailbox_from_email_alter_mailbox_uri.py b/django_mail_admin/migrations/0003_alter_mailbox_from_email_alter_mailbox_uri.py new file mode 100644 index 0000000..6559c13 --- /dev/null +++ b/django_mail_admin/migrations/0003_alter_mailbox_from_email_alter_mailbox_uri.py @@ -0,0 +1,37 @@ +# Generated by Django 5.0.3 on 2024-04-06 16:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("django_mail_admin", "0002_auto_20190709_1139"), + ] + + operations = [ + migrations.AlterField( + model_name="mailbox", + name="from_email", + field=models.CharField( + blank=True, + default=None, + help_text="Example: MailBot <mailbot@yourdomain.com>
'From' header to set for outgoing email.

If you do not use this e-mail inbox for outgoing mail, this setting is unnecessary.
If you send e-mail without setting this, your 'From' header will'be set to match the setting `DEFAULT_FROM_EMAIL`.
Required for Office 365 mailbox", + max_length=255, + null=True, + verbose_name="From email", + ), + ), + migrations.AlterField( + model_name="mailbox", + name="uri", + field=models.CharField( + blank=True, + default=None, + help_text="Example: imap+ssl://myusername:mypassword@someserver

Internet transports include 'imap' and 'pop3'; common local file transports include 'maildir', 'mbox', and less commonly 'babyl', 'mh', and 'mmdf'.


For Office 365 email accounts use: 'office365:username@example.com:/?client_id_key=&client_secret_key=&client_app_id='. Default values of client_id_key and client_secret_key are 'O365_CLIENT_ID', and 'O365_CLIENT_SECRET'. When all 3 are provided it will lookup client_id_key/secret_key values from client_app_id configuration.
supports only on-behalf-of-a-user; thus requires user's auth & consent in a separate authentication flow via console/ or web-browser.
Be sure to urlencode your username and password should they contain illegal characters (like @, :, etc).
", + max_length=255, + null=True, + verbose_name="URI", + ), + ), + ] From 1ac6085990ac47e47d98509cbfbf1bb33160efb6 Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Tue, 9 Apr 2024 17:34:24 -0400 Subject: [PATCH 16/68] OutgoingEmail has message_id, populated post sent --- .../0004_outgoingemail_message_id.py | 25 +++ django_mail_admin/models/outgoing.py | 184 ++++++++++++------ django_mail_admin/o365_utils.py | 20 ++ 3 files changed, 171 insertions(+), 58 deletions(-) create mode 100644 django_mail_admin/migrations/0004_outgoingemail_message_id.py diff --git a/django_mail_admin/migrations/0004_outgoingemail_message_id.py b/django_mail_admin/migrations/0004_outgoingemail_message_id.py new file mode 100644 index 0000000..5727b2d --- /dev/null +++ b/django_mail_admin/migrations/0004_outgoingemail_message_id.py @@ -0,0 +1,25 @@ +# Generated by Django 5.0.3 on 2024-04-09 21:06 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("django_mail_admin", "0003_alter_mailbox_from_email_alter_mailbox_uri"), + ] + + operations = [ + migrations.AddField( + model_name="outgoingemail", + name="message_id", + field=models.CharField( + blank=True, + default=None, + help_text="Internet Message-ID for outgoing draft/email", + max_length=255, + null=True, + verbose_name="OutgoingEmail ID", + ), + ), + ] diff --git a/django_mail_admin/models/outgoing.py b/django_mail_admin/models/outgoing.py index a678a7d..ee63b6c 100644 --- a/django_mail_admin/models/outgoing.py +++ b/django_mail_admin/models/outgoing.py @@ -20,10 +20,17 @@ class OutgoingEmail(models.Model): - PRIORITY_CHOICES = [(PRIORITY.low, _("low")), (PRIORITY.medium, _("medium")), - (PRIORITY.high, _("high")), (PRIORITY.now, _("now"))] - STATUS_CHOICES = [(STATUS.sent, _("sent")), (STATUS.failed, _("failed")), - (STATUS.queued, _("queued"))] + PRIORITY_CHOICES = [ + (PRIORITY.low, _("low")), + (PRIORITY.medium, _("medium")), + (PRIORITY.high, _("high")), + (PRIORITY.now, _("now")), + ] + STATUS_CHOICES = [ + (STATUS.sent, _("sent")), + (STATUS.failed, _("failed")), + (STATUS.queued, _("queued")), + ] class Meta: verbose_name = _("Outgoing email") @@ -32,7 +39,7 @@ class Meta: from_email = models.CharField( verbose_name=_("From email"), max_length=254, - validators=[validate_email_with_name] + validators=[validate_email_with_name], ) to = CommaSeparatedEmailField(_("To email(s)")) @@ -44,45 +51,53 @@ class Meta: verbose_name=_("Template"), null=True, blank=True, - help_text=_("If template is selected, HTML message and " - "subject fields will not be used - they will be populated from template"), - on_delete=models.CASCADE + help_text=_( + "If template is selected, HTML message and " + "subject fields will not be used - they will be populated from template" + ), + on_delete=models.CASCADE, ) - subject = models.CharField( - verbose_name=_("Subject"), - max_length=989, - blank=True - ) + subject = models.CharField(verbose_name=_("Subject"), max_length=989, blank=True) message = models.TextField(_("Message"), blank=True) html_message = models.TextField( verbose_name=_("HTML Message"), blank=True, - help_text=_("Used only if template is not selected") + help_text=_("Used only if template is not selected"), ) created = models.DateTimeField(auto_now_add=True, db_index=True) last_updated = models.DateTimeField(db_index=True, auto_now=True) - scheduled_time = models.DateTimeField(_('The scheduled sending time'), - blank=True, null=True, db_index=True) - headers = JSONField(_('Headers'), blank=True, null=True) + scheduled_time = models.DateTimeField( + _("The scheduled sending time"), blank=True, null=True, db_index=True + ) + headers = JSONField(_("Headers"), blank=True, null=True) + + message_id = models.CharField( + _("OutgoingEmail ID"), + max_length=255, + blank=True, + null=True, + default=None, + help_text=_("Internet Message-ID for outgoing draft/email"), + ) status = models.PositiveSmallIntegerField( - _("Status"), - choices=STATUS_CHOICES, db_index=True, - blank=True, null=True) - priority = models.PositiveSmallIntegerField(_("Priority"), - choices=PRIORITY_CHOICES, - blank=True, null=True) - - send_now = models.BooleanField( - verbose_name=_("Send now"), - default=False + _("Status"), choices=STATUS_CHOICES, db_index=True, blank=True, null=True + ) + priority = models.PositiveSmallIntegerField( + _("Priority"), choices=PRIORITY_CHOICES, blank=True, null=True ) - backend_alias = models.CharField(_('Backend alias'), blank=True, default='', - help_text=get_backend_names_str, - max_length=64) + send_now = models.BooleanField(verbose_name=_("Send now"), default=False) + + backend_alias = models.CharField( + _("Backend alias"), + blank=True, + default="", + help_text=get_backend_names_str, + max_length=64, + ) def __init__(self, *args, **kwargs): super(OutgoingEmail, self).__init__(*args, **kwargs) @@ -118,22 +133,38 @@ def prepare_email_message(self): subject = self.subject html_message = self.html_message - connection = connections[self.backend_alias or 'default'] + connection = connections[self.backend_alias or "default"] if html_message: msg = EmailMultiAlternatives( - subject=subject, body=message, from_email=self.from_email, - to=self.to, bcc=self.bcc, cc=self.cc, - headers=self.headers, connection=connection) + subject=subject, + body=message, + from_email=self.from_email, + to=self.to, + bcc=self.bcc, + cc=self.cc, + headers=self.headers, + connection=connection, + ) msg.attach_alternative(html_message, "text/html") else: msg = EmailMessage( - subject=subject, body=message, from_email=self.from_email, - to=self.to, bcc=self.bcc, cc=self.cc, - headers=self.headers, connection=connection) + subject=subject, + body=message, + from_email=self.from_email, + to=self.to, + bcc=self.bcc, + cc=self.cc, + headers=self.headers, + connection=connection, + ) for attachment in self.attachments.all(): - msg.attach(attachment.name, attachment.file.read(), mimetype=attachment.mimetype or None) + msg.attach( + attachment.name, + attachment.file.read(), + mimetype=attachment.mimetype or None, + ) attachment.file.close() self._cached_email_message = msg @@ -143,6 +174,19 @@ def queue(self): self.status = STATUS.queued self.save() + def _update_message_id(self) -> bool: + """update internet message id if found in cached-email-message""" + retval = False + if not self._cached_email_message: + return retval + internet_message_id = self._cached_email_message.extra_headers.get( + "Message-ID", None + ) + if internet_message_id: + self.message_id = internet_message_id + retval = True + return retval + def dispatch(self, log_level=None, commit=True): """ Sends email and log the result. @@ -154,8 +198,9 @@ def dispatch(self, log_level=None, commit=True): email_message = self.email_message() email_message.send() status = STATUS.sent - message = '' - exception_type = '' + self._update_message_id() + message = "" + exception_type = "" email_sent.send(sender=self, outgoing_email=email_message) except Exception as e: status = STATUS.failed @@ -170,7 +215,7 @@ def dispatch(self, log_level=None, commit=True): if commit: self.status = status - self.save(update_fields=['status']) + self.save(update_fields=["status"]) if log_level is None: log_level = get_log_level() @@ -179,11 +224,13 @@ def dispatch(self, log_level=None, commit=True): # and 2 means log both successes and failures if log_level == 1: if status == STATUS.failed: - self.logs.create(status=status, message=message, - exception_type=exception_type) + self.logs.create( + status=status, message=message, exception_type=exception_type + ) elif log_level == 2: - self.logs.create(status=status, message=message, - exception_type=exception_type) + self.logs.create( + status=status, message=message, exception_type=exception_type + ) def save(self, *args, **kwargs): self.full_clean() @@ -197,11 +244,18 @@ class Attachment(models.Model): """ A model describing an email attachment. """ - file = models.FileField(_('File'), upload_to=get_attachment_save_path) - name = models.CharField(_('Name'), max_length=255, help_text=_("The original filename")) - emails = models.ManyToManyField(OutgoingEmail, related_name='attachments', blank=True, - verbose_name=_('Email addresses')) - mimetype = models.CharField(max_length=255, default='', blank=True) + + file = models.FileField(_("File"), upload_to=get_attachment_save_path) + name = models.CharField( + _("Name"), max_length=255, help_text=_("The original filename") + ) + emails = models.ManyToManyField( + OutgoingEmail, + related_name="attachments", + blank=True, + verbose_name=_("Email addresses"), + ) + mimetype = models.CharField(max_length=255, default="", blank=True) class Meta: verbose_name = _("Attachment") @@ -225,8 +279,8 @@ def create_attachments(attachment_files): for filename, filedata in attachment_files.items(): if isinstance(filedata, dict): - content = filedata.get('file', None) - mimetype = filedata.get('mimetype', None) + content = filedata.get("file", None) + mimetype = filedata.get("mimetype", None) else: content = filedata mimetype = None @@ -235,7 +289,7 @@ def create_attachments(attachment_files): if isinstance(content, str): # `content` is a filename - try to open the file - opened_file = open(content, 'rb') + opened_file = open(content, "rb") content = File(opened_file) attachment = Attachment() @@ -251,8 +305,16 @@ def create_attachments(attachment_files): return attachments -def send_mail(subject, message, from_email, recipient_list, html_message='', - scheduled_time=None, headers=None, priority=PRIORITY.medium): +def send_mail( + subject, + message, + from_email, + recipient_list, + html_message="", + scheduled_time=None, + headers=None, + priority=PRIORITY.medium, +): """ Add a new message to the mail queue. This is a replacement for Django's ``send_mail`` core email method. @@ -264,9 +326,15 @@ def send_mail(subject, message, from_email, recipient_list, html_message='', for address in recipient_list: emails.append( OutgoingEmail.objects.create( - from_email=from_email, to=address, subject=subject, - message=message, html_message=html_message, status=status, - headers=headers, priority=priority, scheduled_time=scheduled_time + from_email=from_email, + to=address, + subject=subject, + message=message, + html_message=html_message, + status=status, + headers=headers, + priority=priority, + scheduled_time=scheduled_time, ) ) if priority == PRIORITY.now: diff --git a/django_mail_admin/o365_utils.py b/django_mail_admin/o365_utils.py index aac6158..7258aa6 100644 --- a/django_mail_admin/o365_utils.py +++ b/django_mail_admin/o365_utils.py @@ -28,6 +28,7 @@ class O365Connection: O365_PROTOCOL = "MSGraphProtocol" DEFAULT_CLIENT_ID_KEY = "O365_CLIENT_ID" DEFAULT_CLIENT_SECRET_KEY = "O365_CLIENT_SECRET" + MESSAGE_ID_KEY = "Message-ID" def __init__( self, @@ -202,6 +203,24 @@ def _get_html_body(self, msg) -> Optional[str]: return alt_content return None + def _update_message_id_header( + self, msg: EmailMessage, internet_message_id: str + ) -> bool: + """add internet message-id of external email message to EmailMessage.extra_headers + + Ensure this is called after the outbound message has had a chance to have an internet message id, e.g. when a message draft is saved in the mail-server. + + This helps mail-admin to associate the OutgoingEmail to the internet-message-id, and tie any replies to the external email back to this OutgoingEmail. + """ + if not msg or not internet_message_id: + return False + msg.extra_headers.update( + { + self.MESSAGE_ID_KEY: internet_message_id, + } + ) + return True + def send_messages(self, email_messages, fail_silently: bool = False) -> int: sent_messages = 0 if not (self.account and self.account.is_authenticated): @@ -223,6 +242,7 @@ def send_messages(self, email_messages, fail_silently: bool = False) -> int: if msg.subject: m.subject = msg.subject m.save_message() + self._update_message_id_header(msg, m.internet_message_id) m.attachments.add( [ self._prepare_attachment_for_dispatch(attachment) From a7b6d6c970e20d9ea7b6d59334133867cf7856da Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Thu, 11 Apr 2024 12:02:53 -0400 Subject: [PATCH 17/68] set outgoingmail's internet_message_id in headers --- .../0004_outgoingemail_message_id.py | 25 ------------------- django_mail_admin/models/configurations.py | 2 +- django_mail_admin/models/outgoing.py | 14 +++-------- django_mail_admin/o365_utils.py | 21 +++++++++++++++- tests/o365/test_commands.py | 2 +- 5 files changed, 25 insertions(+), 39 deletions(-) delete mode 100644 django_mail_admin/migrations/0004_outgoingemail_message_id.py diff --git a/django_mail_admin/migrations/0004_outgoingemail_message_id.py b/django_mail_admin/migrations/0004_outgoingemail_message_id.py deleted file mode 100644 index 5727b2d..0000000 --- a/django_mail_admin/migrations/0004_outgoingemail_message_id.py +++ /dev/null @@ -1,25 +0,0 @@ -# Generated by Django 5.0.3 on 2024-04-09 21:06 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("django_mail_admin", "0003_alter_mailbox_from_email_alter_mailbox_uri"), - ] - - operations = [ - migrations.AddField( - model_name="outgoingemail", - name="message_id", - field=models.CharField( - blank=True, - default=None, - help_text="Internet Message-ID for outgoing draft/email", - max_length=255, - null=True, - verbose_name="OutgoingEmail ID", - ), - ), - ] diff --git a/django_mail_admin/models/configurations.py b/django_mail_admin/models/configurations.py index b299ab8..196a525 100644 --- a/django_mail_admin/models/configurations.py +++ b/django_mail_admin/models/configurations.py @@ -426,7 +426,7 @@ def _process_message(self, message): in_reply_to = message["in-reply-to"].strip() # Hack to work with db-independent JSONField (which is interpreted as string in db) msg.in_reply_to = OutgoingEmail.objects.filter( - headers__contains='"Message-ID":"' + headers__contains='"Message-ID": "' + message["in-reply-to"].strip() + '"' )[0] diff --git a/django_mail_admin/models/outgoing.py b/django_mail_admin/models/outgoing.py index ee63b6c..7811d53 100644 --- a/django_mail_admin/models/outgoing.py +++ b/django_mail_admin/models/outgoing.py @@ -73,15 +73,6 @@ class Meta: ) headers = JSONField(_("Headers"), blank=True, null=True) - message_id = models.CharField( - _("OutgoingEmail ID"), - max_length=255, - blank=True, - null=True, - default=None, - help_text=_("Internet Message-ID for outgoing draft/email"), - ) - status = models.PositiveSmallIntegerField( _("Status"), choices=STATUS_CHOICES, db_index=True, blank=True, null=True ) @@ -183,7 +174,9 @@ def _update_message_id(self) -> bool: "Message-ID", None ) if internet_message_id: - self.message_id = internet_message_id + if not self.headers: + self.headers = {} + self.headers.update({"Message-ID": internet_message_id}) retval = True return retval @@ -191,7 +184,6 @@ def dispatch(self, log_level=None, commit=True): """ Sends email and log the result. """ - email_message = None # Priority is handled in mail.send try: diff --git a/django_mail_admin/o365_utils.py b/django_mail_admin/o365_utils.py index 7258aa6..56d348b 100644 --- a/django_mail_admin/o365_utils.py +++ b/django_mail_admin/o365_utils.py @@ -203,6 +203,18 @@ def _get_html_body(self, msg) -> Optional[str]: return alt_content return None + def _get_draft_message_id( + self, + mailbox: MailBox, + object_id: str, + ) -> Optional[str]: + """helper to get internet_message_id from the saved draft of given object_id message.""" + draft_folder = mailbox.get_folder(folder_name="Drafts") + if not draft_folder: + return None + dm = draft_folder.get_message(object_id=object_id, download_attachments=False) + return dm.internet_message_id if dm else None + def _update_message_id_header( self, msg: EmailMessage, internet_message_id: str ) -> bool: @@ -242,7 +254,6 @@ def send_messages(self, email_messages, fail_silently: bool = False) -> int: if msg.subject: m.subject = msg.subject m.save_message() - self._update_message_id_header(msg, m.internet_message_id) m.attachments.add( [ self._prepare_attachment_for_dispatch(attachment) @@ -253,6 +264,14 @@ def send_messages(self, email_messages, fail_silently: bool = False) -> int: # workaround: avoid NoneType compare w/ int, O365 exception attachment.size = len(attachment.content) m.save_draft() + self._update_message_id_header( + msg, + m.internet_message_id + if m.internet_message_id + else self._get_draft_message_id( + mailbox=mailbox, object_id=m.object_id + ), + ) m.send() sent_messages += 1 except Exception as e: diff --git a/tests/o365/test_commands.py b/tests/o365/test_commands.py index 0e97f18..7dfde3e 100644 --- a/tests/o365/test_commands.py +++ b/tests/o365/test_commands.py @@ -41,7 +41,7 @@ def send_email(self, outgoing_email) -> bool: send_successful = False while True: try: - outgoing_email.dispatch(commit=False) + outgoing_email.dispatch(commit=True) except O365NotAuthenticated as uae: connection = connections[self.O365_BACKEND_ALIAS] print(f"warning: send_mail: {uae};") From d3f804165443717a18cd1750da6c64bf6b0cf8ae Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Thu, 11 Apr 2024 12:29:05 -0400 Subject: [PATCH 18/68] save outgoing message headers --- django_mail_admin/models/outgoing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django_mail_admin/models/outgoing.py b/django_mail_admin/models/outgoing.py index 7811d53..ca9a77a 100644 --- a/django_mail_admin/models/outgoing.py +++ b/django_mail_admin/models/outgoing.py @@ -177,6 +177,7 @@ def _update_message_id(self) -> bool: if not self.headers: self.headers = {} self.headers.update({"Message-ID": internet_message_id}) + self.save(update_fields=["headers"]) retval = True return retval From c8181cded47eba07a00fc4c4beee491148cbfe5f Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Wed, 12 Jun 2024 15:29:30 -0400 Subject: [PATCH 19/68] gmail smtp oauth2 backend w/social-auth --- django_mail_admin/admin.py | 328 +++++++++++++++++--------- django_mail_admin/backends.py | 59 ++++- django_mail_admin/google_utils.py | 42 +++- django_mail_admin/transports/gmail.py | 18 +- example/example/settings.py | 34 ++- example/example/urls.py | 1 + example/templates/mailbox/index.html | 4 +- 7 files changed, 358 insertions(+), 128 deletions(-) diff --git a/django_mail_admin/admin.py b/django_mail_admin/admin.py index 22ceeac..5a6d908 100644 --- a/django_mail_admin/admin.py +++ b/django_mail_admin/admin.py @@ -1,3 +1,5 @@ +import sys +import tempfile import logging from django.conf import settings @@ -10,9 +12,24 @@ from django.utils import timezone from django.utils.safestring import mark_safe from django.utils.translation import gettext_lazy as _ - -from django_mail_admin.models import Mailbox, IncomingAttachment, IncomingEmail, TemplateVariable, OutgoingEmail, \ - Outbox, EmailTemplate, STATUS, Log, Attachment +from django.db import connection +from django.db.models import Q +from django.utils.timezone import now + +from django_mail_admin.mail import send_queued +from django_mail_admin.lockfile import FileLock, FileLocked +from django_mail_admin.models import ( + Mailbox, + IncomingAttachment, + IncomingEmail, + TemplateVariable, + OutgoingEmail, + Outbox, + EmailTemplate, + STATUS, + Log, + Attachment, +) from django_mail_admin.signals import message_received from django_mail_admin.utils import convert_header_to_unicode from .fields import CommaSeparatedEmailField @@ -21,7 +38,7 @@ logger = logging.getLogger(__name__) # Admin row actions -if 'django_admin_row_actions' in settings.INSTALLED_APPS: +if "django_admin_row_actions" in settings.INSTALLED_APPS: try: from django_admin_row_actions import AdminRowActionsMixin except ImportError: @@ -38,9 +55,12 @@ def get_parent(): :return: class to inherit from """ if admin_row_actions: + class BaseAdmin(AdminRowActionsMixin, admin.ModelAdmin): pass + else: + class BaseAdmin(admin.ModelAdmin): pass @@ -49,15 +69,22 @@ class BaseAdmin(admin.ModelAdmin): def get_new_mail(mailbox_admin, request, queryset): for mailbox in queryset.all(): - logger.debug('Receiving mail for %s' % mailbox) + logger.debug("Receiving mail for %s" % mailbox) got_mail = mailbox.get_new_mail() if len(got_mail) > 0: - messages.success(request, _('Got {} new letters for mailbox "{}"').format(str(len(got_mail)), mailbox.name)) + messages.success( + request, + _('Got {} new letters for mailbox "{}"').format( + str(len(got_mail)), mailbox.name + ), + ) else: - messages.info(request, _('No new mail for mailbox "{}"').format(mailbox.name)) + messages.info( + request, _('No new mail for mailbox "{}"').format(mailbox.name) + ) -get_new_mail.short_description = _('Get new mail') +get_new_mail.short_description = _("Get new mail") def switch_active(mailbox_admin, request, queryset): @@ -66,36 +93,78 @@ def switch_active(mailbox_admin, request, queryset): mailbox.save() -switch_active.short_description = _('Switch active status') +switch_active.short_description = _("Switch active status") + + +def send_queued_mail(outbox_admin, request, queryset): + outbox: Outbox = None + for outbox in queryset.all(): + if not outbox.active: + messages.error("Outbox not active!") + return + default_processes = 1 + default_log_level = 1 # 0 - do nothing, 1 - only log errors + default_lockfile = tempfile.gettempdir() + "/django_mail_admin" + logger.info( + "Acquiring lock for sending queued emails at %s.lock" % default_lockfile + ) + try: + with FileLock(default_lockfile): + + while 1: + try: + send_queued(default_processes, default_log_level) + except Exception as e: + logger.error( + e, exc_info=sys.exc_info(), extra={"status_code": 500} + ) + raise + + # Close DB connection to avoid multiprocessing errors + connection.close() + + if ( + not OutgoingEmail.objects.filter(status=STATUS.queued) + .filter(Q(scheduled_time__lte=now()) | Q(scheduled_time=None)) + .exists() + ): + break + except FileLocked: + logger.info("Failed to acquire lock, terminating now.") + + +send_queued_mail.short_description = _("Send queued email") class MailboxAdmin(get_parent()): list_display = ( - 'name', - 'uri', - 'from_email', - 'active', - 'last_polling', + "name", + "uri", + "from_email", + "active", + "last_polling", ) - readonly_fields = ['last_polling', ] + readonly_fields = [ + "last_polling", + ] actions = [get_new_mail, switch_active] class IncomingAttachmentInline(admin.TabularInline): model = IncomingAttachment extra = 0 - readonly_fields = ['headers', ] + readonly_fields = [ + "headers", + ] def resend_message_received_signal(incoming_email_admin, request, queryset): for message in queryset.all(): - logger.debug('Resending \'message_received\' signal for %s' % message) + logger.debug("Resending 'message_received' signal for %s" % message) message_received.send(sender=incoming_email_admin, message=message) -resend_message_received_signal.short_description = ( - _('Re-send message received signal') -) +resend_message_received_signal.short_description = _("Re-send message received signal") def mark_as_unread(incoming_email_admin, request, queryset): @@ -104,7 +173,7 @@ def mark_as_unread(incoming_email_admin, request, queryset): msg.save() -mark_as_unread.short_description = _('Mark as unread') +mark_as_unread.short_description = _("Mark as unread") def mark_as_read(incoming_email_admin, request, queryset): @@ -113,7 +182,7 @@ def mark_as_read(incoming_email_admin, request, queryset): msg.save() -mark_as_read.short_description = _('Mark as read') +mark_as_read.short_description = _("Mark as read") def custom_titled_filter(title): @@ -133,144 +202,164 @@ def html(self, msg): def attachment_count(self, msg): return msg.attachments.count() - attachment_count.short_description = _('Attachment count') + attachment_count.short_description = _("Attachment count") def subject(self, msg): return convert_header_to_unicode(msg.subject) def mailbox_link(self, msg): - return mark_safe('' + msg.mailbox.name + '') + return mark_safe( + '' + + msg.mailbox.name + + "" + ) - mailbox_link.short_description = _('Mailbox') + mailbox_link.short_description = _("Mailbox") def reply_link(self, msg): if msg.in_reply_to: return mark_safe( - '' + msg.in_reply_to.subject + '') + '' + + msg.in_reply_to.subject + + "" + ) else: - return '' + return "" - reply_link.short_description = _('Reply to') + reply_link.short_description = _("Reply to") def from_address(self, msg): f = msg.from_address if len(f) > 0: - return ','.join(f) + return ",".join(f) else: - return '' + return "" - from_address.short_description = _('From') + from_address.short_description = _("From") def envelope_headers(self, msg): email = msg.get_email_object() - return '\n'.join( - [('%s: %s' % (h, v)) for h, v in email.items()] - ) + return "\n".join([("%s: %s" % (h, v)) for h, v in email.items()]) inlines = [ IncomingAttachmentInline, ] fieldsets = [ - (None, {'fields': [('mailbox', 'message_id'), 'read']}), - (None, {'fields': [('from_header', 'in_reply_to'), 'to_header']}), - (None, {'fields': ['text', 'html']}), + (None, {"fields": [("mailbox", "message_id"), "read"]}), + (None, {"fields": [("from_header", "in_reply_to"), "to_header"]}), + (None, {"fields": ["text", "html"]}), ] list_display = ( - 'subject', - 'from_address', - 'processed', - 'read', - 'mailbox_link', - 'attachment_count', - 'reply_link' + "subject", + "from_address", + "processed", + "read", + "mailbox_link", + "attachment_count", + "reply_link", ) - ordering = ['-processed'] + ordering = ["-processed"] list_filter = ( - ('mailbox__name', custom_titled_filter(_('Mailbox name'))), - 'processed', - 'read', - ) - exclude = ( - 'body', - ) - raw_id_fields = ( - 'in_reply_to', + ("mailbox__name", custom_titled_filter(_("Mailbox name"))), + "processed", + "read", ) + exclude = ("body",) + raw_id_fields = ("in_reply_to",) readonly_fields = ( - 'envelope_headers', - 'message_id', - 'text', - 'html', + "envelope_headers", + "message_id", + "text", + "html", ) - search_fields = ['mailbox__name', 'subject', 'from_header', 'in_reply_to__subject'] + search_fields = ["mailbox__name", "subject", "from_header", "in_reply_to__subject"] actions = [resend_message_received_signal, mark_as_unread, mark_as_read] def has_add_permission(self, request): return False - def change_view(self, request, object_id, form_url='', extra_context=None): + def change_view(self, request, object_id, form_url="", extra_context=None): obj = IncomingEmail.objects.filter(id=object_id).first() if obj: if not obj.read: obj.read = timezone.now() obj.save() return super(IncomingEmailAdmin, self).change_view( - request, object_id, form_url, extra_context=extra_context, + request, + object_id, + form_url, + extra_context=extra_context, ) class IncomingAttachmentAdmin(admin.ModelAdmin): - raw_id_fields = ('message',) - list_display = ('message', 'document',) + raw_id_fields = ("message",) + list_display = ( + "message", + "document", + ) if admin_row_actions: + def get_row_actions(self, obj): row_actions = [ { - 'label': _('View emails'), - 'url': reverse('admin:django_mail_admin_incomingemail_changelist') + '?mailbox__name=' + obj.name, - 'tooltip': _('View emails'), - }, { - 'divided': True, - 'label': _('Get new mail'), - 'action': 'get_new_mail', # calls model's get_new_mail + "label": _("View emails"), + "url": reverse("admin:django_mail_admin_incomingemail_changelist") + + "?mailbox__name=" + + obj.name, + "tooltip": _("View emails"), + }, + { + "divided": True, + "label": _("Get new mail"), + "action": "get_new_mail", # calls model's get_new_mail }, ] row_actions += super(MailboxAdmin, self).get_row_actions(obj) return row_actions - MailboxAdmin.get_row_actions = get_row_actions class EmailTemplateAdmin(admin.ModelAdmin): - list_display = ('name', 'description', 'subject') - readonly_fields = ['preview_template_field'] + list_display = ("name", "description", "subject") + readonly_fields = ["preview_template_field"] def preview_template_field(self, o): if o.id: - url = reverse('admin:django_mail_admin_emailtemplate_change', kwargs={'object_id': o.pk}) - url = url + '?preview_template=true' - return mark_safe(f'Preview this template') + url = reverse( + "admin:django_mail_admin_emailtemplate_change", + kwargs={"object_id": o.pk}, + ) + url = url + "?preview_template=true" + return mark_safe( + f'Preview this template' + ) else: - return '---' - preview_template_field.short_description='Preview' + return "---" - def change_view(self, request, object_id, form_url='', extra_context=None): - if request.GET.get('preview_template', '').lower()=='true': + preview_template_field.short_description = "Preview" + + def change_view(self, request, object_id, form_url="", extra_context=None): + if request.GET.get("preview_template", "").lower() == "true": return self.preview_template_view(request, object_id) return super().change_view(request, object_id, form_url, extra_context) def preview_template_view(self, request, object_id): obj = self.get_object(request, object_id) content = obj.render_html_text(Context()) - return HttpResponse(content, content_type='text/html') - + return HttpResponse(content, content_type="text/html") class TemplateVariableInline(admin.TabularInline): @@ -279,11 +368,14 @@ class TemplateVariableInline(admin.TabularInline): def get_message_preview(instance): - return ('{0}...'.format(instance.message[:25]) if len(instance.message) > 25 - else instance.message) + return ( + "{0}...".format(instance.message[:25]) + if len(instance.message) > 25 + else instance.message + ) -get_message_preview.short_description = _('Message') +get_message_preview.short_description = _("Message") class AttachmentInline(admin.TabularInline): @@ -296,15 +388,17 @@ class AttachmentInline(admin.TabularInline): class CommaSeparatedEmailWidget(TextInput): def __init__(self, *args, **kwargs): super(CommaSeparatedEmailWidget, self).__init__(*args, **kwargs) - self.attrs.update({'class': 'vTextField'}) + self.attrs.update({"class": "vTextField"}) def format_value(self, value): # If the value is a string wrap it in a list so it does not get sliced. if not value: - return '' + return "" if isinstance(value, str): - value = [value, ] - return ', '.join([item for item in value]) + value = [ + value, + ] + return ", ".join([item for item in value]) def requeue(modeladmin, request, queryset): @@ -312,7 +406,7 @@ def requeue(modeladmin, request, queryset): queryset.update(status=STATUS.queued) -requeue.short_description = _('Requeue selected emails') +requeue.short_description = _("Requeue selected emails") class LogInline(admin.TabularInline): @@ -320,7 +414,7 @@ class LogInline(admin.TabularInline): can_delete = False def get_queryset(self, request): - return super().get_queryset(request).order_by('date') + return super().get_queryset(request).order_by("date") def has_add_permission(self, request, obj=None): return False @@ -331,53 +425,77 @@ def has_change_permission(self, request, obj=None): class OutgoingEmailAdmin(admin.ModelAdmin): inlines = (TemplateVariableInline, AttachmentInline, LogInline) - list_display = ['id', 'to_display', 'subject', 'template', 'from_email', 'status', 'scheduled_time', 'priority'] + list_display = [ + "id", + "to_display", + "subject", + "template", + "from_email", + "status", + "scheduled_time", + "priority", + ] formfield_overrides = { - CommaSeparatedEmailField: {'widget': CommaSeparatedEmailWidget} + CommaSeparatedEmailField: {"widget": CommaSeparatedEmailWidget} } actions = [requeue] form = OutgoingEmailAdminForm def to_display(self, instance): - return ', '.join(instance.to) + return ", ".join(instance.to) - to_display.short_description = _('To') + to_display.short_description = _("To") def get_form(self, request, obj=None, **kwargs): # Try to get active Outbox and prepopulate from_email field form = super(OutgoingEmailAdmin, self).get_form(request, obj, **kwargs) configurations = Outbox.objects.filter(active=True) if not (len(configurations) > 1 or len(configurations) == 0): - form.base_fields['from_email'].initial = configurations.first().email_host_user + form.base_fields[ + "from_email" + ].initial = configurations.first().email_host_user return form def save_model(self, request, obj, form, change): super(OutgoingEmailAdmin, self).save_model(request, obj, form, change) # If we have an email to reply to, specify replied headers - if form.cleaned_data['reply']: + if form.cleaned_data["reply"]: if not obj.headers: obj.headers = {} - obj.headers.update(form.cleaned_data['reply'].get_reply_headers(obj.headers)) + obj.headers.update( + form.cleaned_data["reply"].get_reply_headers(obj.headers) + ) obj.save() # TODO: add setting to only queue emails after pressing a button/etc. obj.queue() class AttachmentAdmin(admin.ModelAdmin): - list_display = ('name', 'file',) + list_display = ( + "name", + "file", + ) class OutboxAdmin(admin.ModelAdmin): - list_display = ('name', 'email_host', 'email_host_user', 'email_port', 'id', 'active') - list_filter = ('active',) + list_display = ( + "name", + "email_host", + "email_host_user", + "email_port", + "id", + "active", + ) + list_filter = ("active",) + actions = [send_queued_mail] class LogAdmin(admin.ModelAdmin): - list_display = ('email', 'status', 'date', 'message') + list_display = ("email", "status", "date", "message") -if getattr(settings, 'DJANGO_MAILADMIN_ADMIN_ENABLED', True): +if getattr(settings, "DJANGO_MAILADMIN_ADMIN_ENABLED", True): admin.site.register(IncomingEmail, IncomingEmailAdmin) admin.site.register(IncomingAttachment, IncomingAttachmentAdmin) admin.site.register(Mailbox, MailboxAdmin) diff --git a/django_mail_admin/backends.py b/django_mail_admin/backends.py index 12c0d6e..76b49a7 100644 --- a/django_mail_admin/backends.py +++ b/django_mail_admin/backends.py @@ -1,4 +1,5 @@ import logging +import base64 import threading from urllib import parse @@ -10,7 +11,11 @@ from .models import Outbox, create_attachments from .utils import PRIORITY +from social_django.models import UserSocialAuth +from django.core.mail.backends.base import BaseEmailBackend + from django_mail_admin.o365_utils import O365Connection +from django_mail_admin.google_utils import generate_oauth2_string logger = logging.getLogger(__name__) @@ -91,7 +96,7 @@ def send_messages(self, email_messages): return len(email_messages) -class O365Backend(BaseEmailBackend): +class O365Backend(EmailBackend): """ Backend to handle sending emails via o365 connection """ @@ -139,3 +144,55 @@ def send_messages(self, email_messages) -> int: if not self.conn or not self.from_email: raise Exception(f"Backend not yet ready to send_messages") return self.conn.send_messages(email_messages, fail_silently=self.fail_silently) + + +class GmailOAuth2Backend(CustomEmailBackend): + """Override CustomEmailBackend to perform XOAUTH2 for SMTP""" + + def __init__(self, fail_silently: bool = False, **kwargs) -> None: + configuration: Outbox = None + configurations = Outbox.objects.filter(active=True) + if len(configurations) > 1 or len(configurations) == 0: + raise ValueError( + "Got %(l)s active configurations, expected 1" + % {"l": len(configurations)} + ) + else: + configuration = configurations.first() + + super(GmailOAuth2Backend, self).__init__( + host=configuration.email_host + if configuration.email_host + else "smtp.gmail.com", # TODO read default from env/config + port=configuration.email_port + if configuration.email_port + else "587", # TODO read default from env/config + username=configuration.email_host_user, # must be an google powered email address. + password="", # no pwd; just XOAUTH2 + use_tls=configuration.email_use_tls, + fail_silently=False, + use_ssl=configuration.email_use_ssl, + timeout=configuration.email_timeout, + ssl_keyfile=configuration.email_ssl_keyfile, + ssl_certfile=configuration.email_ssl_certfile, + ) + + def open(self): + """override this to refresh token and OAUTH""" + retval = super(GmailOAuth2Backend, self).open() + if self.connection and retval: + # Retrieve the user's social auth credentials + user_social_auth = UserSocialAuth.objects.get( + uid=self.username, provider="google-oauth2" + ) + creds_info = user_social_auth.extra_data + auth_string = generate_oauth2_string( + self.username, creds_info["access_token"], base64_encode=False + ) + self.connection.docmd( + "AUTH", + "XOAUTH2 " + + base64.b64encode(auth_string.encode("utf-8")).decode("utf-8"), + ) + return True + return False diff --git a/django_mail_admin/google_utils.py b/django_mail_admin/google_utils.py index fa53474..9ab3b02 100644 --- a/django_mail_admin/google_utils.py +++ b/django_mail_admin/google_utils.py @@ -1,3 +1,4 @@ +import base64 import logging import requests @@ -29,7 +30,7 @@ def get_google_access_token(email): # TODO: This should be cacheable try: me = UserSocialAuth.objects.get(uid=email, provider="google-oauth2") - return me.extra_data['access_token'] + return me.extra_data["access_token"] except (UserSocialAuth.DoesNotExist, KeyError): raise AccessTokenNotFound @@ -46,7 +47,7 @@ def update_google_extra_data(email, extra_data): def get_google_refresh_token(email): try: me = UserSocialAuth.objects.get(uid=email, provider="google-oauth2") - return me.extra_data['refresh_token'] + return me.extra_data["refresh_token"] except (UserSocialAuth.DoesNotExist, KeyError): raise RefreshTokenNotFound @@ -73,9 +74,11 @@ def google_api_post(email, url, post_data, authorized=True): # TODO: Make this a lot less ugly. especially the 401 handling headers = dict() if authorized is True: - headers.update(dict( - Authorization="Bearer %s" % get_google_access_token(email), - )) + headers.update( + dict( + Authorization="Bearer %s" % get_google_access_token(email), + ) + ) r = requests.post(url, headers=headers, data=post_data) if r.status_code == 401: refresh_authorization(email) @@ -93,20 +96,39 @@ def refresh_authorization(email): refresh_token=refresh_token, client_id=get_google_consumer_key(), client_secret=get_google_consumer_secret(), - grant_type='refresh_token', + grant_type="refresh_token", ) results = google_api_post( email, "https://accounts.google.com/o/oauth2/token?access_type=offline", post_data, - authorized=False) - results.update({'refresh_token': refresh_token}) + authorized=False, + ) + results.update({"refresh_token": refresh_token}) update_google_extra_data(email, results) def fetch_user_info(email): result = google_api_get( - email, - "https://www.googleapis.com/oauth2/v1/userinfo?alt=json" + email, "https://www.googleapis.com/oauth2/v1/userinfo?alt=json" ) return result + + +def generate_oauth2_string(username, access_token, base64_encode=True): + """Generates an IMAP OAuth2 authentication string. + + See https://developers.google.com/google-apps/gmail/oauth2_overview + + Args: + username: the username (email address) of the account to authenticate + access_token: An OAuth2 access token. + base64_encode: Whether to base64-encode the output. + + Returns: + The SASL argument for the OAuth2 mechanism. + """ + auth_string = "user=%s\1auth=Bearer %s\1\1" % (username, access_token) + if base64_encode: + auth_string = base64.b64encode(auth_string.encode("utf-8")) + return auth_string diff --git a/django_mail_admin/transports/gmail.py b/django_mail_admin/transports/gmail.py index f18bf01..05faa0f 100644 --- a/django_mail_admin/transports/gmail.py +++ b/django_mail_admin/transports/gmail.py @@ -27,30 +27,28 @@ def _connect_oauth(self, username): ) except ImportError: raise ValueError( - '''Install python-social-auth/social-app-django to use oauth2 auth for gmail - (pip install social-auth-app-django)''' + """Install python-social-auth/social-app-django to use oauth2 auth for gmail + (pip install social-auth-app-django)""" ) access_token = None while access_token is None: try: access_token = get_google_access_token(username) - google_email_address = fetch_user_info(username)['email'] + google_email_address = fetch_user_info(username)["email"] except TypeError: # This means that the google process took too long # Trying again is the right thing to do pass except AccessTokenNotFound: raise ValueError( - "No Token available in python-social-auth for %s" % ( - username - ) + "No Token available in python-social-auth for %s" % (username) ) - auth_string = 'user=%s\1auth=Bearer %s\1\1' % ( + auth_string = "user=%s\1auth=Bearer %s\1\1" % ( google_email_address, - access_token + access_token, ) - self.server = self.transport(self.hostname, self.port) - self.server.authenticate('XOAUTH2', lambda x: auth_string) + self.server = self.transport("imap.gmail.com", self.port) + self.server.authenticate("XOAUTH2", lambda x: auth_string) self.server.select() diff --git a/example/example/settings.py b/example/example/settings.py index 48b066e..8437550 100644 --- a/example/example/settings.py +++ b/example/example/settings.py @@ -39,6 +39,7 @@ # if your app has other dependencies that need to be added to the site # they should be added here "sslserver", + "social_django", ] MIDDLEWARE = [ @@ -66,6 +67,8 @@ "django.template.context_processors.request", "django.contrib.auth.context_processors.auth", "django.contrib.messages.context_processors.messages", + "social_django.context_processors.backends", + "social_django.context_processors.login_redirect", ], }, }, @@ -120,7 +123,36 @@ STATIC_URL = "/static/" MEDIA_URL = "/media/" INSTALLED_APPS += ("django_admin_row_actions",) -EMAIL_BACKEND = "django_mail_admin.backends.O365Backend" + +DJANGO_MAIL_ADMIN = { + "BACKENDS": { + "default": "django_mail_admin.backends.GmailOAuth2Backend", #'django_mail_admin.backends.CustomEmailBackend', + "smtp": "django.core.mail.backends.smtp.EmailBackend", + "o365": "django_mail_admin.backends.O365Backend", + "gmail": "django_mail_admin.backends.GmailOAuth2Backend", + } +} +AUTHENTICATION_BACKENDS = ( + "social_core.backends.google.GoogleOAuth2", + "django.contrib.auth.backends.ModelBackend", +) + +SOCIAL_AUTH_GOOGLE_OAUTH2 = "abc" +SOCIAL_AUTH_GOOGLE_OAUTH2_LOGIN_REDIRECT_URL = "/example/mailbox" +# Google Keys +SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = config("SOCIAL_AUTH_GOOGLE_OAUTH2_KEY") # Client: ID +SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = config( + "SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET" +) # Client: Secret +SOCIAL_AUTH_GOOGLE_OAUTH2_SCOPE = [ + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/userinfo.profile", + "https://mail.google.com/", +] +SOCIAL_AUTH_GOOGLE_OAUTH2_AUTH_EXTRA_ARGUMENTS = { + "access_type": "offline", + "approval_prompt": "auto", +} O365_ADMIN_SETTINGS = { "TOKEN_BACKEND": "FileSystemTokenBackend", # "AZBlobStorageTokenBackend" diff --git a/example/example/urls.py b/example/example/urls.py index 5950cfb..e1bcfff 100644 --- a/example/example/urls.py +++ b/example/example/urls.py @@ -32,6 +32,7 @@ name="mailbox_auth_step1", ), path(r"example/mailbox/auth2", views.mailbox_auth_step2, name="mailbox_auth_step2"), + re_path(r"^example/", include("social_django.urls", namespace="social")), ] if settings.DEBUG: # static files (images, css, javascript, etc.) diff --git a/example/templates/mailbox/index.html b/example/templates/mailbox/index.html index 44c9364..e1cd083 100644 --- a/example/templates/mailbox/index.html +++ b/example/templates/mailbox/index.html @@ -27,7 +27,9 @@

Mailbox

{{ mbx.uri }} {{ mbx.active }} {{ mbx.last_polling }} - authorize + Authorize w/mbx + Authorize w/Google + {% endfor %} From f7f9efb9ad0f8b67330637246698d24226eef997 Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Sat, 13 Jul 2024 09:10:07 -0400 Subject: [PATCH 20/68] declare google_email_address before use --- django_mail_admin/transports/gmail.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django_mail_admin/transports/gmail.py b/django_mail_admin/transports/gmail.py index 05faa0f..7684239 100644 --- a/django_mail_admin/transports/gmail.py +++ b/django_mail_admin/transports/gmail.py @@ -32,6 +32,7 @@ def _connect_oauth(self, username): ) access_token = None + google_email_address = None while access_token is None: try: access_token = get_google_access_token(username) From 4cae2de70da54bf1e042c42355088d8507a2c4b8 Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Tue, 27 Aug 2024 12:29:46 -0400 Subject: [PATCH 21/68] handle if eml file is missing. --- django_mail_admin/models/incoming.py | 181 +++++++++++++-------------- 1 file changed, 85 insertions(+), 96 deletions(-) diff --git a/django_mail_admin/models/incoming.py b/django_mail_admin/models/incoming.py index 2faa088..10d5865 100644 --- a/django_mail_admin/models/incoming.py +++ b/django_mail_admin/models/incoming.py @@ -13,81 +13,76 @@ from django.utils.translation import gettext_lazy as _ from django_mail_admin.models import Mailbox, OutgoingEmail -from django_mail_admin.settings import get_attachment_interpolation_header, get_altered_message_header -from django_mail_admin.utils import get_body_from_message, get_attachment_save_path, \ - convert_header_to_unicode +from django_mail_admin.settings import ( + get_attachment_interpolation_header, + get_altered_message_header, +) +from django_mail_admin.utils import ( + get_body_from_message, + get_attachment_save_path, + convert_header_to_unicode, +) class UnreadMessageManager(models.Manager): def get_queryset(self): - return super(UnreadMessageManager, self).get_queryset().filter( - read=None - ) + return super(UnreadMessageManager, self).get_queryset().filter(read=None) class IncomingEmail(models.Model): mailbox = models.ForeignKey( Mailbox, - related_name='messages', - verbose_name=_('Mailbox'), - on_delete=models.CASCADE + related_name="messages", + verbose_name=_("Mailbox"), + on_delete=models.CASCADE, ) - subject = models.CharField( - _('Subject'), - max_length=255 - ) + subject = models.CharField(_("Subject"), max_length=255) - message_id = models.CharField( - _('IncomingEmail ID'), - max_length=255 - ) + message_id = models.CharField(_("IncomingEmail ID"), max_length=255) in_reply_to = models.ForeignKey( OutgoingEmail, - related_name='replies', + related_name="replies", blank=True, null=True, - verbose_name=_('In reply to'), - on_delete=models.CASCADE + verbose_name=_("In reply to"), + on_delete=models.CASCADE, ) from_header = models.CharField( - _('From header'), + _("From header"), max_length=255, ) to_header = models.TextField( - _('To header'), + _("To header"), ) body = models.TextField( - _('Body'), + _("Body"), ) encoded = models.BooleanField( - _('Encoded'), + _("Encoded"), default=False, - help_text=_('True if the e-mail body is Base64 encoded'), + help_text=_("True if the e-mail body is Base64 encoded"), ) - processed = models.DateTimeField( - _('Processed'), - auto_now_add=True - ) + processed = models.DateTimeField(_("Processed"), auto_now_add=True) read = models.DateTimeField( - _('Read'), + _("Read"), default=None, blank=True, null=True, ) eml = models.FileField( - _('Raw message contents'), + _("Raw message contents"), null=True, upload_to="messages", - help_text=_('Original full content of message') + help_text=_("Original full content of message"), ) objects = models.Manager() unread_messages = UnreadMessageManager() @@ -128,20 +123,16 @@ def from_address(self): def to_addresses(self): """Returns a list of addresses to which this message was sent.""" addresses = [] - for address in self.to_header.split(','): + for address in self.to_header.split(","): if address: - addresses.append( - parseaddr( - address - )[1].lower() - ) + addresses.append(parseaddr(address)[1].lower()) return addresses def get_reply_headers(self, headers=None): headers = headers or {} - headers['Message-ID'] = make_msgid() - headers['Date'] = formatdate() - headers['In-Reply-To'] = self.message_id.strip() + headers["Message-ID"] = make_msgid() + headers["Date"] = formatdate() + headers["In-Reply-To"] = self.message_id.strip() return headers def reply(self, **kwargs): @@ -154,13 +145,14 @@ def reply(self, **kwargs): """ from django_mail_admin.mail import send - if 'sender' not in kwargs: + + if "sender" not in kwargs: if len(self.from_address) == 0 and not self.mailbox.from_email: - raise ValidationError('No sender address to reply from, %s' % str(self)) + raise ValidationError("No sender address to reply from, %s" % str(self)) else: - kwargs['sender'] = self.from_address[0] or self.mailbox.from_email - headers = self.get_reply_headers(kwargs.get('headers')) - kwargs['headers'] = headers + kwargs["sender"] = self.from_address[0] or self.mailbox.from_email + headers = self.get_reply_headers(kwargs.get("headers")) + kwargs["headers"] = headers return send(**kwargs) @property @@ -168,18 +160,22 @@ def text(self): """ Returns the message body matching content type 'text/plain'. """ - return get_body_from_message( - self.get_email_object(), 'text', 'plain' - ).replace('=\n', '').strip() + return ( + get_body_from_message(self.get_email_object(), "text", "plain") + .replace("=\n", "") + .strip() + ) @property def html(self): """ Returns the message body matching content type 'text/html'. """ - return get_body_from_message( - self.get_email_object(), 'text', 'html' - ).replace('\n', '').strip() + return ( + get_body_from_message(self.get_email_object(), "text", "html") + .replace("\n", "") + .strip() + ) def _rehydrate(self, msg): new = EmailMessage() @@ -188,9 +184,7 @@ def _rehydrate(self, msg): for header, value in msg.items(): new[header] = value for part in msg.get_payload(): - new.attach( - self._rehydrate(part) - ) + new.attach(self._rehydrate(part)) elif get_attachment_interpolation_header() in msg.keys(): try: attachment = IncomingAttachment.objects.get( @@ -198,43 +192,35 @@ def _rehydrate(self, msg): ) for header, value in attachment.items(): new[header] = value - encoding = new['Content-Transfer-Encoding'] - if encoding and encoding.lower() == 'quoted-printable': + encoding = new["Content-Transfer-Encoding"] + if encoding and encoding.lower() == "quoted-printable": # Cannot use `email.encoders.encode_quopri due to # bug 14360: http://bugs.python.org/issue14360 output = BytesIO() encode_quopri( - BytesIO( - attachment.document.read() - ), + BytesIO(attachment.document.read()), output, quotetabs=True, header=False, ) - new.set_payload( - output.getvalue().decode().replace(' ', '=20') - ) - del new['Content-Transfer-Encoding'] - new['Content-Transfer-Encoding'] = 'quoted-printable' + new.set_payload(output.getvalue().decode().replace(" ", "=20")) + del new["Content-Transfer-Encoding"] + new["Content-Transfer-Encoding"] = "quoted-printable" else: - new.set_payload( - attachment.document.read() - ) - del new['Content-Transfer-Encoding'] + new.set_payload(attachment.document.read()) + del new["Content-Transfer-Encoding"] encode_base64(new) except IncomingAttachment.DoesNotExist: - new[get_altered_message_header()] = ( - 'Missing; Attachment %s not found' % ( + new[ + get_altered_message_header() + ] = "Missing; Attachment %s not found" % ( msg[get_attachment_interpolation_header()] ) - ) - new.set_payload('') + new.set_payload("") else: for header, value in msg.items(): new[header] = value - new.set_payload( - msg.get_payload() - ) + new.set_payload(msg.get_payload()) return new def get_body(self): @@ -245,8 +231,8 @@ def get_body(self): """ if self.encoded: - return base64.b64decode(self.body.encode('ascii')) - return self.body.encode('utf-8') + return base64.b64decode(self.body.encode("ascii")) + return self.body.encode("utf-8") def set_body(self, body): """Set the `body` field of this record. @@ -256,9 +242,9 @@ def set_body(self, body): no fields existed for storing arbitrary bytes. """ - body = body.encode('utf-8') + body = body.encode("utf-8") self.encoded = True - self.body = base64.b64encode(body).decode('ascii') + self.body = base64.b64encode(body).decode("ascii") def get_email_object(self): """Returns an `email.message.Message` instance representing the @@ -280,12 +266,15 @@ def get_email_object(self): """ if self.eml: - if self.eml.name.endswith('.gz'): - body = gzip.GzipFile(fileobj=self.eml).read() - else: - self.eml.open() - body = self.eml.file.read() - self.eml.close() + try: + if self.eml.name.endswith(".gz"): + body = gzip.GzipFile(fileobj=self.eml).read() + else: + self.eml.open() + body = self.eml.file.read() + self.eml.close() + except Exception as e: + body = self.get_body() else: body = self.get_body() flat = email.message_from_bytes(body) @@ -299,31 +288,31 @@ def delete(self, *args, **kwargs): return super(IncomingEmail, self).delete(*args, **kwargs) def __str__(self): - return self.subject + ' from ' + ','.join(self.from_address) + return self.subject + " from " + ",".join(self.from_address) class Meta: - verbose_name = _('Incoming email') - verbose_name_plural = _('Incoming emails') + verbose_name = _("Incoming email") + verbose_name_plural = _("Incoming emails") class IncomingAttachment(models.Model): message = models.ForeignKey( IncomingEmail, - related_name='attachments', + related_name="attachments", null=True, blank=True, on_delete=models.CASCADE, - verbose_name=_('IncomingEmail'), + verbose_name=_("IncomingEmail"), ) headers = models.TextField( - _('Headers'), + _("Headers"), null=True, blank=True, ) document = models.FileField( - _('Document'), + _("Document"), upload_to=get_attachment_save_path, ) @@ -368,12 +357,12 @@ def items(self): def __getitem__(self, name): value = self._get_rehydrated_headers()[name] if value is None: - raise KeyError('Header %s does not exist' % name) + raise KeyError("Header %s does not exist" % name) return value def __str__(self): return self.document.url class Meta: - verbose_name = _('IncomingEmail attachment') - verbose_name_plural = _('IncomingEmail attachments') + verbose_name = _("IncomingEmail attachment") + verbose_name_plural = _("IncomingEmail attachments") From f4b6a4551ecc26c3138ad3daae176d20ed8edf36 Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Tue, 27 Aug 2024 13:05:02 -0400 Subject: [PATCH 22/68] handle when/if attachmt files are missing on disk --- django_mail_admin/models/incoming.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/django_mail_admin/models/incoming.py b/django_mail_admin/models/incoming.py index 10d5865..f14c151 100644 --- a/django_mail_admin/models/incoming.py +++ b/django_mail_admin/models/incoming.py @@ -193,12 +193,18 @@ def _rehydrate(self, msg): for header, value in attachment.items(): new[header] = value encoding = new["Content-Transfer-Encoding"] + document_data = "" + try: + document_data = attachment.document.read() + except Exception as e: + # attachment file missing! + document_data = None if encoding and encoding.lower() == "quoted-printable": # Cannot use `email.encoders.encode_quopri due to # bug 14360: http://bugs.python.org/issue14360 output = BytesIO() encode_quopri( - BytesIO(attachment.document.read()), + BytesIO(document_data), output, quotetabs=True, header=False, @@ -207,7 +213,7 @@ def _rehydrate(self, msg): del new["Content-Transfer-Encoding"] new["Content-Transfer-Encoding"] = "quoted-printable" else: - new.set_payload(attachment.document.read()) + new.set_payload(document_data) del new["Content-Transfer-Encoding"] encode_base64(new) except IncomingAttachment.DoesNotExist: From 96c6095779f7d32ae08251d6026f4c315bd793c4 Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Tue, 27 Aug 2024 13:12:07 -0400 Subject: [PATCH 23/68] take 2 handle if attachment files are missing --- django_mail_admin/models/incoming.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_mail_admin/models/incoming.py b/django_mail_admin/models/incoming.py index f14c151..a149ac9 100644 --- a/django_mail_admin/models/incoming.py +++ b/django_mail_admin/models/incoming.py @@ -198,7 +198,7 @@ def _rehydrate(self, msg): document_data = attachment.document.read() except Exception as e: # attachment file missing! - document_data = None + document_data = b"" if encoding and encoding.lower() == "quoted-printable": # Cannot use `email.encoders.encode_quopri due to # bug 14360: http://bugs.python.org/issue14360 From 03bdcd0f3fbfdb7f6cd2fe6dde3fa9905de1f67d Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Tue, 27 Aug 2024 17:03:21 -0400 Subject: [PATCH 24/68] incomingmail uniq together w/ mailbox+message_id --- .../migrations/0004_auto_20240827_1530.py | 17 +++++++++++++++++ django_mail_admin/models/configurations.py | 19 ++++++++++++++++--- django_mail_admin/models/incoming.py | 1 + 3 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 django_mail_admin/migrations/0004_auto_20240827_1530.py diff --git a/django_mail_admin/migrations/0004_auto_20240827_1530.py b/django_mail_admin/migrations/0004_auto_20240827_1530.py new file mode 100644 index 0000000..62bb869 --- /dev/null +++ b/django_mail_admin/migrations/0004_auto_20240827_1530.py @@ -0,0 +1,17 @@ +# Generated by Django 2.2.24 on 2024-08-27 20:30 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("django_mail_admin", "0003_alter_mailbox_from_email_alter_mailbox_uri"), + ] + + operations = [ + migrations.AlterUniqueTogether( + name="incomingemail", + unique_together={("mailbox", "message_id")}, + ), + ] diff --git a/django_mail_admin/models/configurations.py b/django_mail_admin/models/configurations.py index 196a525..b9bdf7c 100644 --- a/django_mail_admin/models/configurations.py +++ b/django_mail_admin/models/configurations.py @@ -393,15 +393,28 @@ def _get_dehydrated_message(self, msg, record): def _process_message(self, message): from django_mail_admin.models import IncomingEmail, OutgoingEmail - msg = IncomingEmail() + message_id = None + if "message-id" in message: + try: + message_id = message["message-id"][0:255].strip() + except Exception as e: + message_id = None + if not message_id: + message_id = uuid.uuid4().hex + + msg, created = IncomingEmail.objects.get_or_create( + mailbox=self, message_id=message_id + ) + + if not created: + return msg if get_store_original_message(): self._process_save_original_message(message, msg) msg.mailbox = self + msg.message_id = message_id if "subject" in message: msg.subject = utils.convert_header_to_unicode(message["subject"])[0:255] - if "message-id" in message: - msg.message_id = message["message-id"][0:255].strip() if "from" in message: msg.from_header = utils.convert_header_to_unicode(message["from"]) if "to" in message: diff --git a/django_mail_admin/models/incoming.py b/django_mail_admin/models/incoming.py index a149ac9..c7ff368 100644 --- a/django_mail_admin/models/incoming.py +++ b/django_mail_admin/models/incoming.py @@ -299,6 +299,7 @@ def __str__(self): class Meta: verbose_name = _("Incoming email") verbose_name_plural = _("Incoming emails") + unique_together = ["mailbox", "message_id"] class IncomingAttachment(models.Model): From b7d9d4fbd6d07336c5b691b24d67119b1b861709 Mon Sep 17 00:00:00 2001 From: Michael Boker Date: Sun, 22 Sep 2024 18:45:37 -0700 Subject: [PATCH 25/68] Update imap.py --- django_mail_admin/transports/imap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_mail_admin/transports/imap.py b/django_mail_admin/transports/imap.py index 07179de..d178b12 100644 --- a/django_mail_admin/transports/imap.py +++ b/django_mail_admin/transports/imap.py @@ -131,5 +131,5 @@ def get_message(self, condition=None): self.server.uid('copy', uid, self.archive) self.server.uid('store', uid, "+FLAGS", "(\\Deleted)") - self.server.expunge() + # self.server.expunge() return From 6560b5a5b26a31cf8d4844a8815b46c7f5b99408 Mon Sep 17 00:00:00 2001 From: mboker Date: Sun, 22 Sep 2024 20:39:51 -0700 Subject: [PATCH 26/68] uncommenting expunge line and adding more logging in attachment processing --- django_mail_admin/models/configurations.py | 5 ++++- django_mail_admin/transports/imap.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/django_mail_admin/models/configurations.py b/django_mail_admin/models/configurations.py index b9bdf7c..0ee8b2f 100644 --- a/django_mail_admin/models/configurations.py +++ b/django_mail_admin/models/configurations.py @@ -344,6 +344,7 @@ def _get_dehydrated_message(self, msg, record): ): filename = None raw_filename = msg.get_filename() + logger.info(f"Processing attachment for incoming email id {record.pk} with raw_filename {raw_filename}") if raw_filename: filename = utils.convert_header_to_unicode(raw_filename) if not filename: @@ -354,15 +355,17 @@ def _get_dehydrated_message(self, msg, record): extension = ".bin" attachment = IncomingAttachment() - + logger.info(f"Saving attachment document for incoming email id {record.pk} with filename {filename}") attachment.document.save( uuid.uuid4().hex + extension, ContentFile(BytesIO(msg.get_payload(decode=True)).getvalue()), ) + logger.info(f"Attachment document saved for incoming email id {record.pk} with filename {filename}") attachment.message = record for key, value in msg.items(): attachment[key] = value attachment.save() + logger.info(f"Attachment created and saved for incoming email id {record.pk} with filename {filename}") placeholder = EmailMessage() placeholder[get_attachment_interpolation_header()] = str(attachment.pk) diff --git a/django_mail_admin/transports/imap.py b/django_mail_admin/transports/imap.py index d178b12..07179de 100644 --- a/django_mail_admin/transports/imap.py +++ b/django_mail_admin/transports/imap.py @@ -131,5 +131,5 @@ def get_message(self, condition=None): self.server.uid('copy', uid, self.archive) self.server.uid('store', uid, "+FLAGS", "(\\Deleted)") - # self.server.expunge() + self.server.expunge() return From 3c1a6dc58b38520a72da88414099353dbf56266f Mon Sep 17 00:00:00 2001 From: mboker Date: Mon, 23 Sep 2024 11:01:20 -0700 Subject: [PATCH 27/68] trying to convert email message payload to bytes --- django_mail_admin/models/configurations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_mail_admin/models/configurations.py b/django_mail_admin/models/configurations.py index 0ee8b2f..a713866 100644 --- a/django_mail_admin/models/configurations.py +++ b/django_mail_admin/models/configurations.py @@ -460,7 +460,7 @@ def _process_save_original_message(self, message, msg): else: msg.eml.save( - "%s.eml" % uuid.uuid4(), ContentFile(message.as_string()), save=False + "%s.eml" % uuid.uuid4(), ContentFile(BytesIO(message.as_string()).getvalue()), save=False ) def get_new_mail(self, condition=None): From 56d142503c59ca9ef957a38a47be42f4c40ce444 Mon Sep 17 00:00:00 2001 From: mboker Date: Mon, 23 Sep 2024 12:56:50 -0700 Subject: [PATCH 28/68] trying to use as_bytes on email obj --- django_mail_admin/models/configurations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_mail_admin/models/configurations.py b/django_mail_admin/models/configurations.py index a713866..73c77c9 100644 --- a/django_mail_admin/models/configurations.py +++ b/django_mail_admin/models/configurations.py @@ -460,7 +460,7 @@ def _process_save_original_message(self, message, msg): else: msg.eml.save( - "%s.eml" % uuid.uuid4(), ContentFile(BytesIO(message.as_string()).getvalue()), save=False + "%s.eml" % uuid.uuid4(), ContentFile(BytesIO(message.as_bytes()).getvalue()), save=False ) def get_new_mail(self, condition=None): From e141e14204a3dec488e65a938cfc5e8b59ac976f Mon Sep 17 00:00:00 2001 From: mboker Date: Mon, 23 Sep 2024 18:21:16 -0700 Subject: [PATCH 29/68] inserting reply-to as 'from address' when present --- django_mail_admin/models/incoming.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/django_mail_admin/models/incoming.py b/django_mail_admin/models/incoming.py index c7ff368..4a58880 100644 --- a/django_mail_admin/models/incoming.py +++ b/django_mail_admin/models/incoming.py @@ -114,8 +114,17 @@ def from_address(self): `to_addresses`. """ - if self.from_header: - return [parseaddr(self.from_header)[1].lower()] + + email_obj = self.get_email_object() + reply_to = email_obj.get('Reply-To') + + from_address = parseaddr(self.from_header)[1].lower() if self.from_header else None + reply_to_address = parseaddr(reply_to)[1].lower() if reply_to else None + + if reply_to_address: + return [reply_to_address] + elif from_address: + return [from_address] else: return [] From 43cd9c7002ed74ca23e66bd25373747f82dc3786 Mon Sep 17 00:00:00 2001 From: mboker Date: Tue, 24 Sep 2024 08:19:28 -0700 Subject: [PATCH 30/68] fixing google oauth issue where refreshed token is not being applied --- django_mail_admin/google_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/django_mail_admin/google_utils.py b/django_mail_admin/google_utils.py index 9ab3b02..63e5ef5 100644 --- a/django_mail_admin/google_utils.py +++ b/django_mail_admin/google_utils.py @@ -61,8 +61,7 @@ def google_api_get(email, url): if r.status_code == 401: # Go use the refresh token refresh_authorization(email) - r = requests.get(url, headers=headers) - logger.info("I got a %s", r.status_code) + return google_api_get(email, url) if r.status_code == 200: try: return r.json() From 4a5596b04d93969ad72bf76260ff8678a1144d9a Mon Sep 17 00:00:00 2001 From: mboker Date: Mon, 21 Oct 2024 21:28:29 -0400 Subject: [PATCH 31/68] more logging in google refresh --- django_mail_admin/google_utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/django_mail_admin/google_utils.py b/django_mail_admin/google_utils.py index 63e5ef5..7aebad4 100644 --- a/django_mail_admin/google_utils.py +++ b/django_mail_admin/google_utils.py @@ -57,7 +57,7 @@ def google_api_get(email, url): Authorization="Bearer %s" % get_google_access_token(email), ) r = requests.get(url, headers=headers) - logger.info("I got a %s", r.status_code) + logger.info("google_api_get got a %s", r.status_code) if r.status_code == 401: # Go use the refresh token refresh_authorization(email) @@ -79,6 +79,7 @@ def google_api_post(email, url, post_data, authorized=True): ) ) r = requests.post(url, headers=headers, data=post_data) + logger.info("google_api_post got a %s", r.status_code) if r.status_code == 401: refresh_authorization(email) r = requests.post(url, headers=headers, data=post_data) @@ -87,6 +88,9 @@ def google_api_post(email, url, post_data, authorized=True): return r.json() except ValueError: return r.text + else: + logger.error("google_api_post ended with a %s", r.status_code) + raise Exception("google_api_post ended with a %s" % r.status_code) def refresh_authorization(email): From 6d7d98ed024f3380bbc9ff7d3724e7b33d3ebd62 Mon Sep 17 00:00:00 2001 From: mboker Date: Mon, 21 Oct 2024 22:13:57 -0400 Subject: [PATCH 32/68] trying different URL for google token refresh --- django_mail_admin/google_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_mail_admin/google_utils.py b/django_mail_admin/google_utils.py index 7aebad4..aba90f5 100644 --- a/django_mail_admin/google_utils.py +++ b/django_mail_admin/google_utils.py @@ -103,7 +103,7 @@ def refresh_authorization(email): ) results = google_api_post( email, - "https://accounts.google.com/o/oauth2/token?access_type=offline", + "https://oauth2.googleapis.com/token", post_data, authorized=False, ) From 54d0688c741023f21260b512edca4895c0b2daa0 Mon Sep 17 00:00:00 2001 From: mboker Date: Wed, 23 Oct 2024 15:52:28 -0400 Subject: [PATCH 33/68] adding sent time on incomingemail, and populating it --- django_mail_admin/google_utils.py | 4 +- .../0005_incomingemail_send_datetime.py | 39 +++++++++++++++++++ django_mail_admin/models/incoming.py | 6 +++ example/example/settings.py | 5 ++- requirements.txt | 1 + tests/settings.py | 5 ++- 6 files changed, 56 insertions(+), 4 deletions(-) create mode 100644 django_mail_admin/migrations/0005_incomingemail_send_datetime.py diff --git a/django_mail_admin/google_utils.py b/django_mail_admin/google_utils.py index aba90f5..9b28033 100644 --- a/django_mail_admin/google_utils.py +++ b/django_mail_admin/google_utils.py @@ -79,7 +79,7 @@ def google_api_post(email, url, post_data, authorized=True): ) ) r = requests.post(url, headers=headers, data=post_data) - logger.info("google_api_post got a %s", r.status_code) + logger.error(f"google_api_post got a {r.status_code}, url: {url}, authorized?: {authorized}") if r.status_code == 401: refresh_authorization(email) r = requests.post(url, headers=headers, data=post_data) @@ -89,7 +89,7 @@ def google_api_post(email, url, post_data, authorized=True): except ValueError: return r.text else: - logger.error("google_api_post ended with a %s", r.status_code) + logger.error(f"google_api_post ended with a {r.status_code}, url: {url}, authorized?: {authorized}") raise Exception("google_api_post ended with a %s" % r.status_code) diff --git a/django_mail_admin/migrations/0005_incomingemail_send_datetime.py b/django_mail_admin/migrations/0005_incomingemail_send_datetime.py new file mode 100644 index 0000000..af1fbf8 --- /dev/null +++ b/django_mail_admin/migrations/0005_incomingemail_send_datetime.py @@ -0,0 +1,39 @@ +# Generated by Django 5.0.9 on 2024-10-23 19:43 + +from django.db import migrations, models +import dateparser + +def populate_sent_datetime(apps, schema_editor): + # Import the model using the historical model `apps.get_model` + IncomingEmail = apps.get_model('django_mail_admin', 'IncomingEmail') + ImapTransport = apps.get_model('django_mail_admin', 'ImapTransport') + + for mail_obj in IncomingEmail.objects.all(): + try: + message_contents = mail_obj.eml.read() + msg = ImapTransport('').get_email_from_bytes(message_contents) # assuming message_contents is needed here + date = msg['Date'] + if date: + mail_obj.sent_datetime = dateparser.parse(date) + mail_obj.save() + except Exception as e: + print(f"Error updating mail_obj {mail_obj.id}: {e}") + mail_obj.sent_datetime = mail_obj.processed + mail_obj.save() + continue + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_mail_admin', '0004_auto_20240827_1530'), + ] + + operations = [ + migrations.AddField( + model_name='incomingemail', + name='sent_datetime', + field=models.DateTimeField(blank=True, null=True, verbose_name='Sent datetime'), + ), + migrations.RunPython(populate_sent_datetime), + ] diff --git a/django_mail_admin/models/incoming.py b/django_mail_admin/models/incoming.py index 4a58880..e97a637 100644 --- a/django_mail_admin/models/incoming.py +++ b/django_mail_admin/models/incoming.py @@ -87,6 +87,12 @@ class IncomingEmail(models.Model): objects = models.Manager() unread_messages = UnreadMessageManager() + sent_datetime = models.DateTimeField( + _("Sent datetime"), + blank=True, + null=True, + ) + @property def address(self): """Property allowing one to get the relevant address(es). diff --git a/example/example/settings.py b/example/example/settings.py index 8437550..210ea89 100644 --- a/example/example/settings.py +++ b/example/example/settings.py @@ -11,7 +11,7 @@ """ import os -from decouple import config +from decouple import config as _config # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -26,6 +26,9 @@ ALLOWED_HOSTS = ["*"] +def config(key): + return _config(key, default="") + # Application definition INSTALLED_APPS = [ diff --git a/requirements.txt b/requirements.txt index e5d59d8..50006d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ asgiref==3.8.0 O365 azure-core==1.30.1 azure-storage-blob==12.19.1 +dateparser==1.2.0 \ No newline at end of file diff --git a/tests/settings.py b/tests/settings.py index 0e8ece2..6dd8b04 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -4,11 +4,14 @@ import django import os -from decouple import config +from decouple import config as _config DEBUG = True USE_TZ = True +def config(key): + return _config(key, default="") + # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) From 91541090a90419aa27e46c622c2ee52f08f497fd Mon Sep 17 00:00:00 2001 From: mboker Date: Wed, 23 Oct 2024 15:57:45 -0400 Subject: [PATCH 34/68] adding index on message for attachments, to speed up admin hopefully --- ...attachment_django_mail_message_dea8fb_idx.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 django_mail_admin/migrations/0006_incomingattachment_django_mail_message_dea8fb_idx.py diff --git a/django_mail_admin/migrations/0006_incomingattachment_django_mail_message_dea8fb_idx.py b/django_mail_admin/migrations/0006_incomingattachment_django_mail_message_dea8fb_idx.py new file mode 100644 index 0000000..f152748 --- /dev/null +++ b/django_mail_admin/migrations/0006_incomingattachment_django_mail_message_dea8fb_idx.py @@ -0,0 +1,17 @@ +# Generated by Django 5.0.9 on 2024-10-23 19:57 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_mail_admin', '0005_incomingemail_send_datetime'), + ] + + operations = [ + migrations.AddIndex( + model_name='incomingattachment', + index=models.Index(fields=['message'], name='django_mail_message_dea8fb_idx'), + ), + ] From 3e5ec11f1edd05b74fe013a0e355eb1b691853f3 Mon Sep 17 00:00:00 2001 From: mboker Date: Wed, 23 Oct 2024 16:32:34 -0400 Subject: [PATCH 35/68] fixing transport import --- .../migrations/0005_incomingemail_send_datetime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_mail_admin/migrations/0005_incomingemail_send_datetime.py b/django_mail_admin/migrations/0005_incomingemail_send_datetime.py index af1fbf8..1b8b9a0 100644 --- a/django_mail_admin/migrations/0005_incomingemail_send_datetime.py +++ b/django_mail_admin/migrations/0005_incomingemail_send_datetime.py @@ -1,12 +1,12 @@ # Generated by Django 5.0.9 on 2024-10-23 19:43 from django.db import migrations, models +from django_mail_admin.transports.imap import ImapTransport import dateparser def populate_sent_datetime(apps, schema_editor): # Import the model using the historical model `apps.get_model` IncomingEmail = apps.get_model('django_mail_admin', 'IncomingEmail') - ImapTransport = apps.get_model('django_mail_admin', 'ImapTransport') for mail_obj in IncomingEmail.objects.all(): try: From 8761f7d0f8a0bafbf910b0da606b9e4999c74456 Mon Sep 17 00:00:00 2001 From: mboker Date: Wed, 23 Oct 2024 16:49:29 -0400 Subject: [PATCH 36/68] adding sent time on newly processed emails as well --- django_mail_admin/models/configurations.py | 13 +++++++++++++ django_mail_admin/models/incoming.py | 3 +++ 2 files changed, 16 insertions(+) diff --git a/django_mail_admin/models/configurations.py b/django_mail_admin/models/configurations.py index 73c77c9..8bbfbd8 100644 --- a/django_mail_admin/models/configurations.py +++ b/django_mail_admin/models/configurations.py @@ -7,6 +7,7 @@ from io import BytesIO from tempfile import NamedTemporaryFile from urllib.parse import parse_qs, unquote, urlparse +import dateparser from django.core.exceptions import ValidationError from django.core.files.base import ContentFile, File @@ -424,6 +425,18 @@ def _process_message(self, message): msg.to_header = utils.convert_header_to_unicode(message["to"]) elif "Delivered-To" in message: msg.to_header = utils.convert_header_to_unicode(message["Delivered-To"]) + + if "Date" in message: + try: + date_str = message["Date"] + sent_datetime = dateparser.parse(date_str) + if sent_datetime: + msg.sent_datetime = sent_datetime + except Exception as e: + logger.warning(f"Failed to parse date for incoming email {msg.pk}: {e}") + if msg.sent_datetime is None: + msg.sent_datetime = msg.processed + msg.save() message = self._get_dehydrated_message(message, msg) try: diff --git a/django_mail_admin/models/incoming.py b/django_mail_admin/models/incoming.py index e97a637..e0c62dc 100644 --- a/django_mail_admin/models/incoming.py +++ b/django_mail_admin/models/incoming.py @@ -388,3 +388,6 @@ def __str__(self): class Meta: verbose_name = _("IncomingEmail attachment") verbose_name_plural = _("IncomingEmail attachments") + indexes = [ + models.Index(fields=["message"]), + ] From 47ff137773738def279fbf0c3c48eed2a151816c Mon Sep 17 00:00:00 2001 From: mboker Date: Wed, 23 Oct 2024 22:45:41 -0400 Subject: [PATCH 37/68] trying to fix one date that failed to parse by using a different library --- .../migrations/0005_incomingemail_send_datetime.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/django_mail_admin/migrations/0005_incomingemail_send_datetime.py b/django_mail_admin/migrations/0005_incomingemail_send_datetime.py index 1b8b9a0..e5887b3 100644 --- a/django_mail_admin/migrations/0005_incomingemail_send_datetime.py +++ b/django_mail_admin/migrations/0005_incomingemail_send_datetime.py @@ -2,6 +2,7 @@ from django.db import migrations, models from django_mail_admin.transports.imap import ImapTransport +from dateutil import parser import dateparser def populate_sent_datetime(apps, schema_editor): @@ -14,7 +15,10 @@ def populate_sent_datetime(apps, schema_editor): msg = ImapTransport('').get_email_from_bytes(message_contents) # assuming message_contents is needed here date = msg['Date'] if date: - mail_obj.sent_datetime = dateparser.parse(date) + sent_datetime = parser.parse(date) + if sent_datetime is None: + sent_datetime = dateparser.parse(date) + mail_obj.sent_datetime = sent_datetime mail_obj.save() except Exception as e: print(f"Error updating mail_obj {mail_obj.id}: {e}") From 145de5b83e5e7442ae5effbd3d7ea018a5739d8e Mon Sep 17 00:00:00 2001 From: mboker Date: Wed, 23 Oct 2024 22:53:27 -0400 Subject: [PATCH 38/68] applying date parser change to configurations file also --- django_mail_admin/models/configurations.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/django_mail_admin/models/configurations.py b/django_mail_admin/models/configurations.py index 8bbfbd8..b7697a5 100644 --- a/django_mail_admin/models/configurations.py +++ b/django_mail_admin/models/configurations.py @@ -8,6 +8,7 @@ from tempfile import NamedTemporaryFile from urllib.parse import parse_qs, unquote, urlparse import dateparser +from dateutil import parser from django.core.exceptions import ValidationError from django.core.files.base import ContentFile, File @@ -429,7 +430,9 @@ def _process_message(self, message): if "Date" in message: try: date_str = message["Date"] - sent_datetime = dateparser.parse(date_str) + sent_datetime = parser.parse(date_str) + if sent_datetime is None: + sent_datetime = dateparser.parse(date_str) if sent_datetime: msg.sent_datetime = sent_datetime except Exception as e: From ee2c3396ceb4eb1251a5ca1ac669fb31b55b8f1c Mon Sep 17 00:00:00 2001 From: mboker Date: Fri, 25 Oct 2024 12:03:47 -0400 Subject: [PATCH 39/68] removing new line characters in subjects before saving, as a belt-and-suspenders with the same on the django app --- django_mail_admin/models/incoming.py | 5 +++++ django_mail_admin/models/outgoing.py | 2 ++ 2 files changed, 7 insertions(+) diff --git a/django_mail_admin/models/incoming.py b/django_mail_admin/models/incoming.py index e0c62dc..543d578 100644 --- a/django_mail_admin/models/incoming.py +++ b/django_mail_admin/models/incoming.py @@ -316,6 +316,11 @@ class Meta: verbose_name_plural = _("Incoming emails") unique_together = ["mailbox", "message_id"] + def save(self, *args, **kwargs): + # Clean the `subject` field + self.subject = self.subject.replace('\r','').replace('\n','') + super(IncomingEmail, self).save(*args, **kwargs) + class IncomingAttachment(models.Model): message = models.ForeignKey( diff --git a/django_mail_admin/models/outgoing.py b/django_mail_admin/models/outgoing.py index ca9a77a..88a252d 100644 --- a/django_mail_admin/models/outgoing.py +++ b/django_mail_admin/models/outgoing.py @@ -227,6 +227,8 @@ def dispatch(self, log_level=None, commit=True): def save(self, *args, **kwargs): self.full_clean() + # Clean the `subject` field + self.subject = self.subject.replace('\r','').replace('\n','') super(OutgoingEmail, self).save(*args, **kwargs) def __str__(self): From 9103d56ee4289384872aa71564b75197dcb718ac Mon Sep 17 00:00:00 2001 From: mboker Date: Sun, 27 Oct 2024 17:41:10 -0400 Subject: [PATCH 40/68] reordering gmail operations so token is refreshed before trying to retrieve emails - instead of the opposite --- django_mail_admin/google_utils.py | 2 +- django_mail_admin/transports/gmail.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/django_mail_admin/google_utils.py b/django_mail_admin/google_utils.py index 9b28033..a45d834 100644 --- a/django_mail_admin/google_utils.py +++ b/django_mail_admin/google_utils.py @@ -79,7 +79,7 @@ def google_api_post(email, url, post_data, authorized=True): ) ) r = requests.post(url, headers=headers, data=post_data) - logger.error(f"google_api_post got a {r.status_code}, url: {url}, authorized?: {authorized}") + logger.info(f"google_api_post got a {r.status_code}, url: {url}, authorized?: {authorized}") if r.status_code == 401: refresh_authorization(email) r = requests.post(url, headers=headers, data=post_data) diff --git a/django_mail_admin/transports/gmail.py b/django_mail_admin/transports/gmail.py index 7684239..650688a 100644 --- a/django_mail_admin/transports/gmail.py +++ b/django_mail_admin/transports/gmail.py @@ -35,8 +35,8 @@ def _connect_oauth(self, username): google_email_address = None while access_token is None: try: - access_token = get_google_access_token(username) google_email_address = fetch_user_info(username)["email"] + access_token = get_google_access_token(username) except TypeError: # This means that the google process took too long # Trying again is the right thing to do From dafcb34a031d99686298e87b4ee9bbb28522ea24 Mon Sep 17 00:00:00 2001 From: mboker Date: Mon, 28 Oct 2024 11:20:34 -0400 Subject: [PATCH 41/68] removing new lines from incoming subjects --- django_mail_admin/models/configurations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_mail_admin/models/configurations.py b/django_mail_admin/models/configurations.py index b7697a5..c80a843 100644 --- a/django_mail_admin/models/configurations.py +++ b/django_mail_admin/models/configurations.py @@ -419,7 +419,7 @@ def _process_message(self, message): msg.mailbox = self msg.message_id = message_id if "subject" in message: - msg.subject = utils.convert_header_to_unicode(message["subject"])[0:255] + msg.subject = utils.convert_header_to_unicode(message["subject"]).replace("\n","").replace("\r","")[0:255] if "from" in message: msg.from_header = utils.convert_header_to_unicode(message["from"]) if "to" in message: From a2a741dae39f12b57a81863f869eb25ef1eeb150 Mon Sep 17 00:00:00 2001 From: Camden Parsons Date: Tue, 10 Dec 2024 17:23:24 -0500 Subject: [PATCH 42/68] fix o365 backend --- django_mail_admin/backends.py | 136 +++++++++++++++++++++++++--------- 1 file changed, 101 insertions(+), 35 deletions(-) diff --git a/django_mail_admin/backends.py b/django_mail_admin/backends.py index 76b49a7..bebec44 100644 --- a/django_mail_admin/backends.py +++ b/django_mail_admin/backends.py @@ -1,7 +1,7 @@ import logging import base64 import threading - +from typing import Optional from urllib import parse from django.core.mail.backends.base import BaseEmailBackend @@ -100,50 +100,116 @@ class O365Backend(EmailBackend): """ Backend to handle sending emails via o365 connection """ - def __init__(self, fail_silently: bool = False, **kwargs) -> None: super().__init__(fail_silently, **kwargs) - self.conn: O365Connection = None - self.from_email: str = None + self.conn: Optional[O365Connection] = None + self.from_email: Optional[str] = None self.fail_silently: bool = fail_silently + self.configuration_id: Optional[int] = None self._lock = threading.RLock() + def close(self): + """Closes and cleans up the current connection""" + with self._lock: + if self.conn: + self.conn = None + self.from_email = None + self.configuration_id = None + super().close() + + def _validate_configuration(self, configuration: Outbox) -> None: + """Validates if we need to create a new connection based on configuration""" + if not configuration: + raise ValueError("No active email configuration found") + + # If configuration has changed, close existing connection + if self.configuration_id != configuration.id: + self.close() + + # If connection exists, validate it's still using correct email + if self.conn and self.from_email != configuration.email_host_user: + self.close() + def open(self): - configuration: Outbox = None + """Opens a new O365 connection using the active configuration""" + with self._lock: + # Get active configuration + configuration = Outbox.objects.filter(active=True).first() + + # Validate configuration and connection state + self._validate_configuration(configuration) + + # If we already have a valid connection, return + if self.conn and self.from_email: + return + + self.from_email = configuration.email_host_user + self.configuration_id = configuration.id + + # Parse O365 connection details from email_host + parseresult = parse.urlparse(configuration.email_host) + if not O365Connection.SCHEME == parseresult.scheme.lower(): + raise ValueError( + f'Invalid EMAIL_HOST scheme, expected "{O365Connection.SCHEME}", got "{parseresult.scheme}"' + ) + + # Extract connection parameters from query string + query_dict = dict(parse.parse_qsl(parseresult.query)) + client_app_id = query_dict.get("client_app_id", "") + client_id_key = query_dict.get("client_id_key", "") + client_secret_key = query_dict.get("client_secret_key", "") + + try: + # Create new connection with current configuration + self.conn = O365Connection( + from_email=self.from_email, + client_app_id=client_app_id, + client_id_key=client_id_key, + client_secret_key=client_secret_key, + ) + except Exception as e: + self.close() # Clean up on failure + raise - configurations = Outbox.objects.filter(active=True) - if len(configurations) > 1 or len(configurations) == 0: - raise ValueError( - "Got %(l)s active configurations, expected 1" - % {"l": len(configurations)} - ) - else: - configuration = configurations.first() + def send_messages(self, email_messages) -> int: + """Sends email messages via O365 connection matching the from_email""" + if not email_messages: + return 0 + + with self._lock: + sent_count = 0 + + for msg in email_messages: + try: + # Try to get configuration matching this message's from_email + configuration = Outbox.objects.filter( + active=True, + email_host_user=msg.from_email + ).first() + + if not configuration: + raise ValueError( + f"No active configuration found for sender: {msg.from_email}" + ) - self.from_email = configuration.email_host_user + # If connection doesn't match this message's configuration, create new connection + if (not self.conn or + self.from_email != msg.from_email or + self.configuration_id != configuration.id): + self.close() + self.from_email = msg.from_email + self.open() - self.connection = None - parseresult = parse.urlparse(configuration.email_host) - if not O365Connection.SCHEME == parseresult.scheme.lower(): - raise ValueError( - f'Invalid EMAIL_HOST scheme, expected "{O365Connection.SCHEME}", got "{parseresult.scheme}"' - ) - query_dict = dict(parse.parse_qsl(parseresult.query)) - client_app_id = query_dict.get("client_app_id", "") - client_id_key = query_dict.get("client_id_key", "") - client_secret_key = query_dict.get("client_secret_key", "") - - self.conn = O365Connection( - from_email=self.from_email, - client_app_id=client_app_id, - client_id_key=client_id_key, - client_secret_key=client_secret_key, - ) + # Send the message + if self.conn.send_messages([msg], fail_silently=self.fail_silently): + sent_count += 1 - def send_messages(self, email_messages) -> int: - if not self.conn or not self.from_email: - raise Exception(f"Backend not yet ready to send_messages") - return self.conn.send_messages(email_messages, fail_silently=self.fail_silently) + except Exception as e: + if not self.fail_silently: + raise + logger.error(f"Failed to send message from {msg.from_email}: {str(e)}") + + return sent_count class GmailOAuth2Backend(CustomEmailBackend): From c19eaef9c28bb15cdf2c5a99e70ef7a885621d7e Mon Sep 17 00:00:00 2001 From: Camden Parsons Date: Tue, 17 Dec 2024 18:09:01 -0500 Subject: [PATCH 43/68] fix gmail backend --- django_mail_admin/backends.py | 166 ++++++++++++++++++++++++++-------- 1 file changed, 127 insertions(+), 39 deletions(-) diff --git a/django_mail_admin/backends.py b/django_mail_admin/backends.py index bebec44..16a10c8 100644 --- a/django_mail_admin/backends.py +++ b/django_mail_admin/backends.py @@ -1,11 +1,15 @@ import logging import base64 +import smtplib +import ssl import threading from typing import Optional from urllib import parse from django.core.mail.backends.base import BaseEmailBackend from django.core.mail.backends.smtp import EmailBackend +from django.core.mail.message import sanitize_address +from django_mail_admin import settings from .mail import create from .models import Outbox, create_attachments @@ -212,53 +216,137 @@ def send_messages(self, email_messages) -> int: return sent_count -class GmailOAuth2Backend(CustomEmailBackend): - """Override CustomEmailBackend to perform XOAUTH2 for SMTP""" +class GmailOAuth2Backend(BaseEmailBackend): + """Email backend that uses XOAUTH2 for SMTP authentication""" def __init__(self, fail_silently: bool = False, **kwargs) -> None: - configuration: Outbox = None - configurations = Outbox.objects.filter(active=True) - if len(configurations) > 1 or len(configurations) == 0: - raise ValueError( - "Got %(l)s active configurations, expected 1" - % {"l": len(configurations)} - ) - else: - configuration = configurations.first() + super().__init__(fail_silently=fail_silently) + self.connection = None + self._lock = threading.RLock() + self.from_email = None + self.configuration_id = None + # Initialize these to None - they'll be set when sending + self.host = None + self.port = None + self.username = None + self.password = None + self.use_tls = None + self.use_ssl = None + self.timeout = None + self.ssl_keyfile = None + self.ssl_certfile = None - super(GmailOAuth2Backend, self).__init__( - host=configuration.email_host - if configuration.email_host - else "smtp.gmail.com", # TODO read default from env/config - port=configuration.email_port - if configuration.email_port - else "587", # TODO read default from env/config - username=configuration.email_host_user, # must be an google powered email address. - password="", # no pwd; just XOAUTH2 - use_tls=configuration.email_use_tls, - fail_silently=False, - use_ssl=configuration.email_use_ssl, - timeout=configuration.email_timeout, - ssl_keyfile=configuration.email_ssl_keyfile, - ssl_certfile=configuration.email_ssl_certfile, - ) + def _initialize_connection(self, from_email: str) -> None: + """Initialize connection settings based on from_email""" + configuration = Outbox.objects.filter( + active=True, + email_host_user=from_email + ).first() + + if not configuration: + raise ValueError(f"No active configuration found for sender: {from_email}") + + self.from_email = from_email + self.configuration_id = configuration.id + + # Initialize settings from configuration + self.host = configuration.email_host or "smtp.gmail.com" + self.port = configuration.email_port or "587" + self.username = configuration.email_host_user # Important: this is used for OAuth lookup + self.password = "" # No password needed for XOAUTH2 + self.use_tls = configuration.email_use_tls + self.use_ssl = configuration.email_use_ssl + self.timeout = configuration.email_timeout + self.ssl_keyfile = configuration.email_ssl_keyfile + self.ssl_certfile = configuration.email_ssl_certfile + + def open(self,from_email=None): + """Override to use OAuth instead of password authentication""" + + try: + # First establish the SMTP connection + if self.use_ssl: + self.connection = smtplib.SMTP_SSL(self.host, self.port, timeout=self.timeout) + else: + self.connection = smtplib.SMTP(self.host, self.port, timeout=self.timeout) + + if self.use_tls: + self.connection.starttls() + + if from_email: + user_social_auth = UserSocialAuth.objects.get( + uid=from_email, # This is configuration.email_host_user + provider="google-oauth2" + ) + else: + # Get OAuth credentials using email_host_user from configuration + user_social_auth = UserSocialAuth.objects.get( + uid=self.username, # This is configuration.email_host_user + provider="google-oauth2" + ) - def open(self): - """override this to refresh token and OAUTH""" - retval = super(GmailOAuth2Backend, self).open() - if self.connection and retval: - # Retrieve the user's social auth credentials - user_social_auth = UserSocialAuth.objects.get( - uid=self.username, provider="google-oauth2" - ) creds_info = user_social_auth.extra_data auth_string = generate_oauth2_string( - self.username, creds_info["access_token"], base64_encode=False + self.username, # This is configuration.email_host_user + creds_info["access_token"], + base64_encode=False ) self.connection.docmd( "AUTH", - "XOAUTH2 " - + base64.b64encode(auth_string.encode("utf-8")).decode("utf-8"), + "XOAUTH2 " + base64.b64encode(auth_string.encode("utf-8")).decode("utf-8"), ) return True - return False + except Exception: + logger.exception("gmail failed to open connection") + return None + + def close(self): + """Close the connection to the email server.""" + if self.connection is None: + return + try: + try: + self.connection.quit() + except (ssl.SSLError, smtplib.SMTPServerDisconnected): + self.connection.close() + except smtplib.SMTPException: + if self.fail_silently: + return + raise + finally: + self.connection = None + + def send_messages(self, email_messages): + """Send messages, reinitializing connection if from_email changes""" + if not email_messages: + return 0 + + with self._lock: + num_sent = 0 + + for message in email_messages: + try: + # Check if we need to reinitialize for a different from_email + if (not self.connection or + self.from_email != message.from_email or + not self.configuration_id): + + self.close() + self._initialize_connection(message.from_email) + new_conn_created = self.open(from_email=message.from_email) + + if not self.connection or new_conn_created is None: + continue + + if self._send(message): + num_sent += 1 + + except Exception as e: + if not self.fail_silently: + raise + logger.error(f"Failed to send message from {message.from_email}: {str(e)}") + + if self.connection: + self.close() + + return num_sent From 8189e13aca586ae151690ea9bb99e083e7b30e36 Mon Sep 17 00:00:00 2001 From: Camden Parsons Date: Wed, 18 Dec 2024 14:59:01 -0500 Subject: [PATCH 44/68] fix gmail send issue --- django_mail_admin/backends.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_mail_admin/backends.py b/django_mail_admin/backends.py index 16a10c8..040a7c9 100644 --- a/django_mail_admin/backends.py +++ b/django_mail_admin/backends.py @@ -216,7 +216,7 @@ def send_messages(self, email_messages) -> int: return sent_count -class GmailOAuth2Backend(BaseEmailBackend): +class GmailOAuth2Backend(EmailBackend): """Email backend that uses XOAUTH2 for SMTP authentication""" def __init__(self, fail_silently: bool = False, **kwargs) -> None: From 2df7ecd577644700b6e64e81943193accfe4f5b6 Mon Sep 17 00:00:00 2001 From: Camden Parsons Date: Wed, 18 Dec 2024 17:12:20 -0500 Subject: [PATCH 45/68] try fix for send_as_email --- django_mail_admin/backends.py | 50 +++++++++++-------- .../0007_create_emailaddressoauthmapping.py | 23 +++++++++ django_mail_admin/models/outgoing.py | 18 +++++++ 3 files changed, 69 insertions(+), 22 deletions(-) create mode 100644 django_mail_admin/migrations/0007_create_emailaddressoauthmapping.py diff --git a/django_mail_admin/backends.py b/django_mail_admin/backends.py index 040a7c9..92a9ba4 100644 --- a/django_mail_admin/backends.py +++ b/django_mail_admin/backends.py @@ -10,6 +10,7 @@ from django.core.mail.backends.smtp import EmailBackend from django.core.mail.message import sanitize_address from django_mail_admin import settings +from django_mail_admin.models.outgoing import EmailAddressOAuthMapping from .mail import create from .models import Outbox, create_attachments @@ -185,10 +186,18 @@ def send_messages(self, email_messages) -> int: for msg in email_messages: try: + + # Use EmailAddressOAuthMapping to find oauth_username + oauth_username = ( + EmailAddressOAuthMapping.objects.filter(send_as_email=msg.from_email) + .values_list('oauth_username', flat=True) + .first() + ) or msg.from_email + # Try to get configuration matching this message's from_email configuration = Outbox.objects.filter( active=True, - email_host_user=msg.from_email + email_host_user=oauth_username ).first() if not configuration: @@ -198,10 +207,9 @@ def send_messages(self, email_messages) -> int: # If connection doesn't match this message's configuration, create new connection if (not self.conn or - self.from_email != msg.from_email or - self.configuration_id != configuration.id): + self.from_email != msg.from_email): self.close() - self.from_email = msg.from_email + self.from_email = oauth_username self.open() # Send the message @@ -246,13 +254,12 @@ def _initialize_connection(self, from_email: str) -> None: if not configuration: raise ValueError(f"No active configuration found for sender: {from_email}") - self.from_email = from_email self.configuration_id = configuration.id # Initialize settings from configuration self.host = configuration.email_host or "smtp.gmail.com" self.port = configuration.email_port or "587" - self.username = configuration.email_host_user # Important: this is used for OAuth lookup + self.username = from_email # Important: this is used for OAuth lookup self.password = "" # No password needed for XOAUTH2 self.use_tls = configuration.email_use_tls self.use_ssl = configuration.email_use_ssl @@ -260,7 +267,7 @@ def _initialize_connection(self, from_email: str) -> None: self.ssl_keyfile = configuration.email_ssl_keyfile self.ssl_certfile = configuration.email_ssl_certfile - def open(self,from_email=None): + def open(self,auth_uid=None): """Override to use OAuth instead of password authentication""" try: @@ -273,21 +280,14 @@ def open(self,from_email=None): if self.use_tls: self.connection.starttls() - if from_email: - user_social_auth = UserSocialAuth.objects.get( - uid=from_email, # This is configuration.email_host_user - provider="google-oauth2" - ) - else: - # Get OAuth credentials using email_host_user from configuration - user_social_auth = UserSocialAuth.objects.get( - uid=self.username, # This is configuration.email_host_user - provider="google-oauth2" - ) + user_social_auth = UserSocialAuth.objects.get( + uid=auth_uid, + provider="google-oauth2" + ) creds_info = user_social_auth.extra_data auth_string = generate_oauth2_string( - self.username, # This is configuration.email_host_user + auth_uid, creds_info["access_token"], base64_encode=False ) @@ -330,10 +330,16 @@ def send_messages(self, email_messages): if (not self.connection or self.from_email != message.from_email or not self.configuration_id): - + #hack way to get the original username + oauth_username = ( + EmailAddressOAuthMapping.objects.filter(send_as_email=message.from_email) + .values_list('oauth_username', flat=True) + .first() + ) or message.from_email + self.close() - self._initialize_connection(message.from_email) - new_conn_created = self.open(from_email=message.from_email) + self._initialize_connection(oauth_username) + new_conn_created = self.open(auth_uid=oauth_username) if not self.connection or new_conn_created is None: continue diff --git a/django_mail_admin/migrations/0007_create_emailaddressoauthmapping.py b/django_mail_admin/migrations/0007_create_emailaddressoauthmapping.py new file mode 100644 index 0000000..dbf28a3 --- /dev/null +++ b/django_mail_admin/migrations/0007_create_emailaddressoauthmapping.py @@ -0,0 +1,23 @@ +from django.db import migrations, models +import django.utils.translation + +class Migration(migrations.Migration): + + dependencies = [ + ('django_mail_admin', '0006_incomingattachment_django_mail_message_dea8fb_idx'), + ] + + operations = [ + migrations.CreateModel( + name='EmailAddressOAuthMapping', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('send_as_email', models.EmailField(max_length=254, unique=True, verbose_name=django.utils.translation.gettext_lazy('Send As Email Address'))), + ('oauth_username', models.EmailField(max_length=254, verbose_name=django.utils.translation.gettext_lazy('OAuth Username'))), + ], + options={ + 'verbose_name': django.utils.translation.gettext_lazy('Email OAuth Mapping'), + 'verbose_name_plural': django.utils.translation.gettext_lazy('Email OAuth Mappings'), + }, + ), + ] diff --git a/django_mail_admin/models/outgoing.py b/django_mail_admin/models/outgoing.py index 88a252d..197a445 100644 --- a/django_mail_admin/models/outgoing.py +++ b/django_mail_admin/models/outgoing.py @@ -339,3 +339,21 @@ def send_mail( for email in emails: email_queued.send(email) return emails + +class EmailAddressOAuthMapping(models.Model): + """Maps send-as email addresses to their OAuth usernames""" + send_as_email = models.EmailField( + unique=True, + verbose_name=_("Send As Email Address") + ) + oauth_username = models.EmailField( + verbose_name=_("OAuth Username") + ) + + class Meta: + app_label = 'django_mail_admin' + verbose_name = _("Email OAuth Mapping") + verbose_name_plural = _("Email OAuth Mappings") + + def __str__(self): + return f"{self.send_as_email} -> {self.oauth_username}" \ No newline at end of file From c251cdad489efe71147e8fb65e2cdb3e5611ca4f Mon Sep 17 00:00:00 2001 From: csabantych Date: Tue, 4 Mar 2025 14:46:09 -0500 Subject: [PATCH 46/68] WIP to ensure mail admin uses the right outbox connection + get rid of the active concept. we dont use it or need it --- django_mail_admin/admin.py | 44 ++++----- django_mail_admin/backends.py | 89 +++++++++++-------- django_mail_admin/connections.py | 32 +++++-- .../management/commands/get_new_mail.py | 3 +- django_mail_admin/models/configurations.py | 19 ++-- django_mail_admin/models/outgoing.py | 34 +++++-- 6 files changed, 140 insertions(+), 81 deletions(-) diff --git a/django_mail_admin/admin.py b/django_mail_admin/admin.py index 5a6d908..05f20a2 100644 --- a/django_mail_admin/admin.py +++ b/django_mail_admin/admin.py @@ -87,21 +87,21 @@ def get_new_mail(mailbox_admin, request, queryset): get_new_mail.short_description = _("Get new mail") -def switch_active(mailbox_admin, request, queryset): - for mailbox in queryset.all(): - mailbox.active = not mailbox.active - mailbox.save() - - -switch_active.short_description = _("Switch active status") +# def switch_active(mailbox_admin, request, queryset): +# for mailbox in queryset.all(): +# mailbox.active = not mailbox.active +# mailbox.save() +# +# +# switch_active.short_description = _("Switch active status") def send_queued_mail(outbox_admin, request, queryset): outbox: Outbox = None for outbox in queryset.all(): - if not outbox.active: - messages.error("Outbox not active!") - return + # if not outbox.active: + # messages.error("Outbox not active!") + # return default_processes = 1 default_log_level = 1 # 0 - do nothing, 1 - only log errors default_lockfile = tempfile.gettempdir() + "/django_mail_admin" @@ -147,7 +147,7 @@ class MailboxAdmin(get_parent()): readonly_fields = [ "last_polling", ] - actions = [get_new_mail, switch_active] + actions = [get_new_mail]#, switch_active] class IncomingAttachmentInline(admin.TabularInline): @@ -446,15 +446,19 @@ def to_display(self, instance): to_display.short_description = _("To") - def get_form(self, request, obj=None, **kwargs): - # Try to get active Outbox and prepopulate from_email field - form = super(OutgoingEmailAdmin, self).get_form(request, obj, **kwargs) - configurations = Outbox.objects.filter(active=True) - if not (len(configurations) > 1 or len(configurations) == 0): - form.base_fields[ - "from_email" - ].initial = configurations.first().email_host_user - return form + # March 2025: commenting out the get_form override + # given how active accounts worked, I don't think this worked + # and I think it's more dangerous to pick a random default + + # def get_form(self, request, obj=None, **kwargs): + # # Try to get active Outbox and prepopulate from_email field + # form = super(OutgoingEmailAdmin, self).get_form(request, obj, **kwargs) + # configurations = Outbox.objects.filter(active=True) + # if not (len(configurations) > 1 or len(configurations) == 0): + # form.base_fields[ + # "from_email" + # ].initial = configurations.first().email_host_user + # return form def save_model(self, request, obj, form, change): super(OutgoingEmailAdmin, self).save_model(request, obj, form, change) diff --git a/django_mail_admin/backends.py b/django_mail_admin/backends.py index 92a9ba4..89ab58f 100644 --- a/django_mail_admin/backends.py +++ b/django_mail_admin/backends.py @@ -42,6 +42,9 @@ def __init__( ): super(CustomEmailBackend, self).__init__(fail_silently=fail_silently) # TODO: implement choosing backend for a letter as a param + # March2025 ChargeUp note: keeping active=True in case we use this to prevent false sends + # if we copy this we should do from user/email filtering like O365/Gmail backends + configurations = Outbox.objects.filter(active=True) if len(configurations) > 1 or len(configurations) == 0: raise ValueError( @@ -108,17 +111,26 @@ class O365Backend(EmailBackend): def __init__(self, fail_silently: bool = False, **kwargs) -> None: super().__init__(fail_silently, **kwargs) self.conn: Optional[O365Connection] = None - self.from_email: Optional[str] = None + + # from_email or from_oauth_user might be hydrated by us (ChargeUp) after init (since Django does the init in places) + self.from_email = None + self.from_oauth_user = None + self.fail_silently: bool = fail_silently self.configuration_id: Optional[int] = None self._lock = threading.RLock() + @property + def lookup_email(self) -> str | None: + return self.from_oauth_user or self.from_email + def close(self): """Closes and cleans up the current connection""" with self._lock: if self.conn: self.conn = None self.from_email = None + self.from_oauth_user = None self.configuration_id = None super().close() @@ -126,48 +138,51 @@ def _validate_configuration(self, configuration: Outbox) -> None: """Validates if we need to create a new connection based on configuration""" if not configuration: raise ValueError("No active email configuration found") - + # If configuration has changed, close existing connection if self.configuration_id != configuration.id: self.close() - + # If connection exists, validate it's still using correct email - if self.conn and self.from_email != configuration.email_host_user: + if self.conn and self.lookup_email != configuration.email_host_user: self.close() def open(self): - """Opens a new O365 connection using the active configuration""" + """Opens a new O365 connection using the relevant configuration""" + # If we already have a valid connection, we're good with self._lock: - # Get active configuration - configuration = Outbox.objects.filter(active=True).first() - + if self.conn and self.lookup_email: + return + + configuration = Outbox.objects.filter( + email_host__icontains="office365", + email_host_user=self.lookup_email + ).first() + + if not configuration: + raise ValueError(f"Unable to find an Outbox with email_host__icontains=office365 email_host_user={self.lookup_email} from_oauth_user={self.from_oauth_user} from_email={self.from_email}") + # Validate configuration and connection state self._validate_configuration(configuration) - - # If we already have a valid connection, return - if self.conn and self.from_email: - return - - self.from_email = configuration.email_host_user self.configuration_id = configuration.id - + # Parse O365 connection details from email_host parseresult = parse.urlparse(configuration.email_host) if not O365Connection.SCHEME == parseresult.scheme.lower(): raise ValueError( f'Invalid EMAIL_HOST scheme, expected "{O365Connection.SCHEME}", got "{parseresult.scheme}"' ) - + # Extract connection parameters from query string query_dict = dict(parse.parse_qsl(parseresult.query)) client_app_id = query_dict.get("client_app_id", "") client_id_key = query_dict.get("client_id_key", "") client_secret_key = query_dict.get("client_secret_key", "") - + try: # Create new connection with current configuration self.conn = O365Connection( - from_email=self.from_email, + from_email=self.lookup_email, client_app_id=client_app_id, client_id_key=client_id_key, client_secret_key=client_secret_key, @@ -180,10 +195,10 @@ def send_messages(self, email_messages) -> int: """Sends email messages via O365 connection matching the from_email""" if not email_messages: return 0 - + with self._lock: sent_count = 0 - + for msg in email_messages: try: @@ -196,17 +211,16 @@ def send_messages(self, email_messages) -> int: # Try to get configuration matching this message's from_email configuration = Outbox.objects.filter( - active=True, email_host_user=oauth_username ).first() - + if not configuration: raise ValueError( f"No active configuration found for sender: {msg.from_email}" ) # If connection doesn't match this message's configuration, create new connection - if (not self.conn or + if (not self.conn or self.from_email != msg.from_email): self.close() self.from_email = oauth_username @@ -220,7 +234,7 @@ def send_messages(self, email_messages) -> int: if not self.fail_silently: raise logger.error(f"Failed to send message from {msg.from_email}: {str(e)}") - + return sent_count @@ -231,7 +245,11 @@ def __init__(self, fail_silently: bool = False, **kwargs) -> None: super().__init__(fail_silently=fail_silently) self.connection = None self._lock = threading.RLock() + + # from_email or from_oauth_user might be hydrated by us (ChargeUp) after init (since Django does the init in places) self.from_email = None + self.from_oauth_user = None + self.configuration_id = None # Initialize these to None - they'll be set when sending self.host = None @@ -247,15 +265,14 @@ def __init__(self, fail_silently: bool = False, **kwargs) -> None: def _initialize_connection(self, from_email: str) -> None: """Initialize connection settings based on from_email""" configuration = Outbox.objects.filter( - active=True, email_host_user=from_email ).first() - + if not configuration: raise ValueError(f"No active configuration found for sender: {from_email}") self.configuration_id = configuration.id - + # Initialize settings from configuration self.host = configuration.email_host or "smtp.gmail.com" self.port = configuration.email_port or "587" @@ -281,13 +298,13 @@ def open(self,auth_uid=None): self.connection.starttls() user_social_auth = UserSocialAuth.objects.get( - uid=auth_uid, + uid=auth_uid, provider="google-oauth2" ) creds_info = user_social_auth.extra_data auth_string = generate_oauth2_string( - auth_uid, + auth_uid, creds_info["access_token"], base64_encode=False ) @@ -323,12 +340,12 @@ def send_messages(self, email_messages): with self._lock: num_sent = 0 - + for message in email_messages: try: # Check if we need to reinitialize for a different from_email - if (not self.connection or - self.from_email != message.from_email or + if (not self.connection or + self.from_email != message.from_email or not self.configuration_id): #hack way to get the original username oauth_username = ( @@ -340,19 +357,19 @@ def send_messages(self, email_messages): self.close() self._initialize_connection(oauth_username) new_conn_created = self.open(auth_uid=oauth_username) - + if not self.connection or new_conn_created is None: continue if self._send(message): num_sent += 1 - + except Exception as e: if not self.fail_silently: raise logger.error(f"Failed to send message from {message.from_email}: {str(e)}") - + if self.connection: self.close() - + return num_sent diff --git a/django_mail_admin/connections.py b/django_mail_admin/connections.py index 24cbd4b..7427d9b 100644 --- a/django_mail_admin/connections.py +++ b/django_mail_admin/connections.py @@ -16,23 +16,39 @@ class ConnectionHandler(object): def __init__(self): self._connections = local() - def __getitem__(self, alias): + def __getitem__(self, maybe_hacked_alias): try: - return self._connections.connections[alias] + return self._connections.connections[maybe_hacked_alias] except AttributeError: self._connections.connections = {} except KeyError: pass + # as a hack other places are using backend_alias;;;from_oauth_user + # or backend_alias???from_email + real_alias = maybe_hacked_alias + from_oauth_user = None + from_email = None + if "???" in maybe_hacked_alias: + real_alias,from_email = maybe_hacked_alias.split("???") + elif ";;;" in maybe_hacked_alias: + real_alias,from_oauth_user = maybe_hacked_alias.split(";;;") + try: - backend = get_backend(alias) + backend = get_backend(real_alias) except KeyError: - raise KeyError('%s is not a valid backend alias' % alias) + raise KeyError('%s is not a valid backend alias' % real_alias) + + # backend_class is a EmailBackend subclass like O365Backend or GmailOAuth2Backend + backend_class = get_connection(backend) + + # now mutate the backend class after init since get_connection is within django + backend_class.from_email = from_email + backend_class.from_oauth_user = from_oauth_user - connection = get_connection(backend) - connection.open() - self._connections.connections[alias] = connection - return connection + backend_class.open() + self._connections.connections[maybe_hacked_alias] = backend_class + return backend_class def all(self): return getattr(self._connections, 'connections', {}).values() diff --git a/django_mail_admin/management/commands/get_new_mail.py b/django_mail_admin/management/commands/get_new_mail.py index cb26d8b..2656810 100644 --- a/django_mail_admin/management/commands/get_new_mail.py +++ b/django_mail_admin/management/commands/get_new_mail.py @@ -10,7 +10,8 @@ class Command(BaseCommand): def handle(self, *args, **options): - mailboxes = Mailbox.active_mailboxes.all() + #mailboxes = Mailbox.active_mailboxes.all() + mailboxes = Mailbox.objects.all() if args: mailboxes = mailboxes.filter( name=' '.join(args) diff --git a/django_mail_admin/models/configurations.py b/django_mail_admin/models/configurations.py index c80a843..e5b11f4 100644 --- a/django_mail_admin/models/configurations.py +++ b/django_mail_admin/models/configurations.py @@ -59,19 +59,22 @@ class Outbox(models.Model): email_timeout = models.PositiveSmallIntegerField( "EMAIL_TIMEOUT", null=True, blank=True ) + # deprecated field us at ChargeUp or not using anymore. keeping for compatability. active = models.BooleanField(_("Active"), default=False) def save(self, *args, **kwargs): # Only one item can be active at a time - if self.active: - # select all other active items - qs = type(self).objects.filter(active=True) - # except self (if self already exists) - if self.pk: - qs = qs.exclude(pk=self.pk) - # and deactive them - qs.update(active=False) + # ChargeUp Note: this was a previous side effect on Outbox.save + + # if self.active: + # # select all other active items + # qs = type(self).objects.filter(active=True) + # # except self (if self already exists) + # if self.pk: + # qs = qs.exclude(pk=self.pk) + # # and deactive them + # qs.update(active=False) super(Outbox, self).save(*args, **kwargs) diff --git a/django_mail_admin/models/outgoing.py b/django_mail_admin/models/outgoing.py index 197a445..20de98a 100644 --- a/django_mail_admin/models/outgoing.py +++ b/django_mail_admin/models/outgoing.py @@ -101,16 +101,16 @@ def _get_context(self): return Context(context) - def email_message(self): + def email_message(self, outbox=None): """ Returns Django EmailMessage object for sending. """ if self._cached_email_message: return self._cached_email_message - return self.prepare_email_message() + return self.prepare_email_message(outbox=outbox) - def prepare_email_message(self): + def prepare_email_message(self, outbox=None): """ Returns a django ``EmailMessage`` or ``EmailMultiAlternatives`` object, depending on whether html_message is empty. @@ -124,7 +124,23 @@ def prepare_email_message(self): subject = self.subject html_message = self.html_message - connection = connections[self.backend_alias or "default"] + # hack: OG version commented out below + #connection = connections[self.backend_alias or "default"] + + # this adds a connection for the outbox we're intending to send from rather than something like: + # "any 0365 outbox" or "any active outbox" + # we should also likely always have a backend alias + if not self.backend_alias: + raise ValueError(f"Outgoing emails should always have a backend alias. It was: {self.backend_alias} for from_email: {self.from_email}") + + if outbox: + hack_alias = self.backend_alias + ";;;" + outbox.email_host_user + else: + # we effectively always want an outbox passed in but keeping for compatability + # let's fallback to the from email in this case, which might cause more misses + hack_alias = self.backend_alias + "???" + self.from_email + + connection = connections[hack_alias] if html_message: msg = EmailMultiAlternatives( @@ -181,14 +197,16 @@ def _update_message_id(self) -> bool: retval = True return retval - def dispatch(self, log_level=None, commit=True): + def dispatch(self, outbox=None, log_level=None, commit=True): """ Sends email and log the result. + + outbox: Attach an outbox to ensure the right outbox is used for connections/validation. """ email_message = None # Priority is handled in mail.send try: - email_message = self.email_message() + email_message = self.email_message(outbox=outbox) email_message.send() status = STATUS.sent self._update_message_id() @@ -351,9 +369,9 @@ class EmailAddressOAuthMapping(models.Model): ) class Meta: - app_label = 'django_mail_admin' + app_label = 'django_mail_admin' verbose_name = _("Email OAuth Mapping") verbose_name_plural = _("Email OAuth Mappings") def __str__(self): - return f"{self.send_as_email} -> {self.oauth_username}" \ No newline at end of file + return f"{self.send_as_email} -> {self.oauth_username}" From b5a4368f91d6ed67d0269a617fd46c09cc9abe02 Mon Sep 17 00:00:00 2001 From: csabantych Date: Tue, 4 Mar 2025 15:08:54 -0500 Subject: [PATCH 47/68] assorted improvements --- django_mail_admin/backends.py | 27 +++++++++++++++------------ django_mail_admin/connections.py | 7 ++++--- django_mail_admin/models/outgoing.py | 8 +++++--- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/django_mail_admin/backends.py b/django_mail_admin/backends.py index 89ab58f..c894a6a 100644 --- a/django_mail_admin/backends.py +++ b/django_mail_admin/backends.py @@ -6,10 +6,7 @@ from typing import Optional from urllib import parse -from django.core.mail.backends.base import BaseEmailBackend from django.core.mail.backends.smtp import EmailBackend -from django.core.mail.message import sanitize_address -from django_mail_admin import settings from django_mail_admin.models.outgoing import EmailAddressOAuthMapping from .mail import create @@ -41,10 +38,8 @@ def __init__( **kwargs, ): super(CustomEmailBackend, self).__init__(fail_silently=fail_silently) - # TODO: implement choosing backend for a letter as a param # March2025 ChargeUp note: keeping active=True in case we use this to prevent false sends # if we copy this we should do from user/email filtering like O365/Gmail backends - configurations = Outbox.objects.filter(active=True) if len(configurations) > 1 or len(configurations) == 0: raise ValueError( @@ -192,7 +187,7 @@ def open(self): raise def send_messages(self, email_messages) -> int: - """Sends email messages via O365 connection matching the from_email""" + """Sends email messages via O365 connection matching the from_email for each message""" if not email_messages: return 0 @@ -221,9 +216,9 @@ def send_messages(self, email_messages) -> int: # If connection doesn't match this message's configuration, create new connection if (not self.conn or - self.from_email != msg.from_email): + self.lookup_email != msg.from_email): self.close() - self.from_email = oauth_username + self.from_oauth_user = oauth_username self.open() # Send the message @@ -262,21 +257,27 @@ def __init__(self, fail_silently: bool = False, **kwargs) -> None: self.ssl_keyfile = None self.ssl_certfile = None + @property + def lookup_email(self) -> str | None: + return self.from_oauth_user or self.from_email + def _initialize_connection(self, from_email: str) -> None: """Initialize connection settings based on from_email""" configuration = Outbox.objects.filter( - email_host_user=from_email + email_host__icontains="gmail", + email_host_user=self.lookup_email ).first() if not configuration: - raise ValueError(f"No active configuration found for sender: {from_email}") + raise ValueError( + f"Unable to find an Outbox with email_host__icontains=gmail email_host_user={self.lookup_email} from_oauth_user={self.from_oauth_user} from_email={self.from_email}") self.configuration_id = configuration.id # Initialize settings from configuration self.host = configuration.email_host or "smtp.gmail.com" self.port = configuration.email_port or "587" - self.username = from_email # Important: this is used for OAuth lookup + self.username = self.lookup_email # Important: this is used for OAuth lookup self.password = "" # No password needed for XOAUTH2 self.use_tls = configuration.email_use_tls self.use_ssl = configuration.email_use_ssl @@ -332,6 +333,8 @@ def close(self): raise finally: self.connection = None + self.from_email = None + self.from_oauth_user = None def send_messages(self, email_messages): """Send messages, reinitializing connection if from_email changes""" @@ -345,7 +348,7 @@ def send_messages(self, email_messages): try: # Check if we need to reinitialize for a different from_email if (not self.connection or - self.from_email != message.from_email or + self.lookup_email != message.from_email or not self.configuration_id): #hack way to get the original username oauth_username = ( diff --git a/django_mail_admin/connections.py b/django_mail_admin/connections.py index 7427d9b..2678760 100644 --- a/django_mail_admin/connections.py +++ b/django_mail_admin/connections.py @@ -24,11 +24,12 @@ def __getitem__(self, maybe_hacked_alias): except KeyError: pass - # as a hack other places are using backend_alias;;;from_oauth_user + # as a hack other places are using backend_alias;;;from_oauth_user. e.g. o365;;;email@example.com # or backend_alias???from_email + # previously it just used any outbox for the alias real_alias = maybe_hacked_alias - from_oauth_user = None - from_email = None + from_oauth_user: str | None = None + from_email: str | None = None if "???" in maybe_hacked_alias: real_alias,from_email = maybe_hacked_alias.split("???") elif ";;;" in maybe_hacked_alias: diff --git a/django_mail_admin/models/outgoing.py b/django_mail_admin/models/outgoing.py index 20de98a..118b690 100644 --- a/django_mail_admin/models/outgoing.py +++ b/django_mail_admin/models/outgoing.py @@ -14,6 +14,7 @@ from django_mail_admin.signals import email_sent, email_failed_to_send, email_queued from django_mail_admin.utils import get_attachment_save_path, PRIORITY, STATUS from django_mail_admin.validators import validate_email_with_name +from . import Outbox from .templates import EmailTemplate logger = logging.getLogger(__name__) @@ -110,7 +111,7 @@ def email_message(self, outbox=None): return self.prepare_email_message(outbox=outbox) - def prepare_email_message(self, outbox=None): + def prepare_email_message(self, outbox: Outbox | None=None): """ Returns a django ``EmailMessage`` or ``EmailMultiAlternatives`` object, depending on whether html_message is empty. @@ -128,7 +129,7 @@ def prepare_email_message(self, outbox=None): #connection = connections[self.backend_alias or "default"] # this adds a connection for the outbox we're intending to send from rather than something like: - # "any 0365 outbox" or "any active outbox" + # "any o365 outbox" or "any active outbox" # we should also likely always have a backend alias if not self.backend_alias: raise ValueError(f"Outgoing emails should always have a backend alias. It was: {self.backend_alias} for from_email: {self.from_email}") @@ -137,9 +138,10 @@ def prepare_email_message(self, outbox=None): hack_alias = self.backend_alias + ";;;" + outbox.email_host_user else: # we effectively always want an outbox passed in but keeping for compatability - # let's fallback to the from email in this case, which might cause more misses + # lets fallback to the message from_email in this case (this might cause more Outbox lookup failures) hack_alias = self.backend_alias + "???" + self.from_email + # this will open and cache a connection to the alias above connection = connections[hack_alias] if html_message: From f90e8419164b5dbab71c4c0a8b5e1a311ecdf333 Mon Sep 17 00:00:00 2001 From: csabantych Date: Tue, 4 Mar 2025 15:17:26 -0500 Subject: [PATCH 48/68] keep active mailboxes. other updates --- django_mail_admin/admin.py | 22 +++++++++---------- django_mail_admin/backends.py | 10 ++++----- .../management/commands/get_new_mail.py | 3 +-- django_mail_admin/models/configurations.py | 5 +++-- django_mail_admin/models/outgoing.py | 3 +-- 5 files changed, 21 insertions(+), 22 deletions(-) diff --git a/django_mail_admin/admin.py b/django_mail_admin/admin.py index 05f20a2..923d494 100644 --- a/django_mail_admin/admin.py +++ b/django_mail_admin/admin.py @@ -87,17 +87,17 @@ def get_new_mail(mailbox_admin, request, queryset): get_new_mail.short_description = _("Get new mail") -# def switch_active(mailbox_admin, request, queryset): -# for mailbox in queryset.all(): -# mailbox.active = not mailbox.active -# mailbox.save() -# -# -# switch_active.short_description = _("Switch active status") +def switch_active(mailbox_admin, request, queryset): + for mailbox in queryset.all(): + mailbox.active = not mailbox.active + mailbox.save() + + +switch_active.short_description = _("Switch active status") def send_queued_mail(outbox_admin, request, queryset): - outbox: Outbox = None + outbox: Outbox | None = None for outbox in queryset.all(): # if not outbox.active: # messages.error("Outbox not active!") @@ -147,7 +147,7 @@ class MailboxAdmin(get_parent()): readonly_fields = [ "last_polling", ] - actions = [get_new_mail]#, switch_active] + actions = [get_new_mail, switch_active] class IncomingAttachmentInline(admin.TabularInline): @@ -489,9 +489,9 @@ class OutboxAdmin(admin.ModelAdmin): "email_host_user", "email_port", "id", - "active", + #"active", ) - list_filter = ("active",) + #list_filter = ("active",) actions = [send_queued_mail] diff --git a/django_mail_admin/backends.py b/django_mail_admin/backends.py index c894a6a..354c204 100644 --- a/django_mail_admin/backends.py +++ b/django_mail_admin/backends.py @@ -38,12 +38,12 @@ def __init__( **kwargs, ): super(CustomEmailBackend, self).__init__(fail_silently=fail_silently) - # March2025 ChargeUp note: keeping active=True in case we use this to prevent false sends - # if we copy this we should do from user/email filtering like O365/Gmail backends + # March2025 Note: keeping active=True in case we use this. + # if we copy this we should add from user/email filtering like O365/Gmail backends configurations = Outbox.objects.filter(active=True) if len(configurations) > 1 or len(configurations) == 0: raise ValueError( - "Got %(l)s active configurations, expected 1" + "Got %(l)s active Outboxes, expected 1" % {"l": len(configurations)} ) else: @@ -132,7 +132,7 @@ def close(self): def _validate_configuration(self, configuration: Outbox) -> None: """Validates if we need to create a new connection based on configuration""" if not configuration: - raise ValueError("No active email configuration found") + raise ValueError("No Outbox found") # If configuration has changed, close existing connection if self.configuration_id != configuration.id: @@ -211,7 +211,7 @@ def send_messages(self, email_messages) -> int: if not configuration: raise ValueError( - f"No active configuration found for sender: {msg.from_email}" + f"No Outbox found for sender: {msg.from_email}" ) # If connection doesn't match this message's configuration, create new connection diff --git a/django_mail_admin/management/commands/get_new_mail.py b/django_mail_admin/management/commands/get_new_mail.py index 2656810..cb26d8b 100644 --- a/django_mail_admin/management/commands/get_new_mail.py +++ b/django_mail_admin/management/commands/get_new_mail.py @@ -10,8 +10,7 @@ class Command(BaseCommand): def handle(self, *args, **options): - #mailboxes = Mailbox.active_mailboxes.all() - mailboxes = Mailbox.objects.all() + mailboxes = Mailbox.active_mailboxes.all() if args: mailboxes = mailboxes.filter( name=' '.join(args) diff --git a/django_mail_admin/models/configurations.py b/django_mail_admin/models/configurations.py index e5b11f4..10b3291 100644 --- a/django_mail_admin/models/configurations.py +++ b/django_mail_admin/models/configurations.py @@ -59,13 +59,14 @@ class Outbox(models.Model): email_timeout = models.PositiveSmallIntegerField( "EMAIL_TIMEOUT", null=True, blank=True ) - # deprecated field us at ChargeUp or not using anymore. keeping for compatability. + # deprecated field we're no longer actively using. keeping for test/code compatability. active = models.BooleanField(_("Active"), default=False) def save(self, *args, **kwargs): # Only one item can be active at a time - # ChargeUp Note: this was a previous side effect on Outbox.save + # March2025 Note: this was a previous side effect on Outbox.save + # we're not using the concept of a singleton Active Outbox # if self.active: # # select all other active items diff --git a/django_mail_admin/models/outgoing.py b/django_mail_admin/models/outgoing.py index 118b690..1792e50 100644 --- a/django_mail_admin/models/outgoing.py +++ b/django_mail_admin/models/outgoing.py @@ -14,7 +14,6 @@ from django_mail_admin.signals import email_sent, email_failed_to_send, email_queued from django_mail_admin.utils import get_attachment_save_path, PRIORITY, STATUS from django_mail_admin.validators import validate_email_with_name -from . import Outbox from .templates import EmailTemplate logger = logging.getLogger(__name__) @@ -111,7 +110,7 @@ def email_message(self, outbox=None): return self.prepare_email_message(outbox=outbox) - def prepare_email_message(self, outbox: Outbox | None=None): + def prepare_email_message(self, outbox=None): """ Returns a django ``EmailMessage`` or ``EmailMultiAlternatives`` object, depending on whether html_message is empty. From 21fb8a05ec489daf75c268cbb2e119c680f02f5d Mon Sep 17 00:00:00 2001 From: csabantych Date: Tue, 4 Mar 2025 15:44:21 -0500 Subject: [PATCH 49/68] gmail fixes --- django_mail_admin/backends.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/django_mail_admin/backends.py b/django_mail_admin/backends.py index 354c204..8ab03c9 100644 --- a/django_mail_admin/backends.py +++ b/django_mail_admin/backends.py @@ -261,11 +261,11 @@ def __init__(self, fail_silently: bool = False, **kwargs) -> None: def lookup_email(self) -> str | None: return self.from_oauth_user or self.from_email - def _initialize_connection(self, from_email: str) -> None: + def _initialize_connection(self, from_oauth_email: str) -> None: """Initialize connection settings based on from_email""" configuration = Outbox.objects.filter( email_host__icontains="gmail", - email_host_user=self.lookup_email + email_host_user=from_oauth_email or self.lookup_email ).first() if not configuration: @@ -273,6 +273,7 @@ def _initialize_connection(self, from_email: str) -> None: f"Unable to find an Outbox with email_host__icontains=gmail email_host_user={self.lookup_email} from_oauth_user={self.from_oauth_user} from_email={self.from_email}") self.configuration_id = configuration.id + self.from_oauth_user = from_oauth_email # Initialize settings from configuration self.host = configuration.email_host or "smtp.gmail.com" From 238715a1fd93d372011a58114e6e5ecacf969aef Mon Sep 17 00:00:00 2001 From: csabantych Date: Tue, 4 Mar 2025 16:26:54 -0500 Subject: [PATCH 50/68] try to simplify --- django_mail_admin/backends.py | 92 ++++++++++------------------ django_mail_admin/connections.py | 9 +-- django_mail_admin/models/outgoing.py | 5 +- 3 files changed, 39 insertions(+), 67 deletions(-) diff --git a/django_mail_admin/backends.py b/django_mail_admin/backends.py index 8ab03c9..3140764 100644 --- a/django_mail_admin/backends.py +++ b/django_mail_admin/backends.py @@ -107,59 +107,43 @@ def __init__(self, fail_silently: bool = False, **kwargs) -> None: super().__init__(fail_silently, **kwargs) self.conn: Optional[O365Connection] = None - # from_email or from_oauth_user might be hydrated by us (ChargeUp) after init (since Django does the init in places) - self.from_email = None - self.from_oauth_user = None + # from_email might be hydrated by us (ChargeUp) in a few scenarios: + # 1. after class __init__ (Django will create this class so we can't add to __init__) + # 2. during email sending when a new connection needs to be opened + self.from_email: str | None = None self.fail_silently: bool = fail_silently self.configuration_id: Optional[int] = None self._lock = threading.RLock() - @property - def lookup_email(self) -> str | None: - return self.from_oauth_user or self.from_email - def close(self): """Closes and cleans up the current connection""" with self._lock: if self.conn: self.conn = None self.from_email = None - self.from_oauth_user = None self.configuration_id = None super().close() - def _validate_configuration(self, configuration: Outbox) -> None: - """Validates if we need to create a new connection based on configuration""" - if not configuration: - raise ValueError("No Outbox found") - - # If configuration has changed, close existing connection - if self.configuration_id != configuration.id: - self.close() - - # If connection exists, validate it's still using correct email - if self.conn and self.lookup_email != configuration.email_host_user: - self.close() - def open(self): """Opens a new O365 connection using the relevant configuration""" - # If we already have a valid connection, we're good with self._lock: - if self.conn and self.lookup_email: - return - configuration = Outbox.objects.filter( email_host__icontains="office365", - email_host_user=self.lookup_email + email_host_user=self.from_email ).first() if not configuration: - raise ValueError(f"Unable to find an Outbox with email_host__icontains=office365 email_host_user={self.lookup_email} from_oauth_user={self.from_oauth_user} from_email={self.from_email}") + raise ValueError(f"Unable to find an Outbox with email_host__icontains=office365 email_host_user={self.from_email}") - # Validate configuration and connection state - self._validate_configuration(configuration) - self.configuration_id = configuration.id + # existing saved connection is valid. no need for a new one + if self.conn and self.configuration_id == configuration.id: + return + elif self.conn: + # close the existing invalid connection + self.close() + self.from_email = configuration.email_host_user + self.configuration_id = configuration.id # Parse O365 connection details from email_host parseresult = parse.urlparse(configuration.email_host) @@ -177,7 +161,7 @@ def open(self): try: # Create new connection with current configuration self.conn = O365Connection( - from_email=self.lookup_email, + from_email=self.from_email, client_app_id=client_app_id, client_id_key=client_id_key, client_secret_key=client_secret_key, @@ -194,9 +178,11 @@ def send_messages(self, email_messages) -> int: with self._lock: sent_count = 0 + # sort by from_email to try to reduce connection closing/opening + email_messages = sorted(email_messages, key=lambda m: m.from_email) + for msg in email_messages: try: - # Use EmailAddressOAuthMapping to find oauth_username oauth_username = ( EmailAddressOAuthMapping.objects.filter(send_as_email=msg.from_email) @@ -204,21 +190,10 @@ def send_messages(self, email_messages) -> int: .first() ) or msg.from_email - # Try to get configuration matching this message's from_email - configuration = Outbox.objects.filter( - email_host_user=oauth_username - ).first() - - if not configuration: - raise ValueError( - f"No Outbox found for sender: {msg.from_email}" - ) - # If connection doesn't match this message's configuration, create new connection - if (not self.conn or - self.lookup_email != msg.from_email): + if (not self.conn or self.from_email != oauth_username): self.close() - self.from_oauth_user = oauth_username + self.from_email = oauth_username self.open() # Send the message @@ -241,9 +216,10 @@ def __init__(self, fail_silently: bool = False, **kwargs) -> None: self.connection = None self._lock = threading.RLock() - # from_email or from_oauth_user might be hydrated by us (ChargeUp) after init (since Django does the init in places) + # from_email might be hydrated by us (ChargeUp) in a few scenarios: + # 1. after class init (not currently really used for anything... a shared feature with Outlook) + # 2. during email sending after connection close self.from_email = None - self.from_oauth_user = None self.configuration_id = None # Initialize these to None - they'll be set when sending @@ -257,28 +233,25 @@ def __init__(self, fail_silently: bool = False, **kwargs) -> None: self.ssl_keyfile = None self.ssl_certfile = None - @property - def lookup_email(self) -> str | None: - return self.from_oauth_user or self.from_email - - def _initialize_connection(self, from_oauth_email: str) -> None: + def _initialize_connection(self, from_email: str) -> None: """Initialize connection settings based on from_email""" configuration = Outbox.objects.filter( email_host__icontains="gmail", - email_host_user=from_oauth_email or self.lookup_email + email_host_user=from_email ).first() if not configuration: raise ValueError( - f"Unable to find an Outbox with email_host__icontains=gmail email_host_user={self.lookup_email} from_oauth_user={self.from_oauth_user} from_email={self.from_email}") + f"Unable to find an Outbox with email_host__icontains=gmail email_host_user={from_email}" + ) self.configuration_id = configuration.id - self.from_oauth_user = from_oauth_email + self.from_email = from_email # Initialize settings from configuration self.host = configuration.email_host or "smtp.gmail.com" self.port = configuration.email_port or "587" - self.username = self.lookup_email # Important: this is used for OAuth lookup + self.username = from_email # Important: this is used for OAuth lookup self.password = "" # No password needed for XOAUTH2 self.use_tls = configuration.email_use_tls self.use_ssl = configuration.email_use_ssl @@ -335,7 +308,7 @@ def close(self): finally: self.connection = None self.from_email = None - self.from_oauth_user = None + self.configuration_id = None def send_messages(self, email_messages): """Send messages, reinitializing connection if from_email changes""" @@ -345,11 +318,14 @@ def send_messages(self, email_messages): with self._lock: num_sent = 0 + # sort by from_email to reduce connection opening + email_messages = sorted(email_messages, key=lambda m: m.from_email) + for message in email_messages: try: # Check if we need to reinitialize for a different from_email if (not self.connection or - self.lookup_email != message.from_email or + self.from_email != message.from_email or not self.configuration_id): #hack way to get the original username oauth_username = ( diff --git a/django_mail_admin/connections.py b/django_mail_admin/connections.py index 2678760..26d3d43 100644 --- a/django_mail_admin/connections.py +++ b/django_mail_admin/connections.py @@ -24,15 +24,11 @@ def __getitem__(self, maybe_hacked_alias): except KeyError: pass - # as a hack other places are using backend_alias;;;from_oauth_user. e.g. o365;;;email@example.com - # or backend_alias???from_email + # as a hack other places are using backend_alias;;;from_email. e.g. o365;;;email@example.com # previously it just used any outbox for the alias real_alias = maybe_hacked_alias - from_oauth_user: str | None = None from_email: str | None = None - if "???" in maybe_hacked_alias: - real_alias,from_email = maybe_hacked_alias.split("???") - elif ";;;" in maybe_hacked_alias: + if ";;;" in maybe_hacked_alias: real_alias,from_oauth_user = maybe_hacked_alias.split(";;;") try: @@ -45,7 +41,6 @@ def __getitem__(self, maybe_hacked_alias): # now mutate the backend class after init since get_connection is within django backend_class.from_email = from_email - backend_class.from_oauth_user = from_oauth_user backend_class.open() self._connections.connections[maybe_hacked_alias] = backend_class diff --git a/django_mail_admin/models/outgoing.py b/django_mail_admin/models/outgoing.py index 1792e50..349fe98 100644 --- a/django_mail_admin/models/outgoing.py +++ b/django_mail_admin/models/outgoing.py @@ -136,11 +136,12 @@ def prepare_email_message(self, outbox=None): if outbox: hack_alias = self.backend_alias + ";;;" + outbox.email_host_user else: - # we effectively always want an outbox passed in but keeping for compatability + # we effectively always want an outbox passed in but keeping this for compatability # lets fallback to the message from_email in this case (this might cause more Outbox lookup failures) - hack_alias = self.backend_alias + "???" + self.from_email + hack_alias = self.backend_alias + ";;;" + self.from_email # this will open and cache a connection to the alias above + # maybe remove this and only open a connection on send connection = connections[hack_alias] if html_message: From dd9412514d6c9facf7ab2bc2015e0dc444413136 Mon Sep 17 00:00:00 2001 From: csabantych Date: Tue, 4 Mar 2025 16:33:36 -0500 Subject: [PATCH 51/68] more self-nits --- django_mail_admin/backends.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/django_mail_admin/backends.py b/django_mail_admin/backends.py index 3140764..c31b384 100644 --- a/django_mail_admin/backends.py +++ b/django_mail_admin/backends.py @@ -39,7 +39,7 @@ def __init__( ): super(CustomEmailBackend, self).__init__(fail_silently=fail_silently) # March2025 Note: keeping active=True in case we use this. - # if we copy this we should add from user/email filtering like O365/Gmail backends + # if we copy this we should add Outbox.email_host_user filtering like O365/Gmail backends configurations = Outbox.objects.filter(active=True) if len(configurations) > 1 or len(configurations) == 0: raise ValueError( @@ -108,7 +108,7 @@ def __init__(self, fail_silently: bool = False, **kwargs) -> None: self.conn: Optional[O365Connection] = None # from_email might be hydrated by us (ChargeUp) in a few scenarios: - # 1. after class __init__ (Django will create this class so we can't add to __init__) + # 1. after class __init__ (Django will create this class so we can't add directly to __init__) # 2. during email sending when a new connection needs to be opened self.from_email: str | None = None @@ -194,6 +194,7 @@ def send_messages(self, email_messages) -> int: if (not self.conn or self.from_email != oauth_username): self.close() self.from_email = oauth_username + # TODO: we could leverage a connection cache rather than re-init each time self.open() # Send the message @@ -217,8 +218,9 @@ def __init__(self, fail_silently: bool = False, **kwargs) -> None: self._lock = threading.RLock() # from_email might be hydrated by us (ChargeUp) in a few scenarios: - # 1. after class init (not currently really used for anything... a shared feature with Outlook) - # 2. during email sending after connection close + # 1. after class __init__ (Django will create this class so we can't add directly to __init__) + # this is currently not useful for Gmail but mirrors what's done with Outlook + # 2. during email sending when a new connection needs to be opened self.from_email = None self.configuration_id = None @@ -335,6 +337,7 @@ def send_messages(self, email_messages): ) or message.from_email self.close() + # TODO: we could leverage a connection cache rather than re-init each time self._initialize_connection(oauth_username) new_conn_created = self.open(auth_uid=oauth_username) From 70ec8905dd7f97cb67f5246ce57076394ad42aa0 Mon Sep 17 00:00:00 2001 From: csabantych Date: Tue, 4 Mar 2025 16:40:11 -0500 Subject: [PATCH 52/68] use oauth for connection check --- django_mail_admin/backends.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/django_mail_admin/backends.py b/django_mail_admin/backends.py index c31b384..fc001bd 100644 --- a/django_mail_admin/backends.py +++ b/django_mail_admin/backends.py @@ -323,18 +323,24 @@ def send_messages(self, email_messages): # sort by from_email to reduce connection opening email_messages = sorted(email_messages, key=lambda m: m.from_email) + from_email_to_oauth_cache = {} for message in email_messages: try: + # hack way to get the original username + if message.from_email in from_email_to_oauth_cache: + oauth_username = from_email_to_oauth_cache[message.from_email] + else: + oauth_username = ( + EmailAddressOAuthMapping.objects.filter(send_as_email=message.from_email) + .values_list('oauth_username', flat=True) + .first() + ) or message.from_email + from_email_to_oauth_cache[message.from_email] = oauth_username + # Check if we need to reinitialize for a different from_email if (not self.connection or - self.from_email != message.from_email or + self.from_email != oauth_username or not self.configuration_id): - #hack way to get the original username - oauth_username = ( - EmailAddressOAuthMapping.objects.filter(send_as_email=message.from_email) - .values_list('oauth_username', flat=True) - .first() - ) or message.from_email self.close() # TODO: we could leverage a connection cache rather than re-init each time From 9fbe0839a702b34f378ac782ad2cbe53046377e5 Mon Sep 17 00:00:00 2001 From: csabantych Date: Tue, 4 Mar 2025 16:45:58 -0500 Subject: [PATCH 53/68] nix cache for simpler diff --- django_mail_admin/backends.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/django_mail_admin/backends.py b/django_mail_admin/backends.py index fc001bd..cd2307b 100644 --- a/django_mail_admin/backends.py +++ b/django_mail_admin/backends.py @@ -323,19 +323,15 @@ def send_messages(self, email_messages): # sort by from_email to reduce connection opening email_messages = sorted(email_messages, key=lambda m: m.from_email) - from_email_to_oauth_cache = {} for message in email_messages: try: # hack way to get the original username - if message.from_email in from_email_to_oauth_cache: - oauth_username = from_email_to_oauth_cache[message.from_email] - else: - oauth_username = ( - EmailAddressOAuthMapping.objects.filter(send_as_email=message.from_email) - .values_list('oauth_username', flat=True) - .first() - ) or message.from_email - from_email_to_oauth_cache[message.from_email] = oauth_username + # TODO: could do a cache here + oauth_username = ( + EmailAddressOAuthMapping.objects.filter(send_as_email=message.from_email) + .values_list('oauth_username', flat=True) + .first() + ) or message.from_email # Check if we need to reinitialize for a different from_email if (not self.connection or From 4925d4f95de62ac8e651527b4308ff99750ffb23 Mon Sep 17 00:00:00 2001 From: csabantych Date: Tue, 4 Mar 2025 16:51:55 -0500 Subject: [PATCH 54/68] fix --- django_mail_admin/connections.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/django_mail_admin/connections.py b/django_mail_admin/connections.py index 26d3d43..293d88c 100644 --- a/django_mail_admin/connections.py +++ b/django_mail_admin/connections.py @@ -29,22 +29,22 @@ def __getitem__(self, maybe_hacked_alias): real_alias = maybe_hacked_alias from_email: str | None = None if ";;;" in maybe_hacked_alias: - real_alias,from_oauth_user = maybe_hacked_alias.split(";;;") + real_alias,from_email = maybe_hacked_alias.split(";;;") try: - backend = get_backend(real_alias) + backend_class = get_backend(real_alias) except KeyError: raise KeyError('%s is not a valid backend alias' % real_alias) - # backend_class is a EmailBackend subclass like O365Backend or GmailOAuth2Backend - backend_class = get_connection(backend) + # backend_instance is a EmailBackend subclass like O365Backend or GmailOAuth2Backend + backend_instance = get_connection(backend_class) # now mutate the backend class after init since get_connection is within django - backend_class.from_email = from_email + backend_instance.from_email = from_email - backend_class.open() - self._connections.connections[maybe_hacked_alias] = backend_class - return backend_class + backend_instance.open() + self._connections.connections[maybe_hacked_alias] = backend_instance + return backend_instance def all(self): return getattr(self._connections, 'connections', {}).values() From 18592f7052435215b6e75f87289795388c2e67fe Mon Sep 17 00:00:00 2001 From: csabantych Date: Fri, 7 Mar 2025 10:05:35 -0500 Subject: [PATCH 55/68] log uid. fallback to from_email --- django_mail_admin/backends.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/django_mail_admin/backends.py b/django_mail_admin/backends.py index cd2307b..ce2c51e 100644 --- a/django_mail_admin/backends.py +++ b/django_mail_admin/backends.py @@ -274,14 +274,16 @@ def open(self,auth_uid=None): if self.use_tls: self.connection.starttls() + query_uid = auth_uid or self.from_email + user_social_auth = UserSocialAuth.objects.get( - uid=auth_uid, + uid=query_uid, provider="google-oauth2" ) creds_info = user_social_auth.extra_data auth_string = generate_oauth2_string( - auth_uid, + query_uid, creds_info["access_token"], base64_encode=False ) @@ -291,7 +293,7 @@ def open(self,auth_uid=None): ) return True except Exception: - logger.exception("gmail failed to open connection") + logger.exception(f"gmail failed to open connection: auth_uid={auth_uid} from_email={self.from_email}") return None def close(self): From ee58d142c5cdb3d8c5ae978c829471becaf24903 Mon Sep 17 00:00:00 2001 From: csabantych Date: Fri, 7 Mar 2025 14:12:51 -0500 Subject: [PATCH 56/68] try verbosely logging things --- django_mail_admin/backends.py | 48 ++++++++++++++++++++++++++++ django_mail_admin/models/outgoing.py | 3 +- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/django_mail_admin/backends.py b/django_mail_admin/backends.py index ce2c51e..f2824b4 100644 --- a/django_mail_admin/backends.py +++ b/django_mail_admin/backends.py @@ -247,6 +247,16 @@ def _initialize_connection(self, from_email: str) -> None: f"Unable to find an Outbox with email_host__icontains=gmail email_host_user={from_email}" ) + logger.info( + "Found a Gmail Outbox with: " + f"email_use_tls={configuration.email_use_tls}" + f"email_use_ssl={configuration.email_use_ssl}" + f"email_host={configuration.email_host}" + f"email_host_user={configuration.email_host_user}" + f"email_port={configuration.email_port}" + f"email_timeout={configuration.email_timeout}" + ) + self.configuration_id = configuration.id self.from_email = from_email @@ -329,6 +339,29 @@ def send_messages(self, email_messages): try: # hack way to get the original username # TODO: could do a cache here + # not all accounts are in EmailAddressOAuthMapping. but here's how one: + # + # EmailAddressOAuthMapping: oauth_username=company_oauth@g.company.ai send_as_email=person@customer.com + # UserSocialAuth: uid=company_oauth@g.company.ai provider=google-oauth2 created=2024-09-22 16:32:46.641390+00:00 modified=2025-03-07 17:52:45.714140+00:00 + # OutgoingEmail: id=545 from_email=person@customer.com created=2025-03-07 01:00:00.824912+00:00 last_updated=2025-03-07 01:00:00.824923+00:00 + # Outbox: email_host_user=company_oauth@g.company.ai email_host=smtp.gmail.com + # + # so for a OutgoingEmail from person@customer.com + # we get EmailAddressOAuthMapping with send_as_email from person@customer.com + # which has an EmailAddressOAuthMapping.oauth_username of company_oauth@g.company.ai, which maps to Outbox.email_host_user=company_oauth@g.company.ai + # and UserSocialAuth: uid=company_oauth@g.company.ai + # + # another example without EmailAddressOAuthMapping: + # + # UserSocialAuth: uid=customer2@g.company.ai provider=google-oauth2 created=2025-02-04 17:45:13.116982+00:00 modified=2025-03-07 17:52:00.138284+00:00 + # UserSocialAuth: uid=company@customer2.com provider=google-oauth2 created=2025-02-03 18:47:19.157579+00:00 modified=2025-03-07 17:52:01.547400+00:00 + # OutgoingEmail: id=546 from_email=company@customer2.com created=2025-03-07 01:00:00.898387+00:00 last_updated=2025-03-07 16:25:05.930577+00:00 + # Outbox: email_host_user=customer2@g.company.ai email_host=smtp.gmail.com + # Outbox: email_host_user=company@customer2.com email_host=smtp.gmail.com + # + # OutgoingEmail is from company@customer2.com + # which maps to the Outbox of company@customer2.com + # and UserSocialAuth: uid=company@customer2.com oauth_username = ( EmailAddressOAuthMapping.objects.filter(send_as_email=message.from_email) .values_list('oauth_username', flat=True) @@ -340,12 +373,27 @@ def send_messages(self, email_messages): self.from_email != oauth_username or not self.configuration_id): + logger.info( + f"Opening a new connection in send_messages:" + f"self.from_email={self.from_email}" + f"oauth_username={oauth_username} " + f"connection_exists={bool(self.connection)}" + f"configuration_id_exists={bool(self.configuration_id)}" + ) + self.close() # TODO: we could leverage a connection cache rather than re-init each time self._initialize_connection(oauth_username) new_conn_created = self.open(auth_uid=oauth_username) if not self.connection or new_conn_created is None: + logger.info( + f"No connection available (skipping):" + f"self.from_email={self.from_email}" + f"oauth_username={oauth_username} " + f"connection_exists={bool(self.connection)}" + f"new_conn_created={bool(new_conn_created)}" + ) continue if self._send(message): diff --git a/django_mail_admin/models/outgoing.py b/django_mail_admin/models/outgoing.py index 349fe98..24bd27e 100644 --- a/django_mail_admin/models/outgoing.py +++ b/django_mail_admin/models/outgoing.py @@ -1,4 +1,5 @@ import logging +import traceback from django.core.files import File from django.core.mail import EmailMessage, EmailMultiAlternatives @@ -217,7 +218,7 @@ def dispatch(self, outbox=None, log_level=None, commit=True): email_sent.send(sender=self, outgoing_email=email_message) except Exception as e: status = STATUS.failed - message = str(e) + message = str(e) + " -- " + traceback.format_exc() exception_type = type(e).__name__ if email_message: email_failed_to_send.send(sender=self, outgoing_email=email_message) From f582b5f063f6e3d8284b44d94183035bfd2c1173 Mon Sep 17 00:00:00 2001 From: csabantych Date: Fri, 7 Mar 2025 14:23:04 -0500 Subject: [PATCH 57/68] whitespace/close logs --- django_mail_admin/backends.py | 36 +++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/django_mail_admin/backends.py b/django_mail_admin/backends.py index f2824b4..7c2750b 100644 --- a/django_mail_admin/backends.py +++ b/django_mail_admin/backends.py @@ -249,12 +249,12 @@ def _initialize_connection(self, from_email: str) -> None: logger.info( "Found a Gmail Outbox with: " - f"email_use_tls={configuration.email_use_tls}" - f"email_use_ssl={configuration.email_use_ssl}" - f"email_host={configuration.email_host}" - f"email_host_user={configuration.email_host_user}" - f"email_port={configuration.email_port}" - f"email_timeout={configuration.email_timeout}" + f"email_use_tls={configuration.email_use_tls} " + f"email_use_ssl={configuration.email_use_ssl} " + f"email_host={configuration.email_host} " + f"email_host_user={configuration.email_host_user} " + f"email_port={configuration.email_port} " + f"email_timeout={configuration.email_timeout} " ) self.configuration_id = configuration.id @@ -313,9 +313,13 @@ def close(self): try: try: self.connection.quit() - except (ssl.SSLError, smtplib.SMTPServerDisconnected): + logger.info(f"Connection quit for {self.from_email}") + except (ssl.SSLError, smtplib.SMTPServerDisconnected) as e: + logger.exception(f"Exception while quitting a connection for {self.from_email}: {e}") self.connection.close() - except smtplib.SMTPException: + logger.info(f"Connection closed for {self.from_email}") + except smtplib.SMTPException as e: + logger.exception(f"SMTPException while quitting a connection for {self.from_email}: {e}") if self.fail_silently: return raise @@ -374,11 +378,11 @@ def send_messages(self, email_messages): not self.configuration_id): logger.info( - f"Opening a new connection in send_messages:" - f"self.from_email={self.from_email}" + f"Opening a new connection in send_messages: " + f"self.from_email={self.from_email} " f"oauth_username={oauth_username} " - f"connection_exists={bool(self.connection)}" - f"configuration_id_exists={bool(self.configuration_id)}" + f"connection_exists={bool(self.connection)} " + f"configuration_id_exists={bool(self.configuration_id)} " ) self.close() @@ -388,11 +392,11 @@ def send_messages(self, email_messages): if not self.connection or new_conn_created is None: logger.info( - f"No connection available (skipping):" - f"self.from_email={self.from_email}" + f"No connection available (skipping): " + f"self.from_email={self.from_email} " f"oauth_username={oauth_username} " - f"connection_exists={bool(self.connection)}" - f"new_conn_created={bool(new_conn_created)}" + f"connection_exists={bool(self.connection)} " + f"new_conn_created={bool(new_conn_created)} " ) continue From c6b3d5ad5df45e9dd2dde64a1c0b95bfffbcc674 Mon Sep 17 00:00:00 2001 From: csabantych Date: Fri, 7 Mar 2025 14:28:43 -0500 Subject: [PATCH 58/68] log social auth --- django_mail_admin/backends.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/django_mail_admin/backends.py b/django_mail_admin/backends.py index 7c2750b..01a38be 100644 --- a/django_mail_admin/backends.py +++ b/django_mail_admin/backends.py @@ -291,6 +291,8 @@ def open(self,auth_uid=None): provider="google-oauth2" ) + logger.info(f"Found UserSocialAuth with pk={user_social_auth.pk} uid={user_social_auth.uid}") + creds_info = user_social_auth.extra_data auth_string = generate_oauth2_string( query_uid, From 8c8ad381d0d9e0235dcba585b3f346b925962695 Mon Sep 17 00:00:00 2001 From: csabantych Date: Fri, 7 Mar 2025 16:04:08 -0500 Subject: [PATCH 59/68] refresh tokens on send --- django_mail_admin/backends.py | 37 +++++++++++++++++++------------ django_mail_admin/google_utils.py | 9 ++++---- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/django_mail_admin/backends.py b/django_mail_admin/backends.py index 01a38be..0f72d5a 100644 --- a/django_mail_admin/backends.py +++ b/django_mail_admin/backends.py @@ -17,7 +17,7 @@ from django.core.mail.backends.base import BaseEmailBackend from django_mail_admin.o365_utils import O365Connection -from django_mail_admin.google_utils import generate_oauth2_string +from django_mail_admin.google_utils import generate_oauth2_string, refresh_authorization logger = logging.getLogger(__name__) @@ -271,6 +271,18 @@ def _initialize_connection(self, from_email: str) -> None: self.ssl_keyfile = configuration.email_ssl_keyfile self.ssl_certfile = configuration.email_ssl_certfile + def _connect_for_social_auth(self, user_social_auth: UserSocialAuth) -> None: + creds_info = user_social_auth.extra_data + auth_string = generate_oauth2_string( + user_social_auth.uid, + creds_info["access_token"], + base64_encode=False + ) + self.connection.docmd( + "AUTH", + "XOAUTH2 " + base64.b64encode(auth_string.encode("utf-8")).decode("utf-8"), + ) + def open(self,auth_uid=None): """Override to use OAuth instead of password authentication""" @@ -284,25 +296,22 @@ def open(self,auth_uid=None): if self.use_tls: self.connection.starttls() - query_uid = auth_uid or self.from_email - user_social_auth = UserSocialAuth.objects.get( - uid=query_uid, + uid=auth_uid or self.from_email, provider="google-oauth2" ) logger.info(f"Found UserSocialAuth with pk={user_social_auth.pk} uid={user_social_auth.uid}") - creds_info = user_social_auth.extra_data - auth_string = generate_oauth2_string( - query_uid, - creds_info["access_token"], - base64_encode=False - ) - self.connection.docmd( - "AUTH", - "XOAUTH2 " + base64.b64encode(auth_string.encode("utf-8")).decode("utf-8"), - ) + try: + self._connect_for_social_auth(user_social_auth) + except Exception as e: + # TODO: we can avoid an except on all exceptions + # and we can proactively refresh based on expiring soon + # but we currently don't store access token expiration date + logger.exception(f"Failed connecting via OAuth. Attemping a token refresh for UserSocialAuth.uid={user_social_auth.uid}") + updated_social_auth = refresh_authorization(user_social_auth.uid) + self._connect_for_social_auth(updated_social_auth) return True except Exception: logger.exception(f"gmail failed to open connection: auth_uid={auth_uid} from_email={self.from_email}") diff --git a/django_mail_admin/google_utils.py b/django_mail_admin/google_utils.py index a45d834..d843dd1 100644 --- a/django_mail_admin/google_utils.py +++ b/django_mail_admin/google_utils.py @@ -35,11 +35,12 @@ def get_google_access_token(email): raise AccessTokenNotFound -def update_google_extra_data(email, extra_data): +def update_google_extra_data(email_or_auth_uid: str, extra_data) -> UserSocialAuth: try: - me = UserSocialAuth.objects.get(uid=email, provider="google-oauth2") + me = UserSocialAuth.objects.get(uid=email_or_auth_uid, provider="google-oauth2") me.extra_data = extra_data me.save() + return me except (UserSocialAuth.DoesNotExist, KeyError): raise AccessTokenNotFound @@ -93,7 +94,7 @@ def google_api_post(email, url, post_data, authorized=True): raise Exception("google_api_post ended with a %s" % r.status_code) -def refresh_authorization(email): +def refresh_authorization(email: str) -> UserSocialAuth: refresh_token = get_google_refresh_token(email) post_data = dict( refresh_token=refresh_token, @@ -108,7 +109,7 @@ def refresh_authorization(email): authorized=False, ) results.update({"refresh_token": refresh_token}) - update_google_extra_data(email, results) + return update_google_extra_data(email, results) def fetch_user_info(email): From b052ca76af9549e9be49a2f88e7c46eb32696e96 Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Fri, 9 May 2025 16:40:55 -0400 Subject: [PATCH 60/68] SMTPOutbox backend and imap send support --- django_mail_admin/backends.py | 132 +++++++++++++++++++++++++++++++++- 1 file changed, 130 insertions(+), 2 deletions(-) diff --git a/django_mail_admin/backends.py b/django_mail_admin/backends.py index 0f72d5a..8180315 100644 --- a/django_mail_admin/backends.py +++ b/django_mail_admin/backends.py @@ -8,9 +8,12 @@ from django.core.mail.backends.smtp import EmailBackend from django_mail_admin.models.outgoing import EmailAddressOAuthMapping +from django_mail_admin.transports import ( + ImapTransport, +) from .mail import create -from .models import Outbox, create_attachments +from .models import Outbox, Mailbox, create_attachments from .utils import PRIORITY from social_django.models import UserSocialAuth @@ -67,6 +70,131 @@ def __init__( self._lock = threading.RLock() +class SMTPOutboxBackend(EmailBackend): + def __init__( + self, + host=None, + port=None, + username=None, + password=None, + use_tls=None, + fail_silently=False, + use_ssl=None, + timeout=None, + ssl_keyfile=None, + ssl_certfile=None, + **kwargs, + ): + super(SMTPOutboxBackend, self).__init__(fail_silently=fail_silently) + self.host = None + self.port = None + self.username = None + self.password = None + self.use_tls = None + self.use_ssl = None + self.timeout = None + self.ssl_keyfile = None + self.ssl_certfile = None + self.conn: Optional[SMTPOutboxBackend] | None = None + self.connection = None + # from_email might be hydrated by us (ChargeUp) in a few scenarios: + # 1. after class __init__ (Django will create this class so we can't add directly to __init__) + # 2. during email sending when a new connection needs to be opened + self.from_email: str | None = None + self.configuration_id: Optional[int] = None + + self._lock = threading.RLock() + + def close(self): + """Closes and cleans up the current connection""" + with self._lock: + if self.conn: + self.conn = None + self.from_email = None + self.configuration_id = None + super().close() + + def open(self): + with self._lock: + configuration = Outbox.objects.filter( + email_host_user__iexact=self.from_email #ignore case + ).first() + + if not configuration: + raise ValueError(f"Unable to find an Outbox with email_host_user={self.from_email}") + + # existing saved connection is valid. no need for a new one + if self.configuration_id and self.configuration_id == configuration.id: + return + elif self.configuration_id: + # close the existing invalid connection + self.from_email = configuration.email_host_user + self.configuration_id = configuration.id + + self.host = configuration.email_host + self.port = configuration.email_port + self.username = configuration.email_host_user + self.password = configuration.email_host_password + self.use_tls = configuration.email_use_tls + self.use_ssl = configuration.email_use_ssl + self.timeout = configuration.email_timeout + self.ssl_keyfile = configuration.email_ssl_keyfile + self.ssl_certfile = configuration.email_ssl_certfile + + return super().open() + + def _get_imap_sent_folder_name(self, imap_transport:ImapTransport|None) -> str | None: + sent_folder: str | None = None + try: + if imap_transport: + sent_folder = imap_transport.get_sent_folder_name() + except Exception as e: + logger.error(f"Error retrieving sent folder name: imap_transport: {imap_transport},error: {e}") + sent_folder = None + return sent_folder + + def _get_imap_transport(self) -> ImapTransport | None: + imap_transport: ImapTransport | None = None + try: + imap_mailbox: Mailbox | None = None + for imap_mailbox in Mailbox.objects.filter( + from_email__iexact=self.from_email + ).all(): + if imap_mailbox and "imap" == imap_mailbox.type: + break + if imap_mailbox and "imap" == imap_mailbox.type: + imap_transport = imap_mailbox.get_connection() + except Exception as e: + logger.error(f"Error retrieving imap_transport: {e}") + imap_transport = None + return imap_transport + + def send_messages(self, email_messages) -> int: + """ + Sends email messages via underlying SMTP connection and saves the same to Sent folder + + SMTP only sends messages, do not deal with ensuring a copy is stored in sent folder + """ + if not email_messages: + return 0 + num_sent_ok=0 + with self._lock: + imap_transport: ImapTransport | None = self._get_imap_transport() + sent_folder: str | None = self._get_imap_sent_folder_name(imap_transport) + for email_message in email_messages: + """ + num_sent = super().send_messages([email_message]) + if 1 != num_sent: + continue + """ + if imap_transport and sent_folder: + imap_transport.store_message_in_folder( + sent_folder, + email_message.message() + ) # not sending any flags + num_sent_ok+=1 + return num_sent_ok + class OutboxEmailBackend(BaseEmailBackend): def send_messages(self, email_messages): for msg in email_messages: @@ -130,7 +258,7 @@ def open(self): with self._lock: configuration = Outbox.objects.filter( email_host__icontains="office365", - email_host_user=self.from_email + email_host_user__iexact=self.from_email #ignore case ).first() if not configuration: From 217186d2f96ad6a20371034a30652c9130af0707 Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Fri, 9 May 2025 16:41:25 -0400 Subject: [PATCH 61/68] SMTPOutbox backend and imap send support --- django_mail_admin/admin.py | 4 +- django_mail_admin/mail.py | 18 ++++--- django_mail_admin/models/outgoing.py | 4 +- django_mail_admin/transports/imap.py | 74 +++++++++++++++++++++++++++- example/example/settings.py | 3 +- 5 files changed, 90 insertions(+), 13 deletions(-) diff --git a/django_mail_admin/admin.py b/django_mail_admin/admin.py index 923d494..b4faa32 100644 --- a/django_mail_admin/admin.py +++ b/django_mail_admin/admin.py @@ -113,7 +113,7 @@ def send_queued_mail(outbox_admin, request, queryset): while 1: try: - send_queued(default_processes, default_log_level) + send_queued(outbox, default_processes, default_log_level) except Exception as e: logger.error( e, exc_info=sys.exc_info(), extra={"status_code": 500} @@ -124,7 +124,7 @@ def send_queued_mail(outbox_admin, request, queryset): connection.close() if ( - not OutgoingEmail.objects.filter(status=STATUS.queued) + not OutgoingEmail.objects.filter(status=STATUS.queued, from_email__iexact=outbox.email_host_user) .filter(Q(scheduled_time__lte=now()) | Q(scheduled_time=None)) .exists() ): diff --git a/django_mail_admin/mail.py b/django_mail_admin/mail.py index dd835ba..32446c0 100644 --- a/django_mail_admin/mail.py +++ b/django_mail_admin/mail.py @@ -14,6 +14,7 @@ from .signals import email_queued from .utils import (parse_emails, parse_priority, split_emails) +from .models import Outbox logger = setup_loghandlers("INFO") @@ -141,23 +142,26 @@ def send_many(kwargs_list): OutgoingEmail.objects.bulk_create(emails) -def get_queued(): +def get_queued(outbox:Outbox|None=None): """ Returns a list of emails that should be sent: - Status is queued - Has scheduled_time lower than the current time or None """ - return OutgoingEmail.objects.filter(status=STATUS.queued) \ + outgoing_emails = OutgoingEmail.objects.filter(status=STATUS.queued) \ .select_related('template') \ - .filter(Q(scheduled_time__lte=now()) | Q(scheduled_time=None)) \ - .order_by(*get_sending_order()).prefetch_related('attachments')[:get_batch_size()] + .filter(Q(scheduled_time__lte=now()) | Q(scheduled_time=None)) + if outbox: + outgoing_emails = outgoing_emails.filter(from_email__iexact=outbox.email_host_user) + return outgoing_emails.order_by(*get_sending_order()).prefetch_related('attachments')[:get_batch_size()] - -def send_queued(processes=1, log_level=None): +def send_queued(outbox=None, processes=1, log_level=None): """ Sends out all queued mails that has scheduled_time less than now or None + + filter by outbox if provided """ - queued_emails = get_queued() + queued_emails = get_queued(outbox=outbox) total_sent, total_failed = 0, 0 total_email = len(queued_emails) diff --git a/django_mail_admin/models/outgoing.py b/django_mail_admin/models/outgoing.py index 24bd27e..4671651 100644 --- a/django_mail_admin/models/outgoing.py +++ b/django_mail_admin/models/outgoing.py @@ -9,6 +9,7 @@ from django.utils.translation import gettext_lazy as _ from jsonfield import JSONField +from django_mail_admin.models.configurations import Outbox from django_mail_admin.connections import connections from django_mail_admin.fields import CommaSeparatedEmailField from django_mail_admin.settings import get_log_level, get_backend_names_str @@ -139,7 +140,8 @@ def prepare_email_message(self, outbox=None): else: # we effectively always want an outbox passed in but keeping this for compatability # lets fallback to the message from_email in this case (this might cause more Outbox lookup failures) - hack_alias = self.backend_alias + ";;;" + self.from_email + outbox = Outbox.objects.filter(email_host_user__iexact=self.from_email).first() + hack_alias = self.backend_alias + ";;;" + outbox.email_host_user if outbox else self.from_email # this will open and cache a connection to the alias above # maybe remove this and only open a connection on send diff --git a/django_mail_admin/transports/imap.py b/django_mail_admin/transports/imap.py index 07179de..076f259 100644 --- a/django_mail_admin/transports/imap.py +++ b/django_mail_admin/transports/imap.py @@ -1,5 +1,6 @@ import imaplib import logging +import time from django.conf import settings @@ -130,6 +131,75 @@ def get_message(self, condition=None): if self.archive: self.server.uid('copy', uid, self.archive) - self.server.uid('store', uid, "+FLAGS", "(\\Deleted)") - self.server.expunge() + #self.server.uid('store', uid, "+FLAGS", "(\\Deleted)") # do not delete + #self.server.expunge() # do not worry about deleted message handling. return + + def store_message_in_folder(self, folder_name, msg, flags='') -> bool: + if not all([folder_name, msg]): + return False + try: + status, response = self.server.create(folder_name) + if status == "OK": + logger.info(f"Folder created: {folder_name}") + else: + pass + #logger.info(f"Failed to create folder: {folder_name}. Server said: {response}") + typ, data = self.server.append( + folder_name, + flags, + imaplib.Time2Internaldate(time.time()), + msg.as_bytes() + ) + return True + except Exception as e: + logger.error(f"Error storing msg to imap folder_name: {folder_name}: error: {e}") + return False + + def _detect_imap_folder_path_separator(self, decoded_folder_name:str) -> str: + parts = decoded_folder_name.split(" ") + if len(parts) < 3: + sep_char = "/" + else: + sep_char = parts[1].strip('"') + return f' "{sep_char}" ' + + def get_sent_folder_name(self) -> str | None: + path_separators = "/" + status, folders = self.server.list() + sent_candidates = [] + separator = self._detect_imap_folder_path_separator(folders[0].decode()) if folders else None + + for folder_raw in folders: + decoded = folder_raw.decode() + + # Example line1: '(\HasNoChildren \Sent) "/" "Sent Items"' + if "\\Sent" in decoded: + # Extract folder name (quoted at the end) + folder_name = decoded.split(separator)[-1].strip('"') + sent_candidates.append(folder_name) + + # Fallback: try common names if \Sent flag was not found + if not sent_candidates: + # Example line2: '(\HasNoChildren \\Exists) "." "INBBOX.Sent"' + common_names = [ + "Sent", "Sent Mail", "Sent Messages", + "[Gmail]/Sent Mail", "INBOX.Sent", "Sent Items" + ] + found_name = False + for folder_raw in folders: + if found_name: + break + folder_line = folder_raw.decode() + for name in common_names: + if name in folder_line: + folder_name = folder_line.split(separator)[-1].strip('"') + # If not already quoted and contains spaces, quote it + if not folder_name.startswith('"') and ' ' in folder_name: + folder_name = f'"{folder_name}"' + sent_candidates.append(folder_name) + found_name = True + break + + # Pick first match, or none + return sent_candidates[0] if sent_candidates else None diff --git a/example/example/settings.py b/example/example/settings.py index 210ea89..211bf40 100644 --- a/example/example/settings.py +++ b/example/example/settings.py @@ -129,7 +129,8 @@ def config(key): DJANGO_MAIL_ADMIN = { "BACKENDS": { - "default": "django_mail_admin.backends.GmailOAuth2Backend", #'django_mail_admin.backends.CustomEmailBackend', + "default": "django_mail_admin.backends.GmailOAuth2Backend", + "smtpoutbox":"django_mail_admin.backends.SMTPOutboxBackend", "smtp": "django.core.mail.backends.smtp.EmailBackend", "o365": "django_mail_admin.backends.O365Backend", "gmail": "django_mail_admin.backends.GmailOAuth2Backend", From 8d861fb9f6e6ff1d9cbdcf77ae23ef41e4ff7cc0 Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Fri, 30 May 2025 13:48:48 -0400 Subject: [PATCH 62/68] test connection method, and actually send email through smtp outbox --- HISTORY.rst | 7 ++ django_mail_admin/admin.py | 46 ++++++- django_mail_admin/backends.py | 112 ++++++++++------- .../management/commands/test_connections.py | 113 ++++++++++++++++++ django_mail_admin/models/configurations.py | 106 +++++++++++++++- docs/index.rst | 1 + docs/testing_connections.rst | 109 +++++++++++++++++ example/example/settings.py | 6 +- example/test_connections.py | 107 +++++++++++++++++ tests/test_connection_methods.py | 96 +++++++++++++++ 10 files changed, 647 insertions(+), 56 deletions(-) create mode 100644 django_mail_admin/management/commands/test_connections.py create mode 100644 docs/testing_connections.rst create mode 100644 example/test_connections.py create mode 100644 tests/test_connection_methods.py diff --git a/HISTORY.rst b/HISTORY.rst index 8a0aa66..75cd4c2 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -4,6 +4,13 @@ History ------- +0.3.2 (2025-05-29) +++++++++++++++++++ + +* Added test_connection methods to Mailbox and Outbox models +* Added test_connections management command +* Added documentation for testing email connections + 0.3.1 (2022-05-05) ++++++++++++++++++ diff --git a/django_mail_admin/admin.py b/django_mail_admin/admin.py index b4faa32..9c2867c 100644 --- a/django_mail_admin/admin.py +++ b/django_mail_admin/admin.py @@ -96,6 +96,39 @@ def switch_active(mailbox_admin, request, queryset): switch_active.short_description = _("Switch active status") +def test_mailbox_connection(mailbox_admin, request, queryset): + for mailbox in queryset.all(): + success, message = mailbox.test_connection() + if success: + messages.success( + request, + f"Connection successful for mailbox '{mailbox.name}': {message}", + ) + else: + messages.error( + request, f"Connection failed for mailbox '{mailbox.name}': {message}" + ) + + +test_mailbox_connection.short_description = _("Test connection") + + +def test_outbox_connection(outbox_admin, request, queryset): + for outbox in queryset.all(): + success, message = outbox.test_connection() + if success: + messages.success( + request, f"Connection successful for outbox '{outbox.name}': {message}" + ) + else: + messages.error( + request, f"Connection failed for outbox '{outbox.name}': {message}" + ) + + +test_outbox_connection.short_description = _("Test connection") + + def send_queued_mail(outbox_admin, request, queryset): outbox: Outbox | None = None for outbox in queryset.all(): @@ -124,7 +157,10 @@ def send_queued_mail(outbox_admin, request, queryset): connection.close() if ( - not OutgoingEmail.objects.filter(status=STATUS.queued, from_email__iexact=outbox.email_host_user) + not OutgoingEmail.objects.filter( + status=STATUS.queued, + from_email__iexact=outbox.email_host_user, + ) .filter(Q(scheduled_time__lte=now()) | Q(scheduled_time=None)) .exists() ): @@ -147,7 +183,7 @@ class MailboxAdmin(get_parent()): readonly_fields = [ "last_polling", ] - actions = [get_new_mail, switch_active] + actions = [get_new_mail, switch_active, test_mailbox_connection] class IncomingAttachmentInline(admin.TabularInline): @@ -489,10 +525,10 @@ class OutboxAdmin(admin.ModelAdmin): "email_host_user", "email_port", "id", - #"active", + # "active", ) - #list_filter = ("active",) - actions = [send_queued_mail] + # list_filter = ("active",) + actions = [send_queued_mail, test_outbox_connection] class LogAdmin(admin.ModelAdmin): diff --git a/django_mail_admin/backends.py b/django_mail_admin/backends.py index 8180315..633c06b 100644 --- a/django_mail_admin/backends.py +++ b/django_mail_admin/backends.py @@ -46,8 +46,7 @@ def __init__( configurations = Outbox.objects.filter(active=True) if len(configurations) > 1 or len(configurations) == 0: raise ValueError( - "Got %(l)s active Outboxes, expected 1" - % {"l": len(configurations)} + "Got %(l)s active Outboxes, expected 1" % {"l": len(configurations)} ) else: configuration = configurations.first() @@ -117,11 +116,13 @@ def close(self): def open(self): with self._lock: configuration = Outbox.objects.filter( - email_host_user__iexact=self.from_email #ignore case + email_host_user__iexact=self.from_email # ignore case ).first() if not configuration: - raise ValueError(f"Unable to find an Outbox with email_host_user={self.from_email}") + raise ValueError( + f"Unable to find an Outbox with email_host_user={self.from_email}" + ) # existing saved connection is valid. no need for a new one if self.configuration_id and self.configuration_id == configuration.id: @@ -134,7 +135,7 @@ def open(self): self.host = configuration.email_host self.port = configuration.email_port self.username = configuration.email_host_user - self.password = configuration.email_host_password + self.password = configuration.email_host_password self.use_tls = configuration.email_use_tls self.use_ssl = configuration.email_use_ssl self.timeout = configuration.email_timeout @@ -143,13 +144,17 @@ def open(self): return super().open() - def _get_imap_sent_folder_name(self, imap_transport:ImapTransport|None) -> str | None: + def _get_imap_sent_folder_name( + self, imap_transport: ImapTransport | None + ) -> str | None: sent_folder: str | None = None - try: + try: if imap_transport: sent_folder = imap_transport.get_sent_folder_name() except Exception as e: - logger.error(f"Error retrieving sent folder name: imap_transport: {imap_transport},error: {e}") + logger.error( + f"Error retrieving sent folder name: imap_transport: {imap_transport},error: {e}" + ) sent_folder = None return sent_folder @@ -177,24 +182,22 @@ def send_messages(self, email_messages) -> int: """ if not email_messages: return 0 - num_sent_ok=0 + num_sent_ok = 0 with self._lock: imap_transport: ImapTransport | None = self._get_imap_transport() sent_folder: str | None = self._get_imap_sent_folder_name(imap_transport) for email_message in email_messages: - """ num_sent = super().send_messages([email_message]) if 1 != num_sent: continue - """ if imap_transport and sent_folder: imap_transport.store_message_in_folder( - sent_folder, - email_message.message() - ) # not sending any flags - num_sent_ok+=1 + sent_folder, email_message.message() + ) # not sending any flags + num_sent_ok += 1 return num_sent_ok + class OutboxEmailBackend(BaseEmailBackend): def send_messages(self, email_messages): for msg in email_messages: @@ -231,6 +234,7 @@ class O365Backend(EmailBackend): """ Backend to handle sending emails via o365 connection """ + def __init__(self, fail_silently: bool = False, **kwargs) -> None: super().__init__(fail_silently, **kwargs) self.conn: Optional[O365Connection] = None @@ -258,11 +262,13 @@ def open(self): with self._lock: configuration = Outbox.objects.filter( email_host__icontains="office365", - email_host_user__iexact=self.from_email #ignore case + email_host_user__iexact=self.from_email, # ignore case ).first() if not configuration: - raise ValueError(f"Unable to find an Outbox with email_host__icontains=office365 email_host_user={self.from_email}") + raise ValueError( + f"Unable to find an Outbox with email_host__icontains=office365 email_host_user={self.from_email}" + ) # existing saved connection is valid. no need for a new one if self.conn and self.configuration_id == configuration.id: @@ -313,13 +319,15 @@ def send_messages(self, email_messages) -> int: try: # Use EmailAddressOAuthMapping to find oauth_username oauth_username = ( - EmailAddressOAuthMapping.objects.filter(send_as_email=msg.from_email) - .values_list('oauth_username', flat=True) + EmailAddressOAuthMapping.objects.filter( + send_as_email=msg.from_email + ) + .values_list("oauth_username", flat=True) .first() ) or msg.from_email # If connection doesn't match this message's configuration, create new connection - if (not self.conn or self.from_email != oauth_username): + if not self.conn or self.from_email != oauth_username: self.close() self.from_email = oauth_username # TODO: we could leverage a connection cache rather than re-init each time @@ -332,7 +340,9 @@ def send_messages(self, email_messages) -> int: except Exception as e: if not self.fail_silently: raise - logger.error(f"Failed to send message from {msg.from_email}: {str(e)}") + logger.error( + f"Failed to send message from {msg.from_email}: {str(e)}" + ) return sent_count @@ -366,8 +376,7 @@ def __init__(self, fail_silently: bool = False, **kwargs) -> None: def _initialize_connection(self, from_email: str) -> None: """Initialize connection settings based on from_email""" configuration = Outbox.objects.filter( - email_host__icontains="gmail", - email_host_user=from_email + email_host__icontains="gmail", email_host_user=from_email ).first() if not configuration: @@ -402,34 +411,37 @@ def _initialize_connection(self, from_email: str) -> None: def _connect_for_social_auth(self, user_social_auth: UserSocialAuth) -> None: creds_info = user_social_auth.extra_data auth_string = generate_oauth2_string( - user_social_auth.uid, - creds_info["access_token"], - base64_encode=False + user_social_auth.uid, creds_info["access_token"], base64_encode=False ) self.connection.docmd( "AUTH", "XOAUTH2 " + base64.b64encode(auth_string.encode("utf-8")).decode("utf-8"), ) - def open(self,auth_uid=None): + def open(self, auth_uid=None): """Override to use OAuth instead of password authentication""" try: # First establish the SMTP connection if self.use_ssl: - self.connection = smtplib.SMTP_SSL(self.host, self.port, timeout=self.timeout) + self.connection = smtplib.SMTP_SSL( + self.host, self.port, timeout=self.timeout + ) else: - self.connection = smtplib.SMTP(self.host, self.port, timeout=self.timeout) + self.connection = smtplib.SMTP( + self.host, self.port, timeout=self.timeout + ) if self.use_tls: self.connection.starttls() user_social_auth = UserSocialAuth.objects.get( - uid=auth_uid or self.from_email, - provider="google-oauth2" + uid=auth_uid or self.from_email, provider="google-oauth2" ) - logger.info(f"Found UserSocialAuth with pk={user_social_auth.pk} uid={user_social_auth.uid}") + logger.info( + f"Found UserSocialAuth with pk={user_social_auth.pk} uid={user_social_auth.uid}" + ) try: self._connect_for_social_auth(user_social_auth) @@ -437,12 +449,16 @@ def open(self,auth_uid=None): # TODO: we can avoid an except on all exceptions # and we can proactively refresh based on expiring soon # but we currently don't store access token expiration date - logger.exception(f"Failed connecting via OAuth. Attemping a token refresh for UserSocialAuth.uid={user_social_auth.uid}") + logger.exception( + f"Failed connecting via OAuth. Attemping a token refresh for UserSocialAuth.uid={user_social_auth.uid}" + ) updated_social_auth = refresh_authorization(user_social_auth.uid) self._connect_for_social_auth(updated_social_auth) return True except Exception: - logger.exception(f"gmail failed to open connection: auth_uid={auth_uid} from_email={self.from_email}") + logger.exception( + f"gmail failed to open connection: auth_uid={auth_uid} from_email={self.from_email}" + ) return None def close(self): @@ -454,11 +470,15 @@ def close(self): self.connection.quit() logger.info(f"Connection quit for {self.from_email}") except (ssl.SSLError, smtplib.SMTPServerDisconnected) as e: - logger.exception(f"Exception while quitting a connection for {self.from_email}: {e}") + logger.exception( + f"Exception while quitting a connection for {self.from_email}: {e}" + ) self.connection.close() logger.info(f"Connection closed for {self.from_email}") except smtplib.SMTPException as e: - logger.exception(f"SMTPException while quitting a connection for {self.from_email}: {e}") + logger.exception( + f"SMTPException while quitting a connection for {self.from_email}: {e}" + ) if self.fail_silently: return raise @@ -506,15 +526,19 @@ def send_messages(self, email_messages): # which maps to the Outbox of company@customer2.com # and UserSocialAuth: uid=company@customer2.com oauth_username = ( - EmailAddressOAuthMapping.objects.filter(send_as_email=message.from_email) - .values_list('oauth_username', flat=True) - .first() + EmailAddressOAuthMapping.objects.filter( + send_as_email=message.from_email + ) + .values_list("oauth_username", flat=True) + .first() ) or message.from_email # Check if we need to reinitialize for a different from_email - if (not self.connection or - self.from_email != oauth_username or - not self.configuration_id): + if ( + not self.connection + or self.from_email != oauth_username + or not self.configuration_id + ): logger.info( f"Opening a new connection in send_messages: " @@ -545,7 +569,9 @@ def send_messages(self, email_messages): except Exception as e: if not self.fail_silently: raise - logger.error(f"Failed to send message from {message.from_email}: {str(e)}") + logger.error( + f"Failed to send message from {message.from_email}: {str(e)}" + ) if self.connection: self.close() diff --git a/django_mail_admin/management/commands/test_connections.py b/django_mail_admin/management/commands/test_connections.py new file mode 100644 index 0000000..f2eeeda --- /dev/null +++ b/django_mail_admin/management/commands/test_connections.py @@ -0,0 +1,113 @@ +import logging +from django.core.management.base import BaseCommand, CommandError +from django_mail_admin.models import Mailbox, Outbox + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "Test connections for Mailboxes and Outboxes" + + def add_arguments(self, parser): + parser.add_argument("--mailbox", type=int, help="ID of the mailbox to test") + parser.add_argument("--outbox", type=int, help="ID of the outbox to test") + parser.add_argument( + "--all", action="store_true", help="Test all mailboxes and outboxes" + ) + + def handle(self, *args, **options): + mailbox_id = options.get("mailbox") + outbox_id = options.get("outbox") + test_all = options.get("all") + + if not any([mailbox_id, outbox_id, test_all]): + self.stdout.write( + self.style.WARNING("Please specify --mailbox, --outbox, or --all") + ) + return + + if mailbox_id: + self.test_mailbox(mailbox_id) + elif outbox_id: + self.test_outbox(outbox_id) + elif test_all: + self.test_all_mailboxes() + self.test_all_outboxes() + + def test_mailbox(self, mailbox_id): + """Test connection for a specific mailbox""" + try: + mailbox = Mailbox.objects.get(id=mailbox_id) + self.stdout.write( + f"Testing connection for Mailbox: {mailbox.name} (ID: {mailbox.id})" + ) + success, message = mailbox.test_connection() + + if success: + self.stdout.write(self.style.SUCCESS(f" SUCCESS: {message}")) + else: + self.stdout.write(self.style.ERROR(f" FAILED: {message}")) + + except Mailbox.DoesNotExist: + self.stdout.write( + self.style.ERROR(f"Mailbox with ID {mailbox_id} does not exist.") + ) + + def test_outbox(self, outbox_id): + """Test connection for a specific outbox""" + try: + outbox = Outbox.objects.get(id=outbox_id) + self.stdout.write( + f"Testing connection for Outbox: {outbox.name} (ID: {outbox.id})" + ) + success, message = outbox.test_connection() + + if success: + self.stdout.write(self.style.SUCCESS(f" SUCCESS: {message}")) + else: + self.stdout.write(self.style.ERROR(f" FAILED: {message}")) + + except Outbox.DoesNotExist: + self.stdout.write( + self.style.ERROR(f"Outbox with ID {outbox_id} does not exist.") + ) + + def test_all_mailboxes(self): + """Test connections for all mailboxes""" + mailboxes = Mailbox.objects.all() + + if not mailboxes: + self.stdout.write(self.style.WARNING("No mailboxes found.")) + return + + self.stdout.write(self.style.NOTICE("Testing all mailboxes:")) + for mailbox in mailboxes: + self.stdout.write( + f"Testing connection for Mailbox: {mailbox.name} (ID: {mailbox.id})" + ) + success, message = mailbox.test_connection() + + if success: + self.stdout.write(self.style.SUCCESS(f" SUCCESS: {message}")) + else: + self.stdout.write(self.style.ERROR(f" FAILED: {message}")) + + def test_all_outboxes(self): + """Test connections for all outboxes""" + outboxes = Outbox.objects.all() + + if not outboxes: + self.stdout.write(self.style.WARNING("No outboxes found.")) + return + + self.stdout.write(self.style.NOTICE("Testing all outboxes:")) + for outbox in outboxes: + self.stdout.write( + f"Testing connection for Outbox: {outbox.name} (ID: {outbox.id})" + ) + success, message = outbox.test_connection() + + if success: + self.stdout.write(self.style.SUCCESS(f" SUCCESS: {message}")) + else: + self.stdout.write(self.style.ERROR(f" FAILED: {message}")) diff --git a/django_mail_admin/models/configurations.py b/django_mail_admin/models/configurations.py index 10b3291..1d1d34f 100644 --- a/django_mail_admin/models/configurations.py +++ b/django_mail_admin/models/configurations.py @@ -62,6 +62,51 @@ class Outbox(models.Model): # deprecated field we're no longer actively using. keeping for test/code compatability. active = models.BooleanField(_("Active"), default=False) + def test_connection(self): + """ + Test the connection to this outbox using configured credentials. + + Returns: + tuple: (success, message) where success is a boolean indicating if + the connection was successful, and message contains details + about the connection attempt. + """ + from django_mail_admin.connections import connections + + try: + # Create a backend alias based on the email host type + if "office365" in self.email_host.lower(): + backend_alias = "o365;;;" + self.email_host_user + elif "gmail" in self.email_host.lower(): + backend_alias = "gmail;;;" + self.email_host_user + else: + backend_alias = "smtp;;;" + self.email_host_user + + # Get a connection using the ConnectionHandler + connection = connections[backend_alias] + + # Test the connection - this will vary by backend type + if hasattr(connection, "connection") and connection.connection: + # For SMTP-based backends, we can use the noop() method + if hasattr(connection.connection, "noop"): + connection.connection.noop() + + # For O365Backend, check if authenticated + if hasattr(connection, "conn") and connection.conn: + if ( + hasattr(connection.conn, "is_authenticated") + and not connection.conn.is_authenticated + ): + return False, "Office365 connection not authenticated" + + return True, f"Successfully connected to {self.email_host}" + except Exception as e: + return False, f"Connection failed: {str(e)}" + finally: + # Close the connection to clean up + if "connection" in locals() and connection: + connection.close() + def save(self, *args, **kwargs): # Only one item can be active at a time @@ -141,6 +186,41 @@ class Mailbox(models.Model): default=None, ) + def test_connection(self): + """ + Test the connection to this mailbox using configured credentials. + + Returns: + tuple: (success, message) where success is a boolean indicating if + the connection was successful, and message contains details + about the connection attempt. + """ + try: + connection = self.get_connection() + if not connection: + return False, "Could not establish connection - invalid configuration" + + # Test the connection based on the transport type + if self.type == "imap" or self.type == "gmail": + # IMAP connections have a noop() method to test the connection + connection.server.noop() + elif self.type == "pop3": + # POP3 connections have a noop() method + connection.server.noop() + elif self.type == O365Transport.SCHEME: # 'office365' + # For Office365, we can check if the connection is authenticated + if not connection.is_authenticated: + return False, "Office365 connection not authenticated" + # For local file transports, just check if the connection exists + elif self.type in ["maildir", "mbox", "babyl", "mh", "mmdf"]: + # These are local file transports, so just check if the path exists + if not connection: + return False, f"Could not access local transport at {self.location}" + + return True, f"Successfully connected to {self.name}" + except Exception as e: + return False, f"Connection failed: {str(e)}" + from_email = models.CharField( _("From email"), max_length=255, @@ -350,7 +430,9 @@ def _get_dehydrated_message(self, msg, record): ): filename = None raw_filename = msg.get_filename() - logger.info(f"Processing attachment for incoming email id {record.pk} with raw_filename {raw_filename}") + logger.info( + f"Processing attachment for incoming email id {record.pk} with raw_filename {raw_filename}" + ) if raw_filename: filename = utils.convert_header_to_unicode(raw_filename) if not filename: @@ -361,17 +443,23 @@ def _get_dehydrated_message(self, msg, record): extension = ".bin" attachment = IncomingAttachment() - logger.info(f"Saving attachment document for incoming email id {record.pk} with filename {filename}") + logger.info( + f"Saving attachment document for incoming email id {record.pk} with filename {filename}" + ) attachment.document.save( uuid.uuid4().hex + extension, ContentFile(BytesIO(msg.get_payload(decode=True)).getvalue()), ) - logger.info(f"Attachment document saved for incoming email id {record.pk} with filename {filename}") + logger.info( + f"Attachment document saved for incoming email id {record.pk} with filename {filename}" + ) attachment.message = record for key, value in msg.items(): attachment[key] = value attachment.save() - logger.info(f"Attachment created and saved for incoming email id {record.pk} with filename {filename}") + logger.info( + f"Attachment created and saved for incoming email id {record.pk} with filename {filename}" + ) placeholder = EmailMessage() placeholder[get_attachment_interpolation_header()] = str(attachment.pk) @@ -423,7 +511,11 @@ def _process_message(self, message): msg.mailbox = self msg.message_id = message_id if "subject" in message: - msg.subject = utils.convert_header_to_unicode(message["subject"]).replace("\n","").replace("\r","")[0:255] + msg.subject = ( + utils.convert_header_to_unicode(message["subject"]) + .replace("\n", "") + .replace("\r", "")[0:255] + ) if "from" in message: msg.from_header = utils.convert_header_to_unicode(message["from"]) if "to" in message: @@ -480,7 +572,9 @@ def _process_save_original_message(self, message, msg): else: msg.eml.save( - "%s.eml" % uuid.uuid4(), ContentFile(BytesIO(message.as_bytes()).getvalue()), save=False + "%s.eml" % uuid.uuid4(), + ContentFile(BytesIO(message.as_bytes()).getvalue()), + save=False, ) def get_new_mail(self, condition=None): diff --git a/docs/index.rst b/docs/index.rst index 731f577..55467f7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,6 +14,7 @@ Contents: readme usage settings + testing_connections contributing authors history diff --git a/docs/testing_connections.rst b/docs/testing_connections.rst new file mode 100644 index 0000000..9319e2a --- /dev/null +++ b/docs/testing_connections.rst @@ -0,0 +1,109 @@ +Testing Email Connections +======================== + +Django Mail Admin now provides methods to test both Mailbox (incoming) and Outbox (outgoing) connections using the credentials already configured on the model instances. + +Testing Connections Programmatically +----------------------------------- + +You can test connections directly in your Python code: + +.. code-block:: python + + from django_mail_admin.models import Mailbox, Outbox + + # Test a specific mailbox + mailbox = Mailbox.objects.get(id=1) + success, message = mailbox.test_connection() + if success: + print(f"Connection successful: {message}") + else: + print(f"Connection failed: {message}") + + # Test a specific outbox + outbox = Outbox.objects.get(id=1) + success, message = outbox.test_connection() + if success: + print(f"Connection successful: {message}") + else: + print(f"Connection failed: {message}") + +Using the Management Command +--------------------------- + +Django Mail Admin provides a management command to test connections from the command line: + +.. code-block:: bash + + # Test a specific mailbox by ID + python manage.py test_connections --mailbox=1 + + # Test a specific outbox by ID + python manage.py test_connections --outbox=1 + + # Test all mailboxes and outboxes + python manage.py test_connections --all + +Using the Example Script +---------------------- + +An example script is provided in the `example` directory that demonstrates how to test connections: + +.. code-block:: bash + + # Test a specific mailbox by ID + python example/test_connections.py --mailbox=1 + + # Test a specific outbox by ID + python example/test_connections.py --outbox=1 + + # Test all mailboxes and outboxes + python example/test_connections.py --all + +How It Works +----------- + +Mailbox Connection Testing +~~~~~~~~~~~~~~~~~~~~~~~~~ + +The `test_connection()` method on the Mailbox model: + +1. Attempts to establish a connection using the configured credentials +2. Tests the connection based on the transport type (IMAP, POP3, Office365, etc.) +3. Returns a tuple of (success, message) where: + - `success` is a boolean indicating if the connection was successful + - `message` contains details about the connection attempt + +Outbox Connection Testing +~~~~~~~~~~~~~~~~~~~~~~~ + +The `test_connection()` method on the Outbox model: + +1. Creates a backend alias based on the email host type (SMTP, Office365, Gmail) +2. Gets a connection using the ConnectionHandler +3. Tests the connection based on the backend type +4. Returns a tuple of (success, message) where: + - `success` is a boolean indicating if the connection was successful + - `message` contains details about the connection attempt + +Supported Connection Types +------------------------- + +The test_connection methods support various connection types: + +For Mailbox: +- IMAP +- POP3 +- Gmail (IMAP) +- Office365 +- Local file transports (maildir, mbox, babyl, mh, mmdf) + +For Outbox: +- SMTP +- Office365 +- Gmail + +Error Handling +------------ + +The test_connection methods include robust error handling to catch various connection issues. If a connection fails, the error message will be returned in the message part of the result tuple. diff --git a/example/example/settings.py b/example/example/settings.py index 211bf40..6c3c2e3 100644 --- a/example/example/settings.py +++ b/example/example/settings.py @@ -26,9 +26,11 @@ ALLOWED_HOSTS = ["*"] + def config(key): return _config(key, default="") + # Application definition INSTALLED_APPS = [ @@ -130,8 +132,8 @@ def config(key): DJANGO_MAIL_ADMIN = { "BACKENDS": { "default": "django_mail_admin.backends.GmailOAuth2Backend", - "smtpoutbox":"django_mail_admin.backends.SMTPOutboxBackend", - "smtp": "django.core.mail.backends.smtp.EmailBackend", + "smtp": "django_mail_admin.backends.SMTPOutboxBackend", + # "smtp": "django.core.mail.backends.smtp.EmailBackend", "o365": "django_mail_admin.backends.O365Backend", "gmail": "django_mail_admin.backends.GmailOAuth2Backend", } diff --git a/example/test_connections.py b/example/test_connections.py new file mode 100644 index 0000000..0731b3f --- /dev/null +++ b/example/test_connections.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +""" +Example script demonstrating how to test Mailbox and Outbox connections +using the new test_connection methods. + +This script can be run from the command line to test connections for +all configured Mailboxes and Outboxes, or specific ones by ID. +""" + +import os +import sys +import django +import argparse + +# Set up Django environment +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "example.settings") +django.setup() + +from django_mail_admin.models import Mailbox, Outbox + + +def test_mailbox_connection(mailbox_id=None): + """ + Test connection for a specific mailbox or all mailboxes. + + Args: + mailbox_id (int, optional): ID of the mailbox to test. If None, test all mailboxes. + """ + if mailbox_id: + try: + mailboxes = [Mailbox.objects.get(id=mailbox_id)] + except Mailbox.DoesNotExist: + print(f"Mailbox with ID {mailbox_id} does not exist.") + return + else: + mailboxes = Mailbox.objects.all() + + if not mailboxes: + print("No mailboxes found.") + return + + for mailbox in mailboxes: + print(f"Testing connection for Mailbox: {mailbox.name} (ID: {mailbox.id})") + success, message = mailbox.test_connection() + status = "SUCCESS" if success else "FAILED" + print(f" Status: {status}") + print(f" Message: {message}") + print() + + +def test_outbox_connection(outbox_id=None): + """ + Test connection for a specific outbox or all outboxes. + + Args: + outbox_id (int, optional): ID of the outbox to test. If None, test all outboxes. + """ + if outbox_id: + try: + outboxes = [Outbox.objects.get(id=outbox_id)] + except Outbox.DoesNotExist: + print(f"Outbox with ID {outbox_id} does not exist.") + return + else: + outboxes = Outbox.objects.all() + + if not outboxes: + print("No outboxes found.") + return + + for outbox in outboxes: + print(f"Testing connection for Outbox: {outbox.name} (ID: {outbox.id})") + success, message = outbox.test_connection() + status = "SUCCESS" if success else "FAILED" + print(f" Status: {status}") + print(f" Message: {message}") + print() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Test email connections for django_mail_admin." + ) + parser.add_argument( + "--mailbox", type=int, help="ID of the mailbox to test", default=None + ) + parser.add_argument( + "--outbox", type=int, help="ID of the outbox to test", default=None + ) + parser.add_argument( + "--all", action="store_true", help="Test all mailboxes and outboxes" + ) + + args = parser.parse_args() + + if args.mailbox: + test_mailbox_connection(args.mailbox) + elif args.outbox: + test_outbox_connection(args.outbox) + elif args.all: + print("Testing all mailboxes:") + test_mailbox_connection() + print("\nTesting all outboxes:") + test_outbox_connection() + else: + parser.print_help() diff --git a/tests/test_connection_methods.py b/tests/test_connection_methods.py new file mode 100644 index 0000000..1004898 --- /dev/null +++ b/tests/test_connection_methods.py @@ -0,0 +1,96 @@ +import unittest +from unittest.mock import patch, MagicMock + +from django.test import TestCase + +from django_mail_admin.models import Mailbox, Outbox + + +class MailboxConnectionTestCase(TestCase): + """Test the Mailbox.test_connection method""" + + def setUp(self): + self.mailbox = Mailbox.objects.create( + name="Test IMAP Mailbox", uri="imap://user:password@example.com" + ) + + @patch("django_mail_admin.models.configurations.ImapTransport") + def test_imap_connection_success(self, mock_imap): + """Test successful IMAP connection""" + # Set up the mock + mock_connection = MagicMock() + mock_connection.server.noop.return_value = True + mock_imap.return_value = mock_connection + + # Test the connection + success, message = self.mailbox.test_connection() + + # Verify the result + self.assertTrue(success) + self.assertIn("Successfully connected", message) + mock_connection.server.noop.assert_called_once() + + @patch("django_mail_admin.models.configurations.ImapTransport") + def test_imap_connection_failure(self, mock_imap): + """Test failed IMAP connection""" + # Set up the mock to raise an exception + mock_connection = MagicMock() + mock_connection.server.noop.side_effect = Exception("Connection refused") + mock_imap.return_value = mock_connection + + # Test the connection + success, message = self.mailbox.test_connection() + + # Verify the result + self.assertFalse(success) + self.assertIn("Connection failed", message) + self.assertIn("Connection refused", message) + + +class OutboxConnectionTestCase(TestCase): + """Test the Outbox.test_connection method""" + + def setUp(self): + self.outbox = Outbox.objects.create( + name="Test SMTP Outbox", + email_host="smtp.example.com", + email_host_user="user@example.com", + email_host_password="password", + email_port=587, + ) + + @patch("django_mail_admin.models.configurations.connections") + def test_smtp_connection_success(self, mock_connections): + """Test successful SMTP connection""" + # Set up the mock + mock_connection = MagicMock() + mock_connection.connection.noop.return_value = True + mock_connections.__getitem__.return_value = mock_connection + + # Test the connection + success, message = self.outbox.test_connection() + + # Verify the result + self.assertTrue(success) + self.assertIn("Successfully connected", message) + mock_connection.connection.noop.assert_called_once() + + @patch("django_mail_admin.models.configurations.connections") + def test_smtp_connection_failure(self, mock_connections): + """Test failed SMTP connection""" + # Set up the mock to raise an exception + mock_connection = MagicMock() + mock_connection.connection.noop.side_effect = Exception("Authentication failed") + mock_connections.__getitem__.return_value = mock_connection + + # Test the connection + success, message = self.outbox.test_connection() + + # Verify the result + self.assertFalse(success) + self.assertIn("Connection failed", message) + self.assertIn("Authentication failed", message) + + +if __name__ == "__main__": + unittest.main() From 601f8895c996a988d3512b416ef5a2c06503d41e Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Wed, 4 Jun 2025 17:04:47 -0400 Subject: [PATCH 63/68] password retrieval hook for SMTPOutbox --- django_mail_admin/backends.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/django_mail_admin/backends.py b/django_mail_admin/backends.py index 633c06b..519bf6d 100644 --- a/django_mail_admin/backends.py +++ b/django_mail_admin/backends.py @@ -113,34 +113,37 @@ def close(self): self.configuration_id = None super().close() + def get_password(self, outbox: Outbox): + return outbox.email_host_password + def open(self): with self._lock: - configuration = Outbox.objects.filter( + outbox = Outbox.objects.filter( email_host_user__iexact=self.from_email # ignore case ).first() - if not configuration: + if not outbox: raise ValueError( f"Unable to find an Outbox with email_host_user={self.from_email}" ) # existing saved connection is valid. no need for a new one - if self.configuration_id and self.configuration_id == configuration.id: + if self.configuration_id and self.configuration_id == outbox.id: return elif self.configuration_id: # close the existing invalid connection - self.from_email = configuration.email_host_user - self.configuration_id = configuration.id - - self.host = configuration.email_host - self.port = configuration.email_port - self.username = configuration.email_host_user - self.password = configuration.email_host_password - self.use_tls = configuration.email_use_tls - self.use_ssl = configuration.email_use_ssl - self.timeout = configuration.email_timeout - self.ssl_keyfile = configuration.email_ssl_keyfile - self.ssl_certfile = configuration.email_ssl_certfile + self.from_email = outbox.email_host_user + self.configuration_id = outbox.id + + self.host = outbox.email_host + self.port = outbox.email_port + self.username = outbox.email_host_user + self.password = self.get_password(outbox) + self.use_tls = outbox.email_use_tls + self.use_ssl = outbox.email_use_ssl + self.timeout = outbox.email_timeout + self.ssl_keyfile = outbox.email_ssl_keyfile + self.ssl_certfile = outbox.email_ssl_certfile return super().open() From 672ae455d78c4fd27c77016eae75a68189b94551 Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Mon, 16 Jun 2025 13:46:00 -0400 Subject: [PATCH 64/68] reverted changes to imap transport that flags messages for deleted and expunges from server for Gmail fetch to work correctly --- django_mail_admin/transports/imap.py | 77 +++++++++++++++------------- 1 file changed, 41 insertions(+), 36 deletions(-) diff --git a/django_mail_admin/transports/imap.py b/django_mail_admin/transports/imap.py index 076f259..645fd45 100644 --- a/django_mail_admin/transports/imap.py +++ b/django_mail_admin/transports/imap.py @@ -17,18 +17,19 @@ class ImapTransport(EmailTransport): def __init__( - self, hostname, port=None, ssl=False, tls=False, - archive='', folder=None, + self, + hostname, + port=None, + ssl=False, + tls=False, + archive="", + folder=None, ): self.max_message_size = getattr( - settings, - 'DJANGO_MAILBOX_MAX_MESSAGE_SIZE', - False + settings, "DJANGO_MAILBOX_MAX_MESSAGE_SIZE", False ) self.integration_testing_subject = getattr( - settings, - 'DJANGO_MAILBOX_INTEGRATION_TESTING_SUBJECT', - None + settings, "DJANGO_MAILBOX_INTEGRATION_TESTING_SUBJECT", None ) self.hostname = hostname self.port = port @@ -57,13 +58,13 @@ def connect(self, username, password): def _get_all_message_ids(self): # Fetch all the message uids - response, message_ids = self.server.uid('search', None, 'ALL') + response, message_ids = self.server.uid("search", None, "ALL") message_id_string = message_ids[0].strip() # Usually `message_id_string` will be a list of space-separated # ids; we must make sure that it isn't an empty string before # splitting into individual UIDs. if message_id_string: - return message_id_string.decode().split(' ') + return message_id_string.decode().split(" ") return [] def _get_small_message_ids(self, message_ids): @@ -72,23 +73,17 @@ def _get_small_message_ids(self, message_ids): # limit safe_message_ids = [] - status, data = self.server.uid( - 'fetch', - ','.join(message_ids), - '(RFC822.SIZE)' - ) + status, data = self.server.uid("fetch", ",".join(message_ids), "(RFC822.SIZE)") for each_msg in data: each_msg = each_msg.decode() try: - uid = each_msg.split(' ')[2] - size = each_msg.split(' ')[4].rstrip(')') + uid = each_msg.split(" ")[2] + size = each_msg.split(" ")[4].rstrip(")") if int(size) <= int(self.max_message_size): safe_message_ids.append(uid) except ValueError as e: - logger.warning( - "ValueError: %s working on %s" % (e, each_msg[0]) - ) + logger.warning("ValueError: %s working on %s" % (e, each_msg[0])) pass return safe_message_ids @@ -110,7 +105,7 @@ def get_message(self, condition=None): for uid in message_ids: try: - typ, msg_contents = self.server.uid('fetch', uid, '(RFC822)') + typ, msg_contents = self.server.uid("fetch", uid, "(RFC822)") if not msg_contents: continue try: @@ -129,13 +124,13 @@ def get_message(self, condition=None): continue if self.archive: - self.server.uid('copy', uid, self.archive) + self.server.uid("copy", uid, self.archive) - #self.server.uid('store', uid, "+FLAGS", "(\\Deleted)") # do not delete - #self.server.expunge() # do not worry about deleted message handling. + self.server.uid("store", uid, "+FLAGS", "(\\Deleted)") + self.server.expunge() return - def store_message_in_folder(self, folder_name, msg, flags='') -> bool: + def store_message_in_folder(self, folder_name, msg, flags="") -> bool: if not all([folder_name, msg]): return False try: @@ -144,19 +139,21 @@ def store_message_in_folder(self, folder_name, msg, flags='') -> bool: logger.info(f"Folder created: {folder_name}") else: pass - #logger.info(f"Failed to create folder: {folder_name}. Server said: {response}") + # logger.info(f"Failed to create folder: {folder_name}. Server said: {response}") typ, data = self.server.append( - folder_name, + folder_name, flags, - imaplib.Time2Internaldate(time.time()), - msg.as_bytes() + imaplib.Time2Internaldate(time.time()), + msg.as_bytes(), ) return True except Exception as e: - logger.error(f"Error storing msg to imap folder_name: {folder_name}: error: {e}") + logger.error( + f"Error storing msg to imap folder_name: {folder_name}: error: {e}" + ) return False - def _detect_imap_folder_path_separator(self, decoded_folder_name:str) -> str: + def _detect_imap_folder_path_separator(self, decoded_folder_name: str) -> str: parts = decoded_folder_name.split(" ") if len(parts) < 3: sep_char = "/" @@ -168,11 +165,15 @@ def get_sent_folder_name(self) -> str | None: path_separators = "/" status, folders = self.server.list() sent_candidates = [] - separator = self._detect_imap_folder_path_separator(folders[0].decode()) if folders else None + separator = ( + self._detect_imap_folder_path_separator(folders[0].decode()) + if folders + else None + ) for folder_raw in folders: decoded = folder_raw.decode() - + # Example line1: '(\HasNoChildren \Sent) "/" "Sent Items"' if "\\Sent" in decoded: # Extract folder name (quoted at the end) @@ -183,8 +184,12 @@ def get_sent_folder_name(self) -> str | None: if not sent_candidates: # Example line2: '(\HasNoChildren \\Exists) "." "INBBOX.Sent"' common_names = [ - "Sent", "Sent Mail", "Sent Messages", - "[Gmail]/Sent Mail", "INBOX.Sent", "Sent Items" + "Sent", + "Sent Mail", + "Sent Messages", + "[Gmail]/Sent Mail", + "INBOX.Sent", + "Sent Items", ] found_name = False for folder_raw in folders: @@ -195,7 +200,7 @@ def get_sent_folder_name(self) -> str | None: if name in folder_line: folder_name = folder_line.split(separator)[-1].strip('"') # If not already quoted and contains spaces, quote it - if not folder_name.startswith('"') and ' ' in folder_name: + if not folder_name.startswith('"') and " " in folder_name: folder_name = f'"{folder_name}"' sent_candidates.append(folder_name) found_name = True From 0ae910f22475497bc7e45616c5f4de645bd04eb8 Mon Sep 17 00:00:00 2001 From: Cam <42305922+CamdenParsons@users.noreply.github.com> Date: Tue, 23 Sep 2025 09:26:28 -0400 Subject: [PATCH 65/68] fixing attachments bug --- django_mail_admin/utils.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/django_mail_admin/utils.py b/django_mail_admin/utils.py index e51706c..0fc4da2 100644 --- a/django_mail_admin/utils.py +++ b/django_mail_admin/utils.py @@ -82,17 +82,24 @@ def get_body_from_message(message, maintype, subtype): def get_attachment_save_path(instance, filename): + + # Keep original filename as display name (what users see in email) if hasattr(instance, 'name'): if not instance.name: - instance.name = filename # set original filename + instance.name = filename + + # Get base upload path path = get_attachment_upload_to() if '%' in path: - path = datetime.datetime.utcnow().strftime(path) + path = datetime.datetime.now(datetime.UTC).strftime(path) + + # Create unique UUID subdirectory to prevent race conditions + unique_subdir = str(uuid.uuid4())[:8] + unique_path = os.path.join(path, unique_subdir) + + # Return: path/uuid_subdir/original_filename.ext + return os.path.join(unique_path, filename) - return os.path.join( - path, - filename, - ) def parse_priority(priority): From dba5e0f94a48c2d584a99430c77774463fee2a6a Mon Sep 17 00:00:00 2001 From: Cam <42305922+CamdenParsons@users.noreply.github.com> Date: Tue, 23 Sep 2025 10:49:23 -0400 Subject: [PATCH 66/68] Update utils.py --- django_mail_admin/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django_mail_admin/utils.py b/django_mail_admin/utils.py index 0fc4da2..8808c7f 100644 --- a/django_mail_admin/utils.py +++ b/django_mail_admin/utils.py @@ -3,6 +3,7 @@ import logging import os from collections import namedtuple +import uuid from django.core.exceptions import ValidationError From 14837e2c22f1d54f4cd0a1264ab343d78dcd8fe8 Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Fri, 10 Oct 2025 13:06:42 -0400 Subject: [PATCH 67/68] get_messages uses no limit to force a batched retrieval of all available emails --- django_mail_admin/o365_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django_mail_admin/o365_utils.py b/django_mail_admin/o365_utils.py index 56d348b..d388bd1 100644 --- a/django_mail_admin/o365_utils.py +++ b/django_mail_admin/o365_utils.py @@ -294,7 +294,8 @@ def get_messages(self, owner_email: str, last_polled: datetime, condition): # ISO 8601 format AND in UTC time. # For example, midnight UTC on Jan 1, 2022 is 2022-01-01T00:00:00Z. qstr = f"receivedDateTime gt {last_polled.replace(microsecond=0).isoformat()[:-6]}Z" - for mail in mail_folder.get_messages(order_by=order_by, query=qstr): + # limit=None will force batch retrieval of all emails that satisfies the query. + for mail in mail_folder.get_messages(order_by=order_by, query=qstr, limit=None): yield (mail.get_mime_content()) From eaf5996f70fe8ce2686c487801a5db11135315ce Mon Sep 17 00:00:00 2001 From: Abhijit Savarkar Date: Fri, 3 Apr 2026 13:14:29 -0400 Subject: [PATCH 68/68] o365 - set from_email value, chgup ai app settings --- django_mail_admin/o365_utils.py | 1 + example/example/settings.py | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/django_mail_admin/o365_utils.py b/django_mail_admin/o365_utils.py index d388bd1..4628e9d 100644 --- a/django_mail_admin/o365_utils.py +++ b/django_mail_admin/o365_utils.py @@ -185,6 +185,7 @@ def _prepare_new_message(self, mailbox, msg: EmailMessage): new_draft_message = ( reply_to_message.reply() if reply_to_message else mailbox.new_message() ) + new_draft_message.sender = msg.from_email return new_draft_message def _prepare_attachment_for_dispatch(self, attachment) -> dict: diff --git a/example/example/settings.py b/example/example/settings.py index 6c3c2e3..850be97 100644 --- a/example/example/settings.py +++ b/example/example/settings.py @@ -185,6 +185,13 @@ def config(key): "O365_CLIENT_ID": config("O365_CLIENT_ID"), "O365_CLIENT_SECRET": config("O365_CLIENT_SECRET"), }, + "chargeup.ai_app": { + # TBD certificate support + "auth": "token_secret", #'certificate' + "TOKEN_BACKEND": "AZBlobStorageTokenBackend", + "O365_CLIENT_ID": config("O365_CLIENT_ID"), + "O365_CLIENT_SECRET": config("O365_CLIENT_SECRET"), + }, } O365_TOKEN_BACKENDS = {