Enhance Automatic Email Submission & Reply

+ add rfc3834 support
+ model changes
  + track creator of message, one of:
    human,auto-self,auto-custom,auto-generated,auto-replied,auto-notified
  + track external replies with in_reply_to
  + track smtp outgoing and smtp incoming msgid
+ tag outgoing email messages with "Auto-Submitted" = auto-generated or auto-replied
+ only forward human and auto-self messages via smtp forward
+ add email send filter exception list settings.EMAIL_UNFILTERED_INDIVIDUALS
+ set the default reply_receiver for all root@system.local messages to get_office_user(submission)
+ change default for SMTPD_CONFIG['store_exceptions'] to True
+ modify testcase test_plain to check for creator="human"
+ add testcases
    + test_forward_auto_submitted
    + test_incoming_auto_submitted
    + test_DEFAULT_FROM_EMAIL_is_marked_automatic_and_reply_to_default_contact
+ cleanup code, fix broken testcases

Author: wuxxin

ecs/checklists/workflow.py

@@ -113,7 +113,9 @@ def pre_perform(self, choice):
             if not c.pdf_document:
                 c.render_pdf_document()
             presenting_parties = c.submission.current_submission_form.get_presenting_parties()
-            presenting_parties.send_message(_('External Review'), 'checklists/external_review_publish.txt',
+            presenting_parties.send_message(
+                _('External Review'),
+                'checklists/external_review_publish.txt',
                 {'checklist': c, 'ABSOLUTE_URL_PREFIX': settings.ABSOLUTE_URL_PREFIX},
                 submission=c.submission)
         elif c.status == 'review_fail':

ecs/communication/mailutils.py

@@ -18,23 +18,29 @@ def html2text(htmltext):
 
 
 def create_mail(subject, message, from_email, recipient, message_html=None,
-    attachments=None, rfc2822_headers=None):
+                attachments=None, rfc2822_headers=None):
 
-    headers = {'Message-ID': make_msgid()}
+    from ecs.core.models import AdvancedSettings
+
+    headers = {'Message-ID': make_msgid(domain=settings.DOMAIN)}
+    if from_email == settings.DEFAULT_FROM_EMAIL:
+        # if empty, set Auto-Submitted and Reply-To for mails from DEFAULT_FROM_EMAIL
+        if not rfc2822_headers or not rfc2822_headers.get('Auto-Submitted', None):
+            headers.update({'Auto-Submitted': 'auto-generated'})
+        if not rfc2822_headers or not rfc2822_headers.get('Reply-To', None):
+            headers.update({'Reply-To': AdvancedSettings.objects.get(pk=1).default_contact})
 
     if rfc2822_headers:
         headers.update(rfc2822_headers)
 
-    if message is None: # make text version out of html if text version is missing
+    if message is None:  # make text version out of html if text version is missing
         message = html2text(message_html)
 
     if message_html:
-        msg = EmailMultiAlternatives(subject, message, from_email, [recipient],
-            headers=headers)
+        msg = EmailMultiAlternatives(subject, message, from_email, [recipient], headers=headers)
         msg.attach_alternative(message_html, "text/html")
     else:
-        msg = EmailMessage(subject, message, from_email, [recipient],
-            headers=headers)
+        msg = EmailMessage(subject, message, from_email, [recipient], headers=headers)
 
     if attachments:
         for filename, content, mimetype in attachments:
@@ -62,12 +68,13 @@ def deliver(recipient_list, *args, **kwargs):
 def deliver_to_recipient(recipient, subject, message, from_email,
     message_html=None, attachments=None, nofilter=False, rfc2822_headers=None):
 
-    msg = create_mail(subject, message, from_email, recipient, message_html,
-        attachments, rfc2822_headers)
+    msg = create_mail(subject, message, from_email, recipient,
+                      message_html, attachments, rfc2822_headers)
     msgid = msg.extra_headers['Message-ID']
 
     backend = settings.EMAIL_BACKEND
-    if nofilter or recipient.split('@')[1] in settings.EMAIL_UNFILTERED_DOMAINS:
+    if (nofilter or recipient.split('@')[1] in settings.EMAIL_UNFILTERED_DOMAINS or
+            recipient in settings.EMAIL_UNFILTERED_INDIVIDUALS):
         backend = settings.EMAIL_BACKEND_UNFILTERED
 
     connection = mail.get_connection(backend=backend)

ecs/communication/migrations/0014_auto_reply.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('communication', '0013_search_tsvector'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='message',
+            old_name='rawmsg_msgid',
+            new_name='outgoing_msgid',
+        ),
+        migrations.AddField(
+            model_name='message',
+            name='creator',
+            field=models.CharField(choices=[('human', 'human'), ('auto-self', 'auto-self'), ('auto-custom', 'auto-custom'), ('auto-generated', 'auto-generated'), ('auto-replied', 'auto-replied'), ('auto-notified', 'auto-notified')], max_length=14, default='human'),
+        ),
+        migrations.AddField(
+            model_name='message',
+            name='in_reply_to',
+            field=models.ForeignKey(related_name='is_replied_in', null=True, default=None, to='communication.Message'),
+        ),
+        migrations.AddField(
+            model_name='message',
+            name='incoming_msgid',
+            field=models.CharField(max_length=250, null=True),
+        ),
+    ]

ecs/communication/migrations/0015_outgoing_msgid_remove_index.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('communication', '0014_auto_reply'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='message',
+            name='outgoing_msgid',
+            field=models.CharField(max_length=250, null=True),
+        ),
+    ]

ecs/communication/models.py

@@ -1,9 +1,14 @@
 import uuid
+import traceback
 
 from django.conf import settings
 from django.db import models
 from django.db.models import Q
 from django.contrib.auth.models import User
+from django.utils.translation import ugettext as _
+
+from ecs.communication.mailutils import deliver_to_recipient
+from ecs.users.utils import get_full_name
 
 
 DELIVERY_STATES = (
@@ -13,6 +18,18 @@
     ("failure", "failure"),
 )
 
+# https://tools.ietf.org/html/rfc3834
+# https://tools.ietf.org/html/rfc5436
+CREATOR_CHOICES = (
+    ("human", "human"),
+    ("auto-self", "auto-self"),
+    ("auto-custom", "auto-custom"),
+    # rfc3834 compatible
+    ("auto-generated", "auto-generated"),
+    ("auto-replied", "auto-replied"),
+    ("auto-notified", "auto-notified"),
+)
+
 
 class ThreadQuerySet(models.QuerySet):
     def by_user(self, user):
@@ -60,7 +77,10 @@ def unstar(self, user):
             self.starred_by_receiver = False
         self.save(update_fields=('starred_by_sender', 'starred_by_receiver'))
 
-    def add_message(self, user, text, rawmsg_msgid=None, rawmsg=None, reply_receiver=None):
+    def add_message(self, user, text,
+                    reply_receiver=None, rawmsg=None,
+                    outgoing_msgid=None, incoming_msgid=None,
+                    in_reply_to=None, creator=None):
         assert user.id in (self.sender_id, self.receiver_id)
 
         if user.id == self.sender_id:
@@ -80,14 +100,20 @@ def add_message(self, user, text, rawmsg_msgid=None, rawmsg=None, reply_receiver
         else:
             self.sender = receiver
 
+        if creator is None:
+            creator = 'human'
+
         msg = self.messages.create(
             sender=user,
             receiver=receiver,
             text=text,
             smtp_delivery_state='new',
             rawmsg=rawmsg,
-            rawmsg_msgid=rawmsg_msgid,
-            reply_receiver=reply_receiver
+            outgoing_msgid=outgoing_msgid,
+            incoming_msgid=incoming_msgid,
+            reply_receiver=reply_receiver,
+            in_reply_to=in_reply_to,
+            creator=creator,
         )
         self.last_message = msg
         self.closed_by_sender = False
@@ -110,24 +136,81 @@ def message_list(self):
 
 
 class Message(models.Model):
+    uuid = models.UUIDField(default=uuid.uuid4, unique=True, db_index=True)
     thread = models.ForeignKey(Thread, related_name='messages')
     sender = models.ForeignKey(User, related_name='outgoing_messages')
     receiver = models.ForeignKey(User, related_name='incoming_messages')
+    reply_receiver = models.ForeignKey(User, null=True,
+        related_name='reply_receiver_for_messages')
     timestamp = models.DateTimeField(auto_now_add=True)
     unread = models.BooleanField(default=True, db_index=True)
     text = models.TextField()
 
     rawmsg = models.TextField(null=True)
-    rawmsg_msgid = models.CharField(max_length=250, null=True, db_index=True)
+    outgoing_msgid = models.CharField(max_length=250, null=True)
+    incoming_msgid = models.CharField(max_length=250, null=True)
 
     smtp_delivery_state = models.CharField(max_length=7,
         choices=DELIVERY_STATES, default='new', db_index=True)
-
-    uuid = models.UUIDField(default=uuid.uuid4, unique=True, db_index=True)
-
-    reply_receiver = models.ForeignKey(User, null=True, related_name='reply_receiver_for_messages')
+    in_reply_to = models.ForeignKey('self', related_name='is_replied_in',
+        null=True, default=None)
+    creator = models.CharField(max_length=14,
+        choices=CREATOR_CHOICES, default='human')
 
     @property
     def return_address(self):
-        return 'ecs-{}@{}'.format(self.uuid.hex,
-            settings.DOMAIN)
+        return 'ecs-{}@{}'.format(self.uuid.hex, settings.DOMAIN)
+
+    @property
+    def smtp_subject(self):
+        submission = self.thread.submission
+        ec_number = ''
+        if submission:
+            ec_number = ' ' + submission.get_ec_number_display()
+        subject = _('[ECS{ec_number}] {subject}.').format(
+            ec_number=ec_number, subject=self.thread.subject)
+        return subject
+
+    def forward_smtp(self):
+        try:
+            forwarded = False
+            subject = self.smtp_subject
+            headers = {}
+
+            if self.creator in ("human", "auto-self"):
+                self.smtp_delivery_state = 'started'
+                self.save()
+
+                if self.creator == "auto-self":
+                    headers.update({"Auto-Submitted": "auto-generated", })
+                else:
+                    headers.update({"Auto-Submitted": "auto-replied", })
+                if self.incoming_msgid:
+                    headers.update({
+                        'In-Reply-To': self.incoming_msgid,
+                        'References': " ".join((self.incoming_msgid,
+                                                self.in_reply_to.outgoing_msgid)),
+                    })
+
+                msgid, rawmsg = deliver_to_recipient(
+                    self.receiver.email,
+                    subject=subject,
+                    message=self.text,
+                    from_email='{0} <{1}>'.format(get_full_name(self.sender), self.return_address),
+                    rfc2822_headers=headers,
+                )
+
+                forwarded = True
+                self.outgoing_msgid = msgid
+                # do not overwrite rawmsg on forward, if it originated from smtp
+                if not self.incoming_msgid:
+                    self.rawmsg = rawmsg.as_string()
+
+            self.smtp_delivery_state = 'success'
+        except:
+            traceback.print_exc()
+            self.smtp_delivery_state = 'failure'
+            raise
+        finally:
+            self.save()
+            return forwarded

ecs/communication/smtpd.py

@@ -38,11 +38,10 @@ def handle_error(self):
 
 def _get_content(message_part):
     payload = message_part.get_payload(decode=True)
-    
+
     if message_part.get_content_charset() is None:
         charset = chardet.detect(payload)['encoding']
-        logger.info(
-            'no content charset declared, detection result: {0}'.format(charset))
+        logger.debug('no content charset declared, detection result: {0}'.format(charset))
     else:
         charset = message_part.get_content_charset()
 
@@ -53,10 +52,9 @@ def _get_content(message_part):
 
     logger.debug('message-part: type: {0} charset: {1}'.format(
         message_part.get_content_type(), charset))
-    content = str(payload, charset, "replace")
+    content = str(payload, charset, 'replace')
     return content
 
-
 class EcsMailReceiver(smtpd.SMTPServer):
     channel_class = EcsSMTPChannel
 
@@ -68,7 +66,7 @@ def __init__(self):
         smtpd.SMTPServer.__init__(self, settings.SMTPD_CONFIG['listen_addr'], None,
             data_size_limit=self.MAX_MSGSIZE, decode_data=False)
         self.logger = logging.getLogger('EcsMailReceiver')
-        self.store_exceptions = settings.SMTPD_CONFIG.get('store_exceptions', False)
+        self.store_exceptions = settings.SMTPD_CONFIG.get('store_exceptions', True)
         if self.store_exceptions:
             self.undeliverable_maildir = mailbox.Maildir(
                 os.path.join(settings.PROJECT_DIR, '..', 'ecs-undeliverable'))
@@ -104,8 +102,8 @@ def _get_text(self, msg):
             else:
                 raise SMTPError(554,
                     'Invalid message format - invalid content type {0}'.format(
-                    part.get_content_type()))
-    
+                        part.get_content_type()))
+
         if not plain and not html:
             raise SMTPError(554, 'Invalid message format - empty message')
 
@@ -121,18 +119,36 @@ def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
             text = self._get_text(msg)
             orig_msg = self._find_msg(rcpttos[0])
             thread = orig_msg.thread
-            # XXX: rawmsg can include multiple content-charsets and should be a binaryfield
-            # as a workaround we convert to base64
+
+            creator = msg.get('Auto-Submitted', None)
+            # XXX email header are case insentitiv matched,
+            # 'auto-submitted' will be matched too it there is no 'Auto-Submitted'
+            if creator in (None, '', 'no'):
+                creator = 'human'
+            elif creator[11] == 'auto-notify':
+                creator = 'auto-notify'
+            elif creator in ('auto-generated', 'auto-replied'):
+                pass
+            else:
+                creator = 'auto-custom'
+
             thread.messages.filter(
                 receiver=orig_msg.receiver).update(unread=False)
+
+            # TODO rawmsg can include multiple content-charsets and should be a binaryfield
+            # as a workaround we convert to base64
             thread_msg = thread.add_message(orig_msg.receiver, text,
                 rawmsg=base64.b64encode(data),
-                rawmsg_msgid=msg['Message-ID'])
+                incoming_msgid=msg['Message-ID'],
+                in_reply_to=orig_msg,
+                creator=creator)
+
             logger.info(
-                'Accepted email from {0} via {1} to {2} id {3} thread {4} orig_msg {5} message {6}'.format(
-                mailfrom, orig_msg.receiver.email, orig_msg.sender.email, 
-                msg['Message-ID'], thread.pk, orig_msg.pk, thread_msg.pk))
-            
+                'Accepted email (creator= {8})from {0} via {1} to {2} id {3} in-reply-to {4} thread {5} orig_msg {6} message {7}'.format(
+                    mailfrom, orig_msg.receiver.email, orig_msg.sender.email,
+                    msg['Message-ID'], orig_msg.outgoing_msgid, thread.pk,
+                    orig_msg.pk, thread_msg.pk, creator))
+
         except SMTPError as e:
             logger.info('Rejected email: {0}'.format(e))
             return str(e)