From e2a70614296aa6c5937a1796d35029a589ee4e79 Mon Sep 17 00:00:00 2001 From: Son NK <> Date: Thu, 26 Nov 2020 17:01:05 +0100 Subject: [PATCH 1/3] add get_encoding() and encode_text() --- app/email_utils.py | 29 +++++++++++++++++++++++++++++ tests/test_email_utils.py | 22 ++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/app/email_utils.py b/app/email_utils.py index f81dec7f..e4cf8594 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -1,5 +1,7 @@ +import base64 import email import os +import quopri import random import re from email.header import decode_header @@ -671,6 +673,33 @@ def is_valid_email(email_address: str) -> bool: ) +def get_encoding(msg: Message) -> str: + """ + Return the message encoding, possible values: + - quoted-printable + - base64: default if unknown + """ + cte = str(msg.get("content-transfer-encoding", "")).lower() + if cte == "": + return "base64" + + if cte == "quoted-printable" or cte == "base64": + return cte + + LOG.exception("Unknown encoding %s", cte) + + return "base64" + + +def encode_text(text: str, encoding: str = "base64") -> str: + if encoding == "quoted-printable": + encoded = quopri.encodestring(text.encode("utf-8")) + return str(encoded, "utf-8") + else: # use base64 by default + encoded = base64.b64encode(text.encode("utf-8")) + return str(encoded, "utf-8") + + def add_header(msg: Message, text_header, html_header) -> Message: if msg.get_content_type() == "text/plain": payload = msg.get_payload() diff --git a/tests/test_email_utils.py b/tests/test_email_utils.py index 2b4b3a97..72276453 100644 --- a/tests/test_email_utils.py +++ b/tests/test_email_utils.py @@ -18,6 +18,8 @@ from app.email_utils import ( to_bytes, generate_reply_email, normalize_reply_email, + get_encoding, + encode_text, ) from app.extensions import db from app.models import User, CustomDomain @@ -416,3 +418,23 @@ def test_generate_reply_email(flask_client): def test_normalize_reply_email(flask_client): assert normalize_reply_email("re+abcd@sl.local") == "re+abcd@sl.local" assert normalize_reply_email('re+"ab cd"@sl.local') == "re+_ab_cd_@sl.local" + + +def test_get_encoding(): + msg = email.message_from_string("") + assert get_encoding(msg) == "base64" + + msg = email.message_from_string("Content-TRANSFER-encoding: Invalid") + assert get_encoding(msg) == "base64" + + msg = email.message_from_string("Content-TRANSFER-encoding: quoted-printable") + assert get_encoding(msg) == "quoted-printable" + + +def test_encode_text(): + assert encode_text("") == "" + assert encode_text("ascii") == "YXNjaWk=" + assert encode_text("ascii", "quoted-printable") == "ascii" + + assert encode_text("mèo méo") == "bcOobyBtw6lv" + assert encode_text("mèo méo", "quoted-printable") == "m=C3=A8o m=C3=A9o" From 1241838b266beb9853ccbc313f4f72a8f07e0048 Mon Sep 17 00:00:00 2001 From: Son NK <> Date: Thu, 26 Nov 2020 17:03:50 +0100 Subject: [PATCH 2/3] take into account message encoding in add_header() --- app/email_utils.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/app/email_utils.py b/app/email_utils.py index e4cf8594..7af8078b 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -702,26 +702,38 @@ def encode_text(text: str, encoding: str = "base64") -> str: def add_header(msg: Message, text_header, html_header) -> Message: if msg.get_content_type() == "text/plain": + encoding = get_encoding(msg) payload = msg.get_payload() if type(payload) is str: clone_msg = copy(msg) - payload = f"{text_header}\n---\n{payload}" + to_append = encode_text(f"{text_header}\n---\n", encoding) + payload = f"{to_append}{payload}" clone_msg.set_payload(payload) return clone_msg elif msg.get_content_type() == "text/html": + encoding = get_encoding(msg) payload = msg.get_payload() if type(payload) is str: - - new_payload = f""" + new_payload = ( + encode_text( + f""" - +
{html_header}
{payload}""", + encoding, + ) + + payload + + encode_text( + """
- """ + """, + encoding, + ) + ) clone_msg = copy(msg) clone_msg.set_payload(new_payload) return clone_msg From d61f45ea8677eda32939cfe625d27ff967e38e80 Mon Sep 17 00:00:00 2001 From: Son NK <> Date: Thu, 26 Nov 2020 17:22:17 +0100 Subject: [PATCH 3/3] use 7bit as default encoding --- app/email_utils.py | 17 ++++++++++------- tests/test_email_utils.py | 13 +++++++++---- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/app/email_utils.py b/app/email_utils.py index 7af8078b..0e519b38 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -677,27 +677,30 @@ def get_encoding(msg: Message) -> str: """ Return the message encoding, possible values: - quoted-printable - - base64: default if unknown + - base64 + - 7bit: default if unknown or empty """ cte = str(msg.get("content-transfer-encoding", "")).lower() - if cte == "": - return "base64" + if cte in ("", "7bit"): + return "7bit" - if cte == "quoted-printable" or cte == "base64": + if cte in ("quoted-printable", "base64"): return cte LOG.exception("Unknown encoding %s", cte) - return "base64" + return "7bit" -def encode_text(text: str, encoding: str = "base64") -> str: +def encode_text(text: str, encoding: str = "7bit") -> str: if encoding == "quoted-printable": encoded = quopri.encodestring(text.encode("utf-8")) return str(encoded, "utf-8") - else: # use base64 by default + elif encoding == "base64": encoded = base64.b64encode(text.encode("utf-8")) return str(encoded, "utf-8") + else: # 7bit - no encoding + return text def add_header(msg: Message, text_header, html_header) -> Message: diff --git a/tests/test_email_utils.py b/tests/test_email_utils.py index 72276453..6aeb571b 100644 --- a/tests/test_email_utils.py +++ b/tests/test_email_utils.py @@ -422,19 +422,24 @@ def test_normalize_reply_email(flask_client): def test_get_encoding(): msg = email.message_from_string("") - assert get_encoding(msg) == "base64" + assert get_encoding(msg) == "7bit" msg = email.message_from_string("Content-TRANSFER-encoding: Invalid") - assert get_encoding(msg) == "base64" + assert get_encoding(msg) == "7bit" msg = email.message_from_string("Content-TRANSFER-encoding: quoted-printable") assert get_encoding(msg) == "quoted-printable" + msg = email.message_from_string("Content-TRANSFER-encoding: base64") + assert get_encoding(msg) == "base64" + def test_encode_text(): assert encode_text("") == "" - assert encode_text("ascii") == "YXNjaWk=" + assert encode_text("ascii") == "ascii" + assert encode_text("ascii", "base64") == "YXNjaWk=" assert encode_text("ascii", "quoted-printable") == "ascii" - assert encode_text("mèo méo") == "bcOobyBtw6lv" + assert encode_text("mèo méo") == "mèo méo" + assert encode_text("mèo méo", "base64") == "bcOobyBtw6lv" assert encode_text("mèo méo", "quoted-printable") == "m=C3=A8o m=C3=A9o"