From 79e41a63de0be386db8ebce2ee8aa7c9a7082734 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jagoda=20=C5=9Al=C4=85zak?= Date: Thu, 15 Jan 2026 20:37:52 +0100 Subject: [PATCH] feat(filtermail): Replace filtermail with rust reimplementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jagoda Ślązak --- .github/workflows/ci.yaml | 3 +- chatmaild/pyproject.toml | 1 - chatmaild/src/chatmaild/filtermail.py | 378 ------------------ .../src/chatmaild/tests/test_filtermail.py | 361 ----------------- cmdeploy/src/cmdeploy/basedeploy.py | 3 +- cmdeploy/src/cmdeploy/deployers.py | 4 +- cmdeploy/src/cmdeploy/filtermail/deployer.py | 52 +++ .../filtermail-incoming.service.j2} | 3 +- .../filtermail.service.j2} | 2 +- .../src/cmdeploy/tests/online/test_1_basic.py | 10 +- 10 files changed, 67 insertions(+), 750 deletions(-) delete mode 100644 chatmaild/src/chatmaild/filtermail.py delete mode 100644 chatmaild/src/chatmaild/tests/test_filtermail.py create mode 100644 cmdeploy/src/cmdeploy/filtermail/deployer.py rename cmdeploy/src/cmdeploy/{service/filtermail-incoming.service.f => filtermail/filtermail-incoming.service.j2} (74%) rename cmdeploy/src/cmdeploy/{service/filtermail.service.f => filtermail/filtermail.service.j2} (74%) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5f6acd824..07bb67366 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -14,7 +14,8 @@ jobs: # Otherwise `test_deployed_state` will be unhappy. with: ref: ${{ github.event.pull_request.head.sha }} - + - name: download filtermail + run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.1.2/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail - name: run chatmaild tests working-directory: chatmaild run: pipx run tox diff --git a/chatmaild/pyproject.toml b/chatmaild/pyproject.toml index 7b4954b63..1eb00c0a0 100644 --- a/chatmaild/pyproject.toml +++ b/chatmaild/pyproject.toml @@ -24,7 +24,6 @@ where = ['src'] [project.scripts] doveauth = "chatmaild.doveauth:main" chatmail-metadata = "chatmaild.metadata:main" -filtermail = "chatmaild.filtermail:main" chatmail-metrics = "chatmaild.metrics:main" chatmail-expire = "chatmaild.expire:main" chatmail-fsreport = "chatmaild.fsreport:main" diff --git a/chatmaild/src/chatmaild/filtermail.py b/chatmaild/src/chatmaild/filtermail.py deleted file mode 100644 index 43c51674f..000000000 --- a/chatmaild/src/chatmaild/filtermail.py +++ /dev/null @@ -1,378 +0,0 @@ -#!/usr/bin/env python3 -import asyncio -import base64 -import binascii -import sys -import time -from email import policy -from email.parser import BytesParser -from email.utils import parseaddr -from smtplib import SMTP as SMTPClient - -from aiosmtpd.controller import Controller -from aiosmtpd.smtp import SMTP - -from .config import read_config - -ENCRYPTION_NEEDED_523 = "523 Encryption Needed: Invalid Unencrypted Mail" - - -def check_openpgp_payload(payload: bytes): - """Checks the OpenPGP payload. - - OpenPGP payload must consist only of PKESK and SKESK packets - terminated by a single SEIPD packet. - - Returns True if OpenPGP payload is correct, - False otherwise. - - May raise IndexError while trying to read OpenPGP packet header - if it is truncated. - """ - i = 0 - while i < len(payload): - # Only OpenPGP format is allowed. - if payload[i] & 0xC0 != 0xC0: - return False - - packet_type_id = payload[i] & 0x3F - i += 1 - - while payload[i] >= 224 and payload[i] < 255: - # Partial body length. - partial_length = 1 << (payload[i] & 0x1F) - i += 1 + partial_length - - if payload[i] < 192: - # One-octet length. - body_len = payload[i] - i += 1 - elif payload[i] < 224: - # Two-octet length. - body_len = ((payload[i] - 192) << 8) + payload[i + 1] + 192 - i += 2 - elif payload[i] == 255: - # Five-octet length. - body_len = ( - (payload[i + 1] << 24) - | (payload[i + 2] << 16) - | (payload[i + 3] << 8) - | payload[i + 4] - ) - i += 5 - else: - # Impossible, partial body length was processed above. - return False - - i += body_len - - if i == len(payload): - # Last packet should be - # Symmetrically Encrypted and Integrity Protected Data Packet (SEIPD) - # - # This is the only place where this function may return `True`. - return packet_type_id == 18 - elif packet_type_id not in [1, 3]: - # All packets except the last one must be either - # Public-Key Encrypted Session Key Packet (PKESK) - # or - # Symmetric-Key Encrypted Session Key Packet (SKESK) - return False - - return False - - -def check_armored_payload(payload: str, outgoing: bool): - """Check the armored PGP message for invalid content. - - :param payload: the armored PGP message - :param outgoing: whether the message is outgoing or incoming - :return: whether the message is a valid PGP message - """ - prefix = "-----BEGIN PGP MESSAGE-----\r\n" - if not payload.startswith(prefix): - return False - payload = payload.removeprefix(prefix) - - while payload.endswith("\r\n"): - payload = payload.removesuffix("\r\n") - suffix = "-----END PGP MESSAGE-----" - if not payload.endswith(suffix): - return False - payload = payload.removesuffix(suffix) - - version_comment = "Version: " - if payload.startswith(version_comment): - if outgoing: # Disallow comments in outgoing messages - return False - # Remove comments from incoming messages - payload = payload.partition("\r\n")[2] - - while payload.startswith("\r\n"): - payload = payload.removeprefix("\r\n") - - # Remove CRC24. - payload = payload.rpartition("=")[0] - - try: - payload = base64.b64decode(payload) - except binascii.Error: - return False - - try: - return check_openpgp_payload(payload) - except IndexError: - return False - - -def is_securejoin(message): - if message.get("secure-join") not in ["vc-request", "vg-request"]: - return False - if not message.is_multipart(): - return False - parts_count = 0 - for part in message.iter_parts(): - parts_count += 1 - if parts_count > 1: - return False - if part.is_multipart(): - return False - if part.get_content_type() != "text/plain": - return False - - payload = part.get_payload().strip().lower() - if payload not in ("secure-join: vc-request", "secure-join: vg-request"): - return False - return True - - -def check_encrypted(message, outgoing=True): - """Check that the message is an OpenPGP-encrypted message. - - MIME structure of the message must correspond to . - """ - if not message.is_multipart(): - return False - if message.get_content_type() != "multipart/encrypted": - return False - parts_count = 0 - for part in message.iter_parts(): - # We explicitly check Content-Type of each part later, - # but this is to be absolutely sure `get_payload()` returns string and not list. - if part.is_multipart(): - return False - - if parts_count == 0: - if part.get_content_type() != "application/pgp-encrypted": - return False - - payload = part.get_payload() - if payload.strip() != "Version: 1": - return False - elif parts_count == 1: - if part.get_content_type() != "application/octet-stream": - return False - - if not check_armored_payload(part.get_payload(), outgoing=outgoing): - return False - else: - return False - parts_count += 1 - return True - - -async def asyncmain_beforequeue(config, mode): - if mode == "outgoing": - port = config.filtermail_smtp_port - handler = OutgoingBeforeQueueHandler(config) - else: - port = config.filtermail_smtp_port_incoming - handler = IncomingBeforeQueueHandler(config) - HackedController( - handler, - hostname="127.0.0.1", - port=port, - data_size_limit=config.max_message_size, - ).start() - - -def recipient_matches_passthrough(recipient, passthrough_recipients): - for addr in passthrough_recipients: - if recipient == addr: - return True - if addr[0] == "@" and recipient.endswith(addr): - return True - return False - - -class HackedController(Controller): - def factory(self): - return SMTPDiscardRCPTO_options(self.handler, **self.SMTP_kwargs) - - -class SMTPDiscardRCPTO_options(SMTP): - def _getparams(self, params): - # Ignore RCPT TO parameters. - # - # Otherwise parameters such as `ORCPT=...` - # or `NOTIFY=DELAY,FAILURE` (generated by Stalwart) - # make aiosmtpd reject the message here: - # - return {} - - -class OutgoingBeforeQueueHandler: - def __init__(self, config): - self.config = config - self.send_rate_limiter = SendRateLimiter() - - async def handle_MAIL(self, server, session, envelope, address, mail_options): - log_info(f"handle_MAIL from {address}") - envelope.mail_from = address - max_sent = self.config.max_user_send_per_minute - if not self.send_rate_limiter.is_sending_allowed(address, max_sent): - return f"450 4.7.1: Too much mail from {address}" - - parts = envelope.mail_from.split("@") - if len(parts) != 2: - return f"500 Invalid from address <{envelope.mail_from!r}>" - - return "250 OK" - - async def handle_DATA(self, server, session, envelope): - loop = asyncio.get_running_loop() - return await loop.run_in_executor(None, self.sync_handle_DATA, envelope) - - def sync_handle_DATA(self, envelope): - log_info("handle_DATA before-queue") - error = self.check_DATA(envelope) - if error: - return error - log_info("re-injecting the mail that passed checks") - client = SMTPClient("localhost", self.config.postfix_reinject_port) - client.sendmail( - envelope.mail_from, envelope.rcpt_tos, envelope.original_content - ) - return "250 OK" - - def check_DATA(self, envelope): - """the central filtering function for e-mails.""" - log_info(f"Processing DATA message from {envelope.mail_from}") - - message = BytesParser(policy=policy.default).parsebytes(envelope.content) - mail_encrypted = check_encrypted(message, outgoing=True) - - _, from_addr = parseaddr(message.get("from").strip()) - - if envelope.mail_from.lower() != from_addr.lower(): - return f"500 Invalid FROM <{from_addr!r}> for <{envelope.mail_from!r}>" - - if mail_encrypted or is_securejoin(message): - print("Outgoing: Filtering encrypted mail.", file=sys.stderr) - return - - print("Outgoing: Filtering unencrypted mail.", file=sys.stderr) - - if envelope.mail_from in self.config.passthrough_senders: - return - - # allow self-sent Autocrypt Setup Message - if envelope.rcpt_tos == [from_addr]: - if message.get("subject") == "Autocrypt Setup Message": - if message.get_content_type() == "multipart/mixed": - return - - passthrough_recipients = self.config.passthrough_recipients - - for recipient in envelope.rcpt_tos: - if recipient_matches_passthrough(recipient, passthrough_recipients): - continue - - print("Rejected unencrypted mail.", file=sys.stderr) - return ENCRYPTION_NEEDED_523 - - -class IncomingBeforeQueueHandler: - def __init__(self, config): - self.config = config - - async def handle_DATA(self, server, session, envelope): - loop = asyncio.get_running_loop() - return await loop.run_in_executor(None, self.sync_handle_DATA, envelope) - - def sync_handle_DATA(self, envelope): - log_info("handle_DATA before-queue") - error = self.check_DATA(envelope) - if error: - return error - log_info("re-injecting the mail that passed checks") - - client = SMTPClient( - "localhost", - self.config.postfix_reinject_port_incoming, - ) - client.sendmail( - envelope.mail_from, envelope.rcpt_tos, envelope.original_content - ) - return "250 OK" - - def check_DATA(self, envelope): - """the central filtering function for e-mails.""" - log_info(f"Processing DATA message from {envelope.mail_from}") - - message = BytesParser(policy=policy.default).parsebytes(envelope.content) - mail_encrypted = check_encrypted(message, outgoing=False) - - if mail_encrypted or is_securejoin(message): - print("Incoming: Filtering encrypted mail.", file=sys.stderr) - return - - print("Incoming: Filtering unencrypted mail.", file=sys.stderr) - - # we want cleartext mailer-daemon messages to pass through - # chatmail core will typically not display them as normal messages - if message.get("auto-submitted"): - _, from_addr = parseaddr(message.get("from").strip()) - if from_addr.lower().startswith("mailer-daemon@"): - if message.get_content_type() == "multipart/report": - return - - for recipient in envelope.rcpt_tos: - user = self.config.get_user(recipient) - if user is None or user.is_incoming_cleartext_ok(): - continue - - print("Rejected unencrypted mail.", file=sys.stderr) - return ENCRYPTION_NEEDED_523 - - -class SendRateLimiter: - def __init__(self): - self.addr2timestamps = {} - - def is_sending_allowed(self, mail_from, max_send_per_minute): - last = self.addr2timestamps.setdefault(mail_from, []) - now = time.time() - last[:] = [ts for ts in last if ts >= (now - 60)] - if len(last) <= max_send_per_minute: - last.append(now) - return True - return False - - -def log_info(msg): - print(msg, file=sys.stderr) - - -def main(): - args = sys.argv[1:] - assert len(args) == 2 - config = read_config(args[0]) - mode = args[1] - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - assert mode in ["incoming", "outgoing"] - task = asyncmain_beforequeue(config, mode) - loop.create_task(task) - log_info("entering serving loop") - loop.run_forever() diff --git a/chatmaild/src/chatmaild/tests/test_filtermail.py b/chatmaild/src/chatmaild/tests/test_filtermail.py deleted file mode 100644 index e39f4a094..000000000 --- a/chatmaild/src/chatmaild/tests/test_filtermail.py +++ /dev/null @@ -1,361 +0,0 @@ -import pytest - -from chatmaild.filtermail import ( - IncomingBeforeQueueHandler, - OutgoingBeforeQueueHandler, - SendRateLimiter, - check_armored_payload, - check_encrypted, - is_securejoin, -) - - -@pytest.fixture -def maildomain(): - # let's not depend on a real chatmail instance for the offline tests below - return "chatmail.example.org" - - -@pytest.fixture -def handler(make_config, maildomain): - config = make_config(maildomain) - return OutgoingBeforeQueueHandler(config) - - -@pytest.fixture -def inhandler(make_config, maildomain): - config = make_config(maildomain) - return IncomingBeforeQueueHandler(config) - - -def test_reject_forged_from(maildata, gencreds, handler): - class env: - mail_from = gencreds()[0] - rcpt_tos = [gencreds()[0]] - - # test that the filter lets good mail through - to_addr = gencreds()[0] - env.content = maildata( - "encrypted.eml", from_addr=env.mail_from, to_addr=to_addr - ).as_bytes() - - assert not handler.check_DATA(envelope=env) - - # test that the filter rejects forged mail - env.content = maildata( - "encrypted.eml", from_addr="forged@c3.testrun.org", to_addr=to_addr - ).as_bytes() - error = handler.check_DATA(envelope=env) - assert "500" in error - - -def test_filtermail_no_encryption_detection(maildata): - msg = maildata( - "plain.eml", from_addr="some@example.org", to_addr="other@example.org" - ) - assert not check_encrypted(msg) - - # https://xkcd.com/1181/ - msg = maildata( - "fake-encrypted.eml", from_addr="some@example.org", to_addr="other@example.org" - ) - assert not check_encrypted(msg) - - -def test_filtermail_securejoin_detection(maildata): - msg = maildata( - "securejoin-vc.eml", from_addr="some@example.org", to_addr="other@example.org" - ) - assert is_securejoin(msg) - - msg = maildata( - "securejoin-vc-fake.eml", - from_addr="some@example.org", - to_addr="other@example.org", - ) - assert not is_securejoin(msg) - - -def test_filtermail_encryption_detection(maildata): - msg = maildata( - "encrypted.eml", - from_addr="1@example.org", - to_addr="2@example.org", - subject="Subject does not matter, will be replaced anyway", - ) - assert check_encrypted(msg) - - -def test_filtermail_no_literal_packets(maildata): - """Test that literal OpenPGP packet is not considered an encrypted mail.""" - msg = maildata("literal.eml", from_addr="1@example.org", to_addr="2@example.org") - assert not check_encrypted(msg) - - -def test_filtermail_unencrypted_mdn(maildata, gencreds): - """Unencrypted MDNs should not pass.""" - from_addr = gencreds()[0] - to_addr = gencreds()[0] + ".other" - msg = maildata("mdn.eml", from_addr=from_addr, to_addr=to_addr) - - assert not check_encrypted(msg) - - -def test_send_rate_limiter(): - limiter = SendRateLimiter() - for i in range(100): - if limiter.is_sending_allowed("some@example.org", 10): - if i <= 10: - continue - pytest.fail("limiter didn't work") - else: - assert i == 11 - break - - -def test_cleartext_excempt_privacy(maildata, gencreds, handler): - from_addr = gencreds()[0] - to_addr = "privacy@testrun.org" - handler.config.passthrough_recipients = [to_addr] - false_to = "privacy@something.org" - - msg = maildata("plain.eml", from_addr=from_addr, to_addr=to_addr) - - class env: - mail_from = from_addr - rcpt_tos = [to_addr] - content = msg.as_bytes() - - # assert that None/no error is returned - assert not handler.check_DATA(envelope=env) - - class env2: - mail_from = from_addr - rcpt_tos = [to_addr, false_to] - content = msg.as_bytes() - - assert "523" in handler.check_DATA(envelope=env2) - - -def test_cleartext_self_send_autocrypt_setup_message(maildata, gencreds, handler): - from_addr = gencreds()[0] - to_addr = from_addr - - msg = maildata("asm.eml", from_addr=from_addr, to_addr=to_addr) - - class env: - mail_from = from_addr - rcpt_tos = [to_addr] - content = msg.as_bytes() - - assert not handler.check_DATA(envelope=env) - - -def test_cleartext_send_fails(maildata, gencreds, handler): - from_addr = gencreds()[0] - to_addr = gencreds()[0] - - msg = maildata("plain.eml", from_addr=from_addr, to_addr=to_addr) - - class env: - mail_from = from_addr - rcpt_tos = [to_addr] - content = msg.as_bytes() - - res = handler.check_DATA(envelope=env) - assert "523 Encryption Needed" in res - - -def test_cleartext_incoming_fails(maildata, gencreds, inhandler): - from_addr = gencreds()[0] - to_addr, password = gencreds() - - msg = maildata("plain.eml", from_addr=from_addr, to_addr=to_addr) - - class env: - mail_from = from_addr - rcpt_tos = [to_addr] - content = msg.as_bytes() - - user = inhandler.config.get_user(to_addr) - user.set_password(password) - res = inhandler.check_DATA(envelope=env) - assert "523 Encryption Needed" in res - - user.allow_incoming_cleartext() - assert not inhandler.check_DATA(envelope=env) - - -def test_cleartext_incoming_mailer_daemon(maildata, gencreds, inhandler): - from_addr = "mailer-daemon@example.org" - to_addr = gencreds()[0] - - msg = maildata("mailer-daemon.eml", from_addr=from_addr, to_addr=to_addr) - - class env: - mail_from = from_addr - rcpt_tos = [to_addr] - content = msg.as_bytes() - - assert not inhandler.check_DATA(envelope=env) - - -def test_cleartext_passthrough_domains(maildata, gencreds, handler): - from_addr = gencreds()[0] - to_addr = "privacy@x.y.z" - handler.config.passthrough_recipients = ["@x.y.z"] - false_to = "something@x.y" - - msg = maildata("plain.eml", from_addr=from_addr, to_addr=to_addr) - - class env: - mail_from = from_addr - rcpt_tos = [to_addr] - content = msg.as_bytes() - - # assert that None/no error is returned - assert not handler.check_DATA(envelope=env) - - class env2: - mail_from = from_addr - rcpt_tos = [to_addr, false_to] - content = msg.as_bytes() - - assert "523" in handler.check_DATA(envelope=env2) - - -def test_cleartext_passthrough_senders(gencreds, handler, maildata): - acc1 = gencreds()[0] - to_addr = "recipient@something.org" - handler.config.passthrough_senders = [acc1] - - msg = maildata("plain.eml", from_addr=acc1, to_addr=to_addr) - - class env: - mail_from = acc1 - rcpt_tos = to_addr - content = msg.as_bytes() - - # assert that None/no error is returned - assert not handler.check_DATA(envelope=env) - - -def test_check_armored_payload(): - prefix = "-----BEGIN PGP MESSAGE-----\r\n" - comment = "Version: ProtonMail\r\n" - payload = """\r -wU4DSqFx0d1yqAoSAQdAYkX/ZN/Az4B0k7X47zKyWrXxlDEdS3WOy0Yf2+GJTFgg\r -Zk5ql0mLG8Ze+ZifCS0XMO4otlemSyJ0K1ZPdFMGzUDBTgNqzkFabxXoXRIBB0AM\r -755wlX41X6Ay3KhnwBq7yEqSykVH6F3x11iHPKraLCAGZoaS8bKKNy/zg5slda1X\r -pt14b4aC1VwtSnYhcRRELNLD/wE2TFif+g7poMmFY50VyMPLYjVP96Z5QCT4+z4H\r -Ikh/pRRN8S3JNMrRJHc6prooSJmLcx47Y5un7VFy390MsJ+LiUJuQMDdYWRAinfs\r -Ebm89Ezjm7F03qbFPXE0X4ZNzVXS/eKO0uhJQdiov/vmbn41rNtHmNpqjaO0vi5+\r -sS9tR7yDUrIXiCUCN78eBLVioxtktsPZm5cDORbQWzv+7nmCEz9/JowCUcBVdCGn\r -1ofOaH82JCAX/cRx08pLaDNj6iolVBsi56Dd+2bGxJOZOG2AMcEyz0pXY0dOAJCD\r -iUThcQeGIdRnU3j8UBcnIEsjLu2+C+rrwMZQESMWKnJ0rnqTk0pK5kXScr6F/L0L\r -UE49ccIexNm3xZvYr5drszr6wz3Tv5fdue87P4etBt90gF/Vzknck+g1LLlkzZkp\r -d8dI0k2tOSPjUbDPnSy1x+X73WGpPZmj0kWT+RGvq0nH6UkJj3AQTG2qf1T8jK+3\r -rTp3LR9vDkMwDjX4R8SA9c0wdnUzzr79OYQC9lTnzcx+fM6BBmgQ2GrS33jaFLp7\r -L6/DFpCl5zhnPjM/2dKvMkw/Kd6XS/vjwsO405FQdjSDiQEEAZA+ZvAfcjdccbbU\r -yCO+x0QNdeBsufDVnh3xvzuWy4CICdTQT4s1AWRPCzjOj+SGmx5WqCLWfsd8Ma0+\r -w/C7SfTYu1FDQILLM+llpq1M/9GPley4QZ8JQjo262AyPXsPF/OW48uuZz0Db1xT\r -Yh4iHBztj4VSdy7l2+IyaIf7cnL4EEBFxv/MwmVDXvDlxyvfAfIsd3D9SvJESzKZ\r -VWDYwaocgeCN+ojKu1p885lu1EfRbX3fr3YO02K5/c2JYDkc0Py0W3wUP/J1XUax\r -pbKpzwlkxEgtmzsGqsOfMJqBV3TNDrOA2uBsa+uBqP5MGYLZ49S/4v/bW9I01Cr1\r -D2ZkV510Y1Vgo66WlP8mRqOTyt/5WRhPD+MxXdk67BNN/PmO6tMlVoJDuk+XwWPR\r -t2TvNaND/yabT9eYI55Og4fzKD6RIjouUX8DvKLkm+7aXxVs2uuLQ3Jco3O82z55\r -dbShU1jYsrw9oouXUz06MHPbkdhNbF/2hfhZ2qA31sNeovJw65iUv7sDKX3LVWgJ\r -10jlywcDwqlU8CO7WC9lGixYTbnOkYZpXCGEl8e6Jbs79l42YFo4ogYpFK1NXFhV\r -kOXRmDf/wmfj+c/ld3L2PkvwlgofhCudOQknZbo3ub1gjiTn7L+lMGHIj/3suMIl\r -ID4EUxAXScIM1ZEz2fjtW5jATlqYcLjLTbf/olw6HFyPNH+9IssqXeZNKnGwPUB9\r -3lTXsg0tpzl+x7F/2WjEw1DSNhjC0KnHt1vEYNMkUGDGFdN9y3ERLqX/FIgiASUb\r -bTvAVupnAK3raBezGmhrs6LsQtLS9P0VvQiLU3uDhMqw8Z4SISLpcD+NnVBHzQqm\r -6W5Qn/8xsCL6av18yUVTi2G3igt3QCNoYx9evt2ZcIkNoyyagUVjfZe5GHXh8Dnz\r -GaBXW/hg3HlXLRGaQu4RYCzBMJILcO25OhZOg6jbkCLiEexQlm2e9krB5cXR49Al\r -UN4fiB0KR9JyG2ayUdNJVkXZSZLnHyRgiaadlpUo16LVvw==\r -=b5Kp\r ------END PGP MESSAGE-----\r -\r -\r -""" - - commented_payload = prefix + comment + payload - assert check_armored_payload(commented_payload, outgoing=False) == True - assert check_armored_payload(commented_payload, outgoing=True) == False - - payload = prefix + payload - assert check_armored_payload(payload, outgoing=False) == True - assert check_armored_payload(payload, outgoing=True) == True - - payload = payload.removesuffix("\r\n") - assert check_armored_payload(payload, outgoing=False) == True - assert check_armored_payload(payload, outgoing=True) == True - - payload = payload.removesuffix("\r\n") - assert check_armored_payload(payload, outgoing=False) == True - assert check_armored_payload(payload, outgoing=True) == True - - payload = payload.removesuffix("\r\n") - assert check_armored_payload(payload, outgoing=False) == True - assert check_armored_payload(payload, outgoing=True) == True - - payload = """-----BEGIN PGP MESSAGE-----\r -\r -HELLOWORLD ------END PGP MESSAGE-----\r -\r -""" - assert check_armored_payload(payload, outgoing=False) == False - assert check_armored_payload(payload, outgoing=True) == False - - payload = """-----BEGIN PGP MESSAGE-----\r -\r -=njUN ------END PGP MESSAGE-----\r -\r -""" - assert check_armored_payload(payload, outgoing=False) == False - assert check_armored_payload(payload, outgoing=True) == False - - # Test payload using partial body length - # as generated by GopenPGP. - payload = """-----BEGIN PGP MESSAGE-----\r -\r -wV4DdCVjRfOT3TQSAQdAY5+pjT6mlCxPGdR3be4w7oJJRUGIPI/Vnh+mJxGSm34w\r -LNlVc89S1g22uQYFif2sUJsQWbpoHpNkuWpkSgOaHmNvrZiY/YU5iv+cZ3LbmtUG\r -0uoBisSHh9O1c+5sYZSbrvYZ1NOwlD7Fv/U5/Mw4E5+CjxfdgNGp5o3DDddzPK78\r -jseDhdSXxnaiIJC93hxNX6R1RPt3G2gukyzx69wciPQShcF8zf3W3o75Ed7B8etV\r -QEeB16xzdFhKa9JxdjTu3osgCs21IO7wpcFkjc7nZzlW6jPnELJJaNmv4yOOCjMp\r -6YAkaN/BkL+jHTznHDuDsT5ilnTXpwHDU1Cm9PIx/KFcNCQnIB+2DcdIHPHUH1ci\r -jvqoeXAVWjKXEjS7PqPFuP/xGbrWG2ugs+toXJOKbgRkExvKs1dwPFKrgghvCVbW\r -AcKejQKAPArLwpkA7aD875TZQShvGt74fNs45XBlGOYOnNOAJ1KAmzrXLIDViyyB\r -kDsmTBk785xofuCkjBpXSe6vsMprPzCteDfaUibh8FHeJjucxPerwuOPEmnogNaf\r -YyL4+iy8H8I9/p7pmUqILprxTG0jTOtlk0bTVzeiF56W1xbtSEMuOo4oFbQTyOM2\r -bKXaYo774Jm+rRtKAnnI2dtf9RpK19cog6YNzfYjesLKbXDsPZbN5rmwyFiCvvxC\r -kQ6JLob+B2fPdY2gzy7LypxktS8Zi1HJcWDHJGVmQodaDLqKUObb4M26bXDe6oxI\r -NS8PJz5exVbM3KhZnUOEn6PJRBBf5a/ZqxlhZPcQo/oBuhKpBRpO5kSDwPIUByu3\r -UlXLSkpMqe9pUarAOEuQjfl2RVY7U+RrQYp4YP5keMO+i8NCefAFbowTTufO1JIq\r -2nVgCi/QVnxZyEc9OYt/8AE3g4cdojE+vsSDifZLSWYIetpfrohHv3dT3StD1QRG\r -0QE6qq6oKpg/IL0cjvuX4c7a7bslv2fXp8t75y37RU6253qdIebhxc/cRhPbc/yu\r -p0YLyD4SrvKTLP2ZV95jT4IPEpqm4AN3QmiOzdtqR2gLyb62L8QfqI/FdwsIiRiM\r -hqydwoqt/lfSqG1WKPh+6EkMkH+TDiCC1BQdbN1MNcyUtcjb35PR2c8Ld2TF3guA\r -jLIqMt/Vb7hBoMb2FcsOYY25ka9oV62OwgKWLXnFzk+modMR5fzb4kxVVAYEqP+D\r -T5KO1Vs76v1fyPGOq6BbBCvLwTqe/e6IZInJles4v5jrhnLcGKmNGivCUDe6X6NY\r -UKNt5RsZllwDQpaAb5dMNhyrk8SgIE7TBI7rvqIdUCE52Vy+0JDxFg5olRpFUfO6\r -/MyTW3Yo/ekk/npHr7iYYqJTCc21bDGLWQcIo/XO7WPxrKNWGBNPFnkRdw0MaKr4\r -+cEM3V8NFnSEpC12xA+RX/CezuJtwXZK5MpG76eYqMO6qyC+c25YcFecEufDZDxx\r -ZLqRszVRyxyWPtk/oIeQK2v9wOqY6N9/ff01gHz69vqYqN5bUw/QKZsmx1zW+gPw\r -6x2tDK2BHeYl182gCbhlKISRFwCtbjqZSkiKWao/VtygHkw0fK34avJuyQ/X9YaN\r -BRy+7Lf3VA53pnB5WJ1xwRXN8VDvmZeXzv2krHveCMemj0OjnRoCLu117xN0A5m9\r -Fm/RoDix5PolDHtWTtr2m1n2hp2LHnj8at9lFEd0SKhAYHVL9KjzycwWODZRXt+x\r -zGDDuooEeTvdY5NLyKcl4gETz1ZP4Ez5jGGjhPSwSpq1mU7UaJ9ZXXdr4KHyifW6\r -ggNzNsGhXTap7IWZpTtqXABydfiBshmH2NjqtNDwBweJVSgP10+r0WhMWlaZs6xl\r -V3o5yskJt6GlkwpJxZrTvN6Tiww/eW7HFV6NGf7IRSWY5tJc/iA7/92tOmkdvJ1q\r -myLbG7cJB787QjplEyVe2P/JBO6xYvbkJLf9Q+HaviTO25rugRSrYsoKMDfO8VlQ\r -1CcnTPVtApPZJEQzAWJEgVAM8uIlkqWJJMgyWT34sTkdBeCUFGloXQFs9Yxd0AGf\r -/zHEkYZSTKpVSvAIGu4=\r -=6iHb\r ------END PGP MESSAGE-----\r -""" - assert check_armored_payload(payload, outgoing=False) == True - assert check_armored_payload(payload, outgoing=True) == True diff --git a/cmdeploy/src/cmdeploy/basedeploy.py b/cmdeploy/src/cmdeploy/basedeploy.py index 7490641c9..dcb17a3ca 100644 --- a/cmdeploy/src/cmdeploy/basedeploy.py +++ b/cmdeploy/src/cmdeploy/basedeploy.py @@ -17,9 +17,8 @@ def configure_remote_units(mail_domain, units) -> None: # install systemd units for fn in units: - execpath = fn if fn != "filtermail-incoming" else "filtermail" params = dict( - execpath=f"{remote_venv_dir}/bin/{execpath}", + execpath=f"{remote_venv_dir}/bin/{fn}", config_path=remote_chatmail_inipath, remote_venv_dir=remote_venv_dir, mail_domain=mail_domain, diff --git a/cmdeploy/src/cmdeploy/deployers.py b/cmdeploy/src/cmdeploy/deployers.py index 5dff76824..0314528ee 100644 --- a/cmdeploy/src/cmdeploy/deployers.py +++ b/cmdeploy/src/cmdeploy/deployers.py @@ -26,6 +26,7 @@ get_resource, ) from .dovecot.deployer import DovecotDeployer +from .filtermail.deployer import FiltermailDeployer from .mtail.deployer import MtailDeployer from .nginx.deployer import NginxDeployer from .opendkim.deployer import OpendkimDeployer @@ -416,8 +417,6 @@ class ChatmailVenvDeployer(Deployer): def __init__(self, config): self.config = config self.units = ( - "filtermail", - "filtermail-incoming", "chatmail-metadata", "lastlogin", "chatmail-expire", @@ -564,6 +563,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) - all_deployers = [ ChatmailDeployer(mail_domain), LegacyRemoveDeployer(), + FiltermailDeployer(), JournaldDeployer(), UnboundDeployer(), TurnDeployer(mail_domain), diff --git a/cmdeploy/src/cmdeploy/filtermail/deployer.py b/cmdeploy/src/cmdeploy/filtermail/deployer.py new file mode 100644 index 000000000..4b13b3d68 --- /dev/null +++ b/cmdeploy/src/cmdeploy/filtermail/deployer.py @@ -0,0 +1,52 @@ +from pyinfra import facts, host +from pyinfra.operations import files, systemd + +from cmdeploy.basedeploy import Deployer, get_resource + + +class FiltermailDeployer(Deployer): + services = ["filtermail", "filtermail-incoming"] + bin_path = "/usr/local/bin/filtermail" + config_path = "/usr/local/lib/chatmaild/chatmail.ini" + + def __init__(self): + self.need_restart = False + + def install(self): + arch = host.get_fact(facts.server.Arch) + url = f"https://github.com/chatmail/filtermail/releases/download/v0.1.2/filtermail-{arch}" + sha256sum = { + "x86_64": "de7de6e011ffc06881d3a05fc9788e327ba2389219e77280ace38b429e11a5ce", + "aarch64": "a78fcdfb81eb3d9c8a8b6f84f6c0a75519b8be01aa25bd4617d72aae543992b4", + }[arch] + self.need_restart |= files.download( + name="Download filtermail", + src=url, + sha256sum=sha256sum, + dest=self.bin_path, + mode="755", + ).changed + + def configure(self): + for service in self.services: + self.need_restart |= files.template( + src=get_resource(f"filtermail/{service}.service.j2"), + dest=f"/etc/systemd/system/{service}.service", + user="root", + group="root", + mode="644", + bin_path=self.bin_path, + config_path=self.config_path, + ).changed + + def activate(self): + for service in self.services: + systemd.service( + name=f"Start and enable {service}", + service=f"{service}.service", + running=True, + enabled=True, + restarted=self.need_restart, + daemon_reload=True, + ) + self.need_restart = False diff --git a/cmdeploy/src/cmdeploy/service/filtermail-incoming.service.f b/cmdeploy/src/cmdeploy/filtermail/filtermail-incoming.service.j2 similarity index 74% rename from cmdeploy/src/cmdeploy/service/filtermail-incoming.service.f rename to cmdeploy/src/cmdeploy/filtermail/filtermail-incoming.service.j2 index 844843e4e..76404dbf5 100644 --- a/cmdeploy/src/cmdeploy/service/filtermail-incoming.service.f +++ b/cmdeploy/src/cmdeploy/filtermail/filtermail-incoming.service.j2 @@ -2,11 +2,10 @@ Description=Incoming Chatmail Postfix before queue filter [Service] -ExecStart={execpath} {config_path} incoming +ExecStart={{ bin_path }} {{ config_path }} incoming Restart=always RestartSec=30 User=vmail [Install] WantedBy=multi-user.target - diff --git a/cmdeploy/src/cmdeploy/service/filtermail.service.f b/cmdeploy/src/cmdeploy/filtermail/filtermail.service.j2 similarity index 74% rename from cmdeploy/src/cmdeploy/service/filtermail.service.f rename to cmdeploy/src/cmdeploy/filtermail/filtermail.service.j2 index 398d5a89a..ce3ea836a 100644 --- a/cmdeploy/src/cmdeploy/service/filtermail.service.f +++ b/cmdeploy/src/cmdeploy/filtermail/filtermail.service.j2 @@ -2,7 +2,7 @@ Description=Outgoing Chatmail Postfix before queue filter [Service] -ExecStart={execpath} {config_path} outgoing +ExecStart={{ bin_path }} {{ config_path }} outgoing Restart=always RestartSec=30 User=vmail diff --git a/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py b/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py index e0350fb99..c7baaa278 100644 --- a/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py +++ b/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py @@ -189,17 +189,23 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config): mail = maildata( "encrypted.eml", from_addr=user1.addr, to_addr=user2.addr ).as_string() - for i in range(chatmail_config.max_user_send_per_minute + 5): + + timestamps = [] + i = 0 + while len(timestamps) <= chatmail_config.max_user_send_per_minute * 1.7: print("Sending mail", str(i)) + i += 1 try: user1.smtp.sendmail(user1.addr, [user2.addr], mail) + timestamps.append(time.time()) except smtplib.SMTPException as e: - if i < chatmail_config.max_user_send_per_minute: + if len(timestamps) < chatmail_config.max_user_send_per_minute: pytest.fail(f"rate limit was exceeded too early with msg {i}") outcome = e.recipients[user2.addr] assert outcome[0] == 450 assert b"4.7.1: Too much mail from" in outcome[1] return + timestamps[:] = [ts for ts in timestamps if ts >= (time.time() - 60)] pytest.fail("Rate limit was not exceeded")