diff --git a/.github/workflows/tests.yml b/.github/workflows/tests-go.yml similarity index 95% rename from .github/workflows/tests.yml rename to .github/workflows/tests-go.yml index b3ae2a4..f946ba6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests-go.yml @@ -1,5 +1,5 @@ --- -name: Run tests +name: Run tests (Go) on: push: diff --git a/.github/workflows/tests-python.yml b/.github/workflows/tests-python.yml new file mode 100644 index 0000000..5d24703 --- /dev/null +++ b/.github/workflows/tests-python.yml @@ -0,0 +1,32 @@ +--- +name: Run tests (Python) +on: + push: + workflow_dispatch: + pull_request: +jobs: + pytest: + strategy: + fail-fast: false + matrix: + cfg: + - component: "lambda/entry_checker" + runs-on: ubuntu-latest + env: + python_version: "3.12" + + steps: + - uses: actions/checkout@v4 + - name: Install poetry + run: pipx install poetry + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.python_version }} + cache: "poetry" + cache-dependency-path: | + ${{ matrix.cfg.component }}/pyproject.toml + ${{ matrix.cfg.component }}/poetry.lock + - name: Run pytest + run: | + cd ${{ matrix.cfg.component }} + poetry run pytest diff --git a/README.md b/README.md index 3b5fae6..c1c4773 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ This repository consists of two parts: Terraform module is provided in `terraform` directory. It can be used to create necessary components on AWS's end. -It's required to set up [SES email receiving](https://docs.aws.amazon.com/ses/latest/dg/receiving-email-setting-up.html) first and +It's required to set up [SES email receiving](https://docs.aws.amazon.com/ses/latest/dg/receiving-email-setting-up.html) first, configuring this part is outside the scope of this project. The minimum to get started is: @@ -55,6 +55,14 @@ The minimum to get started is: recipients = ["some-email@your-domain.com", "yet-another-email@your-domain.com"] ``` +It's also recommended (but optional) to set a regex describing allowed senders. +Emails sent from different addresses will simply be ignored. +It can be set as below: + +``` +senders = ".*@example.com" +``` + 2. run `terraform init` 3. run `terraform apply` 4. plug in the values obtained via `terraform output` as env variables in the following section diff --git a/lambda/entry_checker/.gitignore b/lambda/entry_checker/.gitignore new file mode 100644 index 0000000..f4c749f --- /dev/null +++ b/lambda/entry_checker/.gitignore @@ -0,0 +1,5 @@ +venv +.venv +_pycache_ + +payload diff --git a/lambda/entry_checker/README.md b/lambda/entry_checker/README.md new file mode 100644 index 0000000..e69de29 diff --git a/lambda/entry_checker/entry_checker/__init__.py b/lambda/entry_checker/entry_checker/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lambda/entry_checker/entry_checker/main.py b/lambda/entry_checker/entry_checker/main.py new file mode 100644 index 0000000..8a68f9f --- /dev/null +++ b/lambda/entry_checker/entry_checker/main.py @@ -0,0 +1,45 @@ +import logging +import os +from entry_checker.validate import validate_sender + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +def main(event, context): + allowed_senders_regex = os.environ["ALLOWED_SENDERS_REGEX"] + + ses_notification = event["Records"].pop()["ses"] + receipt = ses_notification["receipt"] + + logger.debug(receipt) + + stop_email = False + + if receipt["spfVerdict"]["status"] == "FAIL": + logger.debug("SPF FAIL detected, cutting off") + stop_email = True + + elif receipt["dkimVerdict"]["status"] == "FAIL": + logger.debug("DKIM FAIL detected, cutting off") + stop_email = True + + elif receipt["spamVerdict"]["status"] == "FAIL": + logger.debug("SPAM FAIL detected, cutting off") + stop_email = True + + elif receipt["virusVerdict"]["status"] == "FAIL": + logger.debug("VIRUS FAIL detected, cutting off") + stop_email = True + + else: + # Such validation is not perfect, but paired with the above conditions + # that rely on Amazon checks, it's definitely better than nothing. + sender = ses_notification["mail"]["commonHeaders"]["from"].pop() + + if not validate_sender(allowed_senders_regex, sender): + logger.debug("sender validation FAIL detected, cutting off") + stop_email = True + + if stop_email: + return {"disposition": "STOP_RULE_SET"} diff --git a/lambda/entry_checker/entry_checker/validate.py b/lambda/entry_checker/entry_checker/validate.py new file mode 100644 index 0000000..bf05ac9 --- /dev/null +++ b/lambda/entry_checker/entry_checker/validate.py @@ -0,0 +1,11 @@ +import re + + +def validate_sender(allowed_senders_regex: str, sender: str): + # Get rid of the "quotation marks" + sender = sender.replace("<", "").replace(">", "") + + if re.search(allowed_senders_regex, sender): + return True + + return False diff --git a/lambda/entry_checker/poetry.lock b/lambda/entry_checker/poetry.lock new file mode 100644 index 0000000..88a0bc7 --- /dev/null +++ b/lambda/entry_checker/poetry.lock @@ -0,0 +1,74 @@ +# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "packaging" +version = "24.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, + {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, +] + +[[package]] +name = "pluggy" +version = "1.5.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pytest" +version = "8.3.4" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-8.3.4-py3-none-any.whl", hash = "sha256:50e16d954148559c9a74109af1eaf0c945ba2d8f30f0a3d3335edde19788b6f6"}, + {file = "pytest-8.3.4.tar.gz", hash = "sha256:965370d062bce11e73868e0335abac31b4d3de0e82f4007408d242b4f8610761"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=1.5,<2" + +[package.extras] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.12" +content-hash = "2d6527c71b22d0e88bbba31dad63ac44bede7d9535395740a0a190eaa31c46f7" diff --git a/lambda/entry_checker/pyproject.toml b/lambda/entry_checker/pyproject.toml new file mode 100644 index 0000000..4e2ca29 --- /dev/null +++ b/lambda/entry_checker/pyproject.toml @@ -0,0 +1,16 @@ +[tool.poetry] +name = "entry-checker" +version = "0.1.0" +description = "" +authors = ["dezeroku "] +readme = "README.md" + +[tool.poetry.dependencies] +python = "^3.12" + +[tool.poetry.group.dev.dependencies] +pytest = "^8.3.4" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/lambda/entry_checker/tests/__init__.py b/lambda/entry_checker/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lambda/entry_checker/tests/validate_test.py b/lambda/entry_checker/tests/validate_test.py new file mode 100644 index 0000000..d5d4d5d --- /dev/null +++ b/lambda/entry_checker/tests/validate_test.py @@ -0,0 +1,17 @@ +from entry_checker.validate import validate_sender + +import pytest + +allowed_senders_regex = ".*@example.com" + + +@pytest.mark.parametrize("sender", ["test@example.com", ""]) +def test_validate_sender_success(sender): + assert validate_sender(allowed_senders_regex, sender) + + +@pytest.mark.parametrize( + "sender", ["test@different-domain.com", ""] +) +def test_validate_sender_fail(sender): + assert not validate_sender(allowed_senders_regex, sender) diff --git a/terraform/.gitignore b/terraform/.gitignore index ed3bc53..7c78b61 100644 --- a/terraform/.gitignore +++ b/terraform/.gitignore @@ -1,3 +1,4 @@ .terraform* terraform* !.terraform.lock.hcl +!terraform.tfvars.development diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl index 9c7989c..4af8768 100644 --- a/terraform/.terraform.lock.hcl +++ b/terraform/.terraform.lock.hcl @@ -1,25 +1,44 @@ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. +provider "registry.terraform.io/hashicorp/archive" { + version = "2.7.0" + hashes = [ + "h1:YkXq4JfcoAW0L4B9ghskZUxYbYAXIPlfSqqVFrAS06U=", + "zh:04e23bebca7f665a19a032343aeecd230028a3822e546e6f618f24c47ff87f67", + "zh:5bb38114238e25c45bf85f5c9f627a2d0c4b98fe44a0837e37d48574385f8dad", + "zh:64584bc1db4c390abd81c76de438d93acf967c8a33e9b923d68da6ed749d55bd", + "zh:697695ab9cce351adf91a1823bdd72ce6f0d219138f5124ef7645cedf8f59a1f", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7edefb1d1e2fead8fd155f7b50a2cb49f2f3fed154ac3ef5f991ccaff93d6120", + "zh:807fb15b75910bf14795f2ad1a2d41b069f9ef52c242131b2964c8527312e235", + "zh:821d9148d261df1d1a8e5a4812df2a6a3ffaf0d2070dad3c785382e489069239", + "zh:a7d92251118fb723048c482154a6ac6368aad583d28d15fffc6f5dafd9507463", + "zh:b627d4cef192b3c12ddaf9cb2c4f98c10d0129883c8c2a9c0049983f9de7030d", + "zh:dfb70306fcc0ad1d512ab7c24765703783cc286062d4849de4fbe23526f5dc8e", + "zh:f21de276f857b7e51fa2593d8fef05a7faafb0a7b62db14ac58a03ce1be7d881", + ] +} + provider "registry.terraform.io/hashicorp/aws" { - version = "5.69.0" + version = "5.82.2" constraints = "~> 5.0" hashes = [ - "h1:Ccpjmuu4G6k6ET0yf9lfhRywN7GBAAxR4rfTl5aY5+U=", - "zh:123af8815a80abfd62eab5f9fc3d9226735cfea3627e834a1b48321cd8d391a6", - "zh:1298f312e239768c1846541e89b4fbec7eb21769c4a488c87181909049219fbe", - "zh:4edc950b39f3653beb8cd3e0b86a7dc9b6a77e90e543ed7be72639107bbc48a9", - "zh:5f24c916d6d2ce51e18210628b3b1aca8b85b383982a920b2a6adc259bdbd4e9", - "zh:66f0b2f5869a4dfed7154444c272022c6d9350dc4dfa0fc6d87ccbfc983ec560", - "zh:67e3be60863cf1c51c5be866d8646d433cc31e07514b9121611f812e73f2400d", - "zh:884672345a1d0362644a4d1588085fd4c4f56d3ca61b10c0d25cd1940d828fec", - "zh:8ab0f92da124171c80a2361beb79822fb0f074ffab74e506f58e953a69b283ce", - "zh:908d879139f2246024b5510a38f00f61489eeee6f3f72be10acc5b424c8fc723", + "h1:RuPaHbllUB8a2TGTyc149wJfoh6zhIEjUvFYKR6iP2E=", + "zh:0262fc96012fb7e173e1b7beadd46dfc25b1dc7eaef95b90e936fc454724f1c8", + "zh:397413613d27f4f54d16efcbf4f0a43c059bd8d827fe34287522ae182a992f9b", + "zh:436c0c5d56e1da4f0a4c13129e12a0b519d12ab116aed52029b183f9806866f3", + "zh:4d942d173a2553d8d532a333a0482a090f4e82a2238acf135578f163b6e68470", + "zh:624aebc549bfbce06cc2ecfd8631932eb874ac7c10eb8466ce5b9a2fbdfdc724", "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:9db6331398d648d9f2f4aa4db1eb9081e9bff584dcfe8f5350e04e6c5d339899", - "zh:a809bbd43bc392e91485b72bd9693874972bc5697b4f24fbcd61b461618ebb6d", - "zh:b9e9464458e7beb9fbf59f8db02f56138f398aaa6173b58a8bfa76aca82106d9", - "zh:cd7f041edaeeb1c4b06152ac8f3ce7b31c39a80a949083255f8fc81bbb11aeac", - "zh:eb71c9b2071ab2caa7aba577902df41c25ded1251c28560f0ac45f5e0f47360e", + "zh:9e632dee2dfdf01b371cca7854b1ec63ceefa75790e619b0642b34d5514c6733", + "zh:a07567acb115b60a3df8f6048d12735b9b3bcf85ec92a62f77852e13d5a3c096", + "zh:ab7002df1a1be6432ac0eb1b9f6f0dd3db90973cd5b1b0b33d2dae54553dfbd7", + "zh:bc1ff65e2016b018b3e84db7249b2cd0433cb5c81dc81f9f6158f2197d6b9fde", + "zh:bcad84b1d767f87af6e1ba3dc97fdb8f2ad5de9224f192f1412b09aba798c0a8", + "zh:cf917dceaa0f9d55d9ff181b5dcc4d1e10af21b6671811b315ae2a6eda866a2a", + "zh:d8e90ecfb3216f3cc13ccde5a16da64307abb6e22453aed2ac3067bbf689313b", + "zh:d9054e0e40705df729682ad34c20db8695d57f182c65963abd151c6aba1ab0d3", + "zh:ecf3a4f3c57eb7e89f71b8559e2a71e4cdf94eea0118ec4f2cb37e4f4d71a069", ] } diff --git a/terraform/bucket.tf b/terraform/bucket.tf index af0b083..de9a231 100644 --- a/terraform/bucket.tf +++ b/terraform/bucket.tf @@ -10,10 +10,10 @@ resource "aws_s3_bucket" "bucket" { resource "aws_s3_bucket_policy" "allow_ses_access" { bucket = aws_s3_bucket.bucket.id - policy = data.aws_iam_policy_document.allow_ses_access.json + policy = data.aws_iam_policy_document.bucket_allow_ses_access.json } -data "aws_iam_policy_document" "allow_ses_access" { +data "aws_iam_policy_document" "bucket_allow_ses_access" { statement { principals { type = "Service" diff --git a/terraform/lambda_entry_checker.tf b/terraform/lambda_entry_checker.tf new file mode 100644 index 0000000..782bd50 --- /dev/null +++ b/terraform/lambda_entry_checker.tf @@ -0,0 +1,88 @@ +data "aws_iam_policy_document" "lambda_entry_checker_policy_assume_role" { + statement { + effect = "Allow" + + principals { + type = "Service" + identifiers = ["lambda.amazonaws.com"] + } + + actions = ["sts:AssumeRole"] + } +} + +resource "aws_iam_role" "lambda_entry_checker" { + name = var.lambda_entry_checker_iam_role_name + assume_role_policy = data.aws_iam_policy_document.lambda_entry_checker_policy_assume_role.json +} + +data "aws_iam_policy_document" "lambda_entry_checker_policy" { + statement { + actions = [ + "logs:CreateLogStream", + "logs:CreateLogGroup" + ] + resources = ["arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:/aws/lambda/${var.lambda_entry_checker_function_name}:*"] + } + + statement { + actions = [ + "logs:PutLogEvents" + ] + resources = [ + "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:/aws/lambda/${var.lambda_entry_checker_function_name}:*:*" + ] + } +} + +resource "aws_iam_policy" "lambda_entry_checker_policy" { + name_prefix = "ses-local-email-lambda-entry-checker-" + policy = data.aws_iam_policy_document.lambda_entry_checker_policy.json +} + +resource "aws_iam_role_policy_attachment" "lambda_entry_checker_policy" { + role = aws_iam_role.lambda_entry_checker.name + policy_arn = aws_iam_policy.lambda_entry_checker_policy.arn +} + +resource "aws_lambda_permission" "lambda_entry_checker_allow_ses" { + statement_id = "AllowExecutionFromSES" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.lambda_entry_checker.function_name + principal = "ses.amazonaws.com" + source_account = data.aws_caller_identity.current.account_id + source_arn = "arn:aws:ses:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:receipt-rule-set/${aws_ses_receipt_rule_set.main.id}:receipt-rule/${var.ses_rule_name}" +} + +data "archive_file" "lambda_entry_checker_payload" { + type = "zip" + source_dir = var.lambda_entry_checker_src_dir + excludes = [ + "venv", + "_pycache_", + "payload" + ] + output_path = "${var.lambda_entry_checker_payload_dir}/payload.zip" +} + +resource "aws_lambda_function" "lambda_entry_checker" { + filename = data.archive_file.lambda_entry_checker_payload.output_path + source_code_hash = data.archive_file.lambda_entry_checker_payload.output_base64sha256 + function_name = var.lambda_entry_checker_function_name + role = aws_iam_role.lambda_entry_checker.arn + handler = "entry_checker/main.main" + runtime = "python3.12" + environment { + variables = { + ALLOWED_SENDERS_REGEX = var.senders_regex + } + } + # Same timeout as the default + timeout = 3 + depends_on = [aws_cloudwatch_log_group.lambda_entry_checker] +} + +resource "aws_cloudwatch_log_group" "lambda_entry_checker" { + name = "/aws/lambda/${var.lambda_entry_checker_function_name}" + retention_in_days = 14 +} diff --git a/terraform/providers.tf b/terraform/providers.tf index f22cb58..97a61b1 100644 --- a/terraform/providers.tf +++ b/terraform/providers.tf @@ -6,6 +6,10 @@ terraform { source = "hashicorp/aws" version = "~> 5.0" } + archive = { + source = "hashicorp/archive" + version = "~> 2.0" + } } } diff --git a/terraform/ses.tf b/terraform/ses.tf index c1f01c7..43f3a3b 100644 --- a/terraform/ses.tf +++ b/terraform/ses.tf @@ -8,7 +8,8 @@ resource "aws_ses_active_receipt_rule_set" "main" { resource "aws_ses_receipt_rule" "store" { depends_on = [ - aws_s3_bucket_policy.allow_ses_access + aws_s3_bucket_policy.allow_ses_access, + aws_lambda_permission.lambda_entry_checker_allow_ses ] name = var.ses_rule_name rule_set_name = aws_ses_receipt_rule_set.main.id @@ -16,8 +17,14 @@ resource "aws_ses_receipt_rule" "store" { enabled = true scan_enabled = true + lambda_action { + function_arn = aws_lambda_function.lambda_entry_checker.arn + invocation_type = "RequestResponse" + position = 1 + } + s3_action { bucket_name = aws_s3_bucket.bucket.id - position = 1 + position = 2 } } diff --git a/terraform/terraform.tfvars.development b/terraform/terraform.tfvars.development new file mode 100644 index 0000000..4cca8e1 --- /dev/null +++ b/terraform/terraform.tfvars.development @@ -0,0 +1,9 @@ +# Example set of variables that can be used for development (requires only domain changes), without conflicting with the prod deployment +# Please note that the SES ruleset will point exclusively to the development resources, +# thus causing downtime for production +recipients = ["develop@incoming.example.com"] +senders = "contact@example.com" +user_name = "ses-local-email-develop" +ses_rule_set_name = "ses-local-email-develop" +lambda_entry_checker_function_name = "ses-local-email-entry-checker-develop" +lambda_entry_checker_iam_role_name = "ses-local-email-entry-checker-develop" diff --git a/terraform/variables.tf b/terraform/variables.tf index c448281..03adc5d 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -36,6 +36,32 @@ variable "user_name" { default = "ses-local-email" } +variable "lambda_entry_checker_src_dir" { + type = string + default = "../lambda/entry_checker" +} + +variable "lambda_entry_checker_payload_dir" { + type = string + default = "../lambda/entry_checker/payload" +} + +variable "lambda_entry_checker_function_name" { + type = string + default = "ses-local-email-entry-checker" +} + +variable "lambda_entry_checker_iam_role_name" { + type = string + default = "ses-local-email-entry-checker" +} + +variable "senders_regex" { + type = string + description = "Regex describing allowed sender emails" + default = ".*" +} + # REQUIRED variable "recipients" { type = list(string)