From b2a28cd2504b18e390e303ee4eb8563190890960 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 22 Feb 2024 18:29:17 +0100 Subject: [PATCH 1/5] Typo --- metrics/fastly/cron.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metrics/fastly/cron.sh b/metrics/fastly/cron.sh index d0dd7f4b..b0ec5de1 100755 --- a/metrics/fastly/cron.sh +++ b/metrics/fastly/cron.sh @@ -7,9 +7,9 @@ now=$(date +%s) prev_week=$((($now / 86400 / 7))) from_date_incl=$(date +%F --date="@$(($prev_week * 86400 * 7 - 2 * 86400))") -to_date_incl=$(date +%F --date="@$(($prev_week * 86400 * 7 + 5 * 86400))") +to_date_excl=$(date +%F --date="@$(($prev_week * 86400 * 7 + 5 * 86400))") -echo "Ingesting [$from_date_incl, $to_date_incl)." +echo "Ingesting [$from_date_incl, $to_date_excl)." marker="$HOME/weeks-done/$prev_week" @@ -21,6 +21,6 @@ fi mkdir -p "$(dirname "$marker")" touch $marker -./ingest-raw-logs.sh "$from_date_incl" "$to_date_incl" +./ingest-raw-logs.sh "$from_date_incl" "$to_date_excl" ./run-queries.sh From 230f4caeb3f198ef5f4fd28cf500e10a8bf6e4f4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 22 Feb 2024 20:52:09 +0100 Subject: [PATCH 2/5] Turn flake.nix into a module --- metrics/fastly/flake.nix | 28 ------------------------- metrics/fastly/process-raw-nix-logs.nix | 25 ++++++++++++++++++++++ 2 files changed, 25 insertions(+), 28 deletions(-) delete mode 100644 metrics/fastly/flake.nix create mode 100644 metrics/fastly/process-raw-nix-logs.nix diff --git a/metrics/fastly/flake.nix b/metrics/fastly/flake.nix deleted file mode 100644 index 8c276421..00000000 --- a/metrics/fastly/flake.nix +++ /dev/null @@ -1,28 +0,0 @@ -{ - outputs = { self, nixpkgs }: { - nixosModules.nix-metrics = { config, pkgs, ... }: { - - users.users.nix-metrics = - { isNormalUser = true; - description = "Nix Metrics Collection"; - }; - - systemd.services.process-raw-nix-logs = { - description = "Process Raw nixos.org Logs"; - serviceConfig.Type = "oneshot"; - serviceConfig.User = "nix-metrics"; - path = - [ pkgs.awscli - pkgs.jq - ]; - script = - '' - cd ${./.} - ./cron.sh - ''; - startAt = "Tue 07:30"; - }; - - }; - }; -} diff --git a/metrics/fastly/process-raw-nix-logs.nix b/metrics/fastly/process-raw-nix-logs.nix new file mode 100644 index 00000000..793b28d2 --- /dev/null +++ b/metrics/fastly/process-raw-nix-logs.nix @@ -0,0 +1,25 @@ +{ config, pkgs, ... }: { + + users.users.nix-metrics = + { isNormalUser = true; + description = "Nix Metrics Collection"; + }; + + systemd.services.process-raw-nix-logs = { + description = "Process Raw nixos.org Logs"; + serviceConfig.Type = "oneshot"; + serviceConfig.User = "nix-metrics"; + path = + [ pkgs.awscli + pkgs.jq + ]; + script = + '' + cd ${./.} + ./cron.sh + ''; + startAt = "Tue 07:30"; + }; + +} + From 0d98ce1453e540f842c1758d7d75dc476372b542 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 22 Feb 2024 20:52:43 +0100 Subject: [PATCH 3/5] Give the fastly-log-processor user access to the Fastly logs --- terraform-iam/fastlylog/main.tf | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/terraform-iam/fastlylog/main.tf b/terraform-iam/fastlylog/main.tf index 79524003..64587852 100644 --- a/terraform-iam/fastlylog/main.tf +++ b/terraform-iam/fastlylog/main.tf @@ -31,19 +31,17 @@ resource "aws_s3_bucket_policy" "logs" { "Sid": "AllowNixOSOrgRead", "Effect": "Allow", "Principal": { - "AWS": "arn:aws:iam::008826681144:user/eelco.dolstra" + "AWS": "arn:aws:iam::008826681144:user/eelco.dolstra", + "AWS": "arn:aws:iam::008826681144:user/fastly-log-processor" }, - "Action": "s3:GetObject", - "Resource": "arn:aws:s3:::${aws_s3_bucket.logs.id}/*" - }, - { - "Sid": "AllowNixOSOrgList", - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam::008826681144:user/eelco.dolstra" - }, - "Action": "s3:ListBucket", - "Resource": "arn:aws:s3:::${aws_s3_bucket.logs.id}" + "Action": [ + "s3:GetObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::${aws_s3_bucket.logs.id}/*", + "arn:aws:s3:::${aws_s3_bucket.logs.id}" + ] } ] } From 5256d866e919e64bf23ff5a34b30a959b2d616c2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 22 Feb 2024 20:55:03 +0100 Subject: [PATCH 4/5] pluto: Enable the Fastly log processor --- delft/pluto/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/delft/pluto/default.nix b/delft/pluto/default.nix index 780ecb20..eb40d7e8 100644 --- a/delft/pluto/default.nix +++ b/delft/pluto/default.nix @@ -9,6 +9,7 @@ ../../modules/netboot-serve.nix ../../modules/rfc39.nix ../../modules/tarball-mirror.nix + ../../metrics/fastly/process-raw-nix-logs.nix ]; networking = { From 2e71b2cc6259b2e0d4a83a6670e2d5f7d9b488b1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 22 Feb 2024 21:00:18 +0100 Subject: [PATCH 5/5] Use the fastly-log-processor user --- metrics/fastly/cron.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/fastly/cron.sh b/metrics/fastly/cron.sh index b0ec5de1..1daa6b1d 100755 --- a/metrics/fastly/cron.sh +++ b/metrics/fastly/cron.sh @@ -1,6 +1,6 @@ #! /bin/sh -e -export AWS_PROFILE=nixos-org +export AWS_PROFILE=fastly-log-processor now=$(date +%s) #now=$((now - 86400))