Migrate the source code from the old repository

.container/Dockerfile

@@ -1,35 +1,47 @@
-FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build-env
+# The `platform` argument here is required, since dotnet-sdk crashes with segmentation fault 
+# in case of arm64 builds, see https://github.com/dotnet/dotnet-docker/issues/4225 for details
+FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0 AS build-env
 
-ARG PROJECT_NAME="Arcane.Ingestion"
+ARG INSTALL_DD_TRACER="true"
+ARG TRACER_VERSION="2.49.0"
+ARG TARGETARCH
 
 WORKDIR /app
 
 # Copy csproj and restore as distinct layers
 COPY src/*.csproj ./
-RUN dotnet restore
+RUN dotnet_arch=$(test "$TARGETARCH" = "amd64" && echo "x64" || echo "$TARGETARCH") && \
+    dotnet restore --runtime "linux-$dotnet_arch"
 
 # Copy everything else and build
 COPY src/. ./
-RUN dotnet publish "$PROJECT_NAME.csproj" -c Release -o out
+RUN dotnet_arch=$(test "$TARGETARCH" = "amd64" && echo "x64" || echo "$TARGETARCH") && \
+    dotnet publish "Arcane.Ingestion.csproj" -c Release -o out --runtime "linux-$dotnet_arch"
 
 # Build runtime image
-FROM mcr.microsoft.com/dotnet/aspnet:6.0-bullseye-slim
-LABEL org.opencontainers.image.source=https://github.com/SneaksAndData/arcane-ingestion
+FROM mcr.microsoft.com/dotnet/aspnet:8.0-bookworm-slim
 
-ARG TRACER_VERSION="2.32.0"
-ARG PROJECT_NAME
-ENV PROJECT_ASSEMBLY=$PROJECT_NAME
+ARG TRACER_VERSION="2.49.0"
+ARG INSTALL_DD_TRACER="true"
+ARG TARGETARCH
 
 RUN apt-get update -y && apt-get install -y curl jq
 
 # Download and install the Datadog Tracer
-RUN mkdir -p /opt/datadog \
-    && mkdir -p /var/log/datadog \
-    && curl -LO https://github.com/DataDog/dd-trace-dotnet/releases/download/v${TRACER_VERSION}/datadog-dotnet-apm_${TRACER_VERSION}_amd64.deb \
-    && dpkg -i ./datadog-dotnet-apm_${TRACER_VERSION}_amd64.deb \
-    && rm ./datadog-dotnet-apm_${TRACER_VERSION}_amd64.deb 
+RUN if [ -z "$INSTALL_DD_TRACER" ]; then \
+      echo "Datadog tracer installation skipped"; \
+    else \
+        mkdir -p /opt/datadog \
+        && echo $TARGETARCH \
+        && mkdir -p /var/log/datadog \
+        && curl -LO https://github.com/DataDog/dd-trace-dotnet/releases/download/v${TRACER_VERSION}/datadog-dotnet-apm_${TRACER_VERSION}_${TARGETARCH}.deb \
+        && dpkg -i ./datadog-dotnet-apm_${TRACER_VERSION}_${TARGETARCH}.deb \
+        && rm ./datadog-dotnet-apm_${TRACER_VERSION}_${TARGETARCH}.deb ; \
+    fi;
 
-
 WORKDIR /app
 COPY --from=build-env /app/out .
-ENTRYPOINT "dotnet" "$PROJECT_ASSEMBLY.dll"
+
+USER app
+
+ENTRYPOINT ["dotnet", "Arcane.Ingestion.dll"]

.github/workflows/build.yaml

@@ -5,35 +5,100 @@ on:
     branches: [ main ]
   pull_request:
     branches: [ main ]
-    
+
 env:
   PROJECT_NAME: Arcane.Ingestion
-
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
 jobs:
   validate_commit:
     name: Validate commit
     runs-on: ubuntu-latest
     if: ${{ github.ref != 'refs/heads/main' }}
+    permissions:
+      pull-requests: write
+      contents: read
     steps:
       - uses: actions/checkout@v4
-        
+
       - name: Setup .NET
         uses: actions/[email protected]
         with:
-          dotnet-version: 6.0.x
-          
+          dotnet-version: 8.0.x
+
       - name: Build
         run: dotnet build
-        
+
       - name: Test
         working-directory: ./test
         run: |
           dotnet add package coverlet.msbuild &&
           dotnet test ${PROJECT_NAME}.Tests.csproj --configuration Debug --runtime linux-x64 /p:CollectCoverage=true /p:CoverletOutput=Coverage/ /p:CoverletOutputFormat=lcov --logger GitHubActions
-          
+
       - name: Publish Code Coverage
         if: ${{ github.event_name == 'pull_request' && always() }}
-        uses: romeovs/lcov-reporter-action@v0.3.1
+        uses: romeovs/lcov-reporter-action@v0.4.0
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           lcov-file: ./test/Coverage/coverage.info
+
+  build_image:
+    name: Build Docker Image and Helm Charts
+    runs-on: ubuntu-latest
+    needs: [ validate_commit ]
+    if: ${{ always() && (needs.validate_commit.result == 'success' || needs.validate_commit.result == 'skipped') }}
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Log in to the Container registry
+        uses: docker/[email protected]
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get project version
+        uses: SneaksAndData/github-actions/[email protected]
+        id: version
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=semver,pattern={{version}},value=${{steps.version.outputs.version}}
+          flavor:
+            latest=false
+
+      - name: Set up Docker Buildx
+        uses: docker/[email protected]
+        with:
+          use: true
+          platforms: linux/arm64,linux/amd64
+
+      - name: Build and push Docker image
+        uses: docker/[email protected]
+        with:
+          context: .
+          file: .container/Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/arm64,linux/amd64
+
+      - name: Build and Push Chart
+        uses: SneaksAndData/github-actions/[email protected]
+        with:
+          application: ${{ github.event.repository.name }}
+          app_version: ${{ steps.meta.outputs.version }}
+          container_registry_user: ${{ github.actor }}
+          container_registry_token: ${{ secrets.GITHUB_TOKEN }}
+          container_registry_address: ghcr.io/sneaksanddata/

.github/workflows/cleanup-repository.yaml

@@ -0,0 +1,35 @@
+name: Remove old artifacts
+on:
+#  schedule:
+#   - cron: '0 12 * * *' # every day at 12:00 UTC
+  workflow_dispatch:
+
+jobs:
+  remove_old_artifacts:
+    name: Remove old artifacts
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      packages: write
+
+    timeout-minutes: 10 # stop the task if it takes longer
+
+    steps:
+      - name: Delete old package versions of ${{ github.repository }}
+        uses: actions/[email protected]
+        with:
+          package-name: ${{ github.repository }}
+          package-type: container
+          token: ${{ secrets.GITHUB_TOKEN }}
+          min-versions-to-keep: 10
+          delete-only-pre-release-versions: "true"
+
+      - name: Delete old package versions of helm/${{ github.repository }}
+        uses: actions/[email protected]
+        with:
+          package-name: helm/${{ github.repository }}
+          package-type: container
+          token: ${{ secrets.GITHUB_TOKEN }}
+          min-versions-to-keep: 10
+          delete-only-pre-release-versions: "true"

.github/workflows/publish-ecr-public.yaml

@@ -0,0 +1,92 @@
+name: Publish Arcane.Stream.BlobStorage to ECR public registry
+run-name: Publish Arcane.Stream.BlobStorage to public.ecr.aws by @${{ github.actor }}
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: |
+          Version number to publish. Defaults to the latest git tag in the repository. 
+          This version MUST exist in the ghcr.io registry.
+        required: false
+        default: "current"
+
+env:
+  PROJECT_NAME: Arcane.Stream.BlobStorage
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  publish_image:
+    name: Publish Docker Image to ECR Public
+    runs-on: ubuntu-latest
+    # if: ${{ startsWith(github.ref, 'refs/tags') }}
+
+    permissions:
+      contents: read
+      id-token: write
+
+    steps:
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+
+      - name: Get project version
+        uses: SneaksAndData/github-actions/[email protected]
+        id: current_version 
+
+      - name: Set up variables
+        env:
+          VERSION: ${{ inputs.version }}
+          CURRENT_VERSION: ${{steps.current_version.outputs.version}}
+        run: |
+          test "$VERSION" == "current" && echo "IMAGE_VERSION=$CURRENT_VERSION" >> ${GITHUB_ENV} || echo "IMAGE_VERSION=$VERSION" >> ${GITHUB_ENV} 
+
+      - name: Import AWS Secrets
+        uses: hashicorp/[email protected]
+        with:
+          url: https://hashicorp-vault.awsp.sneaksanddata.com/
+          role: github
+          method: jwt
+          secrets: |
+            /secret/data/common/package-publishing/aws-ecr-public/production/container-user-public access_key | ACCESS_KEY ;
+            /secret/data/common/package-publishing/aws-ecr-public/production/container-user-public access_key_id | ACCESS_KEY_ID ;
+        id: aws_secrets
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/[email protected]
+        with:
+          aws-access-key-id: ${{ env.ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ env.ACCESS_KEY }}
+          aws-region: us-east-1
+
+      - name: Log in to the GitHub Container Registry
+        uses: docker/[email protected]
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Login to Amazon ECR Public
+        uses: aws-actions/amazon-ecr-login@v2
+        with:
+          registry-type: public
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=semver,pattern={{version}},value=${{ env.IMAGE_VERSION }}
+          flavor:
+            latest=false
+
+      - name: Push image to ECR Public registry
+        uses: akhilerm/[email protected]
+        with:
+          src: ${{ steps.meta.outputs.tags }}
+          dst: public.ecr.aws/s0t1h2z6/arcane/${{ github.event.repository.name }}:${{ steps.meta.outputs.version }}

.github/workflows/release.yaml

@@ -13,7 +13,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Create Release
-        uses: SneaksAndData/github-actions/[email protected].6
+        uses: SneaksAndData/github-actions/[email protected].9
         with:
           major_v: 0
           minor_v: 0

.helm/Chart.yaml

@@ -1,7 +1,8 @@
 apiVersion: v2
 name: arcane-ingestion
 description: |
-  Arcane is a Akka.NET-based data ingestion service that can be used to ingest data from various sources and store it in a data lake.
+  Arcane is a Akka.NET-based data ingestion service that can be used to ingest
+  data from various sources and store it in a data lake.
   This chart deploys the Arcane Ingestion service.
 type: application
 version: 0.0.0

.helm/templates/_helpers.tpl

@@ -56,3 +56,9 @@ Create the name of the service account to use
 {{- default "default" .Values.serviceAccount.name }}
 {{- end }}
 {{- end }}
+
+{{- define "app.tracerIngestionLabels" -}}
+tags.datadoghq.com/env: {{ .Values.environment }}
+tags.datadoghq.com/service: "arcane-ingestion"
+tags.datadoghq.com/version: {{ .Values.image.tag }}
+{{- end -}}

.helm/templates/configmap-appsettings.yaml

@@ -8,12 +8,14 @@ metadata:
 data:
   appsettings.json: |
     {
-      "Logging": {
-        "LogLevel": {
-          "Default": "Information",
-          "Microsoft.AspNetCore": "Warning"
-        }
+      "AllowedHosts": "*",
+      "JsonIngestionConfiguration": {
+        "BufferSize": {{ .Values.settings.bufferSize }},
+        "ThrottleDocumentLimit": {{ .Values.settings.throttleDocumentLimit }},
+        "ThrottleDocumentBurst": {{ .Values.settings.throttleDocumentBurst }},
+        "ThrottleTimespan": {{ .Values.settings.throttleTimespan | quote }},
+        "MaxDocumentsPerFile": {{ .Values.settings.maxDocumentsPerFile }},
+        "GroupingInterval": {{ .Values.settings.groupingInterval | quote }},
+        "IngestionSinkPath": {{ .Values.settings.ingestionSinkPath | quote }}
       },
-      "AllowedHosts": "*"
     }
-