From 60443c754b7dc4d1c70a5254b3fc067f0e0065f8 Mon Sep 17 00:00:00 2001 From: Vitalii Savitskii Date: Wed, 25 Sep 2024 16:28:39 +0200 Subject: [PATCH] Add required project files and GitHub pipelines (#6) * Add required project files and GitHub pipelines * Also fix dockerfile * Fix project name * Fix helm chart name --- .container/Dockerfile | 47 +++--- .github/workflows/build.yaml | 138 +++++++----------- .github/workflows/cleanup-repository.yaml | 35 +++++ .github/workflows/deploy.yaml | 82 ----------- .github/workflows/publish-ecr-public.yaml | 92 ++++++++++++ .github/workflows/release.yaml | 6 +- .helm/Chart.yaml | 2 +- ...Project.sln => Arcane.Stream.SqlServer.sln | 4 +- ....csproj => Arcane.Stream.SqlServer.csproj} | 4 +- src/Models/SqlServerStreamContext.cs | 82 +++++++++++ src/Program.cs | 53 ++++++- src/Services/SqlServerGraphBuilder.cs | 87 +++++++++++ ...j => Arcane.Stream.SqlServer.Tests.csproj} | 4 +- 13 files changed, 438 insertions(+), 198 deletions(-) create mode 100644 .github/workflows/cleanup-repository.yaml delete mode 100644 .github/workflows/deploy.yaml create mode 100644 .github/workflows/publish-ecr-public.yaml rename DotnetProject.sln => Arcane.Stream.SqlServer.sln (74%) rename src/{DotnetProject.csproj => Arcane.Stream.SqlServer.csproj} (56%) create mode 100644 src/Models/SqlServerStreamContext.cs create mode 100644 src/Services/SqlServerGraphBuilder.cs rename test/{DotnetProjectTests.csproj => Arcane.Stream.SqlServer.Tests.csproj} (90%) diff --git a/.container/Dockerfile b/.container/Dockerfile index bdbb63d..73d95be 100644 --- a/.container/Dockerfile +++ b/.container/Dockerfile @@ -1,34 +1,47 @@ -FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build-env -ARG NUGET_TOKEN -ARG PROJECT_NAME +# The `platform` argument here is required, since dotnet-sdk crashes with segmentation fault +# in case of arm64 builds, see https://github.com/dotnet/dotnet-docker/issues/4225 for details +FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0 AS build-env + +ARG INSTALL_DD_TRACER="true" +ARG TRACER_VERSION="2.49.0" +ARG TARGETARCH WORKDIR /app # Copy csproj and restore as distinct layers COPY src/*.csproj ./ -RUN dotnet nuget add source --username USERNAME --password $NUGET_TOKEN --store-password-in-clear-text --name github "https://nuget.pkg.github.com/SneaksAndData/index.json" -RUN dotnet restore +RUN dotnet_arch=$(test "$TARGETARCH" = "amd64" && echo "x64" || echo "$TARGETARCH") && \ + dotnet restore --runtime "linux-$dotnet_arch" # Copy everything else and build COPY src/. ./ -RUN dotnet publish "$PROJECT_NAME.csproj" -c Release -o out +RUN dotnet_arch=$(test "$TARGETARCH" = "amd64" && echo "x64" || echo "$TARGETARCH") && \ + dotnet publish "Arcane.Stream.SqlServer.csproj" -c Release -o out --runtime "linux-$dotnet_arch" # Build runtime image -FROM mcr.microsoft.com/dotnet/aspnet:6.0-bullseye-slim -ARG TRACER_VERSION="2.32.0" -ARG PROJECT_NAME -ENV PROJECT_ASSEMBLY=$PROJECT_NAME +FROM mcr.microsoft.com/dotnet/aspnet:8.0-bookworm-slim + +ARG TRACER_VERSION="2.49.0" +ARG INSTALL_DD_TRACER="true" +ARG TARGETARCH RUN apt-get update -y && apt-get install -y curl jq # Download and install the Datadog Tracer -RUN mkdir -p /opt/datadog \ - && mkdir -p /var/log/datadog \ - && curl -LO https://github.com/DataDog/dd-trace-dotnet/releases/download/v${TRACER_VERSION}/datadog-dotnet-apm_${TRACER_VERSION}_amd64.deb \ - && dpkg -i ./datadog-dotnet-apm_${TRACER_VERSION}_amd64.deb \ - && rm ./datadog-dotnet-apm_${TRACER_VERSION}_amd64.deb +RUN if [ -z "$INSTALL_DD_TRACER" ]; then \ + echo "Datadog tracer installation skipped"; \ + else \ + mkdir -p /opt/datadog \ + && echo $TARGETARCH \ + && mkdir -p /var/log/datadog \ + && curl -LO https://github.com/DataDog/dd-trace-dotnet/releases/download/v${TRACER_VERSION}/datadog-dotnet-apm_${TRACER_VERSION}_${TARGETARCH}.deb \ + && dpkg -i ./datadog-dotnet-apm_${TRACER_VERSION}_${TARGETARCH}.deb \ + && rm ./datadog-dotnet-apm_${TRACER_VERSION}_${TARGETARCH}.deb ; \ + fi; - WORKDIR /app COPY --from=build-env /app/out . -ENTRYPOINT "dotnet" "$PROJECT_ASSEMBLY.dll" + +USER app + +ENTRYPOINT ["dotnet", "Arcane.Stream.SqlServer.dll"] diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9270649..ed13df6 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -6,10 +6,10 @@ on: pull_request: branches: [ main ] -# ! Replace DotnetProject and dotnet-project with project name in real repository env: - PROJECT_NAME: DotnetProject - PROJECT_NAME_LOWER: dotnet-project + PROJECT_NAME: Arcane.Stream.SqlServer + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} jobs: validate_commit: @@ -23,115 +23,83 @@ jobs: steps: - uses: actions/checkout@v4 - name: Setup .NET - uses: actions/setup-dotnet@v4.0.0 + uses: actions/setup-dotnet@v4.0.1 with: dotnet-version: 6.0.x - name: Restore dependencies env: NUGET_TOKEN: ${{ secrets.NUGET_TOKEN }} - run: | - set -e - dotnet nuget add source --username USERNAME --password $NUGET_TOKEN --store-password-in-clear-text --name github "https://nuget.pkg.github.com/SneaksAndData/index.json" - dotnet clean && dotnet nuget locals all --clear - dotnet restore + run: dotnet restore - name: Build run: dotnet build --no-restore - name: Test working-directory: ./test run: | dotnet add package coverlet.msbuild && - dotnet test ${PROJECT_NAME}Tests.csproj --configuration Debug --runtime linux-x64 /p:CollectCoverage=true /p:CoverletOutput=Coverage/ /p:CoverletOutputFormat=lcov --logger GitHubActions + dotnet test ${PROJECT_NAME}.Tests.csproj --configuration Debug --runtime linux-x64 /p:CollectCoverage=true /p:CoverletOutput=Coverage/ /p:CoverletOutputFormat=lcov --logger GitHubActions - name: Publish Code Coverage if: ${{ github.event_name == 'pull_request' && always() }} - uses: romeovs/lcov-reporter-action@v0.3.1 + uses: romeovs/lcov-reporter-action@v0.4.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} lcov-file: ./test/Coverage/coverage.info - build_image_and_chart: + build_image: name: Build Docker Image and Helm Charts runs-on: ubuntu-latest needs: [ validate_commit ] - # Remove the line below and uncomment the next one - if: ${{ false }} - # if: ${{ always() && (needs.validate_commit.result == 'success' || needs.validate_commit.result == 'skipped') }} + if: ${{ always() && (needs.validate_commit.result == 'success' || needs.validate_commit.result == 'skipped') }} permissions: contents: read - id-token: write + packages: write steps: - - uses: actions/checkout@v4 - name: Checkout head commit - if: ${{ github.ref != 'refs/heads/main' && always() }} + - name: Checkout repository + uses: actions/checkout@v4 with: fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} - - uses: actions/checkout@v4 - name: Checkout main - if: ${{ github.ref == 'refs/heads/main' && always() }} + + - name: Log in to the Container registry + uses: docker/login-action@v3.3.0 with: - fetch-depth: 0 - - name: Import Secrets (DEV) - uses: hashicorp/vault-action@v2.7.4 + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Get project version + uses: SneaksAndData/github-actions/generate_version@v0.1.9 + id: version + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 with: - url: https://hashicorp-vault.production.sneaksanddata.com/ - role: github - method: jwt - secrets: | - /secret/data/applications/${{ env.PROJECT_NAME_LOWER }}/test/build acr_user ; - /secret/data/applications/${{ env.PROJECT_NAME_LOWER }}/test/build acr_name ; - /secret/data/applications/${{ env.PROJECT_NAME_LOWER }}/test/build acr_token ; - id: vault_secrets_dev - - name: Build and Push Image (DEV) - env: - AZCR_USER: ${{steps.vault_secrets_dev.outputs.acr_user}} - AZCR_TOKEN: ${{steps.vault_secrets_dev.outputs.acr_token}} - AZCR_REPO: ${{steps.vault_secrets_dev.outputs.acr_name}} - NUGET_TOKEN: ${{ secrets.NUGET_TOKEN }} - run: | - set -e - echo "$AZCR_TOKEN" | docker login $AZCR_REPO.azurecr.io --username $AZCR_USER --password-stdin - version=$(git describe --tags --abbrev=7) - docker build -f .container/Dockerfile . \ - --tag=$AZCR_REPO.azurecr.io/$PROJECT_NAME_LOWER:$version \ - --build-arg NUGET_TOKEN=$NUGET_TOKEN \ - --build-arg PROJECT_NAME=$PROJECT_NAME && \ - docker push $AZCR_REPO.azurecr.io/$PROJECT_NAME_LOWER:$version - - name: Build and Push Chart (DEV) - uses: SneaksAndData/github-actions/build_helm_chart@v0.1.6 + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=semver,pattern={{version}},value=${{steps.version.outputs.version}} + flavor: + latest=false + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.6.1 with: - application: ${{ env.PROJECT_NAME_LOWER }} - container_registry_user: ${{steps.vault_secrets_dev.outputs.acr_user}} - container_registry_token: ${{steps.vault_secrets_dev.outputs.acr_token}} - container_registry_address: ${{steps.vault_secrets_dev.outputs.acr_name}}.azurecr.io - - name: Import Secrets (PROD) - uses: hashicorp/vault-action@v2.7.4 - if: ${{ github.ref == 'refs/heads/main' }} + use: true + platforms: linux/arm64,linux/amd64 + + - name: Build and push Docker image + uses: docker/build-push-action@v6.7.0 with: - url: https://hashicorp-vault.production.sneaksanddata.com/ - role: github - method: jwt - secrets: | - /secret/data/applications/${{ env.PROJECT_NAME_LOWER }}/production/build acr_user ; - /secret/data/applications/${{ env.PROJECT_NAME_LOWER }}/production/build acr_name ; - /secret/data/applications/${{ env.PROJECT_NAME_LOWER }}/production/build acr_token ; - id: vault_secrets_production - - name: Push Image (PROD) - if: ${{ github.ref == 'refs/heads/main' }} - env: - AZCR_USER: ${{steps.vault_secrets_production.outputs.acr_user}} - AZCR_TOKEN: ${{steps.vault_secrets_production.outputs.acr_token}} - AZCR_REPO: ${{steps.vault_secrets_production.outputs.acr_name}} - AZCR_DEV_REPO: ${{steps.vault_secrets_dev.outputs.acr_name}} - run: | - set -e - echo "$AZCR_TOKEN" | docker login $AZCR_REPO.azurecr.io --username $AZCR_USER --password-stdin - version=$(git describe --tags --abbrev=7) - docker tag $AZCR_DEV_REPO.azurecr.io/$PROJECT_NAME_LOWER:$version $AZCR_REPO.azurecr.io/$PROJECT_NAME_LOWER:$version && docker push $AZCR_REPO.azurecr.io/$PROJECT_NAME_LOWER:$version - - name: Build and Push Chart (PROD) - if: ${{ github.ref == 'refs/heads/main' }} - uses: SneaksAndData/github-actions/build_helm_chart@v0.1.6 + context: . + file: .container/Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/arm64,linux/amd64 + + - name: Build and Push Chart + uses: SneaksAndData/github-actions/build_helm_chart@v0.1.9 with: - application: ${{ env.PROJECT_NAME_LOWER }} - container_registry_user: ${{steps.vault_secrets_production.outputs.acr_user}} - container_registry_token: ${{steps.vault_secrets_production.outputs.acr_token}} - container_registry_address: ${{steps.vault_secrets_production.outputs.acr_name}}.azurecr.io + application: arcane-stream-sqlserver + app_version: ${{ steps.meta.outputs.version }} + container_registry_user: ${{ github.actor }} + container_registry_token: ${{ secrets.GITHUB_TOKEN }} + container_registry_address: ghcr.io/sneaksanddata/ diff --git a/.github/workflows/cleanup-repository.yaml b/.github/workflows/cleanup-repository.yaml new file mode 100644 index 0000000..262aefa --- /dev/null +++ b/.github/workflows/cleanup-repository.yaml @@ -0,0 +1,35 @@ +name: Remove old artifacts +on: + # schedule: + # - cron: '0 12 * * *' # every day at 12:00 UTC + workflow_dispatch: + +jobs: + remove_old_artifacts: + name: Remove old artifacts + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + + timeout-minutes: 10 # stop the task if it takes longer + + steps: + - name: Delete old package versions of ${{ github.event.repository.name }} + uses: actions/delete-package-versions@v5.0.0 + with: + package-name: ${{ github.event.repository.name }} + package-type: container + token: ${{ secrets.GITHUB_TOKEN }} + min-versions-to-keep: 10 + delete-only-pre-release-versions: "true" + + - name: Delete old package versions of helm/${{ github.event.repository.name }} + uses: actions/delete-package-versions@v5.0.0 + with: + package-name: helm/${{ github.event.repository.name }} + package-type: container + token: ${{ secrets.GITHUB_TOKEN }} + min-versions-to-keep: 10 + delete-only-pre-release-versions: "true" diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml deleted file mode 100644 index c2e2194..0000000 --- a/.github/workflows/deploy.yaml +++ /dev/null @@ -1,82 +0,0 @@ -name: Deploy to AKS -run-name: Deploy ${{github.ref_name}} to ${{ inputs.environment }} by @${{ github.actor }} - -permissions: - pull-requests: write - contents: read - -on: - workflow_dispatch: - inputs: - environment: - description: Deployment target - required: true - type: environment - default: test -# ! Replace DotnetProject and dotnet-project with project name in real repository -env: - PROJECT_NAME: DotnetProject - PROJECT_NAME_LOWER: dotnet-project - -jobs: - deploy: - name: Deploy - runs-on: ubuntu-latest - environment: ${{ github.event.inputs.environment }} - permissions: - contents: read - id-token: write - # Remove the line below and uncomment the next one - if: ${{ false }} - steps: - - uses: actions/checkout@v4 - if: ${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags') && always() }} - with: - fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} - - uses: actions/checkout@v4 - if: ${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags')) && always() }} - with: - fetch-depth: 0 - - uses: azure/setup-helm@v3 - with: - version: '3.9.2' - id: install_helm - - name: Import Secrets - uses: hashicorp/vault-action@v2.7.4 - with: - url: https://hashicorp-vault.production.sneaksanddata.com/ - role: github - method: jwt - secrets: | - /secret/data/applications/${{ env.PROJECT_NAME_LOWER }}/${{github.event.inputs.environment}}/build acr_user ; - /secret/data/applications/${{ env.PROJECT_NAME_LOWER }}/${{github.event.inputs.environment}}/build acr_name ; - /secret/data/applications/${{ env.PROJECT_NAME_LOWER }}/${{github.event.inputs.environment}}/build aks_name ; - /secret/data/applications/${{ env.PROJECT_NAME_LOWER }}/${{github.event.inputs.environment}}/build cluster_sp_client_id ; - /secret/data/applications/${{ env.PROJECT_NAME_LOWER }}/${{github.event.inputs.environment}}/build cluster_sp_client_password ; - /secret/data/applications/${{ env.PROJECT_NAME_LOWER }}/${{github.event.inputs.environment}}/build acr_token ; - - name: Deployment - working-directory: .helm - env: - DEPLOY_ENVIRONMENT: ${{ github.event.inputs.environment }} - run: | - set -e - echo 'Getting cluster credentials' - az login --service-principal --username $CLUSTER_SP_CLIENT_ID --password $CLUSTER_SP_CLIENT_PASSWORD --tenant 06152121-b4c5-4544-abf5-9268e75db448 - az aks get-credentials --name $AKS_NAME --resource-group $AKS_NAME - chart_version=$(git describe --tags --abbrev=7) - - echo 'Logging to ACR' - helm registry login $ACR_NAME.azurecr.io --username $ACR_NAME --password $ACR_TOKEN - - echo 'Installing chart' - helm pull oci://$ACR_NAME.azurecr.io/helm/$PROJECT_NAME_LOWER --version $chart_version - mkdir -p ./$PROJECT_NAME_LOWER - tar xzf "$PROJECT_NAME_LOWER-${chart_version}.tgz" -C ./$PROJECT_NAME_LOWER - - helm upgrade --install $PROJECT_NAME_LOWER -n $PROJECT_NAME_LOWER --values ./values.yaml \ - --set environment=${DEPLOY_ENVIRONMENT^} \ - --set image.repository=$ACR_NAME.azurecr.io/$PROJECT_NAME_LOWER \ - --set image.tag=$chart_version \ - --set secretStorage.deploymentClusterName=$AKS_NAME \ - ./$PROJECT_NAME_LOWER/$PROJECT_NAME_LOWER diff --git a/.github/workflows/publish-ecr-public.yaml b/.github/workflows/publish-ecr-public.yaml new file mode 100644 index 0000000..3e11e66 --- /dev/null +++ b/.github/workflows/publish-ecr-public.yaml @@ -0,0 +1,92 @@ +name: Publish Arcane.Stream.SqlServer to ECR public registry +run-name: Publish Arcane.Stream.SqlServer to public.ecr.aws by @${{ github.actor }} + +on: + workflow_dispatch: + inputs: + version: + description: | + Version number to publish. Defaults to the latest git tag in the repository. + This version MUST exist in the ghcr.io registry. + required: false + default: "current" + +env: + PROJECT_NAME: Arcane.Stream.SqlServer + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + publish_image: + name: Publish Docker Image to ECR Public + runs-on: ubuntu-latest + if: ${{ startsWith(github.ref, 'refs/tags') }} + + permissions: + contents: read + id-token: write + + steps: + + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + + - name: Get project version + uses: SneaksAndData/github-actions/generate_version@v0.1.9 + id: current_version + + - name: Set up variables + env: + VERSION: ${{ inputs.version }} + CURRENT_VERSION: ${{steps.current_version.outputs.version}} + run: | + test "$VERSION" == "current" && echo "IMAGE_VERSION=$CURRENT_VERSION" >> ${GITHUB_ENV} || echo "IMAGE_VERSION=$VERSION" >> ${GITHUB_ENV} + + - name: Import AWS Secrets + uses: hashicorp/vault-action@v3.0.0 + with: + url: https://hashicorp-vault.awsp.sneaksanddata.com/ + role: github + method: jwt + secrets: | + /secret/data/common/package-publishing/aws-ecr-public/production/container-user-public access_key | ACCESS_KEY ; + /secret/data/common/package-publishing/aws-ecr-public/production/container-user-public access_key_id | ACCESS_KEY_ID ; + id: aws_secrets + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4.0.2 + with: + aws-access-key-id: ${{ env.ACCESS_KEY_ID }} + aws-secret-access-key: ${{ env.ACCESS_KEY }} + aws-region: us-east-1 + + - name: Log in to the GitHub Container Registry + uses: docker/login-action@v3.1.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to Amazon ECR Public + uses: aws-actions/amazon-ecr-login@v2 + with: + registry-type: public + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=semver,pattern={{version}},value=${{ env.IMAGE_VERSION }} + flavor: + latest=false + + - name: Push image to ECR Public registry + uses: akhilerm/tag-push-action@v2.2.0 + with: + src: ${{ steps.meta.outputs.tags }} + dst: public.ecr.aws/ecco-sneaks-and-data/arcane/${{ github.event.repository.name }}:${{ steps.meta.outputs.version }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 82e3d80..e0ea0e9 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -6,16 +6,14 @@ jobs: create_release: name: Create Release runs-on: ubuntu-latest - # Remove the line below and uncomment the next one - if: ${{ false }} - #if: ${{ github.ref == 'refs/heads/main' }} + if: ${{ github.ref == 'refs/heads/main' }} steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Create Release - uses: SneaksAndData/github-actions/semver_release@v0.1.6 + uses: SneaksAndData/github-actions/semver_release@v0.1.9 with: major_v: 0 minor_v: 0 diff --git a/.helm/Chart.yaml b/.helm/Chart.yaml index a6d5ea8..38e12f4 100644 --- a/.helm/Chart.yaml +++ b/.helm/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -name: dotnet-project +name: arcane-stream-sqlserver description: A Helm chart for Kubernetes # A chart can be either an 'application' or a 'library' chart. diff --git a/DotnetProject.sln b/Arcane.Stream.SqlServer.sln similarity index 74% rename from DotnetProject.sln rename to Arcane.Stream.SqlServer.sln index 9e3e7a8..cea108d 100644 --- a/DotnetProject.sln +++ b/Arcane.Stream.SqlServer.sln @@ -1,8 +1,8 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DotnetProject", "src\DotnetProject.csproj", "{67711D50-E64C-4A18-871B-DC6A485DD9E4}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Arcane.Stream.SqlServer", "src\Arcane.Stream.SqlServer.csproj", "{67711D50-E64C-4A18-871B-DC6A485DD9E4}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DotnetProjectTests", "test\DotnetProjectTests.csproj", "{B74A1EEE-CC43-4FFF-A6AB-129387D32F40}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Arcane.Stream.SqlServer.Tests", "test\Arcane.Stream.SqlServer.Tests.csproj", "{B74A1EEE-CC43-4FFF-A6AB-129387D32F40}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/src/DotnetProject.csproj b/src/Arcane.Stream.SqlServer.csproj similarity index 56% rename from src/DotnetProject.csproj rename to src/Arcane.Stream.SqlServer.csproj index 4cc95ce..baaabe1 100644 --- a/src/DotnetProject.csproj +++ b/src/Arcane.Stream.SqlServer.csproj @@ -2,10 +2,10 @@ Exe - net6.0 + net8.0 - + diff --git a/src/Models/SqlServerStreamContext.cs b/src/Models/SqlServerStreamContext.cs new file mode 100644 index 0000000..b4206ea --- /dev/null +++ b/src/Models/SqlServerStreamContext.cs @@ -0,0 +1,82 @@ +using System; +using Arcane.Framework.Services.Base; + +namespace Arcane.Stream.SqlServer.Models; + +public class SqlServerStreamContext : IStreamContext, IStreamContextWriter +{ + /// + /// Sql Server connection string. + /// + public string ConnectionString { get; set; } + + /// + /// Table schema. + /// + public string Schema { get; set; } + + /// + /// Table name. + /// + public string Table { get; set; } + + /// + /// Number of rows per parquet rowgroup. + /// + public int RowsPerGroup { get; set; } + + /// + /// Max time to wait for rowsPerGroup to accumulate. + /// + public TimeSpan GroupingInterval { get; set; } + + /// + /// Number of row groups per file. + /// + public int GroupsPerFile { get; set; } + + /// + /// Data location for parquet files. + /// + public string SinkLocation { get; set; } + + /// + /// Number of seconds to wait for result before sql commands should time out. + /// + public int CommandTimeout { get; set; } + + /// + public string StreamId { get; private set; } + + /// + public bool IsBackfilling { get; private set; } + + /// + public string StreamKind { get; private set; } + + /// + public void SetStreamId(string streamId) + { + this.StreamId = streamId; + } + + /// + public void SetBackfilling(bool isRunningInBackfillMode) + { + this.IsBackfilling = isRunningInBackfillMode; + } + + /// + public void SetStreamKind(string streamKind) + { + this.StreamKind = streamKind; + } + + public void LoadSecretsFromEnvironment() + { + this.ConnectionString = this.GetSecretFromEnvironment("CONNECTIONSTRING"); + } + + private string GetSecretFromEnvironment(string secretName) + => Environment.GetEnvironmentVariable($"{nameof(Arcane)}__{secretName}".ToUpperInvariant()); +} diff --git a/src/Program.cs b/src/Program.cs index 71e7a3a..4157536 100644 --- a/src/Program.cs +++ b/src/Program.cs @@ -1,5 +1,52 @@ -// See https://aka.ms/new-console-template for more information +using System; +using System.Threading.Tasks; +using Akka.Util.Extensions; +using Arcane.Framework.Contracts; +using Arcane.Framework.Providers.Hosting; +using Arcane.Framework.Sources.SqlServer.Exceptions; +using Arcane.Stream.SqlServer.Models; +using Arcane.Stream.SqlServer.Services; +using Microsoft.Extensions.Hosting; +using Serilog; +using Snd.Sdk.Logs.Providers; +using Snd.Sdk.Metrics.Configurations; +using Snd.Sdk.Metrics.Providers; +using Snd.Sdk.Storage.Providers; +using Snd.Sdk.Storage.Providers.Configurations; -using System; +Log.Logger = DefaultLoggingProvider.CreateBootstrapLogger(nameof(Arcane)); -Console.WriteLine("Hello, World!"); \ No newline at end of file +int exitCode; +try +{ + exitCode = await Host.CreateDefaultBuilder(args) + .AddDatadogLogging((_, _, conf) => conf.EnrichWithCustomProperties().WriteTo.Console()) + .ConfigureRequiredServices(services + => services.AddStreamGraphBuilder()) + .ConfigureAdditionalServices((services, context) => + { + services.AddDatadogMetrics(configuration: DatadogConfiguration.UnixDomainSocket(context.ApplicationName)); + services.AddAwsS3Writer(AmazonStorageConfiguration.CreateFromEnv()); + }) + .Build() + .RunStream(Log.Logger, (exception, _)=> + { + Log.Logger.Error(exception, "Fatal exception occured"); + return exception switch + { + SqlServerConnectionException => Task.FromResult(ExitCodes.RESTART.AsOption()), + _ => Task.FromResult(ExitCodes.FATAL.AsOption()) + }; + }); +} +catch (Exception ex) +{ + Log.Fatal(ex, "Host terminated unexpectedly"); + return ExitCodes.FATAL; +} +finally +{ + await Log.CloseAndFlushAsync(); +} + +return exitCode; diff --git a/src/Services/SqlServerGraphBuilder.cs b/src/Services/SqlServerGraphBuilder.cs new file mode 100644 index 0000000..722cd77 --- /dev/null +++ b/src/Services/SqlServerGraphBuilder.cs @@ -0,0 +1,87 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Akka.Streams; +using Akka.Streams.Dsl; +using Arcane.Framework.Contracts; +using Arcane.Framework.Services.Base; +using Arcane.Framework.Sinks.Parquet; +using Arcane.Framework.Sources.Exceptions; +using Arcane.Framework.Sources.SqlServer; +using Arcane.Framework.Sources.SqlServer.Exceptions; +using Arcane.Stream.SqlServer.Models; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Logging; +using Parquet.Data; +using Snd.Sdk.Metrics.Base; +using Snd.Sdk.Storage.Base; + +namespace Arcane.Stream.SqlServer.Services; + +public class SqlServerGraphBuilder( + IBlobStorageWriter blobStorageWriter, + MetricsService metricsService, + ILogger logger + ) : IStreamGraphBuilder +{ + public IRunnableGraph<(UniqueKillSwitch, Task)> BuildGraph(SqlServerStreamContext context) + { + context.LoadSecretsFromEnvironment(); + try + { + + var source = SqlServerSource.Create(context.ConnectionString, + context.Schema, + context.Table, + context.CommandTimeout); + + var schema = source.GetParquetSchema(); + var dimensions = source.GetDefaultTags().GetAsDictionary(context, context.StreamId); + + var parquetSink = ParquetSink.Create( + parquetSchema: schema, + storageWriter: blobStorageWriter, + parquetFilePath: $"{context.SinkLocation}/{context.StreamId}", + rowGroupsPerFile: context.GroupsPerFile, + createSchemaFile: true, + dataSinkPathSegment: context.IsBackfilling ? "backfill" : "data", + dropCompletionToken: context.IsBackfilling); + + return Source + .FromGraph(source) + .GroupedWithin(context.RowsPerGroup, context.GroupingInterval) + .Select(grp => + { + var rows = grp.ToList(); + metricsService.Increment(DeclaredMetrics.ROWS_INCOMING, dimensions, rows.Count); + return rows.AsRowGroup(schema); + }) + .ViaMaterialized(KillSwitches.Single>(), Keep.Right) + .ToMaterialized(parquetSink, Keep.Both); + } + catch (Exception ex) + { + if (ex is SqlException { Number: 4998 or 22105 } rootCause) + { + logger.LogError(ex, "Schema mismatched in attempt to activate stream {streamId}", context.StreamId); + throw new SchemaMismatchException(rootCause); + } + + if (ex is SqlException { Number: 35 or 0 }) + { + throw new SqlServerConnectionException(context.StreamKind, ex); + } + + if (ex is SqlException sqlException) + { + logger.LogError(ex, "Error while creating stream for {streamId}. Got {exception} with {number} and {errorCode}", + context.StreamId, nameof(SqlException), sqlException.Number, sqlException.ErrorCode); + throw; + } + + logger.LogError(ex, "Error while creating stream for {streamId}", context.StreamId); + throw; + } + } +} diff --git a/test/DotnetProjectTests.csproj b/test/Arcane.Stream.SqlServer.Tests.csproj similarity index 90% rename from test/DotnetProjectTests.csproj rename to test/Arcane.Stream.SqlServer.Tests.csproj index 764ee4d..b845233 100644 --- a/test/DotnetProjectTests.csproj +++ b/test/Arcane.Stream.SqlServer.Tests.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 false @@ -26,7 +26,7 @@ - +