Python Check Support

.ci/images/build/Dockerfile

@@ -1,3 +1,7 @@
+ARG DD_AGENT_VERSION=7.58.0-jmx
+ARG DD_AGENT_IMAGE=datadog/agent:${DD_AGENT_VERSION}
+
+FROM ${DD_AGENT_IMAGE} as agent
 FROM registry.ddbuild.io/images/mirror/ubuntu:14.04
 
 ARG RUST_VERSION=1.80.0
@@ -35,7 +39,13 @@ RUN rustup toolchain add nightly && \
 # Pre-install the relevant Cargo tools we use in the build process.
 COPY ./Makefile /
 RUN make cargo-preinstall
-
+
+# Import python bits from Agent image.
+# Do last in the pipeline to avoid polluting the builds of above protobuf/go etc from using these libs
+COPY --from=agent /opt/datadog-agent/embedded/ /opt/datadog-agent/embedded/
+ENV LD_LIBRARY_PATH="/opt/datadog-agent/embedded/lib:${LD_LIBRARY_PATH}"
+ENV PATH="/opt/datadog-agent/embedded/bin:${PATH}"
+
 COPY .ci/images/build/entrypoint.sh /
 RUN chmod +x /entrypoint.sh
 ENTRYPOINT [ "/entrypoint.sh" ]

.ci/images/general/Dockerfile

@@ -1,3 +1,7 @@
+ARG DD_AGENT_VERSION=7.58.0-jmx
+ARG DD_AGENT_IMAGE=datadog/agent:${DD_AGENT_VERSION}
+
+FROM ${DD_AGENT_IMAGE} as agent
 FROM registry.ddbuild.io/images/mirror/ubuntu:20.04
 
 RUN apt-get update && \
@@ -10,6 +14,11 @@ RUN curl -L https://s3.amazonaws.com/dd-package-public/dd-package-$(dpkg --print
     apt-get update && \
     dd-package --bucket binaries.ddbuild.io --package devtools/dd-package-dev --distribution "20.04"
 
+# Import python bits from Agent image.
+COPY --from=agent /opt/datadog-agent/embedded/ /opt/datadog-agent/embedded/
+ENV LD_LIBRARY_PATH="/opt/datadog-agent/embedded/lib:${LD_LIBRARY_PATH}"
+ENV PATH="/opt/datadog-agent/embedded/bin:${PATH}"
+
 COPY .ci/images/general/entrypoint.sh /
 RUN chmod +x /entrypoint.sh
 ENTRYPOINT [ "/entrypoint.sh" ]

.ci/images/smp/Dockerfile

@@ -1,3 +1,8 @@
+ARG DD_AGENT_VERSION=7.58.0-jmx
+ARG DD_AGENT_IMAGE=datadog/agent:${DD_AGENT_VERSION}
+
+FROM ${DD_AGENT_IMAGE} as agent
+
 FROM registry.ddbuild.io/images/mirror/ubuntu:20.04
 
 ENV DEBIAN_FRONTEND=noninteractive \
@@ -18,6 +23,11 @@ RUN curl -L https://s3.amazonaws.com/dd-package-public/dd-package-$(dpkg --print
 COPY .ci/images/smp/install-pr-commenter.sh /
 RUN chmod +x /install-pr-commenter.sh && /install-pr-commenter.sh
 
+# Import python bits from Agent image.
+COPY --from=agent /opt/datadog-agent/embedded/ /opt/datadog-agent/embedded/
+ENV LD_LIBRARY_PATH="/opt/datadog-agent/embedded/lib:${LD_LIBRARY_PATH}"
+ENV PATH="/opt/datadog-agent/embedded/bin:${PATH}"
+
 COPY .ci/images/smp/entrypoint.sh /
 RUN chmod +x /entrypoint.sh
 

.ci/s6/adp-finish

@@ -0,0 +1,17 @@
+#!/usr/bin/execlineb -S1
+
+# Disable the service if it exit(0), else wait 2 seconds before restarting it
+
+# First check if ADP exists
+if { s6-test -x "/opt/datadog-agent/embedded/bin/agent-data-plane" }
+
+ifthenelse
+    { s6-test ${1} -eq 0 }
+    {
+        foreground { /initlog.sh "agent-data-plane exited with code ${1}, disabling" }
+        foreground { /bin/s6-svc -d /var/run/s6/services/adp/ }
+    }
+    {
+        foreground { /initlog.sh "agent-data-plane exited with code ${1}, signal ${2}, restarting in 2 seconds" }
+        foreground { s6-sleep 2 }
+    }

.ci/s6/adp-run

@@ -0,0 +1,12 @@
+#!/usr/bin/execlineb -P
+
+# Check if the system-probe exists before running it
+ifelse -n
+    { s6-test -x "/opt/datadog-agent/embedded/bin/agent-data-plane" }
+    {
+        foreground { /initlog.sh "agent-data-plane not bundled, disabling" }
+        foreground { /bin/s6-svc -d /var/run/s6/services/sysprobe/ }
+    }
+    foreground { /initlog.sh "starting agent-data-plane" }
+
+    exec /opt/datadog-agent/embedded/bin/agent-data-plane

.dockerignore

@@ -2,6 +2,7 @@
 /.gitlab
 /.git
 /.vscode
+/checks_venv
 /docker
 /target
 /test

.gitignore

@@ -5,5 +5,8 @@ test/k8s/charts/
 test/build/
 test/ddprof
 test/lading
+dist/
+checks_venv/
+etc-datadog-agent/
 dhat-heap.json
-.DS_Store
+.DS_Store

.gitlab-ci.yml

@@ -16,16 +16,22 @@ variables:
   # Base repository paths for where our CI images go, whether they're helper images or actual
   # output artifacts like ADP itself.
   SALUKI_IMAGE_REPO_BASE: "${IMAGE_REGISTRY}/saluki"
-  SALUKI_BUILD_CI_IMAGE: "${SALUKI_IMAGE_REPO_BASE}/build-ci:v42220630-6ad1da61"
-  SALUKI_SMP_CI_IMAGE: "${SALUKI_IMAGE_REPO_BASE}/smp-ci:v40512394-da9e8d37"
+  # Build image built from https://gitlab.ddbuild.io/DataDog/saluki/-/jobs/683779080
+  SALUKI_BUILD_CI_IMAGE: "${SALUKI_IMAGE_REPO_BASE}/build-ci:v47392904-45d66c50"
+  # CI image built from https://gitlab.ddbuild.io/DataDog/saluki/-/jobs/683779084
+  SALUKI_SMP_CI_IMAGE: "${SALUKI_IMAGE_REPO_BASE}/smp-ci:v47392904-45d66c50"
+  # 'general' image built with dd-python-libs
+  # registry.ddbuild.io/saluki/general-ci:v47148172-419daeb2
+  # from https://gitlab.ddbuild.io/DataDog/saluki/-/jobs/680099601
+
 
   # ADP-specific variables, controlling how we build ADP images, how we version them, and where we push them.
   ADP_VERSION_BASE: "0.1.0"
   ADP_IMAGE_BASE: "${SALUKI_IMAGE_REPO_BASE}/agent-data-plane"
 
   # Converged Datadog Agent-specific variables, which control how we build the converged Datadog Agent image
   # used internally for testing.
-  DD_AGENT_VERSION: "7-57-0-rc-4-jmx"
+  DD_AGENT_VERSION: "7-58-0-jmx"
   DD_AGENT_IMAGE_BASE: "${IMAGE_REGISTRY}/datadog-agent"
   DD_AGENT_IMAGE: "${DD_AGENT_IMAGE_BASE}:${DD_AGENT_VERSION}"
 

.gitlab/benchmark.yml

@@ -30,12 +30,14 @@ build-adp-baseline-image:
     # Initialize our AWS credentials.
     - ./test/smp/configure-smp-aws-credentials.sh
 
-    # Build baseline from `main`. A more correct approach here would be to check
-    # out all but the dockerfile for saluki from the branch point and build that
-    # way but this should be Good Enough for a while.
+    # Build baseline from `main`
     - git fetch --all
     - git switch main
 
+    # Preserve Dockerfiles and CI configuration files from the current branch
+    # Replace the paths below with the actual paths to your Dockerfiles and CI configs
+    - git checkout origin/"${CI_COMMIT_BRANCH}" -- .gitlab-ci.yml .gitlab/ ./docker/
+
     # Actually build
     - aws ecr get-login-password --region us-west-2 --profile ${AWS_NAMED_PROFILE} | docker login --username AWS --password-stdin ${SMP_ECR_HOST}
     - docker buildx build