Merge remote-tracking branch 'origin/main' into vickenty/pyvenv

DataDog · Nov 1, 2024 · 9ffdcef · 9ffdcef
2 parents 8b2ff7f + 295783a
commit 9ffdcef
Show file tree

Hide file tree

Showing 755 changed files with 25,429 additions and 8,249 deletions.
diff --git a/.circleci/README.md b/.circleci/README.md
@@ -11,12 +11,12 @@ This image is now built alongside other images in [agent-buildimages](https://gi
 Once you have created a new image by building a new version of agent-buildimages, you can test your modification with the associated invoke task:
 
 ```bash
-invoke -e buildimages.update --image-tag v12345678-c0mm1t5
+invoke -e buildimages.update --tag v12345678-c0mm1t5
 ```
-This will update the configuration of circleci and gitlab to use the __test version__ of these images.
-Once your test is successful, you can either move the `_test_version` from files or invoke
+This will update the configuration of circleci and gitlab to use the __test__ version of these images.
+Once your test is successful, you can either move the `_test_only` from files or invoke
 ```bash
-invoke -e buildimages.update --image-tag v12345678-c0mm1t5 --no-test-version
+invoke -e buildimages.update --tag v12345678-c0mm1t5 --no-test
 ```
 
 If everything is green, get a review and merge the PR.
diff --git a/.copyright-overrides.yml b/.copyright-overrides.yml
@@ -365,6 +365,7 @@ github.com/signalfx/sapm-proto/*: Copyright 2019 Splunk, Inc.
 go.mongodb.org/atlas/mongodbatlas: Copyright 2022 MongoDB Inc
 github.com/go-jose/go-jose/v4: Copyright 2014 Square Inc.
 github.com/gocolly/colly: Copyright 2018 Adam Tauber
+github.com/puzpuzpuz/xsync/v3: Copyright (c) 2021 Andrey Pechkurov
 
 github.com/vibrantbyte/go-antpath/antpath: vibrantbyte <[email protected]>|suchao <[email protected]>
 github.com/vibrantbyte/go-antpath/extend: vibrantbyte <[email protected]>|suchao <[email protected]>

diff --git a/.ddqa/config.toml b/.ddqa/config.toml
@@ -57,12 +57,20 @@ jira_statuses = ["To Do", "In Progress", "Done"]
 github_team = "universal-service-monitoring"
 github_labels = ["team/usm"]
 
-[teams."Network Device Monitoring"]
+[teams."Network Device Monitoring - Core"]
 jira_project = "NDMII"
 jira_issue_type = "Task"
 jira_statuses = ["To Do", "In Progress", "Done"]
-github_team = "network-device-monitoring"
-github_labels = ["team/network-device-monitoring"]
+github_team = "ndm-core"
+github_labels = ["team/ndm-core", "team/network-device-monitoring"]
+exclude_members = ["leeavital", "heyronhay"]
+
+[teams."Network Device Monitoring - Integrations"]
+jira_project = "NDINT"
+jira_issue_type = "Task"
+jira_statuses = ["To Do", "In Progress", "Done"]
+github_team = "ndm-integrations"
+github_labels = ["team/ndm-integrations"]
 exclude_members = ["leeavital", "heyronhay"]
 
 [teams."Network Performance Monitoring"]
@@ -203,9 +211,9 @@ jira_statuses = ["💼 To Do", "💡 In Progress", "✅ Done"]
 github_team = "container-app"
 github_labels = ["team/container-app"]
 
-[teams."APM Onboarding"]
-jira_project = "APMON"
-jira_issue_type = "Story"
+[teams."APM Injection Platform"]
+jira_project = "INPLAT"
+jira_issue_type = "Task"
 jira_statuses = ["Selected For Development", "In Progress", "Done"]
 github_team = "apm-onboarding"
 github_labels = ["team/apm-onboarding"]
@@ -222,4 +230,4 @@ jira_project = "DEBUG"
 jira_issue_type = "Task"
 jira_statuses = ["To Do", "In Progress", "Done"]
 github_team = "debugger-go"
-github_labels = ["team/dynamic-instrumentation"]
+github_labels = ["team/dynamic-instrumentation"]
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -190,7 +190,7 @@
 /cmd/agent/subcommands/dogstatsd*       @DataDog/agent-metrics-logs
 /cmd/agent/subcommands/integrations     @DataDog/agent-integrations @DataDog/agent-shared-components
 /cmd/agent/subcommands/remoteconfig     @Datadog/remote-config
-/cmd/agent/subcommands/snmp             @DataDog/network-device-monitoring
+/cmd/agent/subcommands/snmp             @DataDog/ndm-core
 /cmd/agent/subcommands/streamlogs       @DataDog/agent-metrics-logs
 /cmd/agent/subcommands/streamep         @DataDog/container-integrations
 /cmd/agent/subcommands/taggerlist       @DataDog/container-platform
@@ -207,7 +207,7 @@
 /cmd/agent/dist/conf.d/oracle-dbm.d/    @DataDog/database-monitoring
 /cmd/agent/dist/conf.d/network_path.d/  @DataDog/Networks @DataDog/network-device-monitoring
 /cmd/agent/dist/conf.d/sbom.d/          @DataDog/container-integrations
-/cmd/agent/dist/conf.d/snmp.d/          @DataDog/network-device-monitoring
+/cmd/agent/dist/conf.d/snmp.d/          @DataDog/ndm-core
 /cmd/agent/dist/conf.d/win32_event_log.d/ @DataDog/windows-agent
 /cmd/agent/install*.sh                  @DataDog/container-ecosystems @DataDog/agent-delivery
 /cmd/cluster-agent/                     @DataDog/container-platform
@@ -230,7 +230,7 @@
 /cmd/system-probe/modules/eventmonitor* @DataDog/agent-security
 /cmd/system-probe/modules/tcp_queue_tracer* @DataDog/container-integrations
 /cmd/system-probe/modules/traceroute*   @DataDog/network-device-monitoring @Datadog/Networks
-/cmd/system-probe/modules/ping*         @DataDog/network-device-monitoring
+/cmd/system-probe/modules/ping*         @DataDog/ndm-core
 /cmd/system-probe/modules/language_detection* @DataDog/processes @DataDog/universal-service-monitoring
 /cmd/system-probe/modules/dynamic_instrumentation* @DataDog/debugger
 /cmd/system-probe/windows_resources/    @DataDog/windows-kernel-integrations
@@ -274,7 +274,7 @@
 /omnibus/config/software/datadog-security-agent*.rb       @Datadog/agent-security @DataDog/agent-delivery
 /omnibus/config/software/openscap.rb                      @DataDog/agent-cspm
 /omnibus/config/software/sds.rb                           @DataDog/agent-processing-and-routing
-/omnibus/config/software/snmp-traps.rb                    @DataDog/network-device-monitoring
+/omnibus/config/software/snmp-traps.rb                    @DataDog/ndm-core
 /omnibus/resources/*/msi/                                 @DataDog/windows-agent
 
 # The following is managed by `inv lint-components` -- DO NOT EDIT
@@ -290,13 +290,13 @@
 /comp/forwarder @DataDog/agent-processing-and-routing
 /comp/logs @DataDog/agent-metrics-logs
 /comp/metadata @DataDog/agent-shared-components
-/comp/ndmtmp @DataDog/network-device-monitoring
-/comp/netflow @DataDog/network-device-monitoring
+/comp/ndmtmp @DataDog/ndm-core
+/comp/netflow @DataDog/ndm-integrations
 /comp/networkpath @DataDog/Networks @DataDog/network-device-monitoring
 /comp/otelcol @DataDog/opentelemetry
 /comp/process @DataDog/processes
 /comp/remote-config @DataDog/remote-config
-/comp/snmptraps @DataDog/network-device-monitoring
+/comp/snmptraps @DataDog/ndm-core
 /comp/systray @DataDog/windows-agent
 /comp/trace @DataDog/agent-apm
 /comp/updater @DataDog/fleet @DataDog/windows-agent
@@ -316,9 +316,9 @@
 /comp/autoscaling/datadogclient @DataDog/container-integrations
 /comp/etw @DataDog/windows-agent
 /comp/languagedetection/client @DataDog/container-platform
-/comp/rdnsquerier @DataDog/network-device-monitoring
+/comp/rdnsquerier @DataDog/ndm-integrations
 /comp/serializer/compression @DataDog/agent-metrics-logs
-/comp/snmpscan @DataDog/network-device-monitoring
+/comp/snmpscan @DataDog/ndm-core
 # END COMPONENTS
 
 # pkg
@@ -354,7 +354,7 @@
 /pkg/trace/transform/                   @DataDog/opentelemetry
 /comp/core/autodiscovery/listeners/           @DataDog/container-platform
 /comp/core/autodiscovery/listeners/cloudfoundry*.go  @DataDog/platform-integrations
-/comp/core/autodiscovery/listeners/snmp*.go   @DataDog/network-device-monitoring
+/comp/core/autodiscovery/listeners/snmp*.go   @DataDog/ndm-core
 /comp/core/autodiscovery/providers/           @DataDog/container-platform
 /comp/core/autodiscovery/providers/file*.go   @DataDog/agent-metrics-logs
 /comp/core/autodiscovery/providers/config_reader*.go @DataDog/container-platform @DataDog/agent-metrics-logs
@@ -381,13 +381,13 @@
 /pkg/collector/corechecks/embed/apm/               @DataDog/agent-apm
 /pkg/collector/corechecks/embed/process/           @DataDog/processes
 /pkg/collector/corechecks/gpu/                     @DataDog/ebpf-platform
-/pkg/collector/corechecks/network-devices/         @DataDog/network-device-monitoring
+/pkg/collector/corechecks/network-devices/         @DataDog/ndm-integrations
 /pkg/collector/corechecks/orchestrator/   @DataDog/container-app
 /pkg/collector/corechecks/net/          @DataDog/platform-integrations
 /pkg/collector/corechecks/oracle        @DataDog/database-monitoring
 /pkg/collector/corechecks/sbom/         @DataDog/container-integrations
 /pkg/collector/corechecks/servicediscovery/ @DataDog/universal-service-monitoring
-/pkg/collector/corechecks/snmp/         @DataDog/network-device-monitoring
+/pkg/collector/corechecks/snmp/         @DataDog/ndm-core
 /pkg/collector/corechecks/system/                 @DataDog/platform-integrations
 /pkg/collector/corechecks/system/**/*_windows*.go @DataDog/platform-integrations @DataDog/windows-agent
 /pkg/collector/corechecks/system/wincrashdetect/  @DataDog/windows-kernel-integrations
@@ -419,7 +419,7 @@
 /pkg/flare/*_win.go                     @Datadog/windows-agent
 /pkg/flare/*_windows.go                 @Datadog/windows-agent
 /pkg/flare/*_windows_test.go            @Datadog/windows-agent
-/pkg/flare/sysprobe                     @DataDog/Networks @Datadog/windows-agent @DataDog/processes @DataDog/agent-security @DataDog/universal-service-monitoring @DataDog/ebpf-platform
+/pkg/flare/sysprobe                     @DataDog/Networks @Datadog/windows-agent @DataDog/agent-security @DataDog/universal-service-monitoring @DataDog/ebpf-platform
 /pkg/fleet/                             @DataDog/fleet @DataDog/windows-agent
 /pkg/pidfile/                           @DataDog/agent-shared-components
 /pkg/persistentcache/                   @DataDog/agent-metrics-logs
@@ -475,7 +475,7 @@
 /pkg/process/metadata/parser/           @DataDog/universal-service-monitoring @DataDog/processes @DataDog/Networks
 /pkg/process/metadata/parser/*windows*  @DataDog/universal-service-monitoring @DataDog/processes @DataDog/Networks @DataDog/windows-kernel-integrations
 /pkg/process/monitor/                   @DataDog/universal-service-monitoring
-/pkg/process/net/                       @DataDog/universal-service-monitoring @DataDog/processes @DataDog/Networks
+/pkg/process/net/                       @DataDog/universal-service-monitoring @DataDog/Networks
 /pkg/process/net/common_windows.go      @DataDog/windows-agent
 /pkg/process/net/windows_pipe.go        @DataDog/windows-kernel-integrations
 /pkg/process/net/windows_pipe_testutil.go     @DataDog/windows-kernel-integrations
@@ -520,8 +520,8 @@
 /pkg/databasemonitoring                 @DataDog/database-monitoring
 /pkg/kubestatemetrics                   @DataDog/container-integrations
 /pkg/security/                          @DataDog/agent-security
-/pkg/networkdevice/                     @DataDog/network-device-monitoring
-/pkg/snmp/                              @DataDog/network-device-monitoring
+/pkg/networkdevice/                     @DataDog/ndm-core
+/pkg/snmp/                              @DataDog/ndm-core
 /pkg/tagger/                            @DataDog/container-platform
 /pkg/windowsdriver/                     @DataDog/windows-kernel-integrations
 /comp/core/workloadmeta/collectors/internal/cloudfoundry @DataDog/platform-integrations
@@ -576,9 +576,6 @@
 /test/kitchen/                          @DataDog/agent-devx-loops
 /test/kitchen/test-definitions/         @DataDog/container-ecosystems @DataDog/agent-delivery
 /test/kitchen/test/integration/         @DataDog/container-ecosystems @DataDog/agent-delivery
-/test/kitchen/site-cookbooks/dd-security-agent-check/ @DataDog/agent-security
-/test/kitchen/test/integration/security-agent-stress/ @DataDog/agent-security
-/test/kitchen/test/integration/security-agent-test/ @DataDog/agent-security
 /test/kitchen/site-cookbooks/dd-system-probe-check/ @DataDog/windows-kernel-integrations
 /test/kitchen/test/integration/win-all-subservices/ @DataDog/windows-agent
 /test/kitchen/test/integration/win-alt-dir/ @DataDog/windows-agent
@@ -589,8 +586,8 @@
 /test/kitchen/test/integration/win-repair/ @DataDog/windows-agent
 /test/kitchen/test/integration/win-user/ @DataDog/windows-agent
 /test/fakeintake/                             @DataDog/agent-e2e-testing @DataDog/agent-devx-loops
-/test/fakeintake/aggregator/ndmflowAggregator.go @DataDog/network-device-monitoring
-/test/fakeintake/aggregator/ndmflowAggregator_test.go @DataDog/network-device-monitoring
+/test/fakeintake/aggregator/ndmflowAggregator.go @DataDog/ndm-integrations
+/test/fakeintake/aggregator/ndmflowAggregator_test.go @DataDog/ndm-integrations
 /test/fakeintake/aggregator/servicediscovery* @DataDog/universal-service-monitoring
 /test/new-e2e/                                @DataDog/agent-e2e-testing @DataDog/agent-devx-loops
 /test/new-e2e/pkg/components/datadog-installer @DataDog/windows-agent
@@ -603,7 +600,8 @@
 /test/new-e2e/tests/containers                @DataDog/container-integrations @DataDog/container-platform
 /test/new-e2e/tests/discovery                 @DataDog/universal-service-monitoring
 /test/new-e2e/tests/language-detection        @DataDog/processes
-/test/new-e2e/tests/ndm                       @DataDog/network-device-monitoring
+/test/new-e2e/tests/ndm                       @DataDog/ndm-core
+/test/new-e2e/tests/ndm/netflow               @DataDog/ndm-integrations
 /test/new-e2e/tests/npm                       @DataDog/Networks
 /test/new-e2e/tests/npm/ec2_1host_wkit_test.go @DataDog/Networks @DataDog/windows-kernel-integrations
 /test/new-e2e/tests/orchestrator              @DataDog/container-app

diff --git a/.github/workflows/buildimages-update.yml b/.github/workflows/buildimages-update.yml
@@ -83,11 +83,11 @@ jobs:
       - name: Update buildimages IDs and Go version
         id: update_build_images
         env:
-          TEST_VERSION_FLAG: ${{ inputs.test_version && '--test-version' || '--no-test-version' }}
+          TEST_VERSION_FLAG: ${{ inputs.test_version && '--test' || '--no-test' }}
           INCLUDE_OTEL_MODULES: ${{ inputs.include_otel_modules && '--include-otel-modules' || '' }}
         run: |
           if [ "${{ steps.current_go_version.outputs.GO_VERSION }}" = "${{ inputs.go_version }}" ]; then
-            inv -e buildimages.update --image-tag ${{ inputs.images_id }} $TEST_VERSION_FLAG
+            inv -e buildimages.update --tag ${{ inputs.images_id }} $TEST_VERSION_FLAG
             echo 'MESSAGE=Update buildimages ID to ${{ inputs.images_id }}' >> $GITHUB_OUTPUT
           else
             inv -e update-go --image-tag ${{ inputs.images_id }} $TEST_VERSION_FLAG $INCLUDE_OTEL_MODULES -v "${{ inputs.go_version }}"
@@ -140,7 +140,7 @@ jobs:
           ${{ inputs.images_id }} \
           ${{ steps.current_go_version.outputs.GO_VERSION }} \
           ${{ inputs.go_version }} \
-          ${{ inputs.test_version && '--test-version' || '' }} > $TMP_PR_BODY_PATH
+          ${{ inputs.test_version && '--test' || '' }} > $TMP_PR_BODY_PATH
 
           # Create the PR
           gh pr create \

diff --git a/.github/workflows/chase_release_managers.yml b/.github/workflows/chase_release_managers.yml
@@ -32,5 +32,6 @@ jobs:
         ATLASSIAN_USERNAME : ${{ secrets.ATLASSIAN_USERNAME }}
         ATLASSIAN_PASSWORD : ${{ secrets.ATLASSIAN_PASSWORD }}
         SLACK_API_TOKEN : ${{ secrets.SLACK_DATADOG_AGENT_BOT_TOKEN }}
+        VERSION: ${{ github.event.inputs.version }}
       run: |
-        inv -e release.chase-release-managers --version ${{ github.event.inputs.version }}
+        inv -e release.chase-release-managers --version "$VERSION"
diff --git a/.github/workflows/code_review_complexity.yml b/.github/workflows/code_review_complexity.yml
@@ -0,0 +1,36 @@
+---
+name: "PR complexity label"
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    branches:
+      - main
+      - "[0-9]+.[0-9]+.x"
+  pull_request_review_comment:
+    types: [created, deleted]
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+permissions: {}
+jobs:
+  codereview-complexity:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+      - name: Setup python
+        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
+        with:
+          python-version: 3.12
+          cache: 'pip'
+          cache-dependency-path: '**/requirements*.txt'
+      - name: Install dependencies
+        run: pip install -r requirements.txt -r tasks/requirements.txt
+      - name: Check code review complexity
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: inv -e github.assign-codereview-label --pr-id="$PR_NUMBER"
diff --git a/.github/workflows/create_rc_pr.yml b/.github/workflows/create_rc_pr.yml
@@ -82,8 +82,10 @@ jobs:
                 ATLASSIAN_USERNAME: ${{ secrets.ATLASSIAN_USERNAME }}
                 ATLASSIAN_PASSWORD: ${{ secrets.ATLASSIAN_PASSWORD }}
                 SLACK_API_TOKEN : ${{ secrets.SLACK_DATADOG_AGENT_BOT_TOKEN }}
+                MATRIX: ${{ matrix.value }}
+                WARNING: ${{ needs.find_release_branches.outputs.warning }}
               run: |
-                echo "CHANGES=$(inv -e release.check-for-changes -r ${{ matrix.value }} ${{ needs.find_release_branches.outputs.warning }})" >> $GITHUB_OUTPUT
+                echo "CHANGES=$(inv -e release.check-for-changes -r "$MATRIX" "$WARNING")" >> $GITHUB_OUTPUT
 
             - name: Create RC PR
               if: ${{ steps.check_for_changes.outputs.CHANGES == 'true'}}

diff --git a/.github/workflows/create_release_schedule.yml b/.github/workflows/create_release_schedule.yml
@@ -36,5 +36,7 @@ jobs:
       env:
         ATLASSIAN_USERNAME : ${{ secrets.ATLASSIAN_USERNAME }}
         ATLASSIAN_PASSWORD : ${{ secrets.ATLASSIAN_PASSWORD }}
+        INPUT_VERSION: ${{ github.event.inputs.version }}
+        FREEZE_DATE: ${{ github.event.inputs.freeze_date }}
       run: |
-        inv -e release.create-schedule --version ${{ github.event.inputs.version }} --freeze-date ${{ github.event.inputs.freeze_date }}
+        inv -e release.create-schedule --version "$INPUT_VERSION" --freeze-date "$FREEZE_DATE"
diff --git a/.github/workflows/cws-btfhub-sync.yml b/.github/workflows/cws-btfhub-sync.yml
@@ -81,8 +81,11 @@ jobs:
           echo "ARTIFACT_NAME=constants-${{ matrix.cone }}" | tr '/' '-' >> $GITHUB_OUTPUT
 
       - name: Sync constants
+        env:
+          ARTIFACT_NAME: ${{ steps.artifact-name.outputs.ARTIFACT_NAME }}
+          FORCE_REFRESH: ${{ inputs.force_refresh && '--force-refresh' || '' }}
         run: |
-          inv -e security-agent.generate-btfhub-constants --archive-path=./dev/dist/archive --output-path=./${{ steps.artifact-name.outputs.ARTIFACT_NAME }}.json ${{ inputs.force_refresh && '--force-refresh' || '' }}
+          inv -e security-agent.generate-btfhub-constants --archive-path=./dev/dist/archive --output-path=./"$ARTIFACT_NAME".json "$FORCE_REFRESH"
 
       - name: Upload artifact
         uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
@@ -140,6 +143,9 @@ jobs:
           skip_checkout: true
 
       - name: Create Pull Request
+        env:
+          BRANCH_NAME: ${{ steps.branch-name.outputs.BRANCH_NAME }}
+          BASE_BRANCH: ${{ inputs.base_branch || 'main' }}
         uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         if: steps.commit-creator.outputs.changes_detected == 'true'
         with:
@@ -149,8 +155,8 @@ jobs:
               title: 'CWS: sync BTFHub constants',
               owner,
               repo,
-              head: '${{ steps.branch-name.outputs.BRANCH_NAME }}',
-              base: '${{ inputs.base_branch || 'main' }}',
+              head: "$BRANCH_NAME",
+              base: "$BASE_BRANCH",
               body: [
                 '### What does this PR do?',
                 'This PR syncs the BTFHub constants used by CWS',

diff --git a/.github/workflows/go_mod_tidy.yml b/.github/workflows/go_mod_tidy.yml
@@ -22,9 +22,10 @@ jobs:
       - name: Checkout PR
         # run only if triggered manually, otherwise we are already on the right branch and we won't have `pr_number`
         if: ${{ github.event_name == 'workflow_dispatch' }}
-        run: gh pr checkout ${{ github.event.inputs.pr_number }}
+        run: gh pr checkout "$PR_NUMBER"
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PR_NUMBER: ${{ github.event.inputs.pr_number }}
       - name: Install go
         uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
         with:
@@ -51,5 +52,7 @@ jobs:
         with:
           commit_message: Auto-generate go.sum and LICENSE-3rdparty.csv changes
       - name: changes
+        env:
+          CHANGES: ${{ steps.autocommit.outputs.changes_detected }}
         run: |
-          echo "Changes detected: ${{ steps.autocommit.outputs.changes_detected }}"
+          echo "Changes detected: $CHANGES"