diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 000000000..b3c537517
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,12 @@
+version: 2
+updates:
+ - package-ecosystem: "github-actions"
+ directory: "/"
+ schedule:
+ interval: "weekly"
+ target-branch: "develop"
+ - package-ecosystem: "pip"
+ directory: "/"
+ schedule:
+ interval: "weekly"
+ target-branch: "develop"
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 82e4493f0..8c1517731 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -33,7 +33,7 @@ jobs:
echo "::set-output name=current_version_tag::${CURRENT_VERSION_TAG}"
- name: Auto-generate future version
id: autogenerate-version
- uses: paulhatch/semantic-version@v5.3.0
+ uses: paulhatch/semantic-version@v5.4.0
with:
tag_prefix: "v"
major_pattern: "(major)"
@@ -46,7 +46,7 @@ jobs:
steps:
- uses: actions/checkout@v4
- name: Set up Python
- uses: actions/setup-python@v4
+ uses: actions/setup-python@v5
with:
python-version: 3.10.6
cache: pip
@@ -61,7 +61,8 @@ jobs:
- name: Build
run: VERSION=${{ needs.get-version.outputs.version_tag }} make build
- name: Upload artifact
- uses: actions/upload-artifact@v3
+ uses: actions/upload-artifact@v4
with:
name: dist
path: dist
+ overwrite: true
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index 99522ce90..ffc23a4a8 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -32,7 +32,7 @@ jobs:
runs-on: ubuntu-latest
needs: build
steps:
- - uses: actions/download-artifact@v3
+ - uses: actions/download-artifact@v4
with:
name: dist
path: dist
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index f6ea8ed31..62f14a3b0 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -40,7 +40,7 @@ jobs:
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
- id: set-matrix
name: Set up test matrix
run: |
@@ -59,13 +59,13 @@ jobs:
TEST_MEMORY: 1G
TEST_PROCESSES: 1
steps:
- - uses: actions/checkout@v3
- - uses: actions/download-artifact@v3
+ - uses: actions/checkout@v4
+ - uses: actions/download-artifact@v4
with:
name: dist
path: dist
- name: Set up Python
- uses: actions/setup-python@v4
+ uses: actions/setup-python@v5
with:
python-version: 3.10.6
cache: pip
diff --git a/.github/workflows/veracodescan.yml b/.github/workflows/veracodescan.yml
index ad54e905c..83e253ed3 100644
--- a/.github/workflows/veracodescan.yml
+++ b/.github/workflows/veracodescan.yml
@@ -17,7 +17,7 @@ jobs:
runs-on: ubuntu-latest
needs: build
steps:
- - uses: actions/download-artifact@v3
+ - uses: actions/download-artifact@v4
with:
name: dist
path: dist
diff --git a/Makefile b/Makefile
index 64a2ed87b..dfba7c789 100644
--- a/Makefile
+++ b/Makefile
@@ -5,8 +5,8 @@ TEST_FILES ?= tests/integration/test_*.py
VERSION ?= $(shell git tag --list --sort=-version:refname "v*" | head -n 1)
COMMIT ?= $(shell git rev-parse --short HEAD)
-PIP_TOOLS_VERSION ?= 7.3.0
-PIP_VERSION ?= 23.3.1
+PIP_TOOLS_VERSION ?= 7.4.0
+PIP_VERSION ?= 24.0
PYTHON_PLATFORM ?= manylinux2014_x86_64
PYTHON_VERSION ?= 310
@@ -104,5 +104,5 @@ format:
.PHONY: lint
lint:
- ruff check buildpack lib/m2ee/* tests/*/ --show-source
+ ruff check buildpack lib/m2ee/* tests/*/ --output-format=full
pylint --disable=W,R,C buildpack lib/m2ee/* tests/*/
diff --git a/buildpack/core/runtime.py b/buildpack/core/runtime.py
index 99f61949b..a1caf9b9b 100644
--- a/buildpack/core/runtime.py
+++ b/buildpack/core/runtime.py
@@ -12,6 +12,7 @@
from buildpack import util
from lib.m2ee import M2EE as m2ee_class
from lib.m2ee.version import MXVersion
+from lib.m2ee.util import strtobool
from . import security
@@ -173,6 +174,14 @@ def _activate_license():
"""
+ if(strtobool(os.environ.get("MXRUNTIME_License.UseLicenseServer","false"))):
+ prefs_template = """
+
+ """
+
license_key = os.environ.get(
"FORCED_LICENSE_KEY", os.environ.get("LICENSE_KEY", None)
)
diff --git a/buildpack/telemetry/datadog.py b/buildpack/telemetry/datadog.py
index f08761ded..026e3cdf6 100644
--- a/buildpack/telemetry/datadog.py
+++ b/buildpack/telemetry/datadog.py
@@ -12,6 +12,7 @@
import json
import logging
import os
+import shutil
import socket
import subprocess
from collections import OrderedDict
@@ -30,13 +31,21 @@
ROOT_DIR = os.path.abspath(".local")
SIDECAR_ROOT_DIR = os.path.join(ROOT_DIR, NAMESPACE)
AGENT_USER_CHECKS_DIR = os.path.abspath("/home/vcap/app/datadog_integrations")
-
+DOT_DATADOG_PATH = os.path.abspath("/home/vcap/app/.datadog")
STATSD_PORT = 8125
LOGS_PORT = 9032
def _get_agent_dir(root=ROOT_DIR):
- return os.path.join(root, NAMESPACE, "lib")
+ # DD-CF-buildpack assumes the root DATADOG_DIR to be
+ # /home/vcap/app/.datadog
+ # So once we've downloaded and installed the binaries
+ # return the path with ".datadog/"
+ path_with_lib = os.path.join(root, NAMESPACE, "lib")
+ if os.path.exists(path_with_lib):
+ return os.path.join(root, NAMESPACE, "lib")
+ else:
+ return DOT_DATADOG_PATH
def _get_user_checks_dir():
@@ -389,6 +398,19 @@ def _set_up_environment(model_version, runtime_version):
e["DD_ENABLE_CHECKS"] = str(bool(_is_checks_enabled())).lower()
e["LOGS_CONFIG"] = json.dumps(_get_logging_config())
+ # Append the ruby binary path
+ ruby_path = os.path.join(ROOT_DIR, "ruby", "bin")
+ e["PATH"] += os.pathsep + ruby_path
+
+ # Add an empty .global_env file as the DD buildpack expects a .global_env
+ # This is in parity with how the DD buildpack sets up the environment before
+ # installing the DD agents. Sourced from bin/supply scripts
+ # https://github.com/DataDog/datadog-cloudfoundry-buildpack/blob/master/bin/supply#L31
+ # Keeping it empty as we're already adding the RUBY path as part of process env
+ global_env_file = os.path.join(_get_agent_dir(), ".global_env")
+ with open(global_env_file, mode='a'):
+ pass
+
return e
@@ -492,6 +514,16 @@ def update_config(
def _download(buildpack_dir, build_path, cache_dir):
+ # DD for cflinuxfs4 requires ruby to be available, so installing
+ # the ruby dependency first before we proceed
+ # https://github.com/DataDog/datadog-cloudfoundry-buildpack/pull/174
+ util.resolve_dependency(
+ "ruby",
+ os.path.join(build_path, "ruby"),
+ buildpack_dir=buildpack_dir,
+ cache_dir=cache_dir,
+ )
+
util.resolve_dependency(
f"{NAMESPACE}.buildpack",
os.path.join(build_path, NAMESPACE),
@@ -525,13 +557,48 @@ def _patch_run_datadog_script(script_dir):
lines = file_handler.readlines()
file_handler.seek(0)
for line in lines:
- if "stop_datadog &" in line:
+ if "monit_datadog &" in line:
+ file_handler.write(f"# {line}")
+ else:
+ file_handler.write(line)
+ file_handler.truncate()
+
+
+def _patch_create_logs_config_script(script_dir):
+ # Seems like there is a race condition between the
+ # File.exists check and Dir.mkdir call, as a result we get an exception.
+ # Commenting out the Dir.mkdir call, as it is already created as part of
+ # run-datadog.sh script
+ script = os.path.join(script_dir, "scripts", "create_logs_config.rb")
+ pattern = 'Dir.mkdir(logs_config_dir)'
+ with open(script, "r+") as file_handler:
+ lines = file_handler.readlines()
+ file_handler.seek(0)
+ for line in lines:
+ if pattern in line:
file_handler.write(f"# {line}")
else:
file_handler.write(line)
file_handler.truncate()
+def _prep_datadog_dir(datadog_install_path):
+ # Tries to replicate what the datadog-cloudfoundry-buildpack
+ # supply script does before running the datadog startup script.
+ # The DD buildpack assumes /home/vcap/app/.datadog to be the default
+ # directory for all configs and binaries though it provides DATADOG_DIR
+ # as an environment variable. So here we try to move all files under lib/
+ # to /home/vcap/app/.datadog/
+ shutil.copytree(
+ datadog_install_path,
+ DOT_DATADOG_PATH,
+ copy_function=shutil.move,
+ dirs_exist_ok=True,
+ )
+ # finally cleanup up the lib/ dir
+ shutil.rmtree(datadog_install_path)
+
+
def stage(buildpack_path, build_path, cache_path):
if not is_enabled():
return
@@ -545,6 +612,9 @@ def stage(buildpack_path, build_path, cache_path):
logging.debug("Patching run-datadog.sh...")
_patch_run_datadog_script(_get_agent_dir(build_path))
+ logging.debug("Patching create_logs_config.rb ...")
+ _patch_create_logs_config_script(_get_agent_dir(build_path))
+
def run(model_version, runtime_version):
if not is_enabled():
@@ -557,8 +627,11 @@ def run(model_version, runtime_version):
)
return
+ logging.debug("Prep DATADOG_DIR ... %s", _get_agent_dir())
+ _prep_datadog_dir(_get_agent_dir())
+
logging.debug("Setting Datadog Agent script permissions if required...")
- util.set_executable(f"{_get_agent_dir()}/*.sh")
+ util.set_executable(f"{_get_agent_dir()}/scripts/*.sh")
# Start the run script "borrowed" from the official DD buildpack
# and include settings as environment variables
diff --git a/dependencies.yml b/dependencies.yml
index 11eebdadc..37cffd6d6 100644
--- a/dependencies.yml
+++ b/dependencies.yml
@@ -20,10 +20,10 @@ dependencies:
buildpack:
alias: cf-datadog-sidecar
artifact: datadog/datadog-cloudfoundry-buildpack-{{ version }}.zip
- version: 4.33.0
+ version: 4.37.0
trace-agent:
artifact: datadog/dd-java-agent-{{ version }}.jar
- version: 0.101.0
+ version: 1.27.0
dynatrace:
agent:
artifact: "{{ url }}/e/{{ environment }}/api/v1/deployment/installer/agent/unix/paas/latest?include=java&bitness=64&Api-Token={{ token }}"
@@ -92,6 +92,11 @@ dependencies:
fs: cflinuxfs4
cpe: cpe:2.3:a:f5:nginx:{{ version }}:*:*:*:*:*:*:*
version: 1.22.1
+ ruby:
+ artifact: ruby/ruby_{{ version }}_linux_x64_{{ fs }}_{{ commit }}.tgz
+ commit: 5fed98f8
+ fs: cflinuxfs4
+ version: 3.2.2
telegraf:
agent:
artifact: telegraf/telegraf-{{ version }}_linux_amd64.tar.gz
diff --git a/pyproject.toml b/pyproject.toml
index e73e06c23..a9a12f25e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -10,7 +10,7 @@ exclude = [
"tests/unit/test_databroker.py"
]
-[tool.ruff.mccabe]
+[tool.ruff.lint.mccabe]
# Unlike Flake8, default to a complexity level of 10.
max-complexity = 10
diff --git a/requirements-dev.in b/requirements-dev.in
index dbd7bb33a..d76475d0c 100644
--- a/requirements-dev.in
+++ b/requirements-dev.in
@@ -1,11 +1,11 @@
click==8.1.7
idna==3.6
-pytest==7.4.3
-pytest-timer==0.0.11
+pytest==8.0.1
+pytest-timer==1.0.0
pytest-timeout==2.2.0
-pylint==3.0.2
-pyopenssl==23.3.0
+pylint==3.0.3
+pyopenssl==24.0.0
randomname==0.2.1
requests-mock==1.11.0
-ruff==0.1.6
+ruff==0.2.2
parameterized==0.9.0
diff --git a/requirements.in b/requirements.in
index 230527d90..3236c03bd 100644
--- a/requirements.in
+++ b/requirements.in
@@ -1,11 +1,11 @@
backoff==2.2.1
-certifi==2023.11.17
-cryptography==41.0.7
-distro==1.8.0
+certifi==2024.2.2
+cryptography==42.0.4
+distro==1.9.0
httplib2==0.22.0
-jinja2==3.1.2
+jinja2==3.1.3
omegaconf==2.3.0
psycopg2-binary==2.9.9
pyyaml==6.0.1
requests==2.31.0
-urllib3==2.1.0
\ No newline at end of file
+urllib3==2.2.1
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 8c9b7c45e..6d152a1b6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,7 +8,7 @@ antlr4-python3-runtime==4.9.3
# via omegaconf
backoff==2.2.1
# via -r requirements.in
-certifi==2023.11.17
+certifi==2024.2.2
# via
# -r requirements.in
# requests
@@ -16,15 +16,15 @@ cffi==1.14.4
# via cryptography
charset-normalizer==2.0.3
# via requests
-cryptography==41.0.7
+cryptography==42.0.4
# via -r requirements.in
-distro==1.8.0
+distro==1.9.0
# via -r requirements.in
httplib2==0.22.0
# via -r requirements.in
idna==2.8
# via requests
-jinja2==3.1.2
+jinja2==3.1.3
# via -r requirements.in
markupsafe==2.0.1
# via jinja2
@@ -42,7 +42,7 @@ pyyaml==6.0.1
# omegaconf
requests==2.31.0
# via -r requirements.in
-urllib3==2.1.0
+urllib3==2.2.1
# via
# -r requirements.in
# requests
diff --git a/tests/integration/test_datadog.py b/tests/integration/test_datadog.py
index 739142dbe..a78daeee9 100644
--- a/tests/integration/test_datadog.py
+++ b/tests/integration/test_datadog.py
@@ -59,8 +59,8 @@ def _test_logsubscriber_active(self):
logsubscribers = json.loads(logsubscribers_json.text)
self.assertTrue("DatadogSubscriber" in logsubscribers["feedback"])
- def test_datadog_mx7(self):
- self._test_datadog("BuildpackTestApp-mx-7-16.mda")
-
def test_datadog_mx8(self):
self._test_datadog("Mendix8.1.1.58432_StarterApp.mda")
+
+ def test_datadog_mx9(self):
+ self._test_datadog("BuildpackTestApp-mx9-7.mda")