Upgrade to Dagger 0.13 (#46923)

## What
<!--
* Describe what the change is solving. Link all GitHub issues related to this change.
-->
Upgrade from dagger 0.9.6 to dagger 0.13

## How
<!--
* Describe how code changes achieve the solution.
-->
* change behavior of `with_exec()`. The `skip_entrypoint` keyword arg has been changed to an opposite kwarg: `use_entrypoint`
* The `pipeline()` function has been removed. Remove its use.

## How will I deem the update stable
* [x] `pipelines` tests are 🟢 
* [x] `CAT` tests are 🟢 
* [x] `base_image` tests are 🟢 
* [x] Tests are running for a java and a python connector 🟢  [(Python test)](https://github.com/airbytehq/airbyte/actions/runs/11370125686) [(Java test)](https://github.com/airbytehq/airbyte/actions/runs/11381915439/job/31664363609)
* [x] The publish pipeline still works for  java and python connectors 🟢  [(test)](https://github.com/airbytehq/airbyte/actions/runs/11367885824/job/31621641890)


## User Impact
Should have no impact

## Can this PR be safely reverted and rolled back?
<!--
* If unsure, leave it blank.
-->
- [x] YES 💚
- [ ] NO ❌

airbyte-cdk/python/bin/generate-component-manifest-dagger.sh

@@ -7,5 +7,5 @@
 
 set -e
 
-pip install dagger-io==0.9.6
-python bin/generate_component_manifest_files.py
+pip install dagger-io==0.13.3
+python bin/generate_component_manifest_files.py

airbyte-cdk/python/bin/generate_component_manifest_files.py

@@ -29,7 +29,7 @@ def generate_init_module_content() -> str:
 
 
 async def post_process_codegen(codegen_container: dagger.Container):
-    codegen_container = codegen_container.with_exec(["mkdir", "/generated_post_processed"])
+    codegen_container = codegen_container.with_exec(["mkdir", "/generated_post_processed"], use_entrypoint=True)
     for generated_file in await codegen_container.directory("/generated").entries():
         if generated_file.endswith(".py"):
             original_content = await codegen_container.file(f"/generated/{generated_file}").contents()
@@ -49,8 +49,8 @@ async def main():
         codegen_container = (
             dagger_client.container()
             .from_(PYTHON_IMAGE)
-            .with_exec(["mkdir", "/generated"])
-            .with_exec(["pip", "install", " ".join(PIP_DEPENDENCIES)])
+            .with_exec(["mkdir", "/generated"], use_entrypoint=True)
+            .with_exec(["pip", "install", " ".join(PIP_DEPENDENCIES)], use_entrypoint=True)
             .with_mounted_directory("/yaml", dagger_client.host().directory(LOCAL_YAML_DIR_PATH, include=["*.yaml"]))
             .with_new_file("/generated/__init__.py", contents=init_module_content)
         )
@@ -66,7 +66,8 @@ async def main():
                     "--enum-field-as-literal",
                     "one",
                     "--set-default-enum-member",
-                ]
+                ],
+                use_entrypoint=True,
             )
 
         await ((await post_process_codegen(codegen_container)).directory("/generated_post_processed").export(LOCAL_OUTPUT_DIR_PATH))

airbyte-ci/connectors/base_images/README.md

@@ -94,6 +94,9 @@ poetry run mypy base_images --check-untyped-defs
 ```
 ## CHANGELOG
 
+### 1.0.4
+- Upgrade Dagger to `0.13.3`
+
 ### 1.0.2
 
 - Improved support for images with non-semantic-versioned tags.

airbyte-ci/connectors/base_images/base_images/bases.py

@@ -99,4 +99,4 @@ def get_base_container(self, platform: dagger.Platform) -> dagger.Container:
         Returns:
             dagger.Container: The container using the base python image.
         """
-        return self.dagger_client.pipeline(self.name_with_tag).container(platform=platform).from_(self.root_image.address)
+        return self.dagger_client.container(platform=platform).from_(self.root_image.address)

airbyte-ci/connectors/base_images/base_images/python/bases.py

@@ -45,9 +45,9 @@ def get_nltk_data_dir() -> dagger.Directory:
                     zip_file = self.dagger_client.http(nltk_data_url)
                     data_container = (
                         data_container.with_file("/tmp/data.zip", zip_file)
-                        .with_exec(["mkdir", "-p", full_nltk_data_path], skip_entrypoint=True)
-                        .with_exec(["unzip", "-o", "/tmp/data.zip", "-d", full_nltk_data_path], skip_entrypoint=True)
-                        .with_exec(["rm", "/tmp/data.zip"], skip_entrypoint=True)
+                        .with_exec(["mkdir", "-p", full_nltk_data_path])
+                        .with_exec(["unzip", "-o", "/tmp/data.zip", "-d", full_nltk_data_path])
+                        .with_exec(["rm", "/tmp/data.zip"])
                     )
             return data_container.directory(self.nltk_data_path)
 
@@ -58,7 +58,7 @@ def with_tesseract_and_poppler(container: dagger.Container) -> dagger.Container:
             """
 
             container = container.with_exec(
-                ["sh", "-c", "apt-get update && apt-get install -y tesseract-ocr=5.3.0-2 poppler-utils=22.12.0-2+b1"], skip_entrypoint=True
+                ["sh", "-c", "apt-get update && apt-get install -y tesseract-ocr=5.3.0-2 poppler-utils=22.12.0-2+b1"]
             )
 
             return container
@@ -71,9 +71,7 @@ def with_file_based_connector_dependencies(container: dagger.Container) -> dagge
             - nltk data
             """
             container = with_tesseract_and_poppler(container)
-            container = container.with_exec(["mkdir", self.nltk_data_path], skip_entrypoint=True).with_directory(
-                self.nltk_data_path, get_nltk_data_dir()
-            )
+            container = container.with_exec(["mkdir", self.nltk_data_path]).with_directory(self.nltk_data_path, get_nltk_data_dir())
             return container
 
         return with_file_based_connector_dependencies
@@ -95,19 +93,17 @@ def get_container(self, platform: dagger.Platform) -> dagger.Container:
         return (
             self.get_base_container(platform)
             .with_mounted_cache("/root/.cache/pip", pip_cache_volume)
-            # Set the timezone to UTC
-            .with_exec(["ln", "-snf", "/usr/share/zoneinfo/Etc/UTC", "/etc/localtime"])
-            # Upgrade pip to the expected version
-            .with_exec(["pip", "install", "--upgrade", "pip==24.0", "setuptools==70.0.0"])
+            .with_exec(["ln", "-snf", "/usr/share/zoneinfo/Etc/UTC", "/etc/localtime"], use_entrypoint=True)
+            .with_exec(["pip", "install", "--upgrade", "pip==24.0", "setuptools==70.0.0"], use_entrypoint=True)
             # Declare poetry specific environment variables
             .with_env_variable("POETRY_VIRTUALENVS_CREATE", "false")
             .with_env_variable("POETRY_VIRTUALENVS_IN_PROJECT", "false")
             .with_env_variable("POETRY_NO_INTERACTION", "1")
-            .with_exec(["pip", "install", "poetry==1.6.1"], skip_entrypoint=True)
-            # Upgrade system packages
-            .with_exec(["sh", "-c", "apt-get update && apt-get upgrade -y && apt-get dist-upgrade -y && apt-get clean"])
-            # Install socat 1.7.4.4
-            .with_exec(["sh", "-c", "apt-get install -y socat=1.7.4.4-2"])
+            .with_exec(["pip", "install", "poetry==1.6.1"])
+            .with_exec(
+                ["sh", "-c", "apt-get update && apt-get upgrade -y && apt-get dist-upgrade -y && apt-get clean"], use_entrypoint=True
+            )
+            .with_exec(["sh", "-c", "apt-get install -y socat=1.7.4.4-2"], use_entrypoint=True)
             # Install CDK system dependencies
             .with_(self.install_cdk_system_dependencies())
         )

airbyte-ci/connectors/base_images/base_images/python/sanity_checks.py

@@ -18,7 +18,7 @@ async def check_python_version(container: dagger.Container, expected_python_vers
         errors.SanityCheckError: Raised if the python --version command could not be executed or if the outputted version is not the expected one.
     """
     try:
-        python_version_output: str = await container.with_exec(["python", "--version"], skip_entrypoint=True).stdout()
+        python_version_output: str = await container.with_exec(["python", "--version"]).stdout()
     except dagger.ExecError as e:
         raise errors.SanityCheckError(e)
     if python_version_output != f"Python {expected_python_version}\n":
@@ -36,7 +36,7 @@ async def check_pip_version(container: dagger.Container, expected_pip_version: s
         errors.SanityCheckError: Raised if the pip --version command could not be executed or if the outputted version is not the expected one.
     """
     try:
-        pip_version_output: str = await container.with_exec(["pip", "--version"], skip_entrypoint=True).stdout()
+        pip_version_output: str = await container.with_exec(["pip", "--version"]).stdout()
     except dagger.ExecError as e:
         raise errors.SanityCheckError(e)
     if not pip_version_output.startswith(f"pip {expected_pip_version}"):
@@ -54,7 +54,7 @@ async def check_poetry_version(container: dagger.Container, expected_poetry_vers
         errors.SanityCheckError: Raised if the poetry --version command could not be executed or if the outputted version is not the expected one.
     """
     try:
-        poetry_version_output: str = await container.with_exec(["poetry", "--version"], skip_entrypoint=True).stdout()
+        poetry_version_output: str = await container.with_exec(["poetry", "--version"]).stdout()
     except dagger.ExecError as e:
         raise errors.SanityCheckError(e)
     if not poetry_version_output.startswith(f"Poetry (version {expected_poetry_version}"):
@@ -95,11 +95,10 @@ async def check_nltk_data(python_image_container: dagger.Container):
     Raises:
         errors.SanityCheckError: Raised if the nltk data is not available.
     """
-    with_nltk = await python_image_container.with_exec(["pip", "install", "nltk==3.8.1"], skip_entrypoint=True)
+    with_nltk = await python_image_container.with_exec(["pip", "install", "nltk==3.8.1"])
     try:
         await with_nltk.with_exec(
-            ["python", "-c", 'import nltk;nltk.data.find("taggers/averaged_perceptron_tagger");nltk.data.find("tokenizers/punkt")'],
-            skip_entrypoint=True,
+            ["python", "-c", 'import nltk;nltk.data.find("taggers/averaged_perceptron_tagger");nltk.data.find("tokenizers/punkt")']
         )
     except dagger.ExecError as e:
         raise errors.SanityCheckError(e)
@@ -116,7 +115,7 @@ async def check_tesseract_version(python_image_container: dagger.Container, tess
         errors.SanityCheckError: Raised if the tesseract --version command could not be executed or if the outputted version is not the expected one.
     """
     try:
-        tesseract_version_output = await python_image_container.with_exec(["tesseract", "--version"], skip_entrypoint=True).stdout()
+        tesseract_version_output = await python_image_container.with_exec(["tesseract", "--version"]).stdout()
     except dagger.ExecError as e:
         raise errors.SanityCheckError(e)
     if not tesseract_version_output.startswith(f"tesseract {tesseract_version}"):
@@ -135,7 +134,7 @@ async def check_poppler_utils_version(python_image_container: dagger.Container,
         errors.SanityCheckError: Raised if the pdftotext -v command could not be executed or if the outputted version is not the expected one.
     """
     try:
-        pdf_to_text_version_output = await python_image_container.with_exec(["pdftotext", "-v"], skip_entrypoint=True).stderr()
+        pdf_to_text_version_output = await python_image_container.with_exec(["pdftotext", "-v"]).stderr()
     except dagger.ExecError as e:
         raise errors.SanityCheckError(e)
 

airbyte-ci/connectors/base_images/base_images/sanity_checks.py

@@ -23,7 +23,7 @@ async def check_env_var_with_printenv(
         errors.SanityCheckError: Raised if the environment variable is not defined or if it has an unexpected value.
     """
     try:
-        printenv_output = await container.with_exec(["printenv"], skip_entrypoint=True).stdout()
+        printenv_output = await container.with_exec(["printenv"]).stdout()
     except dagger.ExecError as e:
         raise errors.SanityCheckError(e)
     env_vars = {line.split("=")[0]: line.split("=")[1] for line in printenv_output.splitlines()}
@@ -45,7 +45,7 @@ async def check_timezone_is_utc(container: dagger.Container):
         errors.SanityCheckError: Raised if the date command could not be executed or if the outputted timezone is not UTC.
     """
     try:
-        tz_output: str = await container.with_exec(["date"], skip_entrypoint=True).stdout()
+        tz_output: str = await container.with_exec(["date"]).stdout()
     except dagger.ExecError as e:
         raise errors.SanityCheckError(e)
     if "UTC" not in tz_output:
@@ -63,7 +63,7 @@ async def check_a_command_is_available_using_version_option(container: dagger.Co
         errors.SanityCheckError: Raised if the command could not be executed or if the outputted version is not the expected one.
     """
     try:
-        command_version_output: str = await container.with_exec([command, version_option], skip_entrypoint=True).stdout()
+        command_version_output: str = await container.with_exec([command, version_option]).stdout()
     except dagger.ExecError as e:
         raise errors.SanityCheckError(e)
     if command_version_output == "":
@@ -81,7 +81,7 @@ async def check_socat_version(container: dagger.Container, expected_socat_versio
         errors.SanityCheckError: Raised if the socat --version command could not be executed or if the outputted version is not the expected one.
     """
     try:
-        socat_version_output: str = await container.with_exec(["socat", "-V"], skip_entrypoint=True).stdout()
+        socat_version_output: str = await container.with_exec(["socat", "-V"]).stdout()
     except dagger.ExecError as e:
         raise errors.SanityCheckError(e)
     socat_version_line = None

airbyte-ci/connectors/base_images/base_images/utils/docker.py

@@ -52,20 +52,18 @@ def login(self) -> dagger.Container:
         return (
             self.bare_container.with_secret_variable("DOCKER_HUB_USERNAME", self.docker_hub_username_secret)
             .with_secret_variable("DOCKER_HUB_PASSWORD", self.docker_hub_username_password)
-            .with_exec(
-                ["sh", "-c", "crane auth login index.docker.io -u $DOCKER_HUB_USERNAME -p $DOCKER_HUB_PASSWORD"], skip_entrypoint=True
-            )
+            .with_exec(["sh", "-c", "crane auth login index.docker.io -u $DOCKER_HUB_USERNAME -p $DOCKER_HUB_PASSWORD"])
         )
 
     async def digest(self, repository_and_tag: str) -> str:
         console.log(f"Fetching digest for {repository_and_tag}...")
-        return (await self.authenticated_container.with_exec(["digest", repository_and_tag]).stdout()).strip()
+        return (await self.authenticated_container.with_exec(["digest", repository_and_tag], use_entrypoint=True).stdout()).strip()
 
     async def ls(self, registry_name: str, repository_name: str) -> List[str]:
         repository_address = f"{registry_name}/{repository_name}"
         console.log(f"Fetching published images in {repository_address}...")
         try:
-            crane_ls_output = await self.authenticated_container.with_exec(["ls", repository_address]).stdout()
+            crane_ls_output = await self.authenticated_container.with_exec(["ls", repository_address], use_entrypoint=True).stdout()
             return crane_ls_output.splitlines()
         except dagger.ExecError as exec_error:
             # When the repository does not exist, crane ls returns an error with NAME_UNKNOWN in the stderr.