From 57b7ba16a3d860268f03cd2619e5d029c7994013 Mon Sep 17 00:00:00 2001 From: Akash Sharma <35839624+Adaverse@users.noreply.github.com> Date: Thu, 29 Jun 2023 19:31:25 +0530 Subject: [PATCH] Doc changes: Added Transfers section in Azure provider docs (#32241) --- .../providers/microsoft/azure/provider.yaml | 7 +-- .../index.rst | 1 + .../azure_blob_to_gcs.rst | 0 .../transfer/index.rst | 28 ++++++++++ .../{operators => transfer}/local_to_adls.rst | 0 .../transfer/local_to_wasb.rst | 51 +++++++++++++++++++ .../{operators => transfer}/sftp_to_wasb.rst | 0 .../microsoft/azure/example_local_to_wasb.py | 12 ++++- 8 files changed, 94 insertions(+), 5 deletions(-) rename docs/apache-airflow-providers-microsoft-azure/{operators => transfer}/azure_blob_to_gcs.rst (100%) create mode 100644 docs/apache-airflow-providers-microsoft-azure/transfer/index.rst rename docs/apache-airflow-providers-microsoft-azure/{operators => transfer}/local_to_adls.rst (100%) create mode 100644 docs/apache-airflow-providers-microsoft-azure/transfer/local_to_wasb.rst rename docs/apache-airflow-providers-microsoft-azure/{operators => transfer}/sftp_to_wasb.rst (100%) diff --git a/airflow/providers/microsoft/azure/provider.yaml b/airflow/providers/microsoft/azure/provider.yaml index d57a94837252b..c255da931522b 100644 --- a/airflow/providers/microsoft/azure/provider.yaml +++ b/airflow/providers/microsoft/azure/provider.yaml @@ -230,21 +230,22 @@ triggers: transfers: - source-integration-name: Local target-integration-name: Microsoft Azure Data Lake Storage - how-to-guide: /docs/apache-airflow-providers-microsoft-azure/operators/local_to_adls.rst + how-to-guide: /docs/apache-airflow-providers-microsoft-azure/transfer/local_to_adls.rst python-module: airflow.providers.microsoft.azure.transfers.local_to_adls - source-integration-name: Oracle target-integration-name: Microsoft Azure Data Lake Storage python-module: airflow.providers.microsoft.azure.transfers.oracle_to_azure_data_lake - source-integration-name: Local target-integration-name: Microsoft Azure Blob Storage + how-to-guide: /docs/apache-airflow-providers-microsoft-azure/transfer/local_to_wasb.rst python-module: airflow.providers.microsoft.azure.transfers.local_to_wasb - source-integration-name: Microsoft Azure Blob Storage target-integration-name: Google Cloud Storage (GCS) - how-to-guide: /docs/apache-airflow-providers-microsoft-azure/operators/azure_blob_to_gcs.rst + how-to-guide: /docs/apache-airflow-providers-microsoft-azure/transfer/azure_blob_to_gcs.rst python-module: airflow.providers.microsoft.azure.transfers.azure_blob_to_gcs - source-integration-name: SSH File Transfer Protocol (SFTP) target-integration-name: Microsoft Azure Blob Storage - how-to-guide: /docs/apache-airflow-providers-microsoft-azure/operators/sftp_to_wasb.rst + how-to-guide: /docs/apache-airflow-providers-microsoft-azure/transfer/sftp_to_wasb.rst python-module: airflow.providers.microsoft.azure.transfers.sftp_to_wasb diff --git a/docs/apache-airflow-providers-microsoft-azure/index.rst b/docs/apache-airflow-providers-microsoft-azure/index.rst index 5820216b13dd0..dfc0bf56e14ee 100644 --- a/docs/apache-airflow-providers-microsoft-azure/index.rst +++ b/docs/apache-airflow-providers-microsoft-azure/index.rst @@ -36,6 +36,7 @@ Connection types Operators + Transfers Secrets backends Logging for Tasks diff --git a/docs/apache-airflow-providers-microsoft-azure/operators/azure_blob_to_gcs.rst b/docs/apache-airflow-providers-microsoft-azure/transfer/azure_blob_to_gcs.rst similarity index 100% rename from docs/apache-airflow-providers-microsoft-azure/operators/azure_blob_to_gcs.rst rename to docs/apache-airflow-providers-microsoft-azure/transfer/azure_blob_to_gcs.rst diff --git a/docs/apache-airflow-providers-microsoft-azure/transfer/index.rst b/docs/apache-airflow-providers-microsoft-azure/transfer/index.rst new file mode 100644 index 0000000000000..846e1057a22f5 --- /dev/null +++ b/docs/apache-airflow-providers-microsoft-azure/transfer/index.rst @@ -0,0 +1,28 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + + +Azure Transfer Operators +========================= + + +.. toctree:: + :maxdepth: 1 + :glob: + + * diff --git a/docs/apache-airflow-providers-microsoft-azure/operators/local_to_adls.rst b/docs/apache-airflow-providers-microsoft-azure/transfer/local_to_adls.rst similarity index 100% rename from docs/apache-airflow-providers-microsoft-azure/operators/local_to_adls.rst rename to docs/apache-airflow-providers-microsoft-azure/transfer/local_to_adls.rst diff --git a/docs/apache-airflow-providers-microsoft-azure/transfer/local_to_wasb.rst b/docs/apache-airflow-providers-microsoft-azure/transfer/local_to_wasb.rst new file mode 100644 index 0000000000000..970a6298342ed --- /dev/null +++ b/docs/apache-airflow-providers-microsoft-azure/transfer/local_to_wasb.rst @@ -0,0 +1,51 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + +Upload data from Local Filesystem to Azure Blob Storage +======================================================= +The Blob service stores text and binary data as objects in the cloud. +The Blob service offers the following three resources: the storage account, containers, and blobs. +Within your storage account, containers provide a way to organize sets of blobs. +For more information about the service visit `Azure Blob Storage API documentation `_. +This page shows how to upload data from local filesystem to Azure Blob Storage. + + +.. _howto/operator:LocalFilesystemToWasbOperator: + +LocalFilesystemToWasbOperator +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:class:`~airflow.providers.microsoft.azure.transfers.local_to_wasb.LocalFilesystemToWasbOperator` allows you to +upload data from local filesystem to Azure Blob Storage. + + +Below is an example of using this operator to upload a file to Azure Blob Storage. + +.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_local_to_wasb.py + :language: python + :dedent: 0 + :start-after: [START howto_operator_local_to_wasb] + :end-before: [END howto_operator_local_to_wasb] + + +Reference +--------- + +For further information, look at: + +* `Azure Blob Storage Documentation `__ diff --git a/docs/apache-airflow-providers-microsoft-azure/operators/sftp_to_wasb.rst b/docs/apache-airflow-providers-microsoft-azure/transfer/sftp_to_wasb.rst similarity index 100% rename from docs/apache-airflow-providers-microsoft-azure/operators/sftp_to_wasb.rst rename to docs/apache-airflow-providers-microsoft-azure/transfer/sftp_to_wasb.rst diff --git a/tests/system/providers/microsoft/azure/example_local_to_wasb.py b/tests/system/providers/microsoft/azure/example_local_to_wasb.py index cab99713f964a..7fff09c1358fa 100644 --- a/tests/system/providers/microsoft/azure/example_local_to_wasb.py +++ b/tests/system/providers/microsoft/azure/example_local_to_wasb.py @@ -30,15 +30,23 @@ PATH_TO_UPLOAD_FILE = os.environ.get("AZURE_PATH_TO_UPLOAD_FILE", "example-text.txt") ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") DAG_ID = "example_local_to_wasb" +AZURE_CONTAINER_NAME = os.environ.get("AZURE_CONTAINER_NAME", "mycontainer") +AZURE_BLOB_NAME = os.environ.get("AZURE_BLOB_NAME", "myblob") with DAG( DAG_ID, schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - default_args={"container_name": "mycontainer", "blob_name": "myblob"}, ) as dag: - upload = LocalFilesystemToWasbOperator(task_id="upload_file", file_path=PATH_TO_UPLOAD_FILE) + # [START howto_operator_local_to_wasb] + upload = LocalFilesystemToWasbOperator( + task_id="upload_file", + file_path=PATH_TO_UPLOAD_FILE, + container_name=AZURE_CONTAINER_NAME, + blob_name=AZURE_BLOB_NAME, + ) + # [END howto_operator_local_to_wasb] delete = WasbDeleteBlobOperator(task_id="delete_file") upload >> delete