Doc changes: Added Transfers section in Azure provider docs (apache#3…

…2241)
eladkal · Jun 29, 2023 · 57b7ba1 · 57b7ba1
1 parent 7d2ec76
commit 57b7ba1
Show file tree

Hide file tree

Showing 8 changed files with 94 additions and 5 deletions.
diff --git a/airflow/providers/microsoft/azure/provider.yaml b/airflow/providers/microsoft/azure/provider.yaml
@@ -230,21 +230,22 @@ triggers:
 transfers:
   - source-integration-name: Local
     target-integration-name: Microsoft Azure Data Lake Storage
-    how-to-guide: /docs/apache-airflow-providers-microsoft-azure/operators/local_to_adls.rst
+    how-to-guide: /docs/apache-airflow-providers-microsoft-azure/transfer/local_to_adls.rst
     python-module: airflow.providers.microsoft.azure.transfers.local_to_adls
   - source-integration-name: Oracle
     target-integration-name: Microsoft Azure Data Lake Storage
     python-module: airflow.providers.microsoft.azure.transfers.oracle_to_azure_data_lake
   - source-integration-name: Local
     target-integration-name: Microsoft Azure Blob Storage
+    how-to-guide: /docs/apache-airflow-providers-microsoft-azure/transfer/local_to_wasb.rst
     python-module: airflow.providers.microsoft.azure.transfers.local_to_wasb
   - source-integration-name: Microsoft Azure Blob Storage
     target-integration-name: Google Cloud Storage (GCS)
-    how-to-guide: /docs/apache-airflow-providers-microsoft-azure/operators/azure_blob_to_gcs.rst
+    how-to-guide: /docs/apache-airflow-providers-microsoft-azure/transfer/azure_blob_to_gcs.rst
     python-module: airflow.providers.microsoft.azure.transfers.azure_blob_to_gcs
   - source-integration-name: SSH File Transfer Protocol (SFTP)
     target-integration-name: Microsoft Azure Blob Storage
-    how-to-guide: /docs/apache-airflow-providers-microsoft-azure/operators/sftp_to_wasb.rst
+    how-to-guide: /docs/apache-airflow-providers-microsoft-azure/transfer/sftp_to_wasb.rst
     python-module: airflow.providers.microsoft.azure.transfers.sftp_to_wasb
 
 

diff --git a/docs/apache-airflow-providers-microsoft-azure/index.rst b/docs/apache-airflow-providers-microsoft-azure/index.rst
@@ -36,6 +36,7 @@
 
     Connection types <connections/index>
     Operators <operators/index>
+    Transfers <transfer/index>
     Secrets backends <secrets-backends/azure-key-vault>
     Logging for Tasks <logging/index>
 

diff --git a/...oft-azure/operators/azure_blob_to_gcs.rst → ...soft-azure/transfer/azure_blob_to_gcs.rst b/...oft-azure/operators/azure_blob_to_gcs.rst → ...soft-azure/transfer/azure_blob_to_gcs.rst
diff --git a/docs/apache-airflow-providers-microsoft-azure/transfer/index.rst b/docs/apache-airflow-providers-microsoft-azure/transfer/index.rst
@@ -0,0 +1,28 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+
+Azure Transfer Operators
+=========================
+
+
+.. toctree::
+    :maxdepth: 1
+    :glob:
+
+    *
diff --git a/...crosoft-azure/operators/local_to_adls.rst → ...icrosoft-azure/transfer/local_to_adls.rst b/...crosoft-azure/operators/local_to_adls.rst → ...icrosoft-azure/transfer/local_to_adls.rst
diff --git a/docs/apache-airflow-providers-microsoft-azure/transfer/local_to_wasb.rst b/docs/apache-airflow-providers-microsoft-azure/transfer/local_to_wasb.rst
@@ -0,0 +1,51 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+Upload data from Local Filesystem to Azure Blob Storage
+=======================================================
+The Blob service stores text and binary data as objects in the cloud.
+The Blob service offers the following three resources: the storage account, containers, and blobs.
+Within your storage account, containers provide a way to organize sets of blobs.
+For more information about the service visit `Azure Blob Storage API documentation <https://docs.microsoft.com/en-us/rest/api/storageservices/blob-service-rest-api>`_.
+This page shows how to upload data from local filesystem to Azure Blob Storage.
+
+
+.. _howto/operator:LocalFilesystemToWasbOperator:
+
+LocalFilesystemToWasbOperator
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:class:`~airflow.providers.microsoft.azure.transfers.local_to_wasb.LocalFilesystemToWasbOperator` allows you to
+upload data from local filesystem to Azure Blob Storage.
+
+
+Below is an example of using this operator to upload a file to Azure Blob Storage.
+
+.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_local_to_wasb.py
+    :language: python
+    :dedent: 0
+    :start-after: [START howto_operator_local_to_wasb]
+    :end-before: [END howto_operator_local_to_wasb]
+
+
+Reference
+---------
+
+For further information, look at:
+
+* `Azure Blob Storage Documentation <https://learn.microsoft.com/en-us/azure/storage/blobs/>`__
diff --git a/...icrosoft-azure/operators/sftp_to_wasb.rst → ...microsoft-azure/transfer/sftp_to_wasb.rst b/...icrosoft-azure/operators/sftp_to_wasb.rst → ...microsoft-azure/transfer/sftp_to_wasb.rst
diff --git a/tests/system/providers/microsoft/azure/example_local_to_wasb.py b/tests/system/providers/microsoft/azure/example_local_to_wasb.py
@@ -30,15 +30,23 @@
 PATH_TO_UPLOAD_FILE = os.environ.get("AZURE_PATH_TO_UPLOAD_FILE", "example-text.txt")
 ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
 DAG_ID = "example_local_to_wasb"
+AZURE_CONTAINER_NAME = os.environ.get("AZURE_CONTAINER_NAME", "mycontainer")
+AZURE_BLOB_NAME = os.environ.get("AZURE_BLOB_NAME", "myblob")
 
 with DAG(
     DAG_ID,
     schedule="@once",
     start_date=datetime(2021, 1, 1),
     catchup=False,
-    default_args={"container_name": "mycontainer", "blob_name": "myblob"},
 ) as dag:
-    upload = LocalFilesystemToWasbOperator(task_id="upload_file", file_path=PATH_TO_UPLOAD_FILE)
+    # [START howto_operator_local_to_wasb]
+    upload = LocalFilesystemToWasbOperator(
+        task_id="upload_file",
+        file_path=PATH_TO_UPLOAD_FILE,
+        container_name=AZURE_CONTAINER_NAME,
+        blob_name=AZURE_BLOB_NAME,
+    )
+    # [END howto_operator_local_to_wasb]
     delete = WasbDeleteBlobOperator(task_id="delete_file")
 
     upload >> delete