Updates for agents integrations docs

docs/conf.py

@@ -231,8 +231,8 @@
         "auto_examples/basics/index.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/basics/index.html",
         "auto_examples/basics/task.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/basics/task.html",
         "auto_examples/basics/shell_task.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/basics/shell_task.html",
-        "auto_examples/bigquery_plugin/index.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/bigquery_plugin/index.html",
-        "auto_examples/bigquery_plugin/bigquery.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/bigquery_plugin/bigquery.html",
+        "auto_examples/bigquery_plugin/index.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/bigquery_integration/index.html",
+        "auto_examples/bigquery_plugin/bigquery.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/bigquery_integration/bigquery.html",
         "auto_examples/blast/blastx_example.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/blast/blastx_example.html",
         "auto_examples/blast/index.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/blast/index.html",
         "auto_examples/customizing_dependencies/image_spec.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/customizing_dependencies/image_spec.html",
@@ -248,8 +248,8 @@
         "auto_examples/data_types_and_io/dataclass.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/data_types_and_io/dataclass.html",
         "auto_examples/data_types_and_io/folder.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/data_types_and_io/folder.html",
         "auto_examples/data_types_and_io/pickle_type.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/data_types_and_io/pickle_type.html",
-        "auto_examples/databricks_plugin/databricks_job.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/databricks_plugin/databricks_job.html",
-        "auto_examples/databricks_plugin/index.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/databricks_plugin/index.html",
+        "auto_examples/databricks_plugin/databricks_job.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/databricks_integration/databricks_job.html",
+        "auto_examples/databricks_plugin/index.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/databricks_integration/index.html",
         "auto_examples/dbt_plugin/index.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/dbt_plugin/index.html",
         "auto_examples/dbt_plugin/dbt_example.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/dbt_plugin/dbt_example.html",
         "auto_examples/development_lifecycle/inspecting_executions.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/development_lifecycle/inspecting_executions.html",
@@ -317,8 +317,8 @@
         "auto_examples/kftensorflow_plugin/index.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/kftensorflow_plugin/index.html",
         "auto_examples/mlflow_plugin/index.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/mlflow_plugin/index.html",
         "auto_examples/mlflow_plugin/mlflow_example.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/mlflow_plugin/mlflow_example.html",
-        "auto_examples/mmcloud_plugin/example.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/mmcloud_plugin/example.html",
-        "auto_examples/mmcloud_plugin/index.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/mmcloud_plugin/index.html",
+        "auto_examples/mmcloud_plugin/example.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/mmcloud_integration/example.html",
+        "auto_examples/mmcloud_plugin/index.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/mmcloud_integration/index.html",
         "auto_examples/mnist_classifier/pytorch_single_node_and_gpu.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/mnist_classifier/pytorch_single_node_and_gpu.html",
         "auto_examples/mnist_classifier/index.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/mnist_classifier/index.html",
         "auto_examples/mnist_classifier/pytorch_single_node_multi_gpu.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/mnist_classifier/pytorch_single_node_multi_gpu.html",
@@ -358,8 +358,8 @@
         "auto_examples/sagemaker_training_plugin/sagemaker_custom_training.htmlhttps://docs.flyte.org/en/latest/": "flytesnacks/examples/sagemaker_training_plugin/sagemaker_custom_training.html",
         "auto_examples/sensor/file_sensor_example.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/sensor/file_sensor_example.html",
         "auto_examples/sensor/index.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/sensor/index.html",
-        "auto_examples/snowflake_plugin/snowflake.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/snowflake_plugin/snowflake.html",
-        "auto_examples/snowflake_plugin/index.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/snowflake_plugin/index.html",
+        "auto_examples/snowflake_plugin/snowflake.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/snowflake_integration/snowflake.html",
+        "auto_examples/snowflake_plugin/index.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/snowflake_integration/index.html",
         "auto_examples/sql_plugin/sql_alchemy.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/sql_plugin/sql_alchemy.html",
         "auto_examples/sql_plugin/index.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/sql_plugin/index.html",
         "auto_examples/sql_plugin/sqlite3_integration.html": "https://docs.flyte.org/en/latest/flytesnacks/examples/sql_plugin/sqlite3_integration.html",

docs/index.md

@@ -98,14 +98,15 @@ flyte_lab
 :hidden:
 
 Integrations <integrations>
+auto_examples/airflow_agent/index
 auto_examples/airflow_plugin/index
 auto_examples/athena_plugin/index
 auto_examples/aws_batch_plugin/index
 auto_examples/sagemaker_pytorch_plugin/index
 auto_examples/sagemaker_training_plugin/index
-auto_examples/bigquery_plugin/index
+auto_examples/bigquery_integration/index
 auto_examples/k8s_dask_plugin/index
-auto_examples/databricks_plugin/index
+auto_examples/databricks_integration/index
 auto_examples/dbt_plugin/index
 auto_examples/dolt_plugin/index
 auto_examples/duckdb_plugin/index
@@ -114,7 +115,7 @@ auto_examples/greatexpectations_plugin/index
 auto_examples/hive_plugin/index
 auto_examples/k8s_pod_plugin/index
 auto_examples/mlflow_plugin/index
-auto_examples/mmcloud_plugin/index
+auto_examples/mmcloud_integration/index
 auto_examples/modin_plugin/index
 auto_examples/kfmpi_plugin/index
 auto_examples/onnx_plugin/index
@@ -123,7 +124,7 @@ auto_examples/pandera_plugin/index
 auto_examples/kfpytorch_plugin/index
 auto_examples/ray_plugin/index
 auto_examples/sensor/index
-auto_examples/snowflake_plugin/index
+auto_examples/snowflake_integration/index
 auto_examples/k8s_spark_plugin/index
 auto_examples/sql_plugin/index
 auto_examples/kftensorflow_plugin/index

docs/integrations.md

@@ -80,11 +80,35 @@ orchestrated by Flyte itself, within its provisioned Kubernetes clusters.
   - Run Ray jobs on a K8s Cluster.
 ```
 
+(flyte_agents)=
+
+## Flyte agents
+
+[Flyte agents](https://docs.flyte.org/en/latest/flyte_agents/index.html) are long-running, stateless services that receive execution requests via gRPC and initiate jobs with appropriate external or internal services. Each agent service is a Kubernetes deployment that receives gRPC requests from FlytePropeller when users trigger a particular type of task. (For example, the BigQuery agent handles BigQuery tasks.) The agent service then initiates a job with the appropriate service. If you don't see the agent you need below, see "[Developing agents](https://docs.flyte.org/en/latest/flyte_agents/developing_agents.html)" to learn how to develop a new agent.
+
+```{list-table}
+:header-rows: 0
+:widths: 20 30
+
+* - {ref}`Airflow agent <airflow_agent>`
+  - Run Airflow jobs in your workflows with the Airflow agent.
+* - {ref}`BigQuery agent <bigquery_agent>`
+  - Run BigQuery jobs in your workflows with the BigQuery agent.
+* - {ref}`Databricks <databricks_agent>`
+  - Run Databricks jobs in your workflows with the Databricks agent.
+* - {ref}`Memory Machine Cloud <mmcloud_agent>`
+  - Execute tasks using the MemVerge Memory Machine Cloud agent.
+* - {doc}`Sensor <auto_examples/sensor/index>`
+  - Run sensor jobs in your workflows with the sensor agent.
+* - {ref}`Snowflake <snowflake_agent>`
+  - Run Snowflake jobs in your workflows with the Snowflake agent.
+```
+
 (external_service_backend_plugins)=
 
 ## External Service Backend Plugins
 
-As the term suggests, external service backend plugins relies on external services like
+As the term suggests, external service backend plugins rely on external services like
 [AWS Sagemaker](https://aws.amazon.com/sagemaker),
 [Hive](https://docs.qubole.com/en/latest/user-guide/engines/hive/index.html) or
 [Snowflake](https://www.snowflake.com/) for handling the workload defined in
@@ -94,27 +118,25 @@ the Flyte task that use the respective plugin.
 :header-rows: 0
 :widths: 20 30
 
-* - {doc}`AWS Sagemaker: Model Training <auto_examples/sagemaker_training_plugin/index>`
+* - {doc}`AWS Sagemaker: Model Training plugin <auto_examples/sagemaker_training_plugin/index>`
   - Train models with built-in or define your own custom algorithms.
-* - {doc}`AWS Sagemaker: Pytorch Training <auto_examples/sagemaker_pytorch_plugin/index>`
+* - {doc}`AWS Sagemaker: Pytorch Training plugin <auto_examples/sagemaker_pytorch_plugin/index>`
   - Train Pytorch models using Sagemaker, with support for distributed training.
-* - {doc}`AWS Athena <auto_examples/athena_plugin/index>`
+* - {doc}`AWS Athena plugin <auto_examples/athena_plugin/index>`
   - Execute queries using AWS Athena
-* - {doc}`AWS Batch <auto_examples/aws_batch_plugin/index>`
+* - {doc}`AWS Batch plugin <auto_examples/aws_batch_plugin/index>`
   - Running tasks and workflows on AWS batch service
 * - {doc}`Flyte Interactive <auto_examples/flyteinteractive_plugin/index>`
   - Execute tasks using Flyte Interactive to debug.
-* - {doc}`Hive <auto_examples/hive_plugin/index>`
+* - {doc}`Hive plugin <auto_examples/hive_plugin/index>`
   - Run Hive jobs in your workflows.
-* - {doc}`MMCloud <auto_examples/mmcloud_plugin/index>`
+* - {ref}`MMCloud plugin <mmcloud_plugin>`
   - Execute tasks using MemVerge Memory Machine Cloud
-* - {doc}`Sensor <auto_examples/sensor/index>`
-  - Run Sensor jobs in your workflows.
-* - {doc}`Snowflake <auto_examples/snowflake_plugin/index>`
+* - {ref}`Snowflake <snowflake_plugin>`
   - Run Snowflake jobs in your workflows.
-* - {doc}`Databricks <auto_examples/databricks_plugin/index>`
+* - {ref}`Databricks <databricks_plugin>`
   - Run Databricks jobs in your workflows.
-* - {doc}`BigQuery <auto_examples/bigquery_plugin/index>`
+* - {ref}`BigQuery <bigquery_plugin>`
   - Run BigQuery jobs in your workflows.
 ```
 

examples/airflow_agent/README.md

@@ -0,0 +1,48 @@
+(airflow_agent)=
+
+# Airflow agent
+
+```{note}
+The Airflow agent does not support all [Airflow operators](https://airflow.apache.org/docs/apache-airflow/stable/core-concepts/operators.html). We have tested many, but if you run into issues, please [file a bug report](https://github.com/flyteorg/flyte/issues/new?assignees=&labels=bug%2Cuntriaged&projects=&template=bug_report.yaml&title=%5BBUG%5D+).
+```
+
+## Installation
+
+To install the plugin, run the following command:
+
+`pip install flytekitplugins-airflow`
+
+## Example usage
+
+```{note}
+
+You don't need an Airflow cluster to run Airflow tasks, since Flytekit will
+automatically compile Airflow tasks to Flyte tasks and execute them on the Flyte cluster.
+
+```
+
+For a usage example, see the {doc}`Airflow agent example <airflow_agent_example>` page.
+
+## Local testing
+
+To test an agent locally, create a class for the agent task that inherits from [AsyncAgentExecutorMixin](https://github.com/flyteorg/flytekit/blob/master/flytekit/extend/backend/base_agent.py#L155). This mixin can handle both asynchronous tasks and synchronous tasks and allows flytekit to mimic FlytePropeller's behavior in calling the agent. For more information, see "[Testing agents locally](https://docs.flyte.org/en/latest/flyte_agents/testing_agents_locally.html)".
+
+```{note}
+
+In some cases, you will need to store credentials in your local environment when testing locally.
+
+```
+
+## Flyte deployment configuration
+
+```{note}
+If you are using a managed deployment of Flyte, you will need to contact your deployment administrator to configure agents in your deployment.
+```
+
+To enable the Airflow agent in your Flyte deployment, see the {ref}`Airflow agent deployment guide<deployment-agent-setup-airflow>`.
+
+```{toctree}
+:maxdepth: -1
+:hidden:
+airflow_agent_example
+```

examples/airflow_agent/airflow_agent/airflow_agent_example.py

@@ -0,0 +1,23 @@
+# %% [markdown]
+# (airflow_agent_example)=
+# # Airflow agent example
+#
+# %%
+
+from airflow.sensors.filesystem import FileSensor
+from flytekit import task, workflow
+
+
+@task()
+def t1():
+    print("flyte")
+
+
+@workflow
+def wf():
+    sensor = FileSensor(task_id="id", filepath="/tmp/1234")
+    sensor >> t1()
+
+
+if __name__ == "__main__":
+    wf()

examples/airflow_agent/airflow_agent/requirements.in

@@ -0,0 +1,2 @@
+apache-airflow-providers-apache-beam[google]
+apache-airflow[google]