[DO NOT MERGE] Upload all docs script

airbyte-ci/connectors/metadata_service/lib/metadata_service/commands.py

@@ -5,6 +5,7 @@
 import pathlib
 
 import click
+from connector_ops.utils import get_all_connectors_in_repo
 from metadata_service.constants import METADATA_FILE_NAME
 from metadata_service.gcs_upload import MetadataUploadInfo, upload_metadata_to_gcs
 from metadata_service.validators.metadata_validator import PRE_UPLOAD_VALIDATORS, ValidatorOptions, validate_and_load
@@ -60,3 +61,27 @@ def upload(metadata_file_path: pathlib.Path, docs_path: pathlib.Path, bucket_nam
     else:
         click.secho(f"The metadata file {metadata_file_path} was not uploaded.", color="yellow")
         exit(5)
+
+
+@metadata_service.command(help="Upload docs for all connectors to a GCS bucket.")
+@click.argument("airbyte-repo-path", type=click.Path(exists=True, path_type=pathlib.Path))
+@click.argument("docs-dir", type=click.Path(exists=True, path_type=pathlib.Path))
+@click.argument("bucket-name", type=click.STRING)
+def upload_all_metadata(airbyte_repo_path: pathlib.Path, docs_dir: pathlib.Path, bucket_name: str):
+    connectors = get_all_connectors_in_repo()
+    failed_connectors = []
+    for connector in connectors:
+        # failed_connectors.append(str(connector))
+        try:
+            print(f"~~~~~~ Uploading metadata for {connector}")
+            metadata_file_path = airbyte_repo_path / connector.metadata_file_path
+            upload_metadata_to_gcs(bucket_name, metadata_file_path, ValidatorOptions(docs_path=str(docs_dir)))
+            # This break just makes the script upload docs for a single connector. Comment it out to upload docs for all connectors.
+            # break
+            # raise Exception("This is a test exception")
+        except Exception as e:
+            failed_connectors.append([str(connector), str(e)])
+            pass
+        # break
+    print(f"Failed to upload metadata for the following connectors: {failed_connectors}")
+    exit(0)