Update dependencies and logging (#10)

.flake8

@@ -1,6 +1,6 @@
 [flake8]
 max-line-length = 88
-max-complexity = 8
+max-complexity = 10
 ignore = W503,E501
 builtins = unicode
 tee = True

README.md

@@ -30,7 +30,7 @@ Example: `function-1`
 
 ### `cloud_function_directory` (required, string)
 
-The directory in your repository where are you storing the schemas for your tables and views.
+The directory in your repository where are you storing the code files for cloud function.
 
 Example: `wf-gcp-project/function-1`
 

hooks/command

@@ -1,6 +1,29 @@
 #!/bin/bash
 set -euo pipefail
 
+# retry <number-of-retries> <command>
+function retry {
+	local retries=$1
+	shift
+	local attempts=1
+	local status=0
+
+	until "$@"; do
+		status=$?
+		echo "Exited with $status"
+		if ((retries == "0")); then
+			return $status
+		elif ((attempts == retries)); then
+			echo "Failed $attempts retries"
+			return $status
+		else
+			echo "Retrying $((retries - attempts)) more times..."
+			attempts=$((attempts + 1))
+			sleep $(((attempts - 2) * 2))
+		fi
+	done
+}
+
 PLUGIN_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.."
 
 rm -rf /tmp/service_account.json
@@ -117,6 +140,14 @@ command_string="$(
 args+=("${command_string}")
 display_command+=("'${command_string}'")
 
+echo "--- :docker: Pulling ${image}"
+if ! retry "${BUILDKITE_PLUGIN_CLOUD_FUNCTIONS_DOCKER_PULL_RETRIES:-3}" \
+	docker pull "${image}"; then
+	rv=$?
+	echo "--- :docker: Pull failed."
+	exit $rv
+fi
+
 echo '--- :docker: Logging "docker create" command'
 echo "$ docker create" >&2
 

plugin_scripts/deploy.py

@@ -3,6 +3,7 @@
 import os
 import sys
 import zipfile
+from pprint import pformat
 from tempfile import TemporaryFile
 from urllib.parse import urlparse
 
@@ -53,7 +54,7 @@ def _upload_source_code_using_archive_url(archive_url: str, data):
     bucket = storage_client.bucket(bucket_name)
     blob = bucket.blob(blob_name)
     blob.upload_from_string(data.read())
-    _logger.info(f"Source code object {blob_name} uploaded to bucket {bucket_name}.")
+    _logger.info(f"Source code object {blob_name} uploaded to bucket {bucket_name}. \n")
 
 
 def _upload_source_code_using_upload_url(upload_url: str, data):
@@ -64,10 +65,10 @@ def _upload_source_code_using_upload_url(upload_url: str, data):
         "x-goog-content-length-range": "0,104857600",
     }
     response: Response = requests.put(upload_url, headers=headers, data=data)
-    _logger.info(f"HTTP Status Code for uploading data: {response.status_code}")
+    _logger.info(f"HTTP Status Code for uploading data: {response.status_code} \n")
 
     if os.environ.get("debug_mode", False):
-        _logger.info(f"Response body: {response.text}")
+        _logger.info(f"Response body: {pformat(response.json)} \n")
 
 
 def _validate_env_variables():
@@ -100,7 +101,7 @@ def _validate_if_path_exists():
 
 def _handle_exception(e, debug_mode):
     if debug_mode:
-        _logger.info(f"HTTP Status Code for patching Function: {str(e)}")
+        _logger.info(f"HTTP Status Code for patching Function: {str(e)} \n")
 
 
 def _deploy():
@@ -123,6 +124,9 @@ def _deploy():
         # check if cloud function exists, if it exists execution continues as is otherwise it will raise an exception
         function = cloud_functions.get(name=function_path).execute()
 
+        if debug_mode:
+            _logger.info(f"Function Definition: {pformat(function)} \n")
+
         with TemporaryFile() as data:
             file_handler = zipfile.ZipFile(data, mode="w")
             _zip_directory(file_handler)
@@ -144,11 +148,11 @@ def _deploy():
             response = cloud_functions.patch(
                 name=function_path, body=function
             ).execute()
-            _logger.info("Successfully patched Cloud Function.")
-            _logger.info(f"Operation Name: {response['name']}")
+            _logger.info("Successfully patched Cloud Function. \n")
+            _logger.info(f"Operation Name: {response['name']} \n")
 
             if debug_mode:
-                _logger.info(f"Response: {response}")
+                _logger.info(f"Response: {pformat(response)}")
         except Exception as e:
             deploy_failed = True
             _handle_exception(e, debug_mode)

plugin_scripts/requirements.lock

@@ -1,33 +1,40 @@
 # THIS IS AN AUTOGENERATED LOCKFILE. DO NOT EDIT MANUALLY.
-cachetools==4.1.1
-certifi==2020.6.20
-cffi==1.14.2
-chardet==3.0.4
+cachetools==4.2.4
+certifi==2021.10.8
+cffi==1.15.0
+chardet==4.0.0
+charset-normalizer==2.0.9
 gbq==0.4.0
-google-api-core==1.22.1
-google-api-python-client==1.10.0
-google-auth==1.20.1
-google-auth-httplib2==0.0.4
-google-cloud-bigquery==1.27.2
-google-cloud-core==1.4.1
-google-cloud-storage==1.30.0
-google-crc32c==0.1.0
-google-resumable-media==0.7.1
-googleapis-common-protos==1.52.0
-httplib2==0.18.1
-idna==2.10
+google-api-core==2.3.0
+google-api-python-client==2.33.0
+google-auth==2.3.3
+google-auth-httplib2==0.1.0
+google-cloud-bigquery==2.31.0
+google-cloud-core==2.2.1
+google-cloud-storage==1.43.0
+google-crc32c==1.3.0
+google-resumable-media==2.1.0
+googleapis-common-protos==1.54.0
+grpcio==1.42.0
+grpcio-status==1.42.0
+httplib2==0.20.2
+idna==3.3
+packaging==21.3
 pip==21.2.4
-protobuf==3.13.0
+proto-plus==1.19.8
+protobuf==3.19.1
 pyasn1==0.4.8
 pyasn1-modules==0.2.8
-pycparser==2.20
+pycparser==2.21
 pydantic==1.8.2
-pytz==2020.1
-requests==2.24.0
-rsa==4.6
+pyparsing==3.0.6
+python-dateutil==2.8.2
+pytz==2021.3
+requests==2.26.0
+rsa==4.8
 setuptools==57.5.0
-six==1.15.0
+six==1.16.0
 typing_extensions==4.0.1
-uritemplate==3.0.1
-urllib3==1.25.10
+uritemplate==4.1.1
+urllib3==1.26.7
 wheel==0.37.0

requirements.txt

@@ -1,32 +1,39 @@
-cachetools==4.1.1
-certifi==2020.6.20
-cffi==1.14.2
-chardet==3.0.4
+cachetools==4.2.4
+certifi==2021.10.8
+cffi==1.15.0
+chardet==4.0.0
+charset-normalizer==2.0.9
 gbq==0.4.0
-google-api-core==1.22.1
-google-api-python-client==1.10.0
-google-auth==1.20.1
+google-api-core==2.3.0
+google-api-python-client==2.33.0
+google-auth==2.3.3
 google-auth-httplib2==0.1.0
-google-cloud-bigquery==1.27.2
-google-cloud-core==1.4.1
-google-cloud-storage==1.30.0
-google-crc32c==0.1.0
-google-resumable-media==0.7.1
-googleapis-common-protos==1.52.0
-httplib2==0.18.1
-idna==2.10
+google-cloud-bigquery==2.31.0
+google-cloud-core==2.2.1
+google-cloud-storage==1.43.0
+google-crc32c==1.3.0
+google-resumable-media==2.1.0
+googleapis-common-protos==1.54.0
+grpcio==1.42.0
+grpcio-status==1.42.0
+httplib2==0.20.2
+idna==3.3
+packaging==21.3
 pip==21.2.4
+proto-plus==1.19.8
 protobuf==3.19.1
 pyasn1==0.4.8
 pyasn1-modules==0.2.8
-pycparser==2.20
+pycparser==2.21
 pydantic==1.8.2
-pytz==2020.1
-requests==2.24.0
+pyparsing==3.0.6
+python-dateutil==2.8.2
+pytz==2021.3
+requests==2.26.0
 rsa==4.8
 setuptools==57.5.0
-six==1.15.0
+six==1.16.0
 typing_extensions==4.0.1
-uritemplate==3.0.1
-urllib3==1.25.10
+uritemplate==4.1.1
+urllib3==1.26.7
 wheel==0.37.0