proto: add task to (re)generate proto files (#73)

.github/workflows/pypi.yml

@@ -17,7 +17,7 @@ jobs:
       - name: "Checkout code"
         uses: actions/checkout@v3
       - name: "Generate protobuf files to be included in the distribution"
-        run: ./bin/gen_proto_files.py
+        run: ./faasmctl/bin/gen_proto_files.py
       - name: "Set up Python"
         uses: actions/setup-python@v4
         with:

README.md

@@ -8,7 +8,7 @@ running [Faasm](https://github.com/faasm/faasm) cluster.
 To install `faasmctl` you need a working `pip` (virtual-)environment. Then:
 
 ```bash
-pip install faasmctl==0.34.0
+pip install faasmctl==0.35.0
 ```
 
 ## Usage

bin/create_venv.sh

@@ -21,6 +21,6 @@ pip_cmd install -U pip
 pip_cmd install -r ${PROJ_ROOT}/dev_requirements.txt
 
 # Generate the protobuf files if necessary
-${PROJ_ROOT}/bin/gen_proto_files.py
+${PROJ_ROOT}/faasmctl/bin/gen_proto_files.py
 
 popd >> /dev/null

bin/gen_proto_files.py → faasmctl/bin/gen_proto_files.py

@@ -1,25 +1,45 @@
 #!/usr/bin/env python3
 
-from os import listdir, makedirs
-from os.path import exists, join
+from os import environ, listdir, makedirs
+from os.path import dirname, exists, join, realpath
+from shutil import rmtree
+from sys import argv
 from subprocess import run
 
+FAASMCTL_ROOT = dirname(dirname(realpath(__file__)))
+
 # Unfortunately, we need to duplicate this constants here as we need to be
 # able to run this file in standalone mode, as if the proto files are not
 # generated, some imports in `faasmctl` will fail
-GEN_PROTO_DIR = "./faasmctl/util/gen_proto"
-FAASM_CLI_IMAGE = "faasm.azurecr.io/cli:0.17.0"
+GEN_PROTO_DIR = join(FAASMCTL_ROOT, "util", "gen_proto")
+FAASM_CLI_IMAGE = "faasm.azurecr.io/cli"
 
 PROTO_FILES = [
     "faabric_pb2.py",
     "planner_pb2.py",
 ]
 
 
-def gen_proto_files():
+# We can not import faasmctl here
+def get_faasm_version():
+    ver_file = join(FAASMCTL_ROOT, "util", "faasm.py")
+    cmd = "cat {} | grep 'FAASM_VERSION =' | cut -d' ' -f3".format(ver_file)
+    version = (
+        run(cmd, shell=True, check=True, capture_output=True)
+        .stdout.decode("utf-8")
+        .strip()[1:-1]
+    )
+
+    return version
+
+
+def gen_proto_files(clean=False):
     """
     Generate python proto files to interact with the Faasm cluster
     """
+    if clean:
+        rmtree(GEN_PROTO_DIR)
+
     if not exists(GEN_PROTO_DIR):
         makedirs(GEN_PROTO_DIR)
 
@@ -29,9 +49,16 @@ def gen_proto_files():
     if pb_files == PROTO_FILES:
         return
 
-    print("Generating Faasm protobuf files...")
+    if "FAASM_VERSION" in environ:
+        faasm_ver = environ["FAASM_VERSION"]
+    else:
+        faasm_ver = get_faasm_version()
+
+    print("Generating Faasm (v{}) protobuf files...".format(faasm_ver))
     tmp_ctr_name = "faasm_gen_proto"
-    cm = "docker run -d -i --name {} {}".format(tmp_ctr_name, FAASM_CLI_IMAGE)
+    cm = "docker run -d -i --name {} {}:{}".format(
+        tmp_ctr_name, FAASM_CLI_IMAGE, faasm_ver
+    )
     run(cm, shell=True, check=True)
 
     # Find the right protoc binary
@@ -79,4 +106,5 @@ def gen_proto_files():
 
 
 if __name__ == "__main__":
-    gen_proto_files()
+    clean = len(argv) == 2 and argv[1] == "--clean"
+    gen_proto_files(clean)

faasmctl/tasks/__init__.py

@@ -4,6 +4,7 @@
 from . import delete
 from . import deploy
 from . import flush
+from . import generate
 from . import invoke
 from . import logs
 from . import monitor
@@ -15,6 +16,7 @@
     cli,
     delete,
     deploy,
+    generate,
     flush,
     invoke,
     logs,

faasmctl/tasks/generate.py

@@ -0,0 +1,14 @@
+from invoke import task
+from os.path import join
+from subprocess import run
+from faasmctl.util.env import PROJ_ROOT
+
+
+@task
+def proto(ctx):
+    """
+    (Re)generate the Python protobuf files necessary to interact with Faasm
+    """
+    gen_proto_bin = join(PROJ_ROOT, "bin", "gen_proto_files.py")
+    gen_cmd = "python3 {} --clean".format(gen_proto_bin)
+    run(gen_cmd, shell=True, check=True)

faasmctl/util/version.py

@@ -1,4 +1,4 @@
-FAASMCTL_VERSION = "0.34.0"
+FAASMCTL_VERSION = "0.35.0"
 
 
 def get_version():

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "faasmctl"
-version = "0.34.0"
+version = "0.35.0"
 authors = [
   { name="Faasm Team", email="[email protected]" },
 ]

tasks/git.py

@@ -78,7 +78,6 @@ def bump_dep(ctx, faasm=None):
 
         files_to_check = [
             join(DEV_PROJ_ROOT, "faasmctl", "util", "faasm.py"),
-            join(DEV_PROJ_ROOT, "bin", "gen_proto_files.py"),
         ]
 
         for f in files_to_check: