[WIP] Replace Python gRPC with Rust

flyrs/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "flyrs"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+# The name of the native library. This is the name which will be used in Python to import the
+# library (i.e. `import string_sum`). If you change this, you must also change the name of the
+# `#[pymodule]` in `src/lib.rs`.
+name = "flyrs"
+# "cdylib" is necessary to produce a shared library for Python to import from.
+crate-type = ["cdylib"]
+
+[dependencies]
+prost = "0.12.3"
+tonic = "0.11.0"
+tokio = { version = "1.37.0", features = ["full"] }
+pyo3 = { version = "0.21", features = ["extension-module", "experimental-async"] }
+flyteidl = { git="https://github.com/austin362667/flyte.git", branch = "austin362667/flyteidl/rs" }
+tower = "0.4.13"
+
+[build-dependencies]
+

flyrs/perf.py

@@ -0,0 +1,32 @@
+import timeit
+import matplotlib.pyplot as plt
+
+setup = """
+from flytekit.remote import FlyteRemote;
+from remote import RustFlyteRemote;
+from flytekit.configuration import Config;
+PROJECT = "flytesnacks";
+DOMAIN = "development";
+remote_py = FlyteRemote(Config.auto(), default_project=PROJECT, default_domain=DOMAIN);
+remote_rs = RustFlyteRemote(Config.auto(), default_project=PROJECT, default_domain=DOMAIN);
+"""
+
+fetch_task_in_py = """task_py = remote_py.fetch_task(project=PROJECT, domain=DOMAIN, name="workflows_.say_1", version="WhIAnhpyjrAdaRvrQ9Cjpw")"""
+fetch_task_in_rs = """task_rs = remote_rs.fetch_task(project=PROJECT, domain=DOMAIN, name="workflows_.say_1", version="WhIAnhpyjrAdaRvrQ9Cjpw")"""
+
+r = 10
+Xs = [1, 10, 100, 1000]
+py_elpased, rs_elpased = [], []
+for x in Xs:
+    # Python gRPC
+    py_elpased.append(sum(timeit.repeat(fetch_task_in_py, setup=setup, repeat=r, number=x))/r)
+    print()
+    # Rust gRPC
+    rs_elpased.append(sum(timeit.repeat(fetch_task_in_rs, setup=setup, repeat=r, number=x))/r)
+    print()
+plt.xlabel('# of fetched tasks')
+plt.ylabel('average elapsed time (s)')
+plt.plot(Xs, py_elpased,'r-',label='Python gRPC')
+plt.plot(Xs, rs_elpased,'b-',label='Rust gRPC')
+plt.legend()
+plt.savefig("perf.png")

flyrs/pyproject.toml

@@ -0,0 +1,12 @@
+[build-system]
+requires = ["maturin>=1,<2"]
+build-backend = "maturin"
+
+[project]
+name = "flyrs"
+requires-python = ">=3.7"
+classifiers = [
+    "Programming Language :: Rust",
+    "Programming Language :: Python :: Implementation :: CPython",
+    "Programming Language :: Python :: Implementation :: PyPy",
+]

flyrs/readme.md

@@ -0,0 +1,11 @@
+### How to build Rust FlyteRemote Client?
+1. Instal `matruin` to build Python packages with Rust `PyO3`.
+    - `pip install maturin`
+2. Compile Python extension and instal it into local Python virtual environment root.
+    - `maturin develop`
+    - or `maturin develop --release` (Pass `--release` to `cargo build`)
+
+### How to test Rust FlyteRemote Client?
+1. `pyflyte register ./t.py` to get flyte entity `version` id
+2. Set previous fetched `version` id in `./test_flyte_remote.py`'s `VERSION_ID`
+2. `pytest ./test_flyte_remote.py` inside `flyrs/`