diff --git a/.github/labeler.yml b/.github/labeler.yml index 99c0e6deda..fca2a40588 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -18,6 +18,16 @@ pay: - pnpm-lock.yaml core: - - core/**/* + - core/api/* + - core/api-cron/* + - core/api-exporter/* + - core/api-trigger/* + - core/api-ws-server/* - flake.lock - pnpm-lock.yaml + +api-keys: + - third-party/rust/* + - core/api-keys/* + - flake.lock + - Cargo.lock diff --git a/.github/workflows/buck2-test.yaml b/.github/workflows/buck2-test.yaml index 6f11aed2df..36cf3e9f29 100644 --- a/.github/workflows/buck2-test.yaml +++ b/.github/workflows/buck2-test.yaml @@ -23,7 +23,7 @@ jobs: ${{ toJSON(github.event.pull_request.labels.*.name) }} EOF - DEFAULT_LABELS=("dashboard" "consent" "pay" "core") + DEFAULT_LABELS=("dashboard" "consent" "pay" "core", "api-keys") LABELS=($(jq -r '.[]' < labels.json)) if [ ${#LABELS[@]} -eq 0 ]; then LABELS=("${DEFAULT_LABELS[@]}") @@ -32,10 +32,13 @@ jobs: for LABEL in "${LABELS[@]}"; do case "$LABEL" in dashboard|consent|pay) - ARGS+=" //apps/$LABEL:test-unit" + ARGS+=" //apps/$LABEL:test" ;; core) - ARGS+=" //core/api:test-unit" + ARGS+=" //core/api:test" + ;; + api-keys) + ARGS+=" //core/$LABEL:test" ;; esac done diff --git a/BUCK b/BUCK index 2a5359e348..75a103e54d 100644 --- a/BUCK +++ b/BUCK @@ -12,3 +12,8 @@ pnpm_workspace( ], visibility = ["PUBLIC"], ) + +export_file( + name = "rustfmt.toml", + visibility = ["PUBLIC"], +) diff --git a/apps/consent/BUCK b/apps/consent/BUCK index 22ac5e9695..59a38ffd1c 100644 --- a/apps/consent/BUCK +++ b/apps/consent/BUCK @@ -86,7 +86,7 @@ eslint( ) test_suite( - name = "test-unit", + name = "test", tests = [ ":audit", ":lint", diff --git a/apps/dashboard/BUCK b/apps/dashboard/BUCK index ebd3a7612d..29cece9329 100644 --- a/apps/dashboard/BUCK +++ b/apps/dashboard/BUCK @@ -81,7 +81,7 @@ eslint( ) test_suite( - name = "test-unit", + name = "test", tests = [ ":audit", ":lint", diff --git a/apps/pay/BUCK b/apps/pay/BUCK index 06752e578e..cb1c7a62b1 100644 --- a/apps/pay/BUCK +++ b/apps/pay/BUCK @@ -86,7 +86,7 @@ eslint( ) test_suite( - name = "test-unit", + name = "test", tests = [ ":audit", ":lint", diff --git a/ci/apps/app-template.lib.yml b/ci/apps/app-template.lib.yml index a5f5107cb3..0bb9c8dc86 100644 --- a/ci/apps/app-template.lib.yml +++ b/ci/apps/app-template.lib.yml @@ -27,7 +27,7 @@ plan: - get: #@ app_src_resource_name(app) trigger: true - { get: pipeline-tasks } - - task: buck-test-unit + - task: buck-test config: platform: linux image_resource: #@ task_image_config() @@ -36,7 +36,7 @@ plan: - name: #@ app_src_resource_name(app) path: repo params: - BUCK_TARGET: #@ "//apps/" + app + ":test-unit" + BUCK_TARGET: #@ "//apps/" + app + ":test" BUCK_CMD: test run: path: pipeline-tasks/ci/apps/tasks/buck-task.sh diff --git a/ci/core/pipeline.yml b/ci/core/pipeline.yml index 6a3c576ecb..bec3085965 100644 --- a/ci/core/pipeline.yml +++ b/ci/core/pipeline.yml @@ -1,6 +1,7 @@ #@ load("@ytt:data", "data") #@ load("template.lib.yml", +#@ "component_src_resource", #@ "component_src_resource_name", #@ "core_bundle_src_resource", #@ "buck_test_name", @@ -26,6 +27,8 @@ source: repository: #@ release_pipeline_image() #@ end +#@ components = ["api-keys"] + groups: - name: core-bundle jobs: @@ -41,8 +44,17 @@ groups: - quickstart - release-core-bundle - bump-core-bundle-images-in-chart +#@ for component in components: +- name: #@ component + jobs: + - #@ buck_test_name(component) +#@ end jobs: +#@ for component in components: +- #@ buck_test(component) +#@ end + - #@ buck_test("api") #@ for component in core_bundle_components: - #@ build_edge_image(component) @@ -246,6 +258,11 @@ jobs: resources: - #@ core_bundle_src_resource() + +#@ for component in components: +- #@ component_src_resource(component) +#@ end + - name: repo-out type: git source: diff --git a/ci/core/template.lib.yml b/ci/core/template.lib.yml index 987c7bc4d7..3074d2a1c5 100644 --- a/ci/core/template.lib.yml +++ b/ci/core/template.lib.yml @@ -26,7 +26,7 @@ plan: - get: #@ component_src_resource_name(component) trigger: true - { get: pipeline-tasks } - - task: buck-test-unit + - task: buck-test config: platform: linux image_resource: #@ task_image_config() @@ -35,7 +35,7 @@ plan: - name: #@ component_src_resource_name(component) path: repo params: - BUCK_TARGET: #@ "//core/" + component + ":test-unit" + BUCK_TARGET: #@ "//core/" + component + ":test" BUCK_CMD: test run: path: pipeline-tasks/ci/apps/tasks/buck-task.sh @@ -155,6 +155,27 @@ source: webhook_token: ((webhook.secret)) #@ end +#@ def component_src_resource(component): +name: #@ component_src_resource_name(component) +type: git +source: + paths: + - #@ "core/" + component + - #@ "core/" + component + "/*" + - #@ "core/" + component + "/**/*" + - flake.nix + - flake.lock + - toolchains/ + - toolchains/* + - toolchains/**/* + - third-party/rust/**/* + - Cargo.lock + fetch_tags: true + uri: #@ data.values.git_uri + branch: #@ data.values.git_branch + private_key: #@ data.values.github_private_key +#@ end + #@ def edge_image_resource_name(component): #@ return component + "-edge-image" #@ end diff --git a/core/api-keys/BUCK b/core/api-keys/BUCK index 178c3faef3..bf5c878262 100644 --- a/core/api-keys/BUCK +++ b/core/api-keys/BUCK @@ -1,4 +1,5 @@ load("@toolchains//rover:macros.bzl", "sdl", "diff_check", "dev_update_file") +load("@toolchains//rust:macros.bzl", "rustfmt_check", "clippy_check") sdl( name = "sdl", @@ -74,3 +75,64 @@ rust_library( "SQLX_OFFLINE": "true", } ) + +rust_test( + name = "test-unit", + edition = "2021", + srcs = glob([ + "src/**/*.rs", + ".sqlx/*", + "migrations/*", + "api-keys.yml" + ]), + crate_root = "src/lib.rs", + deps = [ + "//lib/tracing-rs:tracing", + "//third-party/rust:tokio", + "//third-party/rust:anyhow", + "//third-party/rust:async-graphql", + "//third-party/rust:async-graphql-axum", + "//third-party/rust:axum", + "//third-party/rust:jsonwebtoken", + "//third-party/rust:clap", + "//third-party/rust:reqwest", + "//third-party/rust:serde", + "//third-party/rust:serde_yaml", + "//third-party/rust:serde_json", + "//third-party/rust:thiserror", + "//third-party/rust:chrono", + "//third-party/rust:sqlx", + "//third-party/rust:rand", + "//third-party/rust:uuid", + "//third-party/rust:serde_with", + ], + env = { + "CARGO_MANIFEST_DIR": ".", + "SQLX_OFFLINE": "true", + } +) + +rustfmt_check( + name = "check-format-rust", + srcs = glob([ + "src/**/*.rs", + ".sqlx/*", + "migrations/*", + "api-keys.yml" + ]), + crate_root = "src/lib.rs", +) + +clippy_check( + name = "check-lint-rust-lib", + clippy_txt_dep = ":lib-api-keys[clippy.txt]", +) + +test_suite( + name = "test", + tests = [ + ":check-format-rust", + ":check-lint-rust-lib", + ":test-unit" + ], +) diff --git a/core/api-keys/src/app/mod.rs b/core/api-keys/src/app/mod.rs index 270f476820..cefc107ad8 100644 --- a/core/api-keys/src/app/mod.rs +++ b/core/api-keys/src/app/mod.rs @@ -32,7 +32,7 @@ impl ApiKeysApp { &self, key: &str, ) -> Result<(IdentityApiKeyId, String, bool), ApplicationError> { - Ok(self.identities.find_subject_by_key(&key).await?) + Ok(self.identities.find_subject_by_key(key).await?) } #[tracing::instrument(name = "app.create_api_key_for_subject", skip_all)] diff --git a/core/api-keys/src/graphql/schema.rs b/core/api-keys/src/graphql/schema.rs index 44dc1e84bd..9d906202da 100644 --- a/core/api-keys/src/graphql/schema.rs +++ b/core/api-keys/src/graphql/schema.rs @@ -17,9 +17,9 @@ impl From> for Timestamp { } } -impl Into> for Timestamp { - fn into(self) -> DateTime { - self.0 +impl From for DateTime { + fn from(Timestamp(dt): Timestamp) -> Self { + dt } } diff --git a/core/api-keys/src/identity/mod.rs b/core/api-keys/src/identity/mod.rs index 33f69247bc..04c151e3a4 100644 --- a/core/api-keys/src/identity/mod.rs +++ b/core/api-keys/src/identity/mod.rs @@ -210,7 +210,7 @@ impl Identities { match record { Some(record) => Ok(IdentityApiKey { - id: IdentityApiKeyId::from(key_id), + id: key_id, name: record.name, identity_id: IdentityId::from(record.identity_id), created_at: record.created_at, diff --git a/core/api-keys/src/scope.rs b/core/api-keys/src/scope.rs index 77bdab207c..1e2ac3163d 100644 --- a/core/api-keys/src/scope.rs +++ b/core/api-keys/src/scope.rs @@ -2,7 +2,7 @@ pub const READ_SCOPE: &str = "read"; pub const WRITE_SCOPE: &str = "write"; pub fn read_only_scope() -> String { - format!("{READ_SCOPE}") + READ_SCOPE.to_string() } pub fn read_write_scope() -> String { @@ -10,5 +10,5 @@ pub fn read_write_scope() -> String { } pub fn is_read_only(scope: &String) -> bool { - !(scope.as_str().split(" ").any(|s| s == WRITE_SCOPE) || scope.is_empty()) + !(scope.as_str().split(' ').any(|s| s == WRITE_SCOPE) || scope.is_empty()) } diff --git a/core/api/BUCK b/core/api/BUCK index 4e4be88b45..ad519b8599 100644 --- a/core/api/BUCK +++ b/core/api/BUCK @@ -201,7 +201,7 @@ jest_test( ) test_suite( - name = "test-unit", + name = "test", tests = [ ":audit", ":check-lint", diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 0000000000..a311b9da0a --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,2 @@ +edition = "2021" +newline_style = "Unix" diff --git a/toolchains/BUCK b/toolchains/BUCK index fd7f559702..479b0a787f 100644 --- a/toolchains/BUCK +++ b/toolchains/BUCK @@ -47,3 +47,10 @@ rover_toolchain( name = "rover", visibility = ["PUBLIC"], ) + +load("@toolchains//rust:toolchain.bzl", "galoy_rust_toolchain") + +galoy_rust_toolchain( + name = "galoy_rust", + visibility = ["PUBLIC"], +) diff --git a/toolchains/rust/BUCK b/toolchains/rust/BUCK new file mode 100644 index 0000000000..50be3cc32a --- /dev/null +++ b/toolchains/rust/BUCK @@ -0,0 +1,14 @@ +export_file( + name = "clippy_output.py", + visibility = ["PUBLIC"], +) + +export_file( + name = "crate_context.py", + visibility = ["PUBLIC"], +) + +export_file( + name = "rustfmt_check.py", + visibility = ["PUBLIC"], +) diff --git a/toolchains/rust/clippy_output.py b/toolchains/rust/clippy_output.py new file mode 100644 index 0000000000..707a20c8be --- /dev/null +++ b/toolchains/rust/clippy_output.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +""" +Prints output of a Clippy compilation and exits non-zero if appropriate. +""" +import argparse +import os +import sys + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "clippy_txt", + help="The file from a Clippy compilation with its output") + + return parser.parse_args() + + +def main() -> int: + args = parse_args() + + # If output is empty, there are no errors/warnings and we can exit `0` + if os.path.getsize(args.clippy_txt) == 0: + return 0 + + # Otherwise print output and exit non-zero + with open(args.clippy_txt, encoding="utf-8") as f: + print(f.read()) + + return 1 + + +if __name__ == "__main__": + sys.exit(main()) + diff --git a/toolchains/rust/crate_context.py b/toolchains/rust/crate_context.py new file mode 100644 index 0000000000..09e84f7c93 --- /dev/null +++ b/toolchains/rust/crate_context.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +""" +Builds an isolated tree containing all crate sources. +""" +import argparse +import os +import shutil +import sys + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--src", + action="append", + help="Add a source into the source tree", + ) + parser.add_argument( + "out_path", + help="Path to output directory", + ) + + return parser.parse_args() + + +def main() -> int: + args = parse_args() + + for src in args.src or []: + parent_dir = os.path.dirname(src) + if parent_dir: + dst_dir = os.path.join(args.out_path, parent_dir) + if not os.path.isdir(dst_dir): + os.makedirs(dst_dir, exist_ok=True) + abspath_src = os.path.abspath(src) + if os.path.isdir(abspath_src): + print("dir; src={}, dst={}".format( + abspath_src, + os.path.join(args.out_path, src), + )) + shutil.copytree( + abspath_src, + os.path.join(args.out_path, src), + symlinks=True, + dirs_exist_ok=True, + ) + else: + print("file; src={}, dst={}".format( + abspath_src, + os.path.join(args.out_path, src), + )) + shutil.copy( + abspath_src, + os.path.join(args.out_path, src), + ) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) + diff --git a/toolchains/rust/macros.bzl b/toolchains/rust/macros.bzl new file mode 100644 index 0000000000..07a1dcde38 --- /dev/null +++ b/toolchains/rust/macros.bzl @@ -0,0 +1,144 @@ +load( + "@prelude//python:toolchain.bzl", + "PythonToolchainInfo", +) +load( + ":toolchain.bzl", + "GaloyRustToolchainInfo", +) +load( + "@prelude//test/inject_test_run_info.bzl", + "inject_test_run_info", +) +load( + "@prelude//tests:re_utils.bzl", + "get_re_executor_from_props", +) + +def clippy_check_impl(ctx: AnalysisContext) -> list[[ + DefaultInfo, + RunInfo, + ExternalRunnerTestInfo, +]]: + clippy_txt = ctx.attrs.clippy_txt_dep[DefaultInfo].default_outputs + + galoy_rust_toolchain = ctx.attrs._galoy_rust_toolchain[GaloyRustToolchainInfo] + + run_cmd_args = cmd_args( + ctx.attrs._python_toolchain[PythonToolchainInfo].interpreter, + galoy_rust_toolchain.clippy_output[DefaultInfo].default_outputs, + clippy_txt, + ) + + args_file = ctx.actions.write("args.txt", run_cmd_args) + + return inject_test_run_info( + ctx, + ExternalRunnerTestInfo( + type = "clippy", + command = [run_cmd_args] + ), + ) + [ + DefaultInfo(default_output = args_file), + ] + +clippy_check = rule( + impl = clippy_check_impl, + attrs = { + "clippy_txt_dep": attrs.dep( + doc = """Clippy sub target dep from a Rust library or binary""", + ), + "_python_toolchain": attrs.toolchain_dep( + default = "toolchains//:python", + providers = [PythonToolchainInfo], + ), + "_galoy_rust_toolchain": attrs.toolchain_dep( + default = "toolchains//:galoy_rust", + providers = [GaloyRustToolchainInfo], + ), + "_inject_test_env": attrs.default_only( + attrs.dep(default = "prelude//test/tools:inject_test_env"), + ), + }, +) + +def rustfmt_check_impl(ctx: AnalysisContext) -> list[[ + DefaultInfo, + RunInfo, + ExternalRunnerTestInfo, +]]: + galoy_rust_toolchain = ctx.attrs._galoy_rust_toolchain[GaloyRustToolchainInfo] + crate_ctx = crate_context(ctx) + + run_cmd_args = cmd_args( + ctx.attrs._python_toolchain[PythonToolchainInfo].interpreter, + galoy_rust_toolchain.rustfmt_check[DefaultInfo].default_outputs, + "--config-path", + galoy_rust_toolchain.rustfmt_toml[DefaultInfo].default_outputs, + cmd_args( + [crate_ctx.srcs_tree, ctx.label.package, ctx.attrs.crate_root], + delimiter = "/", + ) + ) + + args_file = ctx.actions.write("args.txt", run_cmd_args) + + return inject_test_run_info( + ctx, + ExternalRunnerTestInfo( + type = "rustfmt", + command = [run_cmd_args], + ), + ) + [ + DefaultInfo(default_output = args_file), + ] + +rustfmt_check = rule( + impl = rustfmt_check_impl, + attrs = { + "srcs": attrs.list( + attrs.source(), + default = [], + doc = """The set of Rust source files in the crate.""", + ), + "crate_root": attrs.string( + doc = """Top level source file for the crate.""", + ), + "_python_toolchain": attrs.toolchain_dep( + default = "toolchains//:python", + providers = [PythonToolchainInfo], + ), + "_galoy_rust_toolchain": attrs.toolchain_dep( + default = "toolchains//:galoy_rust", + providers = [GaloyRustToolchainInfo], + ), + "_inject_test_env": attrs.default_only( + attrs.dep(default = "prelude//test/tools:inject_test_env"), + ), + }, +) + +CrateContext = record( + srcs_tree = field(Artifact), +) + +def crate_context(ctx: AnalysisContext) -> CrateContext: + srcs_tree = ctx.actions.declare_output("__src") + + galoy_rust_toolchain = ctx.attrs._galoy_rust_toolchain[GaloyRustToolchainInfo] + + cmd = cmd_args( + ctx.attrs._python_toolchain[PythonToolchainInfo].interpreter, + galoy_rust_toolchain.crate_context[DefaultInfo].default_outputs, + ) + for src in ctx.attrs.srcs: + cmd.add("--src") + cmd.add(src) + cmd.add(srcs_tree.as_output()) + + ctx.actions.run(cmd, category = "crate_context") + + return CrateContext( + srcs_tree = srcs_tree, + ) + diff --git a/toolchains/rust/rustfmt_check.py b/toolchains/rust/rustfmt_check.py new file mode 100644 index 0000000000..8e1b6656aa --- /dev/null +++ b/toolchains/rust/rustfmt_check.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +""" +Runs a rustfmt check. +""" +import argparse +import subprocess +import sys + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--config-path", + help="Path for the rustfmt configuration file", + ) + parser.add_argument( + "crate_root", + help="Path to the top level source file of the crate", + ) + + return parser.parse_args() + + +def main() -> int: + args = parse_args() + + cmd = [ + "rustfmt", + "--check", + ] + if args.config_path: + cmd.append("--config-path") + cmd.append(args.config_path) + cmd.append(args.crate_root) + + result = subprocess.run(cmd) + + return result.returncode + + +if __name__ == "__main__": + sys.exit(main()) + diff --git a/toolchains/rust/toolchain.bzl b/toolchains/rust/toolchain.bzl new file mode 100644 index 0000000000..e25345569c --- /dev/null +++ b/toolchains/rust/toolchain.bzl @@ -0,0 +1,42 @@ +GaloyRustToolchainInfo = provider( + fields = { + "clippy_output": typing.Any, + "crate_context": typing.Any, + "rustfmt_check": typing.Any, + "rustfmt_toml": typing.Any + }, +) + + +def galoy_rust_toolchain_impl(ctx) -> list[[DefaultInfo, GaloyRustToolchainInfo]]: + """ + An extended Rust toolchain. + """ + return [ + DefaultInfo(), + GaloyRustToolchainInfo( + clippy_output = ctx.attrs._clippy_output, + crate_context = ctx.attrs._crate_context, + rustfmt_check = ctx.attrs._rustfmt_check, + rustfmt_toml = ctx.attrs._rustfmt_toml, + ), + ] + +galoy_rust_toolchain = rule( + impl = galoy_rust_toolchain_impl, + attrs = { + "_rustfmt_toml": attrs.dep( + default = "root//:rustfmt.toml" + ), + "_clippy_output": attrs.dep( + default = "toolchains//rust:clippy_output.py", + ), + "_crate_context": attrs.dep( + default = "toolchains//rust:crate_context.py", + ), + "_rustfmt_check": attrs.dep( + default = "toolchains//rust:rustfmt_check.py", + ), + }, + is_toolchain_rule = True, +)