derived_data_service: make ZelosDerivationQueue struct correspond to …

…a single repo and a single derived data config

Summary:
The main struct used in the derived data service and the derivation worker for all interactions with the derivation queue is `ZelosDerivationQueue`. This struct consist of a zeus client and a collection of repos (i.e. `MononokeRepos`), which means that there isn't a struct that represent the derivation queue for a single repo. This complicates adding admin commands for inspecting the derivation queue, complicates all interactions with the queue as we have to check that the repo and the derived data config we want to process actually exists in `MononokeRepos` (e.g. https://fburl.com/code/qdja0m09), and overall doesn't fit with the usual Mononoke style of having repos as the central objects and facets for each repo.

This diff tries to fix this by making `ZelosDerivationQueue` correspond to a single repo and a single derived data config, and adding a repo facet `RepoDerivationQueues` which is a map from config name to `ZelosDerivationQueue`. This diff is large because it touches the main object of the derived data service, but it's mostly the result of following and fixing compiler errors.

Reviewed By: singhsrb

Differential Revision: D56766255

fbshipit-source-id: 41bc8fc6961b757b11248789f9d2013a62be0d31

eden/mononoke/Cargo.toml

@@ -414,8 +414,12 @@ members = [
   "repo_attributes/hook_manager/repo_hook_file_content_provider",
   "repo_attributes/repo_bookmark_attrs",
   "repo_attributes/repo_cross_repo",
+  "repo_attributes/repo_derivation_queues",
+  "repo_attributes/repo_derivation_queues/if",
+  "repo_attributes/repo_derivation_queues/if/clients",
+  "repo_attributes/repo_derivation_queues/if/mocks",
+  "repo_attributes/repo_derivation_queues/if/services",
   "repo_attributes/repo_derived_data",
-  "repo_attributes/repo_derived_data_service",
   "repo_attributes/repo_identity",
   "repo_attributes/repo_lock/repo_lock",
   "repo_attributes/repo_permission_checker",

eden/mononoke/facebook/derived_data_service/TARGETS

@@ -23,8 +23,8 @@ rust_binary(
         "fbsource//third-party/rust:slog",
         "fbsource//third-party/rust:thiserror",
         "fbsource//third-party/rust:tokio",
-        ":derivation_queue",
         "//common/rust/shed/cloned:cloned",
+        "//common/rust/shed/facet:facet",
         "//common/rust/shed/fbinit:fbinit",
         "//common/rust/shed/futures_stats:futures_stats",
         "//common/rust/shed/justknobs_stub:justknobs",
@@ -51,9 +51,8 @@ rust_binary(
         "//eden/mononoke/derived_data/remote/if:derived_data_service_if-rust-services",
         "//eden/mononoke/git/git_types:git_types",
         "//eden/mononoke/metaconfig:metaconfig_types",
-        "//eden/mononoke/mononoke_repos:mononoke_repos",
         "//eden/mononoke/mononoke_types:mononoke_types",
-        "//eden/mononoke/repo_attributes/repo_derived_data_service:repo_derived_data_service",
+        "//eden/mononoke/repo_attributes/repo_derivation_queues:repo_derivation_queues",
         "//eden/mononoke/server/context:context",
         "//eden/mononoke/server/metadata:metadata",
         "//eden/scm/lib/clientinfo:clientinfo",
@@ -115,54 +114,6 @@ rust_binary(
     ],
 )
 
-rust_library(
-    name = "derivation_queue",
-    srcs = glob(["derivation_queue/**/*.rs"]),
-    test_deps = [
-        "fbsource//third-party/rust:assert_matches",
-        "fbsource//third-party/rust:pretty_assertions",
-        "fbsource//third-party/rust:tokio",
-        "//common/rust/shed/facet:facet",
-        "//common/rust/shed/fbinit:fbinit",
-        "//common/rust/shed/fbinit:fbinit-tokio",
-        "//eden/mononoke/blobrepo:repo_blobstore",
-        "//eden/mononoke/bonsai_hg_mapping:bonsai_hg_mapping",
-        "//eden/mononoke/bookmarks:bookmarks",
-        "//eden/mononoke/changesets:changesets",
-        "//eden/mononoke/derived_data:unodes",
-        "//eden/mononoke/filestore:filestore",
-        "//eden/mononoke/repo_attributes/repo_derived_data:repo_derived_data",
-        "//eden/mononoke/repo_factory:test_repo_factory",
-        "//eden/mononoke/server/context:context",
-        "//eden/mononoke/tests/utils:tests_utils",
-    ],
-    deps = [
-        "fbsource//third-party/rust:anyhow",
-        "fbsource//third-party/rust:async-trait",
-        "fbsource//third-party/rust:bytes",
-        "fbsource//third-party/rust:futures",
-        "fbsource//third-party/rust:parking_lot",
-        "fbsource//third-party/rust:slog",
-        "fbsource//third-party/rust:strum",
-        "fbsource//third-party/rust:thiserror",
-        "fbsource//third-party/rust:tokio",
-        "//common/rust/shed/cloned:cloned",
-        "//common/rust/shed/justknobs_stub:justknobs",
-        "//common/rust/zeus:zeus_client",
-        "//eden/mononoke/common/bounded_traversal:bounded_traversal",
-        "//eden/mononoke/derived_data:derived_data_manager",
-        "//eden/mononoke/derived_data:derived_data_utils",
-        "//eden/mononoke/facebook/derived_data_service/derivation_queue/if:derivation_queue_thrift-rust",
-        "//eden/mononoke/mononoke_repos:mononoke_repos",
-        "//eden/mononoke/mononoke_types:mononoke_types",
-        "//eden/mononoke/repo_attributes/commit_graph/commit_graph:commit_graph",
-        "//eden/mononoke/repo_attributes/repo_derived_data_service:repo_derived_data_service",
-        "//eden/mononoke/server/context:context",
-        "//eden/scm/lib/clientinfo:clientinfo",
-        "//thrift/lib/rust:fbthrift",
-    ],
-)
-
 rust_binary(
     name = "derivation_worker",
     srcs = glob(["workers/**/*.rs"]),
@@ -173,8 +124,8 @@ rust_binary(
         "fbsource//third-party/rust:futures",
         "fbsource//third-party/rust:slog",
         "fbsource//third-party/rust:tokio",
-        ":derivation_queue",
         "//common/rust/shed/cloned:cloned",
+        "//common/rust/shed/facet:facet",
         "//common/rust/shed/fbinit:fbinit",
         "//common/rust/shed/futures_stats:futures_stats",
         "//common/rust/shed/justknobs_stub:justknobs",
@@ -188,9 +139,9 @@ rust_binary(
         "//eden/mononoke/derived_data:derived_data_manager",
         "//eden/mononoke/derived_data:derived_data_utils",
         "//eden/mononoke/metaconfig:metaconfig_types",
-        "//eden/mononoke/mononoke_repos:mononoke_repos",
         "//eden/mononoke/mononoke_types:mononoke_types",
-        "//eden/mononoke/repo_attributes/repo_derived_data_service:repo_derived_data_service",
+        "//eden/mononoke/repo_attributes/repo_derivation_queues:repo_derivation_queues",
+        "//eden/mononoke/repo_attributes/repo_identity:repo_identity",
         "//eden/mononoke/server/context:context",
         "//eden/mononoke/server/metadata:metadata",
         "//eden/scm/lib/clientinfo:clientinfo",

eden/mononoke/mononoke_types/serialization/TARGETS

@@ -31,7 +31,6 @@ thrift_library(
         "//eden/mononoke/derived_data:changeset_info-unittest",
         "//eden/mononoke/derived_data/remote/if:",
         "//eden/mononoke/facebook/bookmark_service/if:",
-        "//eden/mononoke/facebook/derived_data_service/derivation_queue/if:",
         "//eden/mononoke/facebook/get_bundle:bundle_store_lib",
         "//eden/mononoke/filenodes/if:",
         "//eden/mononoke/git/git_types:",
@@ -46,5 +45,6 @@ thrift_library(
         "//eden/mononoke/mononoke_types/if:",
         "//eden/mononoke/mutable_renames/if:",
         "//eden/mononoke/repo_attributes/commit_graph/commit_graph_types/if:",
+        "//eden/mononoke/repo_attributes/repo_derivation_queues/if:",
     ],
 )

eden/mononoke/repo_attributes/repo_derivation_queues/Cargo.toml

@@ -0,0 +1,29 @@
+# @generated by autocargo from //eden/mononoke/repo_attributes/repo_derivation_queues:repo_derivation_queues
+
+[package]
+name = "repo_derivation_queues"
+version = "0.1.0"
+authors = ["Facebook"]
+edition = "2021"
+license = "GPLv2+"
+
+[dependencies]
+anyhow = "1.0.75"
+async-trait = "0.1.71"
+bounded_traversal = { version = "0.1.0", path = "../../common/bounded_traversal" }
+bytes = { version = "1.6.0", features = ["serde"] }
+clientinfo = { version = "0.1.0", path = "../../../scm/lib/clientinfo" }
+cloned = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" }
+commit_graph = { version = "0.1.0", path = "../commit_graph/commit_graph" }
+context = { version = "0.1.0", path = "../../server/context" }
+derivation_queue_thrift = { version = "0.1.0", path = "if" }
+derived_data_manager = { version = "0.1.0", path = "../../derived_data/manager" }
+derived_data_utils = { version = "0.1.0", path = "../../derived_data/utils" }
+facet = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" }
+fbthrift = { version = "0.0.1+unstable", git = "https://github.com/facebook/fbthrift.git", branch = "main" }
+futures = { version = "0.3.30", features = ["async-await", "compat"] }
+mononoke_types = { version = "0.1.0", path = "../../mononoke_types" }
+parking_lot = { version = "0.12.1", features = ["send_guard"] }
+slog = { version = "2.7", features = ["max_level_trace", "nested-values"] }
+thiserror = "1.0.49"
+tokio = { version = "1.37.0", features = ["full", "test-util", "tracing"] }

eden/mononoke/repo_attributes/repo_derivation_queues/TARGETS

@@ -0,0 +1,30 @@
+load("@fbcode_macros//build_defs:rust_library.bzl", "rust_library")
+
+oncall("mononoke")
+
+rust_library(
+    name = "repo_derivation_queues",
+    srcs = glob(["src/**/*.rs"]),
+    deps = [
+        "fbsource//third-party/rust:anyhow",
+        "fbsource//third-party/rust:async-trait",
+        "fbsource//third-party/rust:bytes",
+        "fbsource//third-party/rust:futures",
+        "fbsource//third-party/rust:parking_lot",
+        "fbsource//third-party/rust:slog",
+        "fbsource//third-party/rust:thiserror",
+        "fbsource//third-party/rust:tokio",
+        "//common/rust/shed/cloned:cloned",
+        "//common/rust/shed/facet:facet",
+        "//common/rust/zeus:zeus_client",
+        "//eden/mononoke/common/bounded_traversal:bounded_traversal",
+        "//eden/mononoke/derived_data:derived_data_manager",
+        "//eden/mononoke/derived_data:derived_data_utils",
+        "//eden/mononoke/mononoke_types:mononoke_types",
+        "//eden/mononoke/repo_attributes/commit_graph/commit_graph:commit_graph",
+        "//eden/mononoke/repo_attributes/repo_derivation_queues/if:derivation_queue_thrift-rust",
+        "//eden/mononoke/server/context:context",
+        "//eden/scm/lib/clientinfo:clientinfo",
+        "//thrift/lib/rust:fbthrift",
+    ],
+)

eden/mononoke/repo_attributes/repo_derivation_queues/facebook/TARGETS

@@ -0,0 +1,75 @@
+load("@fbcode_macros//build_defs:rust_library.bzl", "rust_library")
+load("@fbcode_macros//build_defs:rust_unittest.bzl", "rust_unittest")
+
+oncall("mononoke")
+
+rust_library(
+    name = "zelos_queue",
+    srcs = glob(["zelos_queue/**/*.rs"]),
+    test_deps = [
+        "fbsource//third-party/rust:assert_matches",
+        "//common/rust/shed/fbinit:fbinit",
+        "//common/rust/shed/fbinit:fbinit-tokio",
+        "//eden/mononoke/blobrepo:repo_blobstore",
+        "//eden/mononoke/bonsai_hg_mapping:bonsai_hg_mapping",
+        "//eden/mononoke/changesets:changesets",
+        "//eden/mononoke/server/context:context",
+    ],
+    deps = [
+        "fbsource//third-party/rust:anyhow",
+        "fbsource//third-party/rust:async-trait",
+        "fbsource//third-party/rust:bytes",
+        "fbsource//third-party/rust:futures",
+        "fbsource//third-party/rust:slog",
+        "fbsource//third-party/rust:strum",
+        "//common/rust/shed/cloned:cloned",
+        "//common/rust/shed/fbinit:fbinit",
+        "//common/rust/shed/justknobs_stub:justknobs",
+        "//common/rust/zeus:zeus_client",
+        "//eden/mononoke/blobrepo:repo_blobstore",
+        "//eden/mononoke/blobstore:cacheblob",
+        "//eden/mononoke/bonsai_git_mapping:bonsai_git_mapping",
+        "//eden/mononoke/bonsai_hg_mapping:bonsai_hg_mapping",
+        "//eden/mononoke/changesets:changesets",
+        "//eden/mononoke/common/scuba_ext:scuba_ext",
+        "//eden/mononoke/derived_data:derived_data_manager",
+        "//eden/mononoke/derived_data:derived_data_utils",
+        "//eden/mononoke/filenodes:filenodes",
+        "//eden/mononoke/metaconfig:metaconfig_types",
+        "//eden/mononoke/mononoke_types:mononoke_types",
+        "//eden/mononoke/repo_attributes/commit_graph/commit_graph:commit_graph",
+        "//eden/mononoke/repo_attributes/repo_derivation_queues:repo_derivation_queues",
+        "//eden/mononoke/server/context:context",
+    ],
+)
+
+rust_unittest(
+    name = "zelos_queue_tests",
+    srcs = glob(["zelos_queue_tests/**/*.rs"]),
+    deps = [
+        "fbsource//third-party/rust:anyhow",
+        "fbsource//third-party/rust:assert_matches",
+        "fbsource//third-party/rust:futures",
+        "fbsource//third-party/rust:pretty_assertions",
+        "fbsource//third-party/rust:tokio",
+        ":zelos_queue",
+        "//common/rust/shed/facet:facet",
+        "//common/rust/shed/fbinit:fbinit",
+        "//common/rust/shed/fbinit:fbinit-tokio",
+        "//common/rust/zeus:zeus_client",
+        "//eden/mononoke/blobrepo:repo_blobstore",
+        "//eden/mononoke/bonsai_hg_mapping:bonsai_hg_mapping",
+        "//eden/mononoke/bookmarks:bookmarks",
+        "//eden/mononoke/changesets:changesets",
+        "//eden/mononoke/derived_data:derived_data_manager",
+        "//eden/mononoke/derived_data:unodes",
+        "//eden/mononoke/filestore:filestore",
+        "//eden/mononoke/mononoke_types:mononoke_types",
+        "//eden/mononoke/repo_attributes/repo_derivation_queues:repo_derivation_queues",
+        "//eden/mononoke/repo_attributes/repo_derived_data:repo_derived_data",
+        "//eden/mononoke/repo_attributes/repo_identity:repo_identity",
+        "//eden/mononoke/repo_factory:test_repo_factory",
+        "//eden/mononoke/server/context:context",
+        "//eden/mononoke/tests/utils:tests_utils",
+    ],
+)

eden/mononoke/repo_attributes/repo_derivation_queues/if/Cargo.toml

@@ -0,0 +1,34 @@
+# @generated by autocargo from //eden/mononoke/repo_attributes/repo_derivation_queues/if:derivation_queue_thrift-rust
+
+[package]
+name = "derivation_queue_thrift"
+version = "0.1.0"
+authors = ["Facebook"]
+edition = "2021"
+license = "GPLv2+"
+build = "thrift_build.rs"
+
+[lib]
+path = "thrift_lib.rs"
+test = false
+doctest = false
+doc = false
+
+[dependencies]
+anyhow = "1.0.75"
+codegen_includer_proc_macro = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" }
+fbthrift = { version = "0.0.1+unstable", git = "https://github.com/facebook/fbthrift.git", branch = "main" }
+futures = { version = "0.3.30", features = ["async-await", "compat"] }
+mononoke_types_serialization = { version = "0.1.0", path = "../../../mononoke_types/serialization" }
+once_cell = "1.12"
+ref-cast = "1.0.18"
+serde = { version = "1.0.185", features = ["derive", "rc"] }
+serde_derive = "1.0.185"
+thiserror = "1.0.49"
+
+[build-dependencies]
+thrift_compiler = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" }
+
+[features]
+default = ["thrift_library_unittests_disabled"]
+thrift_library_unittests_disabled = []

eden/mononoke/repo_attributes/repo_derivation_queues/if/clients/Cargo.toml

@@ -0,0 +1,30 @@
+# @generated by autocargo from //eden/mononoke/repo_attributes/repo_derivation_queues/if:derivation_queue_thrift-rust-clients
+
+[package]
+name = "derivation_queue_thrift_clients"
+version = "0.1.0"
+authors = ["Facebook"]
+edition = "2021"
+license = "GPLv2+"
+build = "thrift_build.rs"
+
+[lib]
+path = "thrift_lib.rs"
+test = false
+doctest = false
+doc = false
+
+[dependencies]
+anyhow = "1.0.75"
+async-trait = "0.1.71"
+codegen_includer_proc_macro = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" }
+const-cstr = "0.3.0"
+derivation_queue_thrift__types = { package = "derivation_queue_thrift", version = "0.1.0", path = ".." }
+fbthrift = { version = "0.0.1+unstable", git = "https://github.com/facebook/fbthrift.git", branch = "main" }
+futures = { version = "0.3.30", features = ["async-await", "compat"] }
+mononoke_types_serialization = { version = "0.1.0", path = "../../../../mononoke_types/serialization" }
+mononoke_types_serialization_clients = { version = "0.1.0", path = "../../../../mononoke_types/serialization/clients" }
+tracing = { version = "0.1.40", features = ["attributes"] }
+
+[build-dependencies]
+thrift_compiler = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" }

eden/mononoke/repo_attributes/repo_derivation_queues/if/clients/thrift_build.rs

@@ -0,0 +1,51 @@
+// @generated by autocargo
+
+use std::env;
+use std::fs;
+use std::path::Path;
+use thrift_compiler::Config;
+use thrift_compiler::GenContext;
+const CRATEMAP: &str = "\
+blame mononoke_types_serialization //eden/mononoke/mononoke_types/serialization:mononoke_types_serialization-rust
+bonsai mononoke_types_serialization //eden/mononoke/mononoke_types/serialization:mononoke_types_serialization-rust
+bssm mononoke_types_serialization //eden/mononoke/mononoke_types/serialization:mononoke_types_serialization-rust
+changeset_info mononoke_types_serialization //eden/mononoke/mononoke_types/serialization:mononoke_types_serialization-rust
+content mononoke_types_serialization //eden/mononoke/mononoke_types/serialization:mononoke_types_serialization-rust
+data mononoke_types_serialization //eden/mononoke/mononoke_types/serialization:mononoke_types_serialization-rust
+deleted_manifest mononoke_types_serialization //eden/mononoke/mononoke_types/serialization:mononoke_types_serialization-rust
+derivation_queue crate //eden/mononoke/repo_attributes/repo_derivation_queues/if:derivation_queue_thrift-rust
+fastlog mononoke_types_serialization //eden/mononoke/mononoke_types/serialization:mononoke_types_serialization-rust
+fsnodes mononoke_types_serialization //eden/mononoke/mononoke_types/serialization:mononoke_types_serialization-rust
+id mononoke_types_serialization //eden/mononoke/mononoke_types/serialization:mononoke_types_serialization-rust
+path mononoke_types_serialization //eden/mononoke/mononoke_types/serialization:mononoke_types_serialization-rust
+raw_bundle2 mononoke_types_serialization //eden/mononoke/mononoke_types/serialization:mononoke_types_serialization-rust
+redaction mononoke_types_serialization //eden/mononoke/mononoke_types/serialization:mononoke_types_serialization-rust
+sharded_map mononoke_types_serialization //eden/mononoke/mononoke_types/serialization:mononoke_types_serialization-rust
+skeleton_manifest mononoke_types_serialization //eden/mononoke/mononoke_types/serialization:mononoke_types_serialization-rust
+test_manifest mononoke_types_serialization //eden/mononoke/mononoke_types/serialization:mononoke_types_serialization-rust
+time mononoke_types_serialization //eden/mononoke/mononoke_types/serialization:mononoke_types_serialization-rust
+unodes mononoke_types_serialization //eden/mononoke/mononoke_types/serialization:mononoke_types_serialization-rust
+";
+#[rustfmt::skip]
+fn main() {
+    println!("cargo:rerun-if-changed=thrift_build.rs");
+    let out_dir = env::var_os("OUT_DIR").expect("OUT_DIR env not provided");
+    let cratemap_path = Path::new(&out_dir).join("cratemap");
+    fs::write(cratemap_path, CRATEMAP).expect("Failed to write cratemap");
+    let mut conf = Config::from_env(GenContext::Clients)
+        .expect("Failed to instantiate thrift_compiler::Config");
+    let cargo_manifest_dir = env::var_os("CARGO_MANIFEST_DIR")
+        .expect("CARGO_MANIFEST_DIR not provided");
+    let mut base_path = Path::new(&cargo_manifest_dir)
+        .join("../../../../../..")
+        .canonicalize()
+        .expect("Failed to canonicalize base_path");
+    if cfg!(windows) {
+        base_path = base_path.to_string_lossy().trim_start_matches(r"\\?\").into();
+    }
+    conf.base_path(base_path);
+    conf.types_crate("derivation_queue_thrift__types");
+    conf.clients_crate("derivation_queue_thrift__clients");
+    let srcs = &["../derivation_queue.thrift"];
+    conf.run(srcs).expect("Failed while running thrift compilation");
+}

eden/mononoke/repo_attributes/repo_derivation_queues/if/clients/thrift_lib.rs

@@ -0,0 +1,3 @@
+// @generated by autocargo
+
+::codegen_includer_proc_macro::include!();