Add the capability to increase harmonizer heap size (#516)

For large to very graphs, it is possible for deno v8 to run out of
memory with the default settings. For us, it happens around `1400MB` of
heap (I wasn't able to find the default v8 heap size)

Similar to the query planner, this vows to allow the harmonizer to
grow up to a specified limit using env variables APOLLO_HARMONIZER_EXPERIMENTAL_V8_MAX_HEAP_SIZE and APOLLO_HARMONIZER_EXPERIMENTAL_V8_INITIAL_HEAP_SIZE.

harmonizer/Cargo.toml

@@ -25,6 +25,7 @@ apollo-federation-types = { version = "0.11.0", path = "../apollo-federation-typ
 deno_core = "0.200.0"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
+tracing = "0.1.33"
 
 [dev-dependencies]
 insta = "1.34.0"

harmonizer/src/lib.rs

@@ -39,6 +39,12 @@ use apollo_federation_types::build::{
     BuildError, BuildErrors, BuildOutput, BuildResult, SubgraphDefinition,
 };
 
+// A reasonable default starting limit for our deno heap.
+const APOLLO_HARMONIZER_EXPERIMENTAL_V8_INITIAL_HEAP_SIZE_DEFAULT: &str = "256";
+// A reasonable default max limit for our deno heap.
+const APOLLO_HARMONIZER_EXPERIMENTAL_V8_MAX_HEAP_SIZE_DEFAULT: &str = "1400";
+
+
 /// The `harmonize` function receives a [`Vec<SubgraphDefinition>`] and invokes JavaScript
 /// composition on it, either returning the successful output, or a list of error messages.
 pub fn harmonize(subgraph_definitions: Vec<SubgraphDefinition>) -> BuildResult {
@@ -55,13 +61,64 @@ pub fn harmonize_limit(
     // The snapshot is created in the build_harmonizer.rs script and included in our binary image
     let buffer = include_bytes!(concat!(env!("OUT_DIR"), "/composition.snap"));
 
+    let initial_heap_size =
+        std::env::var("APOLLO_HARMONIZER_EXPERIMENTAL_V8_INITIAL_HEAP_SIZE").unwrap_or_else(|_e| {
+            APOLLO_HARMONIZER_EXPERIMENTAL_V8_INITIAL_HEAP_SIZE_DEFAULT.to_string()
+        });
+
+    let max_heap_size_maybe = std::env::var("APOLLO_HARMONIZER_EXPERIMENTAL_V8_MAX_HEAP_SIZE").ok();
+    let max_heap_size_provided = max_heap_size_maybe.is_some();
+    let max_heap_size = max_heap_size_maybe.unwrap_or_else(|| {
+            APOLLO_HARMONIZER_EXPERIMENTAL_V8_MAX_HEAP_SIZE_DEFAULT.to_string()
+        });
+
+    // The first flag is argv[0], so provide an ignorable value
+    let flags = vec![
+        "--ignored".to_string(),
+        "--initial_heap_size".to_string(),
+        initial_heap_size.to_string(),
+        "--max-heap-size".to_string(),
+        max_heap_size.to_string(),
+    ];
+
+    // Deno will warn us if we supply flags it doesn't recognise.
+    // We ignore "--ignored" and report any others as warnings
+    let ignored: Vec<_> = deno_core::v8_set_flags(flags)
+        .into_iter()
+        .filter(|x| x != "--ignored")
+        .collect();
+    if !ignored.is_empty() {
+        panic!("deno ignored these flags: {:?}", ignored);
+    }
+
     // Use our snapshot to provision our new runtime
     let options = RuntimeOptions {
         startup_snapshot: Some(Snapshot::Static(buffer)),
         ..Default::default()
     };
     let mut runtime = JsRuntime::new(options);
 
+    // if max_heap_size was not set, we resize the heap every time
+    // we approach the limit. This is a tradeoff as it might cause
+    // an instance to run out of physical memory.
+    if !max_heap_size_provided {
+        // Add a callback that expands our heap by 1.25 each time
+        // it is invoked. There is no limit, since we rely on the
+        // execution environment (OS) to provide that.
+        let name = "harmonize".to_string();
+        runtime.add_near_heap_limit_callback(move |current, initial| {
+            let new = current * 5 / 4;
+            tracing::info!(
+                "deno heap expansion({}): initial: {}, current: {}, new: {}",
+                name,
+                initial,
+                current,
+                new
+            );
+            new
+        });
+    }
+
     // convert the subgraph definitions into JSON
     let service_list_javascript = format!(
         "serviceList = {}",