Use serde_yaml throughout yaml_test_runner instead of yaml-rust (open…

…search-project#151)

Signed-off-by: Thomas Farr <[email protected]>

.ci/opensearch/Dockerfile

@@ -5,7 +5,7 @@ WORKDIR /usr/share/opensearch
 
 COPY --chown=opensearch:opensearch opensearch.yml ./config/
 
-RUN ./opensearch-onetime-setup.sh
+RUN if [[ -f "./opensearch-onetime-setup.sh" ]]; then ./opensearch-onetime-setup.sh ; fi
 
 COPY --chown=opensearch:opensearch *.pem ./config/
 

yaml_test_runner/Cargo.toml

@@ -33,7 +33,6 @@ simple_logger = "4.0.0"
 syn = { version = "~1.0", features = ["full"] }
 sysinfo = "0.28"
 url = "2.1.1"
-yaml-rust = "0.4.3"
 tar = "~0.4"
 flate2 = "~1"
 globset = "~0.4"

yaml_test_runner/src/generator.rs

@@ -28,30 +28,26 @@
  * GitHub history for details.
  */
 
-use crate::{
-    skip::{GlobalSkips, SkippedFeaturesAndTests},
-    step::*,
-};
+use crate::{skip::SkippedFeaturesAndTests, step::*};
 use anyhow::anyhow;
 use api_generator::generator::Api;
 use inflector::Inflector;
 use lazy_static::lazy_static;
 use log::{error, info};
-use opensearch::DEFAULT_ADDRESS;
 use path_slash::PathExt;
 use proc_macro2::{Span, TokenStream};
 use quote::{quote, ToTokens, TokenStreamExt};
 use regex::Regex;
 use semver::Version;
+use serde::Deserialize;
+use serde_yaml::Value;
 use std::{
     collections::HashSet,
     fs,
     fs::{File, OpenOptions},
     io::Write,
     path::{Path, PathBuf},
 };
-use url::Url;
-use yaml_rust::{Yaml, YamlLoader};
 
 /// The test suite to compile
 #[derive(Debug, PartialEq, Eq)]
@@ -370,25 +366,15 @@ impl TestFn {
     }
 }
 
-fn cluster_addr() -> String {
-    match std::env::var("OPENSEARCH_URL") {
-        Ok(server) => server,
-        Err(_) => DEFAULT_ADDRESS.into(),
-    }
-}
-
 pub fn generate_tests_from_yaml(
     api: &Api,
     _suite: &TestSuite,
     version: &semver::Version,
     base_download_dir: &Path,
     download_dir: &Path,
     generated_dir: &Path,
+    skips: &SkippedFeaturesAndTests,
 ) -> anyhow::Result<()> {
-    let url = Url::parse(cluster_addr().as_ref()).unwrap();
-    let global_skips = serde_yaml::from_str::<GlobalSkips>(include_str!("../skip.yml"))?;
-    let skips = global_skips.get_skips_for(version, url.scheme() == "https");
-
     let paths = fs::read_dir(download_dir)?;
     for entry in paths.flatten() {
         if let Ok(file_type) = entry.file_type() {
@@ -400,6 +386,7 @@ pub fn generate_tests_from_yaml(
                     base_download_dir,
                     &entry.path(),
                     generated_dir,
+                    skips,
                 )?;
             } else if file_type.is_file() {
                 let path = entry.path();
@@ -415,34 +402,37 @@ pub fn generate_tests_from_yaml(
                 info!("Generating: {}", relative_path.display());
                 let yaml = fs::read_to_string(&entry.path()).unwrap();
 
-                // a yaml test can contain multiple yaml docs, so use yaml_rust to parse
-                let result = YamlLoader::load_from_str(&yaml);
-                if result.is_err() {
-                    error!(
-                        "skipping {}. cannot read as Yaml struct: {}",
-                        relative_path.to_slash_lossy(),
-                        result.err().unwrap().to_string()
-                    );
-                    continue;
-                }
+                let docs = match serde_yaml::Deserializer::from_str(&yaml)
+                    .map(|doc| Value::deserialize(doc))
+                    .collect::<Result<Vec<_>, _>>()
+                {
+                    Ok(docs) => docs,
+                    Err(err) => {
+                        error!(
+                            "skipping {}. contains one or more malformed YAML documents: {}",
+                            relative_path.display(),
+                            err
+                        );
+                        continue;
+                    }
+                };
 
-                let docs = result.unwrap();
                 let mut test =
                     YamlTests::new(relative_path, version, &skips, test_suite, docs.len());
 
                 let results : Vec<anyhow::Result<()>> = docs
                         .iter()
                         .map(|doc| {
                             let hash = doc
-                                .as_hash()
+                                .as_mapping()
                                 .ok_or_else(|| anyhow!(
                                     "expected hash but found {:?}",
                                     &doc
                                 ))?;
 
                             let (key, value) = hash.iter().next().unwrap();
                             match (key, value) {
-                                (Yaml::String(name), Yaml::Array(steps)) => {
+                                (Value::String(name), Value::Sequence(steps)) => {
                                     let steps = parse_steps(api, steps)?;
                                     let test_fn = TestFn::new(name, steps);
                                     match name.as_str() {

yaml_test_runner/src/main.rs

@@ -48,6 +48,8 @@ mod step;
 
 use generator::TestSuite;
 
+use crate::skip::GlobalSkips;
+
 fn main() -> anyhow::Result<()> {
     simple_logger::SimpleLogger::new()
         .with_level(LevelFilter::Info)
@@ -128,13 +130,17 @@ fn main() -> anyhow::Result<()> {
         }
     }
 
+    let global_skips = serde_yaml::from_str::<GlobalSkips>(include_str!("../skip.yml"))?;
+    let skips = global_skips.get_skips_for(&version, url.starts_with("https://"));
+
     generator::generate_tests_from_yaml(
         &api,
         &suite,
         &version,
         &download_dir,
         &download_dir,
         &generated_dir,
+        &skips,
     )?;
 
     Ok(())

yaml_test_runner/src/step/comparison.rs

@@ -33,13 +33,13 @@ use crate::step::Expr;
 use anyhow::anyhow;
 use proc_macro2::{Span, TokenStream};
 use quote::{quote, ToTokens, TokenStreamExt};
-use yaml_rust::Yaml;
+use serde_yaml::Value;
 
 pub const OPERATORS: [&str; 4] = ["lt", "lte", "gt", "gte"];
 
 pub struct Comparison {
     pub(crate) expr: Expr,
-    value: Yaml,
+    value: Value,
     op: String,
 }
 
@@ -50,9 +50,9 @@ impl From<Comparison> for Step {
 }
 
 impl Comparison {
-    pub fn try_parse(yaml: &Yaml, op: &str) -> anyhow::Result<Comparison> {
+    pub fn try_parse(yaml: &Value, op: &str) -> anyhow::Result<Comparison> {
         let hash = yaml
-            .as_hash()
+            .as_mapping()
             .ok_or_else(|| anyhow!("expected hash but found {:?}", yaml))?;
 
         let (k, v) = hash.iter().next().unwrap();

yaml_test_runner/src/step/contains.rs

@@ -33,11 +33,11 @@ use crate::step::{json_string_from_yaml, Expr};
 use anyhow::anyhow;
 use proc_macro2::TokenStream;
 use quote::{quote, ToTokens, TokenStreamExt};
-use yaml_rust::Yaml;
+use serde_yaml::Value;
 
 pub struct Contains {
     expr: Expr,
-    value: Yaml,
+    value: Value,
 }
 
 impl From<Contains> for Step {
@@ -47,9 +47,9 @@ impl From<Contains> for Step {
 }
 
 impl Contains {
-    pub fn try_parse(yaml: &Yaml) -> anyhow::Result<Contains> {
+    pub fn try_parse(yaml: &Value) -> anyhow::Result<Contains> {
         let hash = yaml
-            .as_hash()
+            .as_mapping()
             .ok_or_else(|| anyhow!("expected hash but found {:?}", yaml))?;
 
         let (k, v) = hash.iter().next().unwrap();
@@ -66,28 +66,35 @@ impl ToTokens for Contains {
         let expr = self.expr.expression();
 
         match &self.value {
-            Yaml::Real(r) => {
-                let f = r.parse::<f64>().unwrap();
-                tokens.append_all(quote! {
-                    assert_contains!(json#expr, json!(#f));
-                });
-            }
-            Yaml::Integer(i) => {
-                tokens.append_all(quote! {
-                    assert_contains!(json#expr, json!(#i));
-                });
+            Value::Number(n) => {
+                if n.is_f64() {
+                    let f = n.as_f64().unwrap();
+                    tokens.append_all(quote! {
+                        assert_contains!(json#expr, json!(#f));
+                    });
+                } else if n.is_u64() {
+                    let u = n.as_u64().unwrap();
+                    tokens.append_all(quote! {
+                        assert_contains!(json#expr, json!(#u));
+                    });
+                } else {
+                    let i = n.as_i64().unwrap();
+                    tokens.append_all(quote! {
+                        assert_contains!(json#expr, json!(#i));
+                    });
+                }
             }
-            Yaml::String(s) => {
+            Value::String(s) => {
                 tokens.append_all(quote! {
                     assert_contains!(json#expr, json!(#s));
                 });
             }
-            Yaml::Boolean(b) => {
+            Value::Bool(b) => {
                 tokens.append_all(quote! {
                     assert_contains!(json#expr, json!(#b));
                 });
             }
-            yaml if yaml.is_array() || yaml.as_hash().is_some() => {
+            yaml if yaml.is_sequence() || yaml.is_mapping() => {
                 let json = syn::parse_str::<TokenStream>(&json_string_from_yaml(yaml)).unwrap();
 
                 tokens.append_all(quote! {