extend lookups feature, create preprocessed & array support (#46)

newrelic · Dec 31, 2019 · 042cbf9 · 042cbf9
1 parent 36efd3e
commit 042cbf9
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 2 deletions.
diff --git a/internal/inputs/http.go b/internal/inputs/http.go
@@ -253,7 +253,7 @@ func handleJSON(dataStore *[]interface{}, body []byte, resp *gorequest.Response,
 			// load.StoreAppend(theSample)
 			*dataStore = append(*dataStore, theSample)
 
-			if theSample["error"] != nil {
+			if theSample["error"] != nil && fmt.Sprintf("%v", theSample["error"]) != "false" {
 				load.Logrus.Debug(fmt.Sprintf("http: request failed %v", theSample["error"]))
 			}
 

diff --git a/internal/processor/data.go b/internal/processor/data.go
@@ -35,6 +35,16 @@ func RunDataHandler(dataSets []interface{}, samplesToMerge *map[string][]interfa
 func processDataSet(dataSet *map[string]interface{}, samplesToMerge *map[string][]interface{}, i int, cfg *load.Config) {
 	ds := (*dataSet)
 
+	if cfg.LookupStore == nil {
+		cfg.LookupStore = map[string][]string{}
+	}
+
+	// perform an early lookup store
+	// useful for arrays of data
+	for k, v := range ds {
+		StoreLookups(cfg.APIs[i].StoreLookups, &k, &cfg.LookupStore, &v)
+	}
+
 	FindStartKey(&ds, cfg.APIs[i].StartKey, cfg.APIs[i].InheritAttributes) // start at a later part in the received data
 	StripKeys(&ds, cfg.APIs[i].StripKeys)                                  // remove before flattening
 	RunLazyFlatten(&ds, cfg, i)                                            // perform lazy flatten if needed

diff --git a/internal/processor/lookups.go b/internal/processor/lookups.go
@@ -21,7 +21,18 @@ func StoreLookups(storeLookups map[string]string, key *string, lookupStore *map[
 				lookupStoreKey:  fmt.Sprintf("%v", *v),
 			}).Debug("create: store lookup")
 
-			(*lookupStore)[lookupStoreKey] = append((*lookupStore)[lookupStoreKey], fmt.Sprintf("%v", *v))
+			switch data := (*v).(type) {
+			case []interface{}:
+				load.Logrus.WithFields(logrus.Fields{
+					"lookupFindKey": lookupFindKey,
+				}).Debug("splitting array")
+
+				for _, dataKey := range data {
+					(*lookupStore)[lookupStoreKey] = append((*lookupStore)[lookupStoreKey], fmt.Sprintf("%v", dataKey))
+				}
+			default:
+				(*lookupStore)[lookupStoreKey] = append((*lookupStore)[lookupStoreKey], fmt.Sprintf("%v", *v))
+			}
 		}
 	}
 }