diff --git a/internal/inputs/http.go b/internal/inputs/http.go index 842da2f2..3fde1319 100644 --- a/internal/inputs/http.go +++ b/internal/inputs/http.go @@ -253,7 +253,7 @@ func handleJSON(dataStore *[]interface{}, body []byte, resp *gorequest.Response, // load.StoreAppend(theSample) *dataStore = append(*dataStore, theSample) - if theSample["error"] != nil { + if theSample["error"] != nil && fmt.Sprintf("%v", theSample["error"]) != "false" { load.Logrus.Debug(fmt.Sprintf("http: request failed %v", theSample["error"])) } diff --git a/internal/processor/data.go b/internal/processor/data.go index e93ae169..6f2d36b4 100644 --- a/internal/processor/data.go +++ b/internal/processor/data.go @@ -35,6 +35,16 @@ func RunDataHandler(dataSets []interface{}, samplesToMerge *map[string][]interfa func processDataSet(dataSet *map[string]interface{}, samplesToMerge *map[string][]interface{}, i int, cfg *load.Config) { ds := (*dataSet) + if cfg.LookupStore == nil { + cfg.LookupStore = map[string][]string{} + } + + // perform an early lookup store + // useful for arrays of data + for k, v := range ds { + StoreLookups(cfg.APIs[i].StoreLookups, &k, &cfg.LookupStore, &v) + } + FindStartKey(&ds, cfg.APIs[i].StartKey, cfg.APIs[i].InheritAttributes) // start at a later part in the received data StripKeys(&ds, cfg.APIs[i].StripKeys) // remove before flattening RunLazyFlatten(&ds, cfg, i) // perform lazy flatten if needed diff --git a/internal/processor/lookups.go b/internal/processor/lookups.go index 44daab44..b070b636 100644 --- a/internal/processor/lookups.go +++ b/internal/processor/lookups.go @@ -21,7 +21,18 @@ func StoreLookups(storeLookups map[string]string, key *string, lookupStore *map[ lookupStoreKey: fmt.Sprintf("%v", *v), }).Debug("create: store lookup") - (*lookupStore)[lookupStoreKey] = append((*lookupStore)[lookupStoreKey], fmt.Sprintf("%v", *v)) + switch data := (*v).(type) { + case []interface{}: + load.Logrus.WithFields(logrus.Fields{ + "lookupFindKey": lookupFindKey, + }).Debug("splitting array") + + for _, dataKey := range data { + (*lookupStore)[lookupStoreKey] = append((*lookupStore)[lookupStoreKey], fmt.Sprintf("%v", dataKey)) + } + default: + (*lookupStore)[lookupStoreKey] = append((*lookupStore)[lookupStoreKey], fmt.Sprintf("%v", *v)) + } } } }