add quality table

install/README.md

@@ -25,6 +25,7 @@ Meta information for the GeoNet equipment network.
 * `channels.csv` - Individual datalogger recording elements including digitiser position, sampling rate, and responses.
 * [`preamps.csv`](#preamps) - site specific settings applied to individual datalogger pre-amplification that may impact overall sensitivities.
 * [`telemetries.csv`](#telemetries) - site specific settings applied to datalogger and sensor connections that may use analogue telemetry.
+* [`quality.csv`](#quality) - site specific settings indicating the quality and usefulness of the recorded data.
 
 * `cameras.csv` - Installed field cameras.
 * `doases.csv` - Installed field DOAS (Differential Optical Absorption Spectrometer) equipment.
@@ -377,6 +378,19 @@ telephone line, or an FM radio link. This table allows this to be documented, an
 | _Start_ | Telemetry start time|
 | _Stop_ | Telemetry stop time|
 
+#### _QUALITIES_ ####
+
+Sometimes the datalogger will record faulty data, possibility due to a sensor fault. This table is a mechanism to indicate and quality issues related to the
+recorded data which cannot be identified from the data itself.
+
+| Field | Description | Units |
+| --- | --- | --- |
+| _Station_ | Datalogger recording _Station_|
+| _Location_ | Recording sensor site _Location_ |
+| _Fault_ | An indication that the data should not be used for processing (An empty entry will default to __false__)
+| _Start_ | Quality start time|
+| _Stop_ | Quality stop time|
+
 ### CAMERA ###
 
 #### _CAMERAS_ ####

install/qualities.csv

@@ -0,0 +1 @@
+Station,Location,Fault,Start Date,End Date

meta/correction.go

@@ -14,6 +14,7 @@ type Correction struct {
 	Telemetry             *Telemetry
 	SensorCalibration     *Calibration
 	DataloggerCalibration *Calibration
+	Quality               *Quality
 }
 
 // Corrections returns a slice of Correction values for a given Collection.
@@ -56,15 +57,24 @@ func (set *Set) Corrections(coll Collection) []Correction {
 								continue
 							}
 
-							corrections = append(corrections, Correction{
-								Span:                  span,
-								Polarity:              polarity.Polarity,
-								Gain:                  gain.Gain,
-								Preamp:                preamp.Preamp,
-								Telemetry:             telemetry.Telemetry,
-								SensorCalibration:     sensor.SensorCalibration,
-								DataloggerCalibration: datalogger.DataloggerCalibration,
-							})
+							for _, quality := range set.QualityCorrections(coll) {
+
+								span, ok := span.Extent(quality.Span)
+								if !ok {
+									continue
+								}
+
+								corrections = append(corrections, Correction{
+									Span:                  span,
+									Polarity:              polarity.Polarity,
+									Gain:                  gain.Gain,
+									Preamp:                preamp.Preamp,
+									Telemetry:             telemetry.Telemetry,
+									SensorCalibration:     sensor.SensorCalibration,
+									DataloggerCalibration: datalogger.DataloggerCalibration,
+									Quality:               quality.Quality,
+								})
+							}
 						}
 					}
 				}
@@ -531,3 +541,75 @@ func (s *Set) TelemetryCorrections(coll Collection) []Correction {
 
 	return res
 }
+
+// TelemetryCorrections returns a slice of Correction values to account for any changes in Preamp settings.
+func (s *Set) QualityCorrections(coll Collection) []Correction {
+
+	var qualities []Quality
+	for _, t := range s.Qualities() {
+		if t.Station != coll.Stream.Station {
+			continue
+		}
+		if t.Location != coll.Stream.Location {
+			continue
+		}
+		if !coll.Span.Overlaps(t.Span) {
+			continue
+		}
+		qualities = append(qualities, t)
+	}
+	sort.Slice(qualities, func(i, j int) bool {
+		return qualities[i].Span.Start.Before(qualities[j].Span.Start)
+	})
+
+	// no telemetries found return an empty span
+	if !(len(qualities) > 0) {
+		return []Correction{{Span: coll.Span}}
+	}
+
+	var res []Correction
+
+	// check prior to the first quality
+	if v := qualities[0]; v.Start.After(coll.Span.Start) {
+		res = append(res, Correction{
+			Span: Span{
+				Start: coll.Span.Start,
+				End:   v.Start,
+			},
+		})
+	}
+
+	// first telemetry
+	res = append(res, Correction{
+		Span:    qualities[0].Span,
+		Quality: &qualities[0],
+	})
+
+	// subsequent telemetries, checking for gaps
+	for i := 1; i < len(qualities); i++ {
+		if qualities[i].Start.After(qualities[i-1].End) {
+			res = append(res, Correction{
+				Span: Span{
+					Start: qualities[i-1].End,
+					End:   qualities[i].Start,
+				},
+			})
+		}
+		res = append(res, Correction{
+			Span:    qualities[i].Span,
+			Quality: &qualities[i],
+		})
+	}
+
+	// check after the last telemetry
+	if v := qualities[len(qualities)-1]; v.End.Before(coll.Span.End) {
+		res = append(res, Correction{
+			Span: Span{
+				Start: v.End,
+				End:   coll.Span.End,
+			},
+		})
+	}
+
+	return res
+}

meta/generate/main.go

@@ -36,6 +36,7 @@ func main() {
 			"placenames":          {"Placename"},
 			"polarities":          {"Polarity"},
 			"preamps":             {"Preamp"},
+			"qualities":           {"Quality"},
 			"samples":             {"Sample"},
 			"sessions":            {"Session"},
 			"sites":               {"Site"},

meta/quality.go

@@ -0,0 +1,155 @@
+package meta
+
+import (
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	qualityStation = iota
+	qualityLocation
+	qualityFault
+	qualityStart
+	qualityEnd
+	qualityLast
+)
+
+var qualityHeaders Header = map[string]int{
+	"Station":    qualityStation,
+	"Location":   qualityLocation,
+	"Fault":      qualityFault,
+	"Start Date": qualityStart,
+	"End Date":   qualityEnd,
+}
+
+// Quality describes when a datalogger is connected to a sensor via analogue quality (e.g. FM radio).
+type Quality struct {
+	Span
+
+	Station  string
+	Location string
+	Fault    bool
+
+	fault string
+}
+
+// String implements the Stringer interface.
+func (q Quality) String() string {
+	return strings.Join([]string{q.Station, q.Location, Format(q.Start)}, " ")
+}
+
+// Id returns a unique string which can be used for sorting or checking.
+func (q Quality) Id() string {
+	return strings.Join([]string{q.Station, q.Location}, ":")
+}
+
+// Less returns whether one Quality sorts before another.
+func (q Quality) Less(quality Quality) bool {
+	switch {
+	case q.Station < quality.Station:
+		return true
+	case q.Station > quality.Station:
+		return false
+	case q.Location < quality.Location:
+		return true
+	case q.Location > quality.Location:
+		return false
+	case q.Span.Start.Before(quality.Span.Start):
+		return true
+	default:
+		return false
+	}
+}
+
+type QualityList []Quality
+
+func (t QualityList) Len() int           { return len(t) }
+func (t QualityList) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
+func (t QualityList) Less(i, j int) bool { return t[i].Less(t[j]) }
+
+func (t QualityList) encode() [][]string {
+	var data [][]string
+
+	data = append(data, qualityHeaders.Columns())
+	for _, v := range t {
+		data = append(data, []string{
+			strings.TrimSpace(v.Station),
+			strings.TrimSpace(v.Location),
+			strings.TrimSpace(v.fault),
+			v.Start.Format(DateTimeFormat),
+			v.End.Format(DateTimeFormat),
+		})
+	}
+
+	return data
+}
+
+// toFloat64 is used in decoding to allow mathematical expressions as well as actual floating point values,
+// if the string parameter is empty the default value will be returned.
+func (q *QualityList) toBool(str string, def bool) (bool, error) {
+	switch s := strings.TrimSpace(str); {
+	case s != "":
+		return strconv.ParseBool(s)
+	default:
+		return def, nil
+	}
+}
+
+func (q *QualityList) decode(data [][]string) error {
+	var telemetries []Quality
+
+	// needs more than a comment line
+	if !(len(data) > 1) {
+		return nil
+	}
+
+	fields := qualityHeaders.Fields(data[0])
+	for _, v := range data[1:] {
+		d := fields.Remap(v)
+
+		fault, err := q.toBool(d[qualityFault], false)
+		if err != nil {
+			return err
+		}
+
+		start, err := time.Parse(DateTimeFormat, d[qualityStart])
+		if err != nil {
+			return err
+		}
+
+		end, err := time.Parse(DateTimeFormat, d[qualityEnd])
+		if err != nil {
+			return err
+		}
+
+		telemetries = append(telemetries, Quality{
+			Span: Span{
+				Start: start,
+				End:   end,
+			},
+			Fault:    fault,
+			Station:  strings.TrimSpace(d[qualityStation]),
+			Location: strings.TrimSpace(d[qualityLocation]),
+
+			fault: strings.TrimSpace(d[qualityFault]),
+		})
+	}
+
+	*q = QualityList(telemetries)
+
+	return nil
+}
+
+func LoadQualities(path string) ([]Quality, error) {
+	var g []Quality
+
+	if err := LoadList(path, (*QualityList)(&g)); err != nil {
+		return nil, err
+	}
+
+	sort.Sort(QualityList(g))
+
+	return g, nil
+}

meta/quality_test.go

@@ -0,0 +1,31 @@
+package meta
+
+import (
+	"testing"
+	"time"
+)
+
+func TestQuality(t *testing.T) {
+
+	t.Run("check quality", testListFunc("testdata/qualities.csv", &QualityList{
+		Quality{
+			Station:  "GISS",
+			Location: "40",
+			Span: Span{
+				Start: time.Date(2016, 12, 3, 22, 0, 0, 0, time.UTC),
+				End:   time.Date(2016, 12, 14, 19, 0, 0, 0, time.UTC),
+			},
+			Fault: true,
+			fault: "true",
+		},
+		Quality{
+			Station:  "GISS",
+			Location: "41",
+			Span: Span{
+				Start: time.Date(2016, 12, 3, 22, 0, 0, 0, time.UTC),
+				End:   time.Date(2016, 12, 14, 19, 0, 0, 0, time.UTC),
+			},
+			Fault: false,
+		},
+	}))
+}

meta/set.go

@@ -33,6 +33,7 @@ const (
 	MetsensorsFile   = "install/metsensors.csv"
 	PolaritiesFile   = "install/polarities.csv"
 	PreampsFile      = "install/preamps.csv"
+	QualitiesFile    = "install/qualities.csv"
 	RadomesFile      = "install/radomes.csv"
 	ReceiversFile    = "install/receivers.csv"
 	RecordersFile    = "install/recorders.csv"
@@ -89,6 +90,7 @@ type Set struct {
 	sessions            SessionList
 	streams             StreamList
 	telemetries         TelemetryList
+	qualities           QualityList
 
 	constituents ConstituentList
 	features     FeatureList
@@ -125,6 +127,7 @@ func (s *Set) files() map[string]List {
 		MetsensorsFile:   &s.installedMetSensors,
 		PolaritiesFile:   &s.polarities,
 		PreampsFile:      &s.preamps,
+		QualitiesFile:    &s.qualities,
 		RadomesFile:      &s.installedRadomes,
 		ReceiversFile:    &s.deployedReceivers,
 		RecordersFile:    &s.installedRecorders,

meta/set_auto.go

@@ -199,6 +199,13 @@ func (s Set) Preamps() []Preamp {
 	return preamps
 }
 
+// Qualities is a helper function to return a slice copy of Quality values.
+func (s Set) Qualities() []Quality {
+	qualities := make([]Quality, len(s.qualities))
+	copy(qualities, s.qualities)
+	return qualities
+}
+
 // Samples is a helper function to return a slice copy of Sample values.
 func (s Set) Samples() []Sample {
 	samples := make([]Sample, len(s.samples))

meta/testdata/qualities.csv

@@ -0,0 +1,3 @@
+Station,Location,Fault,Start Date,End Date
+GISS,40,true,2016-12-03T22:00:00Z,2016-12-14T19:00:00Z
+GISS,41,,2016-12-03T22:00:00Z,2016-12-14T19:00:00Z