Merge go-whosonfirst-spatial-rtree codebase (#41)

* issue #21 - start work to merge rtree package

* snapshot: app/hierarchy/update/ - RunWithOptions

* perform extra checks to see if files have changed, docs

* snapshot: move everything around; rip out old flags code for app/pip, untested

* docs for application/application.go

---------

Co-authored-by: sfomuseumbot <sfomuseumbot@localhost>

Makefile

@@ -0,0 +1,5 @@
+GOMOD=$(shell test -f "go.work" && echo "readonly" || echo "vendor")
+LDFLAGS=-s -w
+
+cli:
+	go build -mod $(GOMOD) -ldflags="$(LDFLAGS)" -o bin/pip cmd/pip/main.go

app/hierarchy/update/options.go

@@ -0,0 +1,61 @@
+package update
+
+import (
+	"context"
+	"flag"
+
+	"github.com/sfomuseum/go-flags/flagset"
+	"github.com/whosonfirst/go-whosonfirst-export/v2"
+	"github.com/whosonfirst/go-whosonfirst-spatial/database"
+	"github.com/whosonfirst/go-whosonfirst-spatial/filter"
+	"github.com/whosonfirst/go-whosonfirst-spatial/hierarchy"
+	hierarchy_filter "github.com/whosonfirst/go-whosonfirst-spatial/hierarchy/filter"
+	"github.com/whosonfirst/go-writer/v3"
+)
+
+type RunOptions struct {
+	Writer             writer.Writer
+	WriterURI          string
+	Exporter           export.Exporter
+	ExporterURI        string
+	MapshaperServerURI string
+	SpatialDatabase    database.SpatialDatabase
+	SpatialDatabaseURI string
+	ToIterator         string
+	FromIterator       string
+	SPRFilterInputs    *filter.SPRInputs
+	SPRResultsFunc     hierarchy_filter.FilterSPRResultsFunc                   // This one chooses one result among many (or nil)
+	PIPUpdateFunc      hierarchy.PointInPolygonHierarchyResolverUpdateCallback // This one constructs a map[string]interface{} to update the target record (or not)
+	To                 []string
+	From               []string
+}
+
+func RunOptionsFromFlagSet(ctx context.Context, fs *flag.FlagSet) (*RunOptions, error) {
+
+	flagset.Parse(fs)
+
+	inputs := &filter.SPRInputs{}
+
+	inputs.IsCurrent = is_current
+	inputs.IsCeased = is_ceased
+	inputs.IsDeprecated = is_deprecated
+	inputs.IsSuperseded = is_superseded
+	inputs.IsSuperseding = is_superseding
+
+	hierarchy_paths := fs.Args()
+
+	opts := &RunOptions{
+		WriterURI:          writer_uri,
+		ExporterURI:        exporter_uri,
+		SpatialDatabaseURI: spatial_database_uri,
+		MapshaperServerURI: mapshaper_server,
+		SPRResultsFunc:     hierarchy_filter.FirstButForgivingSPRResultsFunc, // sudo make me configurable
+		SPRFilterInputs:    inputs,
+		ToIterator:         iterator_uri,
+		FromIterator:       spatial_iterator_uri,
+		To:                 hierarchy_paths,
+		From:               spatial_paths,
+	}
+
+	return opts, nil
+}

app/hierarchy/update/update.go

@@ -6,13 +6,10 @@ import (
 	"fmt"
 	"log"
 
-	"github.com/sfomuseum/go-flags/flagset"
 	"github.com/sfomuseum/go-sfomuseum-mapshaper"
 	"github.com/whosonfirst/go-whosonfirst-export/v2"
 	"github.com/whosonfirst/go-whosonfirst-spatial/database"
-	"github.com/whosonfirst/go-whosonfirst-spatial/filter"
 	"github.com/whosonfirst/go-whosonfirst-spatial/hierarchy"
-	hierarchy_filter "github.com/whosonfirst/go-whosonfirst-spatial/hierarchy/filter"
 	"github.com/whosonfirst/go-writer/v3"
 )
 
@@ -29,33 +26,20 @@ func Run(ctx context.Context, logger *log.Logger) error {
 
 func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) error {
 
-	flagset.Parse(fs)
-
-	inputs := &filter.SPRInputs{}
-
-	inputs.IsCurrent = is_current
-	inputs.IsCeased = is_ceased
-	inputs.IsDeprecated = is_deprecated
-	inputs.IsSuperseded = is_superseded
-	inputs.IsSuperseding = is_superseding
-
-	opts := &UpdateApplicationOptions{
-		WriterURI:          writer_uri,
-		ExporterURI:        exporter_uri,
-		SpatialDatabaseURI: spatial_database_uri,
-		MapshaperServerURI: mapshaper_server,
-		SPRResultsFunc:     hierarchy_filter.FirstButForgivingSPRResultsFunc, // sudo make me configurable
-		SPRFilterInputs:    inputs,
-		ToIterator:         iterator_uri,
-		FromIterator:       spatial_iterator_uri,
+	opts, err := RunOptionsFromFlagSet(ctx, fs)
+
+	if err != nil {
+		return fmt.Errorf("Failed to derive run options, %w", err)
 	}
 
-	hierarchy_paths := fs.Args()
+	return RunWithOptions(ctx, opts, logger)
+}
 
-	paths := &UpdateApplicationPaths{
-		To:   hierarchy_paths,
-		From: spatial_paths,
-	}
+func RunWithOptions(ctx context.Context, opts *RunOptions, logger *log.Logger) error {
+
+	// Note that the bulk of this method is simply taking opts and using it to
+	// instantiate all the different pieces necessary for the updateApplication
+	// type to actually do the work of updating hierarchies.
 
 	var ex export.Exporter
 	var wr writer.Writer
@@ -74,6 +58,17 @@ func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) e
 		ex = _ex
 	}
 
+	// In addition to the "exporter" we also create a default options instance
+	// that is used by the updateApplication instance to do a final check whether
+	// or not records have actually been updated (beyond just incrementing the
+	// lastmodified date).
+
+	export_opts, err := export.NewDefaultOptions(ctx)
+
+	if err != nil {
+		return fmt.Errorf("Failed to create export options, %w", err)
+	}
+
 	if opts.Writer != nil {
 		wr = opts.Writer
 	} else {
@@ -145,17 +140,24 @@ func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) e
 		return fmt.Errorf("Failed to create PIP tool, %v", err)
 	}
 
-	app := &UpdateApplication{
+	// This is where the actual work happens
+
+	app := &updateApplication{
 		to:                  opts.ToIterator,
 		from:                opts.FromIterator,
 		spatial_db:          spatial_db,
-		tool:                resolver,
+		resolver:            resolver,
 		exporter:            ex,
+		export_opts:         export_opts,
 		writer:              wr,
 		sprFilterInputs:     opts.SPRFilterInputs,
 		sprResultsFunc:      opts.SPRResultsFunc,
 		hierarchyUpdateFunc: update_cb,
-		logger:              logger,
+	}
+
+	paths := &updateApplicationPaths{
+		To:   opts.To,
+		From: opts.From,
 	}
 
 	return app.Run(ctx, paths)

app/hierarchy/update/app.go → app/hierarchy/update/update_app.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"log"
+	_ "log/slog"
 
 	"github.com/whosonfirst/go-whosonfirst-export/v2"
 	"github.com/whosonfirst/go-whosonfirst-feature/geometry"
@@ -17,47 +17,35 @@ import (
 	"github.com/whosonfirst/go-writer/v3"
 )
 
-type UpdateApplicationOptions struct {
-	Writer             writer.Writer
-	WriterURI          string
-	Exporter           export.Exporter
-	ExporterURI        string
-	MapshaperServerURI string
-	SpatialDatabase    database.SpatialDatabase
-	SpatialDatabaseURI string
-	ToIterator         string
-	FromIterator       string
-	SPRFilterInputs    *filter.SPRInputs
-	SPRResultsFunc     hierarchy_filter.FilterSPRResultsFunc                   // This one chooses one result among many (or nil)
-	PIPUpdateFunc      hierarchy.PointInPolygonHierarchyResolverUpdateCallback // This one constructs a map[string]interface{} to update the target record (or not)
-}
-
-type UpdateApplicationPaths struct {
+type updateApplicationPaths struct {
 	To   []string
 	From []string
 }
 
-type UpdateApplication struct {
+// updateApplication is a struct to wrap the details of (optionally) populating a spatial
+// database and updating the hierarchies of (n) files derived from an iterator including
+// writing (publishing) the updated records.
+type updateApplication struct {
 	to                  string
 	from                string
-	tool                *hierarchy.PointInPolygonHierarchyResolver
+	resolver            *hierarchy.PointInPolygonHierarchyResolver
 	writer              writer.Writer
 	exporter            export.Exporter
+	export_opts         *export.Options
 	spatial_db          database.SpatialDatabase
 	sprResultsFunc      hierarchy_filter.FilterSPRResultsFunc
 	sprFilterInputs     *filter.SPRInputs
 	hierarchyUpdateFunc hierarchy.PointInPolygonHierarchyResolverUpdateCallback
-	logger              *log.Logger
 }
 
-func (app *UpdateApplication) Run(ctx context.Context, paths *UpdateApplicationPaths) error {
+func (app *updateApplication) Run(ctx context.Context, paths *updateApplicationPaths) error {
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
 	// These are the data we are indexing to HIERARCHY from
 
-	err := app.IndexSpatialDatabase(ctx, paths.From...)
+	err := app.indexSpatialDatabase(ctx, paths.From...)
 
 	if err != nil {
 		return err
@@ -73,7 +61,7 @@ func (app *UpdateApplication) Run(ctx context.Context, paths *UpdateApplicationP
 			return fmt.Errorf("Failed to read '%s', %v", path, err)
 		}
 
-		_, err = app.UpdateAndPublishFeature(ctx, body)
+		_, err = app.updateAndPublishFeature(ctx, body)
 
 		if err != nil {
 			return fmt.Errorf("Failed to update feature for '%s', %v", path, err)
@@ -101,7 +89,7 @@ func (app *UpdateApplication) Run(ctx context.Context, paths *UpdateApplicationP
 	return app.writer.Close(ctx)
 }
 
-func (app *UpdateApplication) IndexSpatialDatabase(ctx context.Context, uris ...string) error {
+func (app *updateApplication) indexSpatialDatabase(ctx context.Context, uris ...string) error {
 
 	from_cb := func(ctx context.Context, path string, fh io.ReadSeeker, args ...interface{}) error {
 
@@ -143,17 +131,28 @@ func (app *UpdateApplication) IndexSpatialDatabase(ctx context.Context, uris ...
 // UpdateAndPublishFeature will invoke the `PointInPolygonAndUpdate` method using the `hierarchy.PointInPolygonHierarchyResolver` instance
 // associated with 'app' using 'body' as its input. If successful and there are changes the result will be published using the `PublishFeature`
 // method.
-func (app *UpdateApplication) UpdateAndPublishFeature(ctx context.Context, body []byte) ([]byte, error) {
+func (app *updateApplication) updateAndPublishFeature(ctx context.Context, body []byte) ([]byte, error) {
 
-	has_changed, new_body, err := app.UpdateFeature(ctx, body)
+	has_changed, new_body, err := app.updateFeature(ctx, body)
 
 	if err != nil {
 		return nil, fmt.Errorf("Failed to update feature, %w", err)
 	}
 
+	// But really, has the record _actually_ changed?
+
+	if has_changed {
+
+		has_changed, err = export.ExportChanged(new_body, body, app.export_opts, io.Discard)
+
+		if err != nil {
+			return nil, fmt.Errorf("Failed to determine if export has changed post update, %w", err)
+		}
+	}
+
 	if has_changed {
 
-		new_body, err = app.PublishFeature(ctx, new_body)
+		new_body, err = app.publishFeature(ctx, new_body)
 
 		if err != nil {
 			return nil, fmt.Errorf("Failed to publish feature, %w", err)
@@ -165,13 +164,13 @@ func (app *UpdateApplication) UpdateAndPublishFeature(ctx context.Context, body
 
 // UpdateFeature will invoke the `PointInPolygonAndUpdate` method using the `hierarchy.PointInPolygonHierarchyResolver` instance
 // associated with 'app' using 'body' as its input.
-func (app *UpdateApplication) UpdateFeature(ctx context.Context, body []byte) (bool, []byte, error) {
+func (app *updateApplication) updateFeature(ctx context.Context, body []byte) (bool, []byte, error) {
 
-	return app.tool.PointInPolygonAndUpdate(ctx, app.sprFilterInputs, app.sprResultsFunc, app.hierarchyUpdateFunc, body)
+	return app.resolver.PointInPolygonAndUpdate(ctx, app.sprFilterInputs, app.sprResultsFunc, app.hierarchyUpdateFunc, body)
 }
 
 // PublishFeature exports 'body' using the `whosonfirst/go-writer/v3` instance associated with 'app'.
-func (app *UpdateApplication) PublishFeature(ctx context.Context, body []byte) ([]byte, error) {
+func (app *updateApplication) publishFeature(ctx context.Context, body []byte) ([]byte, error) {
 
 	new_body, err := app.exporter.Export(ctx, body)