Update to reflect backwards incompatible changes introduced in `whoso…

…nfirst/go-whosonfirst-spatial` v0.11.0 (#10) * update vendor deps * update flags --------- Co-authored-by: sfomuseumbot <sfomuseumbot@localhost>
whosonfirst · Nov 21, 2024 · 606c4c4 · 606c4c4
1 parent 03da921
commit 606c4c4
Show file tree

Hide file tree

Showing 226 changed files with 6,052 additions and 2,823 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,2 +1,4 @@
 *~
-bin
+bin
+work
+go.work*
diff --git a/Makefile b/Makefile
@@ -13,5 +13,4 @@ cli:
 
 debug:
 	go run -mod $(GOMOD) -ldflags="$(LDFLAGS)" cmd/server/main.go \
-		-iterator-uri repo:// \
-		/usr/local/data/sfomuseum-data-maps
+		-iterator-uri 'repo://?strict=false#/$(REPO)'
diff --git a/README.md b/README.md
@@ -4,7 +4,7 @@ gRPC support for the `go-whosonfirst-spatial` interfaces.
 
 ## Documentation
 
-Documentatin is incomplete.
+Documentation is incomplete.
 
 ## Tools
 
@@ -59,7 +59,7 @@ $> ./bin/client -h
 For example:
 
 ```
-$> ./bin/client -latitude 43.873889 -longitude 18.408611 -inception-date 199X | jq
+$> ./bin/client -latitude 43.873889 -longitude 18.408611 -inception 199X | jq
 {
   "places": [
     {
@@ -101,8 +101,8 @@ $> ./bin/server -h
     	The host to listen for requests on (default "localhost")
   -is-wof
     	Input data is WOF-flavoured GeoJSON. (Pass a value of '0' or 'false' if you need to index non-WOF documents. (default true)
-  -iterator-uri string
-    	A valid whosonfirst/go-whosonfirst-iterate/v2 URI. Supported schemes are: directory://, featurecollection://, file://, filelist://, geojsonl://, null://, repo://. (default "repo://")
+  -iterator-uri value
+    	Zero or more URIs denoting data sources to use for indexing the spatial database at startup. URIs take the form of {ITERATOR_URI} + "#" + {PIPE-SEPARATED LIST OF ITERATOR SOURCES}. Where {ITERATOR_URI} is expected to be a registered whosonfirst/go-whosonfirst-iterate/v2 iterator (emitter) URI and {ITERATOR SOURCES} are valid input paths for that iterator. Supported whosonfirst/go-whosonfirst-iterate/v2 iterator schemes are: cwd://, directory://, featurecollection://, file://, filelist://, geojsonl://, null://, repo://.
   -port int
     	The port to listen for requests on (default 8082)
   -properties-reader-uri string
@@ -114,7 +114,7 @@ $> ./bin/server -h
 For example:
 
 ```
-$> ./bin/server -spatial-database-uri rtree:// /usr/local/data/whosonfirst-data-admin-ba/
+$> ./bin/server -spatial-database-uri 'rtree://#/usr/local/data/whosonfirst-data-admin-ba/'
 2021/03/26 08:59:36 Listening on localhost:8082
 08:59:37.890704 [server] STATUS indexing 8792 records indexed
 08:59:38.890291 [server] STATUS indexing 16249 records indexed

diff --git a/app/client/client.go b/app/client/client.go
@@ -5,39 +5,38 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
+	"os"
 
 	"github.com/whosonfirst/go-whosonfirst-spatial-grpc/request"
 	"github.com/whosonfirst/go-whosonfirst-spatial-grpc/spatial"
 	"github.com/whosonfirst/go-whosonfirst-spatial/pip"
 	"google.golang.org/grpc"
-	"io"
-	"log"
-	"os"
 )
 
-func Run(ctx context.Context, logger *log.Logger) error {
+func Run(ctx context.Context) error {
 
 	fs, err := DefaultFlagSet()
 
 	if err != nil {
 		return fmt.Errorf("Failed to create default flagset, %w", err)
 	}
 
-	return RunWithFlagSet(ctx, fs, logger)
+	return RunWithFlagSet(ctx, fs)
 }
 
-func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) error {
+func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet) error {
 
 	opts, err := RunOptionsFromFlagSet(ctx, fs)
 
 	if err != nil {
 		return fmt.Errorf("Failed to derive options from flagset, %w", err)
 	}
 
-	return RunWithOptions(ctx, opts, logger)
+	return RunWithOptions(ctx, opts)
 }
 
-func RunWithOptions(ctx context.Context, opts *RunOptions, logger *log.Logger) error {
+func RunWithOptions(ctx context.Context, opts *RunOptions) error {
 
 	pip_req := &pip.PointInPolygonRequest{
 		Latitude:            opts.Latitude,

diff --git a/app/server/flags.go b/app/server/flags.go
@@ -3,13 +3,11 @@ package server
 import (
 	"flag"
 	"fmt"
-	"sort"
-	"strings"
 
 	"github.com/sfomuseum/go-flags/flagset"
 	"github.com/whosonfirst/go-reader"
-	"github.com/whosonfirst/go-whosonfirst-iterate/v2/emitter"
 	"github.com/whosonfirst/go-whosonfirst-spatial/database"
+	spatial_flags "github.com/whosonfirst/go-whosonfirst-spatial/flags"
 )
 
 var host string
@@ -23,7 +21,7 @@ var is_wof bool
 var enable_custom_placetypes bool
 var custom_placetypes string
 
-var iterator_uri string
+var iterator_uris spatial_flags.MultiCSVIteratorURIFlag
 
 func DefaultFlagSet() (*flag.FlagSet, error) {
 
@@ -50,13 +48,9 @@ func DefaultFlagSet() (*flag.FlagSet, error) {
 
 	// Indexing flags
 
-	modes := emitter.Schemes()
-	sort.Strings(modes)
-
-	valid_modes := strings.Join(modes, ", ")
-	desc_modes := fmt.Sprintf("A valid whosonfirst/go-whosonfirst-iterate/v2 URI. Supported schemes are: %s.", valid_modes)
-
-	fs.StringVar(&iterator_uri, "iterator-uri", "repo://", desc_modes)
+	desc_iter := spatial_flags.IteratorURIFlagDescription()
+	desc_iter = fmt.Sprintf("Zero or more URIs denoting data sources to use for indexing the spatial database at startup. %s", desc_iter)
+	fs.Var(&iterator_uris, "iterator-uri", desc_iter)
 
 	return fs, nil
 }
diff --git a/app/server/options.go b/app/server/options.go
@@ -10,13 +10,12 @@ import (
 type RunOptions struct {
 	Host                   string
 	Port                   int
-	SpatialDatabaseURI     string   `json:"spatial_database_uri"`
-	PropertiesReaderURI    string   `json:"properties_reader_uri"`
-	IteratorURI            string   `json:"iterator_uri"`
-	IteratorSources        []string `json:"iterator_sources"`
-	EnableCustomPlacetypes bool     `json:"enable_custom_placetypes"`
-	CustomPlacetypes       string   `json:"custom_placetypes"`
-	IsWhosOnFirst          bool     `json:"is_whosonfirst"`
+	SpatialDatabaseURI     string              `json:"spatial_database_uri"`
+	PropertiesReaderURI    string              `json:"properties_reader_uri"`
+	IteratorSources        map[string][]string `json:"iterator_sources"`
+	EnableCustomPlacetypes bool                `json:"enable_custom_placetypes"`
+	CustomPlacetypes       string              `json:"custom_placetypes"`
+	IsWhosOnFirst          bool                `json:"is_whosonfirst"`
 }
 
 func RunOptionsFromFlagSet(ctx context.Context, fs *flag.FlagSet) (*RunOptions, error) {
@@ -29,14 +28,13 @@ func RunOptionsFromFlagSet(ctx context.Context, fs *flag.FlagSet) (*RunOptions,
 		return nil, err
 	}
 
-	iterator_sources := fs.Args()
+	iterator_sources := iterator_uris.AsMap()
 
 	opts := &RunOptions{
 		Host:                   host,
 		Port:                   port,
 		SpatialDatabaseURI:     spatial_database_uri,
 		PropertiesReaderURI:    properties_reader_uri,
-		IteratorURI:            iterator_uri,
 		IteratorSources:        iterator_sources,
 		EnableCustomPlacetypes: enable_custom_placetypes,
 		CustomPlacetypes:       custom_placetypes,

diff --git a/app/server/server.go b/app/server/server.go
@@ -4,7 +4,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"log"
+	"log/slog"
 	"net"
 
 	grpc_server "github.com/whosonfirst/go-whosonfirst-spatial-grpc/server"
@@ -13,37 +13,35 @@ import (
 	"google.golang.org/grpc"
 )
 
-func Run(ctx context.Context, logger *log.Logger) error {
+func Run(ctx context.Context) error {
 
 	fs, err := DefaultFlagSet()
 
 	if err != nil {
 		return fmt.Errorf("Failed to derive default flag set, %w", err)
 	}
 
-	return RunWithFlagSet(ctx, fs, logger)
+	return RunWithFlagSet(ctx, fs)
 }
 
-func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) error {
+func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet) error {
 
 	opts, err := RunOptionsFromFlagSet(ctx, fs)
 
 	if err != nil {
 		return fmt.Errorf("Failed to derive options from flagset, %w", err)
 	}
 
-	return RunWithOptions(ctx, opts, logger)
+	return RunWithOptions(ctx, opts)
 }
 
-func RunWithOptions(ctx context.Context, opts *RunOptions, logger *log.Logger) error {
+func RunWithOptions(ctx context.Context, opts *RunOptions) error {
 
 	spatial_opts := &app.SpatialApplicationOptions{
 		SpatialDatabaseURI:     opts.SpatialDatabaseURI,
 		PropertiesReaderURI:    opts.PropertiesReaderURI,
-		IteratorURI:            opts.IteratorURI,
 		EnableCustomPlacetypes: opts.EnableCustomPlacetypes,
 		CustomPlacetypes:       opts.CustomPlacetypes,
-		IsWhosOnFirst:          opts.IsWhosOnFirst,
 	}
 
 	spatial_app, err := app.NewSpatialApplication(ctx, spatial_opts)
@@ -52,14 +50,14 @@ func RunWithOptions(ctx context.Context, opts *RunOptions, logger *log.Logger) e
 		return fmt.Errorf("Failed to create new spatial application, %w", err)
 	}
 
-	if len(opts.IteratorSources) > 0 {
+	go func() {
 
-		err = spatial_app.IndexPaths(ctx, opts.IteratorSources...)
+		err := spatial_app.IndexDatabaseWithIterators(ctx, opts.IteratorSources)
 
 		if err != nil {
-			return fmt.Errorf("Failed to index paths, %v", err)
+			slog.Error("Failed to index database", "error", err)
 		}
-	}
+	}()
 
 	spatial_server, err := grpc_server.NewSpatialServer(spatial_app)
 
@@ -72,7 +70,7 @@ func RunWithOptions(ctx context.Context, opts *RunOptions, logger *log.Logger) e
 	spatial.RegisterSpatialServer(grpc_server, spatial_server)
 
 	addr := fmt.Sprintf("%s:%d", opts.Host, opts.Port)
-	log.Printf("Listening on %s\n", addr)
+	slog.Info("Listening for requests", "address", addr)
 
 	lis, err := net.Listen("tcp", addr)
 

diff --git a/cmd/client/main.go b/cmd/client/main.go
@@ -4,17 +4,15 @@ import (
 	"context"
 	"log"
 
-	"github.com/whosonfirst/go-whosonfirst-spatial-grpc/app/client"	
+	"github.com/whosonfirst/go-whosonfirst-spatial-grpc/app/client"
 )
 
 func main() {
 
 	ctx := context.Background()
-	logger := log.Default()
-
-	err := client.Run(ctx, logger)
+	err := client.Run(ctx)
 
 	if err != nil {
-		logger.Fatalf("Failed to run client, %v", err)
+		log.Fatalf("Failed to run client, %v", err)
 	}
 }
diff --git a/cmd/server/main.go b/cmd/server/main.go
@@ -4,17 +4,15 @@ import (
 	"context"
 	"log"
 
-	"github.com/whosonfirst/go-whosonfirst-spatial-grpc/app/server"	
+	"github.com/whosonfirst/go-whosonfirst-spatial-grpc/app/server"
 )
 
 func main() {
 
 	ctx := context.Background()
-	logger := log.Default()
-
-	err := server.Run(ctx, logger)
+	err := server.Run(ctx)
 
 	if err != nil {
-		logger.Fatalf("Failed to run client, %v", err)
+		log.Fatalf("Failed to run client, %v", err)
 	}
 }
diff --git a/go.mod b/go.mod
@@ -1,23 +1,21 @@
 module github.com/whosonfirst/go-whosonfirst-spatial-grpc
 
-go 1.22.2
-
-toolchain go1.22.4
+go 1.23
 
 require (
 	github.com/paulmach/orb v0.11.1
 	github.com/sfomuseum/go-flags v0.10.0
 	github.com/whosonfirst/go-reader v1.0.2
 	github.com/whosonfirst/go-whosonfirst-flags v0.5.2
-	github.com/whosonfirst/go-whosonfirst-iterate/v2 v2.4.1
-	github.com/whosonfirst/go-whosonfirst-spatial v0.10.0
+	github.com/whosonfirst/go-whosonfirst-iterate/v2 v2.5.0
+	github.com/whosonfirst/go-whosonfirst-spatial v0.10.2
 	github.com/whosonfirst/go-whosonfirst-spr/v2 v2.3.7
-	google.golang.org/grpc v1.65.0
-	google.golang.org/protobuf v1.34.2
+	google.golang.org/grpc v1.68.0
+	google.golang.org/protobuf v1.35.2
 )
 
 require (
-	github.com/aaronland/go-json-query v0.1.4 // indirect
+	github.com/aaronland/go-json-query v0.1.5 // indirect
 	github.com/aaronland/go-roster v1.0.0 // indirect
 	github.com/dhconnelly/rtreego v1.2.0 // indirect
 	github.com/dominikbraun/graph v0.23.0 // indirect
@@ -26,26 +24,26 @@ require (
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/natefinch/atomic v1.0.1 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
-	github.com/sfomuseum/go-edtf v1.1.1 // indirect
-	github.com/sfomuseum/go-timings v1.2.1 // indirect
+	github.com/sfomuseum/go-edtf v1.2.1 // indirect
+	github.com/sfomuseum/go-timings v1.4.0 // indirect
 	github.com/sfomuseum/iso8601duration v1.1.0 // indirect
-	github.com/tidwall/gjson v1.17.1 // indirect
+	github.com/tidwall/gjson v1.18.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tidwall/sjson v1.2.5 // indirect
 	github.com/whosonfirst/go-ioutil v1.0.2 // indirect
 	github.com/whosonfirst/go-sanitize v0.1.0 // indirect
 	github.com/whosonfirst/go-whosonfirst-crawl v0.2.2 // indirect
-	github.com/whosonfirst/go-whosonfirst-feature v0.0.27 // indirect
-	github.com/whosonfirst/go-whosonfirst-placetypes v0.7.2 // indirect
+	github.com/whosonfirst/go-whosonfirst-feature v0.0.28 // indirect
+	github.com/whosonfirst/go-whosonfirst-placetypes v0.7.3 // indirect
 	github.com/whosonfirst/go-whosonfirst-sources v0.1.0 // indirect
 	github.com/whosonfirst/go-whosonfirst-uri v1.3.0 // indirect
 	github.com/whosonfirst/go-writer/v3 v3.1.1 // indirect
 	github.com/whosonfirst/walk v0.0.2 // indirect
 	github.com/whosonfirst/warning v0.1.1 // indirect
 	go.mongodb.org/mongo-driver v1.11.4 // indirect
-	golang.org/x/net v0.25.0 // indirect
-	golang.org/x/sys v0.20.0 // indirect
-	golang.org/x/text v0.15.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect
+	golang.org/x/net v0.29.0 // indirect
+	golang.org/x/sys v0.25.0 // indirect
+	golang.org/x/text v0.18.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect
 )