MLPAB-1785 Add opensearch and pagination to supporter dashboard (#1070)

.github/workflows/ui_test_job.yml

@@ -85,7 +85,7 @@ jobs:
           TAG: ${{ inputs.tag }}
         run: |
           docker compose -f docker/docker-compose.yml -f docker/docker-compose.ci.yml pull
-          docker compose -f docker/docker-compose.yml -f docker/docker-compose.ci.yml up -d
+          docker compose -f docker/docker-compose.yml -f docker/docker-compose.ci.yml up -d --wait
 
       - name: Manage Ingress/Configure AWS Credentials
         if: inputs.run_against_image != true

.mockery.yaml

@@ -21,6 +21,7 @@ packages:
   github.com/ministryofjustice/opg-modernising-lpa/internal/page:
   github.com/ministryofjustice/opg-modernising-lpa/internal/place:
   github.com/ministryofjustice/opg-modernising-lpa/internal/s3:
+  github.com/ministryofjustice/opg-modernising-lpa/internal/search:
   github.com/ministryofjustice/opg-modernising-lpa/internal/secrets:
   github.com/ministryofjustice/opg-modernising-lpa/internal/uid:
   github.com/ministryofjustice/opg-modernising-lpa/internal/validation:

cmd/event-received/cloud_watch_event_handler.go

@@ -19,6 +19,7 @@ import (
 	"github.com/ministryofjustice/opg-modernising-lpa/internal/notify"
 	"github.com/ministryofjustice/opg-modernising-lpa/internal/page"
 	"github.com/ministryofjustice/opg-modernising-lpa/internal/random"
+	"github.com/ministryofjustice/opg-modernising-lpa/internal/search"
 	"github.com/ministryofjustice/opg-modernising-lpa/internal/secrets"
 	"github.com/ministryofjustice/opg-modernising-lpa/internal/uid"
 )
@@ -32,12 +33,18 @@ type cloudWatchEventHandler struct {
 	notifyBaseURL      string
 	appPublicURL       string
 	eventBusName       string
+	searchEndpoint     string
 }
 
 func (h *cloudWatchEventHandler) Handle(ctx context.Context, cloudWatchEvent events.CloudWatchEvent) error {
 	switch cloudWatchEvent.DetailType {
 	case "uid-requested":
-		uidStore := app.NewUidStore(h.dynamoClient, h.now)
+		searchClient, err := search.NewClient(h.cfg, h.searchEndpoint)
+		if err != nil {
+			return err
+		}
+
+		uidStore := app.NewUidStore(h.dynamoClient, searchClient, h.now)
 		uidClient := uid.New(h.uidBaseURL, lambda.New(h.cfg, v4.NewSigner(), &http.Client{Timeout: 10 * time.Second}, time.Now))
 
 		return handleUidRequested(ctx, uidStore, uidClient, cloudWatchEvent)

cmd/event-received/main.go

@@ -35,7 +35,7 @@ type dynamodbClient interface {
 	One(ctx context.Context, pk, sk string, v interface{}) error
 	OneByUID(ctx context.Context, uid string, v interface{}) error
 	Put(ctx context.Context, v interface{}) error
-	Update(ctx context.Context, pk, sk string, values map[string]dynamodbtypes.AttributeValue, expression string) error
+	UpdateReturn(ctx context.Context, pk, sk string, values map[string]dynamodbtypes.AttributeValue, expression string) (map[string]dynamodbtypes.AttributeValue, error)
 	DeleteOne(ctx context.Context, pk, sk string) error
 }
 
@@ -82,6 +82,7 @@ func handler(ctx context.Context, event Event) error {
 		evidenceBucketName = os.Getenv("UPLOADS_S3_BUCKET_NAME")
 		uidBaseURL         = os.Getenv("UID_BASE_URL")
 		eventBusName       = env.Get("EVENT_BUS_NAME", "default")
+		searchEndpoint     = os.Getenv("SEARCH_ENDPOINT")
 	)
 
 	cfg, err := config.LoadDefaultConfig(ctx)
@@ -125,6 +126,7 @@ func handler(ctx context.Context, event Event) error {
 			notifyBaseURL:      notifyBaseURL,
 			appPublicURL:       appPublicURL,
 			eventBusName:       eventBusName,
+			searchEndpoint:     searchEndpoint,
 		}
 
 		if err := handler.Handle(ctx, event.CloudWatchEvent); err != nil {

cmd/mlpa/main.go

@@ -38,6 +38,7 @@ import (
 	"github.com/ministryofjustice/opg-modernising-lpa/internal/pay"
 	"github.com/ministryofjustice/opg-modernising-lpa/internal/place"
 	"github.com/ministryofjustice/opg-modernising-lpa/internal/s3"
+	"github.com/ministryofjustice/opg-modernising-lpa/internal/search"
 	"github.com/ministryofjustice/opg-modernising-lpa/internal/secrets"
 	"github.com/ministryofjustice/opg-modernising-lpa/internal/telemetry"
 	"github.com/ministryofjustice/opg-modernising-lpa/internal/templatefn"
@@ -105,6 +106,7 @@ func run(ctx context.Context, logger *slog.Logger) error {
 		evidenceBucketName    = env.Get("UPLOADS_S3_BUCKET_NAME", "evidence")
 		eventBusName          = env.Get("EVENT_BUS_NAME", "default")
 		mockIdentityPublicKey = env.Get("MOCK_IDENTITY_PUBLIC_KEY", "")
+		searchEndpoint        = env.Get("SEARCH_ENDPOINT", "")
 	)
 
 	staticHash, err := dirhash.HashDir(webDir+"/static", webDir, dirhash.DefaultHash)
@@ -205,6 +207,15 @@ func run(ctx context.Context, logger *slog.Logger) error {
 
 	eventClient := event.NewClient(cfg, eventBusName)
 
+	searchClient, err := search.NewClient(cfg, searchEndpoint)
+	if err != nil {
+		return err
+	}
+
+	if err := searchClient.CreateIndices(ctx); err != nil {
+		return err
+	}
+
 	secretsClient, err := secrets.NewClient(cfg, time.Hour)
 	if err != nil {
 		return err
@@ -308,6 +319,7 @@ func run(ctx context.Context, logger *slog.Logger) error {
 		evidenceS3Client,
 		eventClient,
 		lpaStoreClient,
+		searchClient,
 	)))
 
 	mux.Handle("/", app.App(
@@ -329,6 +341,7 @@ func run(ctx context.Context, logger *slog.Logger) error {
 		evidenceS3Client,
 		eventClient,
 		lpaStoreClient,
+		searchClient,
 	))
 
 	var handler http.Handler = mux

docker/docker-compose.dev.yml

@@ -46,6 +46,7 @@ services:
       - GOVUK_NOTIFY_BASE_URL=http://mock-notify:8080
       - UPLOADS_S3_BUCKET_NAME=evidence
       - UID_BASE_URL=http://mock-uid:8080
+      - SEARCH_ENDPOINT=http://my-domain.eu-west-1.opensearch.localhost.localstack.cloud:4566
     ports:
       - "9000:8080"
     entrypoint: aws-lambda-rie /var/task/event-received

docker/docker-compose.yml

@@ -26,6 +26,7 @@ services:
       - AWS_BASE_URL=http://localstack:4566
       - AWS_REGION=eu-west-1
       - AWS_SECRET_ACCESS_KEY=fakeAccessKey
+      - SEARCH_ENDPOINT=http://my-domain.eu-west-1.opensearch.localhost.localstack.cloud:4566
 
   event-logger:
     build:
@@ -46,6 +47,8 @@ services:
     build:
       context: ..
       dockerfile: docker/localstack/Dockerfile
+    depends_on:
+      - opensearch
     container_name: localstack
     ports:
       - "127.0.0.1:4566:4566"            # LocalStack Gateway
@@ -54,13 +57,43 @@ services:
       - "/var/run/docker.sock:/var/run/docker.sock"
     environment:
       - DOCKER_HOST=unix:///var/run/docker.sock
-      - SERVICES=s3,secretsmanager,sqs,dynamodb,events,kms,lambda
+      - SERVICES=s3,secretsmanager,sqs,dynamodb,events,kms,lambda,opensearch
       - DATA_DIR=/tmp/localstack/data
       - DEBUG=1
       - AWS_ACCESS_KEY_ID=fakeKeyId
       - AWS_BASE_URL=http://localstack:4566
       - AWS_REGION=eu-west-1
       - AWS_SECRET_ACCESS_KEY=fakeAccessKey
+      - OPENSEARCH_CUSTOM_BACKEND=http://opensearch:9200
+    networks:
+      default:
+        aliases:
+          - my-domain.eu-west-1.opensearch.localhost.localstack.cloud
+    healthcheck:
+      test: "curl --silent --fail http://my-domain.eu-west-1.opensearch.localhost.localstack.cloud:4566 | grep 'cluster_name'"
+      interval: 20s
+      retries: 10
+      start_period: 20s
+      timeout: 10s
+
+  opensearch:
+    container_name: opensearch
+    image: opensearchproject/opensearch:1.1.0
+    environment:
+      - node.name=opensearch
+      - cluster.name=opensearch-docker-cluster
+      - discovery.type=single-node
+      - bootstrap.memory_lock=true
+      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
+      - "DISABLE_SECURITY_PLUGIN=true"
+    ports:
+      - "9200:9200"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    volumes:
+      - data01:/usr/share/opensearch/data
 
   mock-lpa-store:
     image: outofcoffee/imposter:latest
@@ -108,3 +141,7 @@ services:
     container_name: mock-uid
     ports:
       - "8080:8080"
+
+volumes:
+  data01:
+    driver: local

docker/localstack/localstack-init.sh

@@ -32,8 +32,11 @@ awslocal events put-rule --region eu-west-1 --name send-events-to-queue-rule --e
 awslocal events put-targets --region eu-west-1 --event-bus-name default --rule send-events-to-queue-rule --targets "Id"="event-queue","Arn"="arn:aws:sqs:eu-west-1:000000000000:event-queue"
 
 echo 'creating lambda'
-awslocal lambda create-function --environment Variables="{LPAS_TABLE=lpas,GOVUK_NOTIFY_IS_PRODUCTION=0,APP_PUBLIC_URL=localhost:5050,GOVUK_NOTIFY_BASE_URL=http://mock-notify:8080,UPLOADS_S3_BUCKET_NAME=evidence,UID_BASE_URL=http://mock-uid:8080}" --region eu-west-1 --function-name event-received --handler event-received --runtime go1.x --role arn:aws:iam::000000000000:role/lambda-role --zip-file fileb:///etc/event-received.zip
+awslocal lambda create-function --environment Variables="{LPAS_TABLE=lpas,GOVUK_NOTIFY_IS_PRODUCTION=0,APP_PUBLIC_URL=localhost:5050,GOVUK_NOTIFY_BASE_URL=http://mock-notify:8080,UPLOADS_S3_BUCKET_NAME=evidence,UID_BASE_URL=http://mock-uid:8080,SEARCH_ENDPOINT=http://my-domain.eu-west-1.opensearch.localhost.localstack.cloud:4566}" --region eu-west-1 --function-name event-received --handler event-received --runtime go1.x --role arn:aws:iam::000000000000:role/lambda-role --zip-file fileb:///etc/event-received.zip
 awslocal lambda wait function-active-v2 --region eu-west-1 --function-name event-received
 
 awslocal events put-rule --region eu-west-1 --name receive-events-mlpa --event-bus-name default --event-pattern '{"source":["opg.poas.makeregister"],"detail-type":["uid-requested"]}'
 awslocal events put-targets --region eu-west-1 --event-bus-name default --rule receive-events-mlpa --targets "Id"="receive-events-sirius","Arn"="arn:aws:lambda:eu-west-1:000000000000:function:event-received"
+
+echo 'configuring opensearch'
+awslocal opensearch create-domain --region eu-west-1 --domain-name my-domain

docs/architecture/decisions/0003-use-opensearch.md

@@ -0,0 +1,37 @@
+# 3. Use AWS OpenSearch service
+
+Date: 2024-02-28
+
+## Status
+
+Accepted
+
+## Context
+
+We currently use DynamoDB to store data. We have a requirement to present
+numbered pages of data to users, which is something we cannot do with DynamoDB
+(it supports cursor based pagination only, i.e. "load next page"). We also have
+future requirements for filtering and searching on other fields, to produce a
+dashboard.
+
+## Decision
+
+Deploying AWS OpenSearch will satisify these use cases, without requiring us to
+rewrite all of our data access layer. We will be able to index the data required
+in addition to our current store. In particular, we will use AWS OpenSearch
+Serverless so we don't have to deal with tuning the setup of search, at least
+for the initial period of use.
+
+An alternative suggestion, we won't pursue now, is to use Postgres. This would
+allow us to use a single store for all uses. But the negative is that we would
+need to change all of our current data layer to use it instead of DynamoDB. It
+also would mean making further decisions on data management: how would we
+migrate schemas, what indexes would we need to build for our queries, and which
+package should we use for data access.
+
+## Consequences
+
+- We will have an additional store of data that needs to be kept in sync with a
+subset of our main data.
+- We will have an additional cost of running OpenSearch.
+- We will be able to satisify any future requirements for searching or filtering.

go.mod

@@ -24,6 +24,7 @@ require (
 	github.com/gorilla/sessions v1.2.2
 	github.com/ministryofjustice/opg-go-common v0.0.0-20240223111039-066f4e682b9b
 	github.com/mitchellh/hashstructure/v2 v2.0.2
+	github.com/opensearch-project/opensearch-go/v3 v3.0.0
 	github.com/pact-foundation/pact-go/v2 v2.0.4
 	github.com/stretchr/testify v1.8.4
 	github.com/vektra/mockery/v2 v2.42.0
@@ -48,7 +49,7 @@ require (
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	cloud.google.com/go/iam v1.1.5 // indirect
 	cloud.google.com/go/storage v1.36.0 // indirect
-	github.com/aws/aws-sdk-go v1.44.298 // indirect
+	github.com/aws/aws-sdk-go v1.48.13 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.3 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.1 // indirect

go.sum

@@ -196,8 +196,8 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd
 github.com/aws/aws-lambda-go v1.46.0 h1:UWVnvh2h2gecOlFhHQfIPQcD8pL/f7pVCutmFl+oXU8=
 github.com/aws/aws-lambda-go v1.46.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A=
 github.com/aws/aws-sdk-go v1.44.122/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
-github.com/aws/aws-sdk-go v1.44.298 h1:5qTxdubgV7PptZJmp/2qDwD2JL187ePL7VOxsSh1i3g=
-github.com/aws/aws-sdk-go v1.44.298/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/aws/aws-sdk-go v1.48.13 h1:6N4GTme6MpxfCisWf5pql8k3TBORiKTmbeutZCDXlG8=
+github.com/aws/aws-sdk-go v1.48.13/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/aws/aws-sdk-go-v2 v1.25.1 h1:P7hU6A5qEdmajGwvae/zDkOq+ULLC9tQBTwqqiwFGpI=
 github.com/aws/aws-sdk-go-v2 v1.25.1/go.mod h1:Evoc5AsmtveRt1komDwIsjHFyrP5tDuF1D1U+6z6pNo=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 h1:gTK2uhtAPtFcdRRJilZPx8uJLL2J85xK11nKtWL0wfU=
@@ -489,6 +489,8 @@ github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4
 github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/opensearch-project/opensearch-go/v3 v3.0.0 h1:KBaZC2qjTMX651JKmTPopW0D1VsZvqydlNBMQWaeI7w=
+github.com/opensearch-project/opensearch-go/v3 v3.0.0/go.mod h1:Au5KA380eWrGAYOYh19Ql7wIjysm5Q+V4BSYUHpXuj0=
 github.com/pact-foundation/pact-go/v2 v2.0.4 h1:KnVmFkqRZfg9aRI5OKp6Ub39nWUUuXCk2mDJwc+I2JE=
 github.com/pact-foundation/pact-go/v2 v2.0.4/go.mod h1:4yy8YWVCuxAgqSmfiEf5lmzfLsa6oLCmVXEEC2AcObw=
 github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
@@ -544,6 +546,8 @@ github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
 github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/vektra/mockery/v2 v2.42.0 h1:xnP1KXjpcc1GD8jHRjgdpRIW4LDK5MdSMrhbJizAmaI=
 github.com/vektra/mockery/v2 v2.42.0/go.mod h1:XNTE9RIu3deGAGQRVjP1VZxGpQNm0YedZx4oDs3prr8=
+github.com/wI2L/jsondiff v0.4.0 h1:iP56F9tK83eiLttg3YdmEENtZnwlYd3ezEpNNnfZVyM=
+github.com/wI2L/jsondiff v0.4.0/go.mod h1:nR/vyy1efuDeAtMwc3AF6nZf/2LD1ID8GTyyJ+K8YB0=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=