Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
add some logs
Browse files Browse the repository at this point in the history
hawx committed Nov 8, 2023

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent ecd1943 commit 4146cab
Showing 26 changed files with 397 additions and 1,149 deletions.
2 changes: 1 addition & 1 deletion cmd/event-received/main.go
Original file line number Diff line number Diff line change
@@ -116,7 +116,7 @@ func handler(ctx context.Context, event Event) error {

if event.isS3Event() {
s3Client := s3.NewClient(cfg, evidenceBucketName)
documentStore := app.NewDocumentStore(dynamoClient, s3Client, random.UuidString)
documentStore := app.NewDocumentStore(dynamoClient, nil, nil, nil, nil)

if err := handleObjectTagsAdded(ctx, dynamoClient, event.S3Event, s3Client, documentStore); err != nil {
return fmt.Errorf("ObjectTagging:Put: %w", err)
8 changes: 4 additions & 4 deletions internal/app/app.go
Original file line number Diff line number Diff line change
@@ -9,7 +9,6 @@ import (
"time"

dynamodbtypes "github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/google/uuid"
"github.com/gorilla/sessions"
"github.com/ministryofjustice/opg-go-common/logging"
@@ -50,15 +49,15 @@ type DynamoClient interface {
DeleteKeys(ctx context.Context, keys []dynamo.Key) error
DeleteOne(ctx context.Context, pk, sk string) error
Update(ctx context.Context, pk, sk string, values map[string]dynamodbtypes.AttributeValue, expression string) error
BatchPut(ctx context.Context, items []interface{}) (int, error)
BatchPut(ctx context.Context, items []interface{}) error
}

//go:generate mockery --testonly --inpackage --name S3Client --structname mockS3Client
type S3Client interface {
PutObject(context.Context, string, []byte) error
PutObjectTagging(context.Context, string, []types.Tag) error
DeleteObject(context.Context, string) error
DeleteObjects(ctx context.Context, keys []string) error
PutObjectTagging(context.Context, string, map[string]string) error
}

//go:generate mockery --testonly --inpackage --name SessionStore --structname mockSessionStore
@@ -87,7 +86,7 @@ func App(
s3Client S3Client,
eventClient *event.Client,
) http.Handler {
documentStore := NewDocumentStore(lpaDynamoClient, s3Client, random.UuidString)
documentStore := NewDocumentStore(lpaDynamoClient, s3Client, eventClient, random.UuidString, time.Now)

donorStore := &donorStore{
dynamoClient: lpaDynamoClient,
@@ -186,6 +185,7 @@ func App(
evidenceReceivedStore,
s3Client,
documentStore,
eventClient,
)

return withAppData(page.ValidateCsrf(rootMux, sessionStore, random.String, errorHandler), localizer, lang, rumConfig, staticHash, oneloginURL)
4 changes: 1 addition & 3 deletions internal/app/app_test.go
Original file line number Diff line number Diff line change
@@ -8,20 +8,18 @@ import (
"github.com/gorilla/sessions"
"github.com/ministryofjustice/opg-go-common/logging"
"github.com/ministryofjustice/opg-go-common/template"
"github.com/ministryofjustice/opg-modernising-lpa/internal/dynamo"
"github.com/ministryofjustice/opg-modernising-lpa/internal/localize"
"github.com/ministryofjustice/opg-modernising-lpa/internal/notify"
"github.com/ministryofjustice/opg-modernising-lpa/internal/onelogin"
"github.com/ministryofjustice/opg-modernising-lpa/internal/page"
"github.com/ministryofjustice/opg-modernising-lpa/internal/pay"
"github.com/ministryofjustice/opg-modernising-lpa/internal/place"
"github.com/ministryofjustice/opg-modernising-lpa/internal/s3"
"github.com/ministryofjustice/opg-modernising-lpa/internal/sesh"
"github.com/stretchr/testify/assert"
)

func TestApp(t *testing.T) {
app := App(&logging.Logger{}, &localize.Localizer{}, localize.En, template.Templates{}, nil, &dynamo.Client{}, "http://public.url", &pay.Client{}, &notify.Client{}, &place.Client{}, page.RumConfig{}, "?%3fNEI0t9MN", page.Paths, &onelogin.Client{}, "http://onelogin.url", &s3.Client{}, nil)
app := App(&logging.Logger{}, &localize.Localizer{}, localize.En, template.Templates{}, nil, nil, "http://public.url", &pay.Client{}, &notify.Client{}, &place.Client{}, page.RumConfig{}, "?%3fNEI0t9MN", page.Paths, &onelogin.Client{}, "http://onelogin.url", nil, nil)

assert.Implements(t, (*http.Handler)(nil), app)
}
80 changes: 47 additions & 33 deletions internal/app/document_store.go
Original file line number Diff line number Diff line change
@@ -3,30 +3,30 @@ package app
import (
"context"
"errors"
"fmt"
"time"

"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/ministryofjustice/opg-modernising-lpa/internal/dynamo"
"github.com/ministryofjustice/opg-modernising-lpa/internal/event"
"github.com/ministryofjustice/opg-modernising-lpa/internal/page"
)

type PartialBatchWriteError struct {
Written int
Expected int
}

func (e PartialBatchWriteError) Error() string {
return fmt.Sprintf("Expected to write %d but %d were written", e.Expected, e.Written)
}

type documentStore struct {
dynamoClient DynamoClient
s3Client S3Client
eventClient EventClient
randomUUID func() string
now func() time.Time
}

func NewDocumentStore(dynamoClient DynamoClient, s3Client S3Client, randomUUID func() string) *documentStore {
return &documentStore{dynamoClient: dynamoClient, s3Client: s3Client, randomUUID: randomUUID}
func NewDocumentStore(dynamoClient DynamoClient, s3Client S3Client, eventClient EventClient, randomUUID func() string, now func() time.Time) *documentStore {
return &documentStore{
dynamoClient: dynamoClient,
s3Client: s3Client,
eventClient: eventClient,
randomUUID: randomUUID,
now: now,
}
}

func (s *documentStore) Create(ctx context.Context, lpa *page.Lpa, filename string, data []byte) (page.Document, error) {
@@ -80,21 +80,12 @@ func (s *documentStore) UpdateScanResults(ctx context.Context, lpaID, s3ObjectKe
}

func (s *documentStore) BatchPut(ctx context.Context, documents []page.Document) error {
var converted []interface{}
var converted []any
for _, d := range documents {
converted = append(converted, d)
}

toWrite := len(converted)
written, err := s.dynamoClient.BatchPut(ctx, converted)

if err != nil {
return err
} else if written != toWrite {
return PartialBatchWriteError{Written: written, Expected: toWrite}
}

return nil
return s.dynamoClient.BatchPut(ctx, converted)
}

func (s *documentStore) Put(ctx context.Context, document page.Document) error {
@@ -103,26 +94,17 @@ func (s *documentStore) Put(ctx context.Context, document page.Document) error {

func (s *documentStore) DeleteInfectedDocuments(ctx context.Context, documents page.Documents) error {
var dynamoKeys []dynamo.Key
var s3Keys []string

for _, d := range documents {
if d.VirusDetected {
dynamoKeys = append(dynamoKeys, dynamo.Key{
PK: d.PK,
SK: d.SK,
})
s3Keys = append(s3Keys, d.Key)
dynamoKeys = append(dynamoKeys, dynamo.Key{PK: d.PK, SK: d.SK})
}
}

if len(dynamoKeys) == 0 {
return nil
}

if err := s.s3Client.DeleteObjects(ctx, s3Keys); err != nil {
return err
}

return s.dynamoClient.DeleteKeys(ctx, dynamoKeys)
}

@@ -134,6 +116,38 @@ func (s *documentStore) Delete(ctx context.Context, document page.Document) erro
return s.dynamoClient.DeleteOne(ctx, document.PK, document.SK)
}

func (s *documentStore) Submit(ctx context.Context, lpa *page.Lpa, documents page.Documents) error {
var unsentDocuments page.Documents

for _, document := range documents {
if document.Sent.IsZero() && !document.VirusDetected {
document.Sent = s.now()
unsentDocuments = append(unsentDocuments, document)

if err := s.s3Client.PutObjectTagging(ctx, document.Key, map[string]string{"replicate": "true"}); err != nil {
return err
}
}
}

if len(unsentDocuments) > 0 {
if err := s.eventClient.SendReducedFeeRequested(ctx, event.ReducedFeeRequested{
UID: lpa.UID,
RequestType: lpa.FeeType.String(),
Evidence: unsentDocuments.Keys(),
EvidenceDelivery: lpa.EvidenceDelivery.String(),
}); err != nil {
return err
}

if err := s.BatchPut(ctx, unsentDocuments); err != nil {
return err
}
}

return nil
}

func documentKey(s3Key string) string {
return "#DOCUMENT#" + s3Key
}
62 changes: 7 additions & 55 deletions internal/app/document_store_test.go
Original file line number Diff line number Diff line change
@@ -15,10 +15,11 @@ import (
func TestNewDocumentStore(t *testing.T) {
dynamoClient := newMockDynamoClient(t)
s3Client := newMockS3Client(t)
eventClient := newMockEventClient(t)

expected := &documentStore{dynamoClient: dynamoClient, s3Client: s3Client, randomUUID: nil}
expected := &documentStore{dynamoClient: dynamoClient, s3Client: s3Client, eventClient: eventClient}

assert.Equal(t, expected, NewDocumentStore(dynamoClient, s3Client, nil))
assert.Equal(t, expected, NewDocumentStore(dynamoClient, s3Client, eventClient, nil, nil))
}

func TestDocumentStoreGetAll(t *testing.T) {
@@ -169,11 +170,6 @@ func TestDocumentStorePutWhenDynamoClientError(t *testing.T) {
func TestDeleteInfectedDocuments(t *testing.T) {
ctx := page.ContextWithSessionData(context.Background(), &page.SessionData{LpaID: "123"})

s3Client := newMockS3Client(t)
s3Client.
On("DeleteObjects", ctx, []string{"a-key", "another-key"}).
Return(nil)

dynamoClient := newMockDynamoClient(t)
dynamoClient.
On("DeleteKeys", ctx, []dynamo.Key{
@@ -182,7 +178,7 @@ func TestDeleteInfectedDocuments(t *testing.T) {
}).
Return(nil)

documentStore := documentStore{s3Client: s3Client, dynamoClient: dynamoClient}
documentStore := documentStore{dynamoClient: dynamoClient}

err := documentStore.DeleteInfectedDocuments(ctx, page.Documents{
{PK: "a-pk", SK: "a-sk", Key: "a-key", VirusDetected: true},
@@ -192,32 +188,9 @@ func TestDeleteInfectedDocuments(t *testing.T) {
assert.Nil(t, err)
}

func TestDeleteInfectedDocumentsWhenS3ClientError(t *testing.T) {
ctx := page.ContextWithSessionData(context.Background(), &page.SessionData{LpaID: "123"})

s3Client := newMockS3Client(t)
s3Client.
On("DeleteObjects", ctx, []string{"a-key", "another-key"}).
Return(expectedError)

documentStore := documentStore{s3Client: s3Client}

err := documentStore.DeleteInfectedDocuments(ctx, page.Documents{
{PK: "a-pk", SK: "a-sk", Key: "a-key", VirusDetected: true},
{PK: "another-pk", SK: "another-sk", Key: "another-key", VirusDetected: true},
})

assert.Equal(t, expectedError, err)
}

func TestDeleteInfectedDocumentsWhenDynamoClientError(t *testing.T) {
ctx := page.ContextWithSessionData(context.Background(), &page.SessionData{LpaID: "123"})

s3Client := newMockS3Client(t)
s3Client.
On("DeleteObjects", ctx, []string{"a-key", "another-key"}).
Return(nil)

dynamoClient := newMockDynamoClient(t)
dynamoClient.
On("DeleteKeys", ctx, []dynamo.Key{
@@ -226,7 +199,7 @@ func TestDeleteInfectedDocumentsWhenDynamoClientError(t *testing.T) {
}).
Return(expectedError)

documentStore := documentStore{s3Client: s3Client, dynamoClient: dynamoClient}
documentStore := documentStore{dynamoClient: dynamoClient}

err := documentStore.DeleteInfectedDocuments(ctx, page.Documents{
{PK: "a-pk", SK: "a-sk", Key: "a-key", VirusDetected: true},
@@ -317,7 +290,7 @@ func TestBatchPut(t *testing.T) {
page.Document{PK: "a-pk", SK: "a-sk", Key: "a-key"},
page.Document{PK: "aanother-pk", SK: "aanother-sk", Key: "aanother-key"},
}).
Return(2, nil)
Return(nil)

documentStore := documentStore{dynamoClient: dynamoClient}

@@ -329,27 +302,6 @@ func TestBatchPut(t *testing.T) {
assert.Nil(t, err)
}

func TestBatchPutPartialWrite(t *testing.T) {
ctx := page.ContextWithSessionData(context.Background(), &page.SessionData{LpaID: "123"})

dynamoClient := newMockDynamoClient(t)
dynamoClient.
On("BatchPut", ctx, []interface{}{
page.Document{PK: "a-pk", SK: "a-sk", Key: "a-key"},
page.Document{PK: "aanother-pk", SK: "aanother-sk", Key: "aanother-key"},
}).
Return(1, nil)

documentStore := documentStore{dynamoClient: dynamoClient}

err := documentStore.BatchPut(ctx, []page.Document{
{PK: "a-pk", SK: "a-sk", Key: "a-key"},
{PK: "aanother-pk", SK: "aanother-sk", Key: "aanother-key"},
})

assert.Equal(t, PartialBatchWriteError{Written: 1, Expected: 2}, err)
}

func TestBatchPutWhenDynamoError(t *testing.T) {
ctx := page.ContextWithSessionData(context.Background(), &page.SessionData{LpaID: "123"})

@@ -359,7 +311,7 @@ func TestBatchPutWhenDynamoError(t *testing.T) {
page.Document{PK: "a-pk", SK: "a-sk", Key: "a-key"},
page.Document{PK: "aanother-pk", SK: "aanother-sk", Key: "aanother-key"},
}).
Return(0, expectedError)
Return(expectedError)

documentStore := documentStore{dynamoClient: dynamoClient}

41 changes: 0 additions & 41 deletions internal/app/donor_store.go
Original file line number Diff line number Diff line change
@@ -30,7 +30,6 @@ type DocumentStore interface {
GetAll(context.Context) (page.Documents, error)
Put(context.Context, page.Document) error
UpdateScanResults(context.Context, string, string, bool) error
BatchPut(context.Context, []page.Document) error
}

type donorStore struct {
@@ -188,46 +187,6 @@ func (s *donorStore) Put(ctx context.Context, lpa *page.Lpa) error {
lpa.HasSentPreviousApplicationLinkedEvent = true
}

if lpa.UID != "" && lpa.Tasks.PayForLpa.IsPending() {
documents, err := s.documentStore.GetAll(ctx)
if err != nil {
s.logger.Print(err)
return s.dynamoClient.Put(ctx, lpa)
}

var unsentKeys []string

for _, document := range documents {
if document.Sent.IsZero() && !document.Scanned {
unsentKeys = append(unsentKeys, document.Key)
}
}

if len(unsentKeys) > 0 {
if err := s.eventClient.SendReducedFeeRequested(ctx, event.ReducedFeeRequested{
UID: lpa.UID,
RequestType: lpa.FeeType.String(),
Evidence: unsentKeys,
EvidenceDelivery: lpa.EvidenceDelivery.String(),
}); err != nil {
return err
}

var updatedDocuments page.Documents

for _, document := range documents {
if document.Sent.IsZero() && !document.Scanned {
document.Sent = s.now()
updatedDocuments = append(updatedDocuments, document)
}
}

if err := s.documentStore.BatchPut(ctx, updatedDocuments); err != nil {
s.logger.Print(err)
}
}
}

return s.dynamoClient.Put(ctx, lpa)
}

Loading

0 comments on commit 4146cab

Please sign in to comment.