From bef68488ceadc0832de4b44f760bc1d22f3fbcde Mon Sep 17 00:00:00 2001 From: fabiante Date: Fri, 2 Feb 2024 11:43:17 +0100 Subject: [PATCH 1/8] Refactor request creation to explicitly state JSON or XML usage --- r_get_record.go | 2 +- r_get_record_attachment.go | 2 +- r_get_store_status.go | 2 +- r_put_store.go | 2 +- r_store_post_record.go | 2 +- r_store_search.go | 2 +- store_client.go | 20 ++++++++++++++++---- 7 files changed, 22 insertions(+), 10 deletions(-) diff --git a/r_get_record.go b/r_get_record.go index 47016ae..1795df6 100644 --- a/r_get_record.go +++ b/r_get_record.go @@ -23,7 +23,7 @@ type RecordAttachment struct { } func (c *StoreClient) GetRecord(ctx context.Context, id uuid.UUID) (*Record, error) { - req, err := c.newRequest(ctx) + req, err := c.newRequestJSON(ctx) if err != nil { return nil, err } diff --git a/r_get_record_attachment.go b/r_get_record_attachment.go index 2fc58fd..0028ab6 100644 --- a/r_get_record_attachment.go +++ b/r_get_record_attachment.go @@ -12,7 +12,7 @@ import ( // // This returns the number of bytes written and an error if any. func (c *StoreClient) GetRecordAttachment(ctx context.Context, writer io.Writer, recordID, attachmentID uuid.UUID) (int64, error) { - req, err := c.newRequest(ctx) + req, err := c.newRequestJSON(ctx) if err != nil { return 0, err } diff --git a/r_get_store_status.go b/r_get_store_status.go index 8b802b1..9c4064d 100644 --- a/r_get_store_status.go +++ b/r_get_store_status.go @@ -46,7 +46,7 @@ type StoreStatus struct { } func (c *StoreClient) GetStoreStatus(ctx context.Context) (*StoreStatus, error) { - req, err := c.newRequest(ctx) + req, err := c.newRequestJSON(ctx) if err != nil { return nil, err } diff --git a/r_put_store.go b/r_put_store.go index 5ee0f89..a865d2f 100644 --- a/r_put_store.go +++ b/r_put_store.go @@ -19,7 +19,7 @@ type PutStoreRequest struct { } func (c *ServerClient) PutStore(ctx context.Context, storeName string, request *PutStoreRequest) error { - req, err := newRequest(ctx, c.c) + req, err := newRequestJSON(ctx, c.c) if err != nil { return err } diff --git a/r_store_post_record.go b/r_store_post_record.go index 9a594c1..d476b3e 100644 --- a/r_store_post_record.go +++ b/r_store_post_record.go @@ -19,7 +19,7 @@ type PostRecordResponse struct { } func (c *StoreClient) PostRecord(ctx context.Context, request *RecordRequest) (*PostRecordResponse, error) { - req, err := c.newRequest(ctx) + req, err := c.newRequestJSON(ctx) if err != nil { return nil, err } diff --git a/r_store_search.go b/r_store_search.go index 98a119c..a1c5726 100644 --- a/r_store_search.go +++ b/r_store_search.go @@ -104,7 +104,7 @@ func (c *StoreClient) SearchQuery(ctx context.Context, url string) (*SearchRespo } func (c *StoreClient) Search(ctx context.Context, request *SearchRequest) (*SearchResponse, error) { - req, err := c.newRequest(ctx) + req, err := c.newRequestJSON(ctx) if err != nil { return nil, err } diff --git a/store_client.go b/store_client.go index 21e771d..1ab1218 100644 --- a/store_client.go +++ b/store_client.go @@ -18,11 +18,23 @@ func NewStoreClient(c *resty.Client) *StoreClient { return &StoreClient{c: c} } -func (c *StoreClient) newRequest(ctx context.Context) (*resty.Request, error) { - return newRequest(ctx, c.c) +func (c *StoreClient) newRequestJSON(ctx context.Context) (*resty.Request, error) { + return newRequestJSON(ctx, c.c) } -func newRequest(ctx context.Context, c *resty.Client) (*resty.Request, error) { +func (c *StoreClient) newRequestXML(ctx context.Context) (*resty.Request, error) { + return newRequestXML(ctx, c.c) +} + +func newRequestJSON(ctx context.Context, c *resty.Client) (*resty.Request, error) { + return newRequest(ctx, c, "application/json") +} + +func newRequestXML(ctx context.Context, c *resty.Client) (*resty.Request, error) { + return newRequest(ctx, c, "application/xml") +} + +func newRequest(ctx context.Context, c *resty.Client, contentType string) (*resty.Request, error) { claims := UserClaimsFromContext(ctx) if claims == nil { return nil, errors.New("missing user claims in context object") @@ -30,7 +42,7 @@ func newRequest(ctx context.Context, c *resty.Client) (*resty.Request, error) { req := c.NewRequest() req.SetContext(ctx) - req.SetHeader("Accept", "application/json") + req.SetHeader("Accept", contentType) claims.SetOnHeader(req.Header) return req, nil From 0c165a4d56faeccaec0db98c597cc4e32c044733 Mon Sep 17 00:00:00 2001 From: fabiante Date: Fri, 2 Feb 2024 11:44:24 +0100 Subject: [PATCH 2/8] Change GetStoreStatus to use XML encoding --- r_get_store_status.go | 61 +++++++++++--------------------------- r_get_store_status_test.go | 31 +++++++++++++++++++ 2 files changed, 49 insertions(+), 43 deletions(-) diff --git a/r_get_store_status.go b/r_get_store_status.go index 9c4064d..1208c38 100644 --- a/r_get_store_status.go +++ b/r_get_store_status.go @@ -2,60 +2,35 @@ package easclient import ( "context" - "time" + "encoding/xml" ) type StoreStatus struct { + XMLName xml.Name `xml:"status"` Registry struct { - AllRecords int `json:"allRecords"` - IndexedRecords int `json:"indexedRecords"` - AllAttachments int `json:"allAttachments"` - IndexedAttachments int `json:"indexedAttachments"` - } `json:"registry"` + Records struct { + All int `xml:"all"` + Indexed int `xml:"indexed"` + } `xml:"records"` + Attachments struct { + All int `xml:"all"` + Indexed int `xml:"indexed"` + } `xml:"attachments"` + } `xml:"registry"` Index struct { - Documents int `json:"documents"` - IsCurrent bool `json:"isCurrent"` - HasDeletions bool `json:"hasDeletions"` - Records int `json:"records"` - Attachments int `json:"attachments"` - } `json:"index"` - Capacity struct { - Maximum int64 `json:"maximum"` - Utilized float64 `json:"utilized"` - GrowthRate float64 `json:"growthRate"` - ExpectedEnd time.Time `json:"expectedEnd"` - Lifetime int `json:"lifetime"` - } `json:"capacity"` - Periods []struct { - Start string `json:"start"` - End string `json:"end"` - Registry struct { - AllRecords int `json:"allRecords"` - IndexedRecords int `json:"indexedRecords"` - AllAttachments int `json:"allAttachments"` - IndexedAttachments int `json:"indexedAttachments"` - } `json:"registry"` - Index struct { - Records int `json:"records"` - Attachments int `json:"attachments"` - } `json:"index"` - Capacity struct { - Utilized float64 `json:"utilized"` - } `json:"capacity"` - } `json:"periods"` + Documents int `xml:"documents"` + IsCurrent bool `xml:"isCurrent"` + HasDeletions bool `xml:"hasDeletions"` + } `xml:"index"` } func (c *StoreClient) GetStoreStatus(ctx context.Context) (*StoreStatus, error) { - req, err := c.newRequestJSON(ctx) + req, err := c.newRequestXML(ctx) if err != nil { return nil, err } - type Res struct { - Status *StoreStatus `json:"status"` - } - - var result Res + var result StoreStatus req.SetResult(&result) res, err := req.Get("/status") @@ -67,5 +42,5 @@ func (c *StoreClient) GetStoreStatus(ctx context.Context) (*StoreStatus, error) return nil, err } - return result.Status, nil + return &result, nil } diff --git a/r_get_store_status_test.go b/r_get_store_status_test.go index ce6f5de..62743d7 100644 --- a/r_get_store_status_test.go +++ b/r_get_store_status_test.go @@ -2,6 +2,7 @@ package easclient_test import ( "context" + "encoding/xml" "testing" "github.com/DEXPRO-Solutions-GmbH/easclient" @@ -22,3 +23,33 @@ func TestStoreClient_GetStoreStatus(t *testing.T) { require.NoError(t, err) require.NotNil(t, status) } + +func Test_UnmarshalStoreStatus(t *testing.T) { + respBody := ` + + + + 34 + 34 + + + 4 + 4 + + + + 38 + true + false + +` + + var resp easclient.StoreStatus + + require.NoError(t, xml.Unmarshal([]byte(respBody), &resp)) + require.Equal(t, 34, resp.Registry.Records.All) + require.Equal(t, 34, resp.Registry.Records.Indexed) + require.Equal(t, 38, resp.Index.Documents) + require.Equal(t, true, resp.Index.IsCurrent) + require.Equal(t, false, resp.Index.HasDeletions) +} From d2af77fafc815a4b5f6b008cbd31afee40886f29 Mon Sep 17 00:00:00 2001 From: fabiante Date: Fri, 2 Feb 2024 11:56:33 +0100 Subject: [PATCH 3/8] Change GetRecord to use XML encoding --- r_get_record.go | 2 +- r_store_search.go | 2 +- record.go | 35 ++++++++++++++++++++++++++--------- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/r_get_record.go b/r_get_record.go index 1795df6..82d4d66 100644 --- a/r_get_record.go +++ b/r_get_record.go @@ -23,7 +23,7 @@ type RecordAttachment struct { } func (c *StoreClient) GetRecord(ctx context.Context, id uuid.UUID) (*Record, error) { - req, err := c.newRequestJSON(ctx) + req, err := c.newRequestXML(ctx) if err != nil { return nil, err } diff --git a/r_store_search.go b/r_store_search.go index a1c5726..1e785c6 100644 --- a/r_store_search.go +++ b/r_store_search.go @@ -80,7 +80,7 @@ type SearchResult struct { HistoryLink Link `json:"historyLink"` VerifyLink Link `json:"verifyLink"` HeaderFields HeaderFields `json:"headerFields"` - RecordFields RecordFields `json:"recordFields"` + // RecordFields RecordFields `json:"recordFields"` // TODO: Re-add when refactoring search to XML } type SearchResponse struct { diff --git a/record.go b/record.go index 2b18977..f6319cf 100644 --- a/record.go +++ b/record.go @@ -1,6 +1,7 @@ package easclient import ( + "encoding/xml" "time" "github.com/google/uuid" @@ -17,15 +18,31 @@ type HeaderFields struct { InitialArchiveDateTime time.Time `json:"_initialArchiveDateTime"` } -type RecordFields map[string]string - -type Record struct { - HeaderFields HeaderFields `json:"headerFields"` - RecordFields RecordFields `json:"recordFields"` - Attachments []*RecordAttachment `json:"attachments"` +type RecordField struct { + Name string `xml:"name,attr"` + Value string `xml:",chardata"` } -// GetHeaderField returns either the value of the given header field or an empty string if the field does not exist. -func (rec *Record) GetHeaderField(name string) string { - return rec.RecordFields[name] +type Record struct { + XMLName xml.Name `xml:"records"` + Record struct { + DocumentType string `xml:"documentType"` + MasterId uuid.UUID `xml:"masterId"` + ArchiveDateTime time.Time `xml:"archiveDateTime"` + ID uuid.UUID `xml:"id"` + Version string `xml:"version"` + ArchiverLogin string `xml:"archiverLogin"` + Archiver string `xml:"archiver"` + InitialArchiver string `xml:"initialArchiver"` + InitialArchiverLogin string `xml:"initialArchiverLogin"` + InitialArchiveDateTime time.Time `xml:"initialArchiveDateTime"` + Field []*RecordField `xml:"field"` + Attachment struct { + Name string `xml:"name"` + Size string `xml:"size"` + Register string `xml:"register"` + Author string `xml:"author"` + ID string `xml:"id"` + } `xml:"attachment"` + } `xml:"record"` } From ddf5616f3a8a8bec3be024fc830fa7e094039a05 Mon Sep 17 00:00:00 2001 From: fabiante Date: Fri, 2 Feb 2024 12:24:26 +0100 Subject: [PATCH 4/8] Fix zero values in search request causing EOF responses --- r_store_search.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/r_store_search.go b/r_store_search.go index 1e785c6..5602bc4 100644 --- a/r_store_search.go +++ b/r_store_search.go @@ -54,13 +54,23 @@ func SearchRequestFromURL(s string) (*SearchRequest, error) { } func (request SearchRequest) ToQuery() map[string]string { - return map[string]string{ + q := map[string]string{ "query": request.Query, "itemsPerPage": strconv.Itoa(request.ItemsPerPage), "startIndex": strconv.Itoa(request.StartIndex), "sort": request.Sort, "sortOrder": request.SortOrder, } + + // delete zero values which would result in an invalid request + if q["itemsPerPage"] == "0" { + delete(q, "itemsPerPage") + } + if q["startIndex"] == "0" { + delete(q, "startIndex") + } + + return q } type Link struct { From 336ca5a715f3ca76342d20d64bf888c79cb367f8 Mon Sep 17 00:00:00 2001 From: fabiante Date: Fri, 2 Feb 2024 12:29:24 +0100 Subject: [PATCH 5/8] Change Search to use XML encoding --- r_store_search.go | 77 +++++++++++++++++++++++++++--------------- r_store_search_test.go | 26 +++++++++++--- 2 files changed, 71 insertions(+), 32 deletions(-) diff --git a/r_store_search.go b/r_store_search.go index 5602bc4..f1fef8f 100644 --- a/r_store_search.go +++ b/r_store_search.go @@ -2,9 +2,11 @@ package easclient import ( "context" + "encoding/xml" "fmt" "net/url" "strconv" + "time" "github.com/google/uuid" ) @@ -74,37 +76,58 @@ func (request SearchRequest) ToQuery() map[string]string { } type Link struct { - Type string `json:"type"` - Title string `json:"title"` - Href string `json:"href"` + Type string `xml:"type,attr"` + Href string `xml:"href,attr"` } -type SearchResult struct { - // TODO: Re-add this field once we can confirm this is either always string or bool. - // Title string `json:"title"` - Score float64 `json:"score"` - Id uuid.UUID `json:"id"` - FileLink Link `json:"fileLink"` - ExplainLink Link `json:"explainLink"` - CheckVersionLink Link `json:"checkVersionLink"` - HistoryLink Link `json:"historyLink"` - VerifyLink Link `json:"verifyLink"` - HeaderFields HeaderFields `json:"headerFields"` - // RecordFields RecordFields `json:"recordFields"` // TODO: Re-add when refactoring search to XML +type SearchResponse struct { + XMLName xml.Name `xml:"rss"` + Version string `xml:"version,attr"` + Channel *SearchResponseChannel `xml:"channel"` } -type SearchResponse struct { - Query string `json:"query"` - TotalHits int `json:"totalHits"` - ItemsPerPage int `json:"itemsPerPage"` - StartIndex int `json:"startIndex"` - Topn int `json:"topn"` - EffectiveResults int `json:"effectiveResults"` - Result []*SearchResult `json:"result"` +type SearchResponseChannel struct { + Title string `xml:"title"` + Link string `xml:"link"` + Description string `xml:"description"` + TotalResults int `xml:"totalResults"` // TODO: Assert in unmarshal test + ItemsPerPage int `xml:"itemsPerPage"` // TODO: Assert in unmarshal test + StartIndex int `xml:"startIndex"` // TODO: Assert in unmarshal test + Query struct { + Role string `xml:"role,attr"` + SearchTerms string `xml:"searchTerms,attr"` + StartPage int `xml:"startPage,attr"` + } `xml:"Query"` + Topn int `xml:"topn"` + EffectiveResults int `xml:"effectiveResults"` + NextPage string `xml:"nextPage"` + Items []*SearchResponseItem `xml:"item"` +} + +type SearchResponseItem struct { + Title string `xml:"title"` + Link string `xml:"link"` + Description string `xml:"description"` + Score float64 `xml:"score"` + ExplainLink Link `xml:"explainLink"` + VersionLink Link `xml:"versionLink"` + HistoryLink Link `xml:"historyLink"` + VerifyLink Link `xml:"verifyLink"` + DocumentType string `xml:"documentType"` + Fields []*RecordField `xml:"field"` // TODO: Assert and check in get attachment response if this is the correct way to handle recurring fields + MasterId uuid.UUID `xml:"masterId"` + ArchiveDateTime time.Time `xml:"archiveDateTime"` + ID uuid.UUID `xml:"id"` + Version string `xml:"version"` + ArchiverLogin string `xml:"archiverLogin"` + Archiver string `xml:"archiver"` + InitialArchiver string `xml:"initialArchiver"` + InitialArchiverLogin string `xml:"initialArchiverLogin"` + InitialArchiveDateTime time.Time `xml:"initialArchiveDateTime"` } // SearchQuery is similar to Search but expects a URL from which SearchRequest is parsed via SearchRequestFromURL. -func (c *StoreClient) SearchQuery(ctx context.Context, url string) (*SearchResponse, error) { +func (c *StoreClient) SearchQuery(ctx context.Context, url string) (*SearchResponseChannel, error) { request, err := SearchRequestFromURL(url) if err != nil { return nil, fmt.Errorf("failed to parse search request: %w", err) @@ -113,8 +136,8 @@ func (c *StoreClient) SearchQuery(ctx context.Context, url string) (*SearchRespo return c.Search(ctx, request) } -func (c *StoreClient) Search(ctx context.Context, request *SearchRequest) (*SearchResponse, error) { - req, err := c.newRequestJSON(ctx) +func (c *StoreClient) Search(ctx context.Context, request *SearchRequest) (*SearchResponseChannel, error) { + req, err := c.newRequestXML(ctx) if err != nil { return nil, err } @@ -133,5 +156,5 @@ func (c *StoreClient) Search(ctx context.Context, request *SearchRequest) (*Sear return nil, err } - return &result, nil + return result.Channel, nil } diff --git a/r_store_search_test.go b/r_store_search_test.go index 9c6a910..f8456cb 100644 --- a/r_store_search_test.go +++ b/r_store_search_test.go @@ -28,25 +28,41 @@ func TestStoreClient_Search(t *testing.T) { require.NoError(t, err) require.NotNil(t, response) - assert.Equal(t, "Amazo*", response.Query) - assert.Greater(t, response.TotalHits, 0) + // assert search result in general + assert.Equal(t, "Amazo*", response.Query.SearchTerms) + assert.Greater(t, response.TotalResults, 0) assert.Greater(t, response.EffectiveResults, 0) + + // assert single hit + hit := response.Items[0] + require.NotNil(t, hit) + require.Greater(t, len(hit.Fields), 0) + require.Equal(t, "creditor", hit.Fields[0].Name) + require.Equal(t, "Amazon", hit.Fields[0].Value) }) t.Run("returns results when using pagination details", func(t *testing.T) { request := &easclient.SearchRequest{ Query: "Amazo*", ItemsPerPage: 25, - StartIndex: 2500, + StartIndex: 1, // this requires at least 2 records to be present } response, err := eastest.DefaultClient().Search(ctx, request) require.NoError(t, err) require.NotNil(t, response) - assert.Equal(t, "Amazo*", response.Query) - assert.Greater(t, response.TotalHits, 0) + // assert search result in general + assert.Equal(t, "Amazo*", response.Query.SearchTerms) + assert.Greater(t, response.TotalResults, 0) assert.Greater(t, response.EffectiveResults, 0) + + // assert single hit + hit := response.Items[0] + require.NotNil(t, hit) + require.Greater(t, len(hit.Fields), 0) + require.Equal(t, "creditor", hit.Fields[0].Name) + require.Equal(t, "Amazon", hit.Fields[0].Value) }) } From 824212acad8401c2aa4ae9fec5356e0472b1ff79 Mon Sep 17 00:00:00 2001 From: fabiante Date: Fri, 2 Feb 2024 13:25:18 +0100 Subject: [PATCH 6/8] Change newRequest helper to ignore empty content type parameter --- store_client.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/store_client.go b/store_client.go index 1ab1218..7bcc36f 100644 --- a/store_client.go +++ b/store_client.go @@ -42,7 +42,9 @@ func newRequest(ctx context.Context, c *resty.Client, contentType string) (*rest req := c.NewRequest() req.SetContext(ctx) - req.SetHeader("Accept", contentType) + if contentType != "" { + req.SetHeader("Accept", contentType) + } claims.SetOnHeader(req.Header) return req, nil From ba1a4c061724e4cf009980ee029e478321b08b49 Mon Sep 17 00:00:00 2001 From: fabiante Date: Fri, 2 Feb 2024 13:25:52 +0100 Subject: [PATCH 7/8] Change GetRecordAttachment to not use explicit content type This is not required since the endpoint responds with the blob in any case. --- r_get_record_attachment.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r_get_record_attachment.go b/r_get_record_attachment.go index 0028ab6..2368d74 100644 --- a/r_get_record_attachment.go +++ b/r_get_record_attachment.go @@ -12,7 +12,7 @@ import ( // // This returns the number of bytes written and an error if any. func (c *StoreClient) GetRecordAttachment(ctx context.Context, writer io.Writer, recordID, attachmentID uuid.UUID) (int64, error) { - req, err := c.newRequestJSON(ctx) + req, err := newRequest(ctx, c.c, "") if err != nil { return 0, err } From 904b1c1fc43bc30ef9e6ff7b8003d5a19b251c9b Mon Sep 17 00:00:00 2001 From: fabiante Date: Fri, 2 Feb 2024 13:26:17 +0100 Subject: [PATCH 8/8] Change PostRecord to use XML encoding --- r_store_post_record.go | 17 ++++++++--------- r_store_post_record_test.go | 5 +---- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/r_store_post_record.go b/r_store_post_record.go index d476b3e..c42079f 100644 --- a/r_store_post_record.go +++ b/r_store_post_record.go @@ -2,24 +2,23 @@ package easclient import ( "context" + "encoding/xml" "strings" "github.com/google/uuid" ) type PostRecordResponse struct { - Records []struct { - Id uuid.UUID `json:"id"` - Link struct { - Type string `json:"type"` - Title string `json:"title"` - Href string `json:"href"` - } `json:"link"` - } `json:"records"` + XMLName xml.Name `xml:"recordArchive"` + ID struct { + Value uuid.UUID `xml:",chardata"` + Type string `xml:"type,attr"` + Href string `xml:"href,attr"` + } `xml:"id"` } func (c *StoreClient) PostRecord(ctx context.Context, request *RecordRequest) (*PostRecordResponse, error) { - req, err := c.newRequestJSON(ctx) + req, err := c.newRequestXML(ctx) if err != nil { return nil, err } diff --git a/r_store_post_record_test.go b/r_store_post_record_test.go index cdc2396..ea8f7b0 100644 --- a/r_store_post_record_test.go +++ b/r_store_post_record_test.go @@ -28,8 +28,5 @@ func TestStoreClient_PostRecord(t *testing.T) { require.NoError(t, err) require.NotNil(t, res) - - require.Len(t, res.Records, 1) - require.NoError(t, err) - require.NotEqual(t, uuid.Nil, res.Records[0].Id) + require.NotEqual(t, uuid.Nil, res.ID.Value) }