From 234f28c7a614d7eaa781807bb00df426cc6ff19f Mon Sep 17 00:00:00 2001 From: RonnyLV Date: Mon, 20 May 2024 15:17:02 +0300 Subject: [PATCH] Add Cursor based pagination support for Keys and Translations APIs (#52) * feat: Add Cursor based pagination support for Keys and Translations APIs * Added additional test cases to pagination tests --- README.md | 38 ++++++++++++++ api.go | 4 +- pagination.go | 28 +++++++--- service.go | 6 +++ svc_key.go | 6 ++- svc_key_test.go | 114 ++++++++++++++++++++++++++++++++++++++++ svc_paymentcard_test.go | 1 + svc_translation.go | 6 ++- svc_translation_test.go | 114 ++++++++++++++++++++++++++++++++++++++++ 9 files changed, 304 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index fba68f9..e2cb378 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,44 @@ t.SetPageOptions(lokalise.PageOptions{ resp, err := t.List() ``` +### Cursor pagination + +The [List Keys](https://developers.lokalise.com/reference/list-all-keys) and [List Translations](https://developers.lokalise.com/reference/list-all-translations) endpoints support cursor pagination, which is recommended for its faster performance compared to traditional "offset" pagination. By default, "offset" pagination is used, so you must explicitly set `pagination` to `"cursor"` to use cursor pagination. + +```go +// This approach is also applicable for `client.Translations()` +keys := Api.Keys() +keys.SetPageOptions(lokalise.PageOptions{ + Pagination: "cursor", + Cursor: "eyIxIjo1MjcyNjU2MTd9" +}) + +resp, err := keys.List() +``` + +After retrieving data from the Lokalise API, you can check for the availability of the next cursor and proceed accordingly: + +```go +cursor := "" + +for { + keys := client.Keys() + keys.SetListOptions(KeyListOptions{ + Pagination: "cursor", + Cursor: cursor, + }) + resp, _ := keys.List(projectId) + + // Do something with the response + + if !resp.Paged.HasNextCursor() { + // no more keys + break + } + cursor = resp.Paged.Cursor +} +``` + ## Queued Processes Some resource actions, such as Files.upload, are subject to intensive processing before request fulfills. These processes got optimised by becoming asynchronous. diff --git a/api.go b/api.go index 6a51ad2..8a648d5 100644 --- a/api.go +++ b/api.go @@ -48,10 +48,10 @@ func New(apiToken string, options ...ClientOption) (*Api, error) { // predefined list options if any prjOpts := ProjectListOptions{Page: c.pageOptions.Page, Limit: c.pageOptions.Limit} - keyOpts := KeyListOptions{Page: c.pageOptions.Page, Limit: c.pageOptions.Limit} + keyOpts := KeyListOptions{Pagination: c.pageOptions.Pagination, Page: c.pageOptions.Page, Limit: c.pageOptions.Limit, Cursor: c.pageOptions.Cursor} taskOpts := TaskListOptions{Page: c.pageOptions.Page, Limit: c.pageOptions.Limit} scOpts := ScreenshotListOptions{Page: c.pageOptions.Page, Limit: c.pageOptions.Limit} - trOpts := TranslationListOptions{Page: c.pageOptions.Page, Limit: c.pageOptions.Limit} + trOpts := TranslationListOptions{Pagination: c.pageOptions.Pagination, Page: c.pageOptions.Page, Limit: c.pageOptions.Limit, Cursor: c.pageOptions.Cursor} fOpts := FileListOptions{Page: c.pageOptions.Page, Limit: c.pageOptions.Limit} c.Projects = func() *ProjectService { return &ProjectService{BaseService: bs, opts: prjOpts} } diff --git a/pagination.go b/pagination.go index e9b5fef..54c1e6b 100644 --- a/pagination.go +++ b/pagination.go @@ -8,16 +8,22 @@ import ( "github.com/google/go-querystring/query" ) +const ( + PaginationOffset = "offset" + PaginationCursor = "cursor" +) + type PageCounter interface { NumberOfPages() int64 CurrentPage() int64 } type Paged struct { - TotalCount int64 `json:"-"` - PageCount int64 `json:"-"` - Limit int64 `json:"-"` - Page int64 `json:"-"` + TotalCount int64 `json:"-"` + PageCount int64 `json:"-"` + Limit int64 `json:"-"` + Page int64 `json:"-"` + Cursor string `json:"-"` } func (p Paged) NumberOfPages() int64 { @@ -28,13 +34,19 @@ func (p Paged) CurrentPage() int64 { return p.Page } +func (p Paged) NextCursor() string { return p.Cursor } + +func (p Paged) HasNextCursor() bool { return p.Cursor != "" } + type OptionsApplier interface { Apply(req *resty.Request) } type PageOptions struct { - Limit uint `url:"limit,omitempty"` - Page uint `url:"page,omitempty"` + Pagination string `url:"pagination,omitempty"` + Limit uint `url:"limit,omitempty"` + Page uint `url:"page,omitempty"` + Cursor string `url:"cursor,omitempty"` } func (options PageOptions) Apply(req *resty.Request) { @@ -47,14 +59,16 @@ const ( headerPageCount = "X-Pagination-Page-Count" headerLimit = "X-Pagination-Limit" headerPage = "X-Pagination-Page" + headerNextCursor = "X-Pagination-Next-Cursor" ) func applyPaged(res *resty.Response, paged *Paged) { headers := res.Header() + paged.Limit = headerInt64(headers, headerLimit) paged.TotalCount = headerInt64(headers, headerTotalCount) paged.PageCount = headerInt64(headers, headerPageCount) - paged.Limit = headerInt64(headers, headerLimit) paged.Page = headerInt64(headers, headerPage) + paged.Cursor = headers.Get(headerNextCursor) } func headerInt64(headers http.Header, headerKey string) int64 { diff --git a/service.go b/service.go index 0883c1a..5771410 100644 --- a/service.go +++ b/service.go @@ -33,6 +33,12 @@ func (s *BaseService) SetPageOptions(opts PageOptions) { if opts.Page != 0 { s.Page = opts.Page } + if opts.Pagination != "" { + s.Pagination = opts.Pagination + } + if opts.Cursor != "" { + s.Cursor = opts.Cursor + } } // ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ diff --git a/svc_key.go b/svc_key.go index f4ea559..7422819 100644 --- a/svc_key.go +++ b/svc_key.go @@ -270,8 +270,10 @@ func (k NewKey) MarshalJSON() ([]byte, error) { // List options type KeyListOptions struct { // page options - Page uint `url:"page,omitempty"` - Limit uint `url:"limit,omitempty"` + Pagination string `url:"pagination,omitempty"` + Page uint `url:"page,omitempty"` + Limit uint `url:"limit,omitempty"` + Cursor string `url:"cursor,omitempty"` // Possible values are 1 and 0. DisableReferences uint8 `url:"disable_references,omitempty"` diff --git a/svc_key_test.go b/svc_key_test.go index 54085c0..8ec6758 100644 --- a/svc_key_test.go +++ b/svc_key_test.go @@ -839,3 +839,117 @@ func TestKeyService_Update_Empty_Tags(t *testing.T) { t.Errorf("Keys.Update returned \n %+v\n want\n %+v", r.Key, want) } } + +func TestKeyService_Retrieve_Paged_offset(t *testing.T) { + client, mux, _, teardown := setup() + defer teardown() + + mux.HandleFunc( + fmt.Sprintf("/projects/%s/keys", testProjectID), + func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Pagination-Total-Count", "1000") + w.Header().Set("X-Pagination-Page-Count", "10") + w.Header().Set("X-Pagination-Limit", "100") + w.Header().Set("X-Pagination-Page", "1") + testMethod(t, r, "GET") + testHeader(t, r, apiTokenHeader, testApiToken) + + _, _ = fmt.Fprint(w, `{}`) + }) + + r, err := client.Keys().List(testProjectID) + if err != nil { + t.Errorf("Keys.List.Paged returned error: %v", err) + } + + want := Paged{ + TotalCount: 1000, + PageCount: 10, + Limit: 100, + Page: 1, + } + + if !reflect.DeepEqual(r.Paged, want) { + t.Errorf("Keys.List.Paged returned %+v, want %+v", r.Paged, want) + } + + if r.Paged.CurrentPage() != 1 { + t.Errorf("Keys.List.Paged.CurrentPage() returned %+v, want %+v", r.Paged.CurrentPage(), 1) + } +} + +func TestKeyService_Retrieve_Paged_cursor(t *testing.T) { + client, mux, _, teardown := setup() + defer teardown() + + mux.HandleFunc( + fmt.Sprintf("/projects/%s/keys", testProjectID), + func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Pagination-Next-Cursor", "eyIxIjo5NzM1NjI0NX0=") + w.Header().Set("X-Pagination-Limit", "100") + testMethod(t, r, "GET") + testHeader(t, r, apiTokenHeader, testApiToken) + + _, _ = fmt.Fprint(w, `{}`) + }) + + r, err := client.Keys().List(testProjectID) + if err != nil { + t.Errorf("Keys.List.Paged returned error: %v", err) + } + + want := Paged{ + TotalCount: -1, + PageCount: -1, + Limit: 100, + Page: -1, + Cursor: "eyIxIjo5NzM1NjI0NX0=", + } + + if !reflect.DeepEqual(r.Paged, want) { + t.Errorf("Keys.List.Paged returned %+v, want %+v", r.Paged, want) + } + + if !r.Paged.HasNextCursor() { + t.Errorf("Keys.List.Paged.HasNextCursor() returned false, want true") + } +} + +func TestKeyService_Retrieve_Paged_cursor_empty(t *testing.T) { + client, mux, _, teardown := setup() + defer teardown() + + mux.HandleFunc( + fmt.Sprintf("/projects/%s/keys", testProjectID), + func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Pagination-Limit", "100") + testMethod(t, r, "GET") + testHeader(t, r, apiTokenHeader, testApiToken) + + _, _ = fmt.Fprint(w, `{}`) + }) + + r, err := client.Keys().List(testProjectID) + if err != nil { + t.Errorf("Keys.List.Paged returned error: %v", err) + } + + want := Paged{ + TotalCount: -1, + PageCount: -1, + Limit: 100, + Page: -1, + Cursor: "", + } + + if !reflect.DeepEqual(r.Paged, want) { + t.Errorf("Keys.List.Paged returned %+v, want %+v", r.Paged, want) + } + + if r.Paged.HasNextCursor() { + t.Errorf("Keys.List.Paged.HasNextCursor() returned true, want false") + } +} diff --git a/svc_paymentcard_test.go b/svc_paymentcard_test.go index 5e09bb6..f0a880c 100644 --- a/svc_paymentcard_test.go +++ b/svc_paymentcard_test.go @@ -132,6 +132,7 @@ func TestPaymentCardService_List(t *testing.T) { PageCount: -1, Page: -1, Limit: -1, + Cursor: "", }, WithUserID: WithUserID{ UserID: 420, diff --git a/svc_translation.go b/svc_translation.go index c011e95..6ebe743 100644 --- a/svc_translation.go +++ b/svc_translation.go @@ -219,8 +219,10 @@ func (c *TranslationService) Update(projectID string, translationID int64, opts type TranslationListOptions struct { // page options - Page uint `url:"page,omitempty"` - Limit uint `url:"limit,omitempty"` + Pagination string `url:"pagination,omitempty"` + Page uint `url:"page,omitempty"` + Limit uint `url:"limit,omitempty"` + Cursor string `url:"cursor,omitempty"` // Possible values are 1 and 0. DisableReferences uint8 `url:"disable_references,omitempty"` diff --git a/svc_translation_test.go b/svc_translation_test.go index cd0994f..e9a3d73 100644 --- a/svc_translation_test.go +++ b/svc_translation_test.go @@ -363,3 +363,117 @@ func JsonCompact(text string) string { } return compactedBuffer.String() } + +func TestTranslationService_List_Paged_offset(t *testing.T) { + client, mux, _, teardown := setup() + defer teardown() + + mux.HandleFunc( + fmt.Sprintf("/projects/%s/translations", testProjectID), + func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Pagination-Total-Count", "1000") + w.Header().Set("X-Pagination-Page-Count", "10") + w.Header().Set("X-Pagination-Limit", "100") + w.Header().Set("X-Pagination-Page", "1") + testMethod(t, r, "GET") + testHeader(t, r, apiTokenHeader, testApiToken) + + _, _ = fmt.Fprint(w, `{}`) + }) + + r, err := client.Translations().List(testProjectID) + if err != nil { + t.Errorf("Translations.List returned error: %v", err) + } + + want := Paged{ + TotalCount: 1000, + PageCount: 10, + Limit: 100, + Page: 1, + } + + if !reflect.DeepEqual(r.Paged, want) { + t.Errorf(assertionTemplate, "Translations.List.Paged", r.Paged, want) + } + + if r.Paged.CurrentPage() != 1 { + t.Errorf(assertionTemplate, "Translations.List.Paged.CurrentPage()", r.Paged.CurrentPage(), 1) + } +} + +func TestTranslationService_List_Paged_cursor(t *testing.T) { + client, mux, _, teardown := setup() + defer teardown() + + mux.HandleFunc( + fmt.Sprintf("/projects/%s/translations", testProjectID), + func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Pagination-Next-Cursor", "eyIxIjo5NzM1NjI0NX0=") + w.Header().Set("X-Pagination-Limit", "100") + testMethod(t, r, "GET") + testHeader(t, r, apiTokenHeader, testApiToken) + + _, _ = fmt.Fprint(w, `{}`) + }) + + r, err := client.Translations().List(testProjectID) + if err != nil { + t.Errorf("Translations.List returned error: %v", err) + } + + want := Paged{ + TotalCount: -1, + PageCount: -1, + Limit: 100, + Page: -1, + Cursor: "eyIxIjo5NzM1NjI0NX0=", + } + + if !reflect.DeepEqual(r.Paged, want) { + t.Errorf(assertionTemplate, "Translations.List.Paged", r.Paged, want) + } + + if !r.Paged.HasNextCursor() { + t.Errorf("Translations.List.Paged.HasNextCursor() returned false, want true") + } +} + +func TestTranslationService_List_Paged_cursor_empty(t *testing.T) { + client, mux, _, teardown := setup() + defer teardown() + + mux.HandleFunc( + fmt.Sprintf("/projects/%s/translations", testProjectID), + func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Pagination-Limit", "100") + testMethod(t, r, "GET") + testHeader(t, r, apiTokenHeader, testApiToken) + + _, _ = fmt.Fprint(w, `{}`) + }) + + r, err := client.Translations().List(testProjectID) + if err != nil { + t.Errorf("Translations.List returned error: %v", err) + } + + want := Paged{ + TotalCount: -1, + PageCount: -1, + Limit: 100, + Page: -1, + Cursor: "", + } + + if !reflect.DeepEqual(r.Paged, want) { + t.Errorf(assertionTemplate, "Translations.List.Paged", r.Paged, want) + } + + if r.Paged.HasNextCursor() { + t.Errorf("Translations.List.Paged.HasNextCursor() returned true, want false") + } +}