From 5db509875df4683c1035df1db8c866ba2c6b2334 Mon Sep 17 00:00:00 2001 From: masv3971 Date: Thu, 19 Dec 2024 09:50:19 +0100 Subject: [PATCH 01/28] Add test for POST document. --- pkg/datastoreclient/endpoint_document_test.go | 131 ++++++++++++++++++ .../testdata/documentGetReplyOK.golden | 43 ++++++ .../testdata/documentListReplyOK .golden | 43 ++++++ 3 files changed, 217 insertions(+) create mode 100644 pkg/datastoreclient/endpoint_document_test.go create mode 100644 pkg/datastoreclient/testdata/documentGetReplyOK.golden create mode 100644 pkg/datastoreclient/testdata/documentListReplyOK .golden diff --git a/pkg/datastoreclient/endpoint_document_test.go b/pkg/datastoreclient/endpoint_document_test.go new file mode 100644 index 00000000..b6aa7585 --- /dev/null +++ b/pkg/datastoreclient/endpoint_document_test.go @@ -0,0 +1,131 @@ +package datastoreclient + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + "vc/pkg/ehic" + "vc/pkg/model" + + "github.com/stretchr/testify/assert" + "gotest.tools/v3/golden" +) + +func mockHappyHttServer(t *testing.T, serverReply []byte) *httptest.Server { + mux := http.NewServeMux() + + mux.HandleFunc("/api/v1/document", func(rw http.ResponseWriter, req *http.Request) { + assert.Equal(t, req.URL.Path, "/api/v1/document") + assert.Equal(t, req.Method, http.MethodPost) + rw.Write(golden.Get(t, "documentGetReplyOK.golden")) + }) + + mux.HandleFunc("/api/v1/document/list", func(rw http.ResponseWriter, req *http.Request) { + assert.Equal(t, req.URL.Path, "/api/v1/document/list") + assert.Equal(t, req.Method, http.MethodPost) + rw.Write(golden.Get(t, "documentListReplyOK.golden")) + }) + + server := httptest.NewServer(mux) + return server +} + +func mockClient(ctx context.Context, t *testing.T, url string) *Client { + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + + client, err := New(&Config{ + URL: url, + }) + if err != nil { + t.FailNow() + return nil + } + + return client +} + +func TestGet(t *testing.T) { + tts := []struct { + name string + query *DocumentGetQuery + expected *model.Document + expectedDocumentData *ehic.Document + }{ + { + name: "success", + query: &DocumentGetQuery{ + AuthenticSource: "test_authentic_source", + DocumentType: "test_document_type", + DocumentID: "test_document_id", + }, + expected: &model.Document{ + Meta: &model.MetaData{ + AuthenticSource: "SUNET", + DocumentVersion: "1.0.0", + DocumentType: "EHIC", + DocumentID: "test_document_id", + RealData: false, + Collect: &model.Collect{ + ID: "test_collect_id", + ValidUntil: 1731767173, + }, + Revocation: &model.Revocation{ + ID: "test_revocation_id", + Revoked: false, + Reference: model.RevocationReference{ + AuthenticSource: "SUNET", + DocumentType: "EHIC", + DocumentID: "test_document_id", + }, + RevokedAt: 0, + Reason: "", + }, + CredentialValidFrom: 695706629, + CredentialValidTo: -1730367911, + DocumentDataValidationRef: "", + }, + }, + expectedDocumentData: &ehic.Document{ + Subject: ehic.Subject{ + Forename: "test_forename", + FamilyName: "test_family_name", + DateOfBirth: "1986-02-23", + }, + SocialSecurityPin: "1234", + PeriodEntitlement: ehic.PeriodEntitlement{ + StartingDate: "1970-01-01", + EndingDate: "2038-01-19", + }, + DocumentID: "test_document_id", + CompetentInstitution: ehic.CompetentInstitution{ + InstitutionID: "SE:1234", + InstitutionName: "Myndigheten", + InstitutionCountry: "SE", + }, + }, + }, + } + + for _, tt := range tts { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + serverReply := golden.Get(t, "documentGetReplyOK.golden") + + httpServer := mockHappyHttServer(t, serverReply) + defer httpServer.Close() + + client := mockClient(ctx, t, httpServer.URL) + got, _, err := client.Document.Get(ctx, tt.query) + assert.NoError(t, err) + + documentData, err := tt.expectedDocumentData.Marshal() + assert.NoError(t, err) + tt.expected.DocumentData = documentData + + assert.Equal(t, tt.expected, got) + }) + } +} diff --git a/pkg/datastoreclient/testdata/documentGetReplyOK.golden b/pkg/datastoreclient/testdata/documentGetReplyOK.golden new file mode 100644 index 00000000..c68ccc21 --- /dev/null +++ b/pkg/datastoreclient/testdata/documentGetReplyOK.golden @@ -0,0 +1,43 @@ +{ + "data": { + "meta": { + "authentic_source": "SUNET", + "document_version": "1.0.0", + "document_type": "EHIC", + "document_id": "test_document_id", + "real_data": false, + "collect": { + "id": "test_collect_id", + "valid_until": 1731767173 + }, + "revocation": { + "id": "test_revocation_id", + "reference": { + "authentic_source": "SUNET", + "document_type": "EHIC", + "document_id": "test_document_id" + } + }, + "credential_valid_from": 695706629, + "credential_valid_to": -1730367911 + }, + "document_data": { + "subject": { + "forename": "test_forename", + "family_name": "test_family_name", + "date_of_birth": "1986-02-23" + }, + "social_security_pin": "1234", + "period_entitlement": { + "starting_date": "1970-01-01", + "ending_date": "2038-01-19" + }, + "document_id": "test_document_id", + "competent_institution": { + "institution_id": "SE:1234", + "institution_name": "Myndigheten", + "institution_country": "SE" + } + } + } +} \ No newline at end of file diff --git a/pkg/datastoreclient/testdata/documentListReplyOK .golden b/pkg/datastoreclient/testdata/documentListReplyOK .golden new file mode 100644 index 00000000..c68ccc21 --- /dev/null +++ b/pkg/datastoreclient/testdata/documentListReplyOK .golden @@ -0,0 +1,43 @@ +{ + "data": { + "meta": { + "authentic_source": "SUNET", + "document_version": "1.0.0", + "document_type": "EHIC", + "document_id": "test_document_id", + "real_data": false, + "collect": { + "id": "test_collect_id", + "valid_until": 1731767173 + }, + "revocation": { + "id": "test_revocation_id", + "reference": { + "authentic_source": "SUNET", + "document_type": "EHIC", + "document_id": "test_document_id" + } + }, + "credential_valid_from": 695706629, + "credential_valid_to": -1730367911 + }, + "document_data": { + "subject": { + "forename": "test_forename", + "family_name": "test_family_name", + "date_of_birth": "1986-02-23" + }, + "social_security_pin": "1234", + "period_entitlement": { + "starting_date": "1970-01-01", + "ending_date": "2038-01-19" + }, + "document_id": "test_document_id", + "competent_institution": { + "institution_id": "SE:1234", + "institution_name": "Myndigheten", + "institution_country": "SE" + } + } + } +} \ No newline at end of file From 6c4ba2cbe6274fb46a4f8c622073fab3946f27b9 Mon Sep 17 00:00:00 2001 From: Mats Kramer Date: Fri, 29 Nov 2024 15:58:18 +0100 Subject: [PATCH 02/28] UI+APIGW: first working embryo-version of search documents when not in production mode. Simple optional filter for AS added. Using common json display of search result in article instead of table with links for actions (future) --- internal/apigw/apiv1/handlers_datastore.go | 25 +++++ internal/apigw/db/methods_vc_datastore.go | 29 ++++++ internal/apigw/httpserver/api.go | 3 + internal/apigw/httpserver/endpoints.go | 17 ++++ internal/apigw/httpserver/service.go | 1 + internal/ui/apiv1/apigw_client.go | 12 ++- internal/ui/apiv1/handlers.go | 8 ++ internal/ui/httpserver/api.go | 1 + internal/ui/httpserver/endpoints.go | 13 +++ internal/ui/httpserver/service.go | 1 + internal/ui/static/index.html | 5 + internal/ui/static/ui.js | 102 ++++++++++++--------- 12 files changed, 172 insertions(+), 45 deletions(-) diff --git a/internal/apigw/apiv1/handlers_datastore.go b/internal/apigw/apiv1/handlers_datastore.go index 15df625e..6417e7f7 100644 --- a/internal/apigw/apiv1/handlers_datastore.go +++ b/internal/apigw/apiv1/handlers_datastore.go @@ -2,6 +2,7 @@ package apiv1 import ( "context" + "errors" "time" "vc/internal/apigw/db" "vc/pkg/helpers" @@ -479,3 +480,27 @@ func (c *Client) RevokeDocument(ctx context.Context, req *RevokeDocumentRequest) return nil } + +type SearchDocumentsRequest struct { + AuthenticSource string `json:"authentic_source"` +} + +type SearchDocumentsReply struct { + Documents []*model.CompleteDocument +} + +func (c *Client) SearchDocuments(ctx context.Context, req *SearchDocumentsRequest) (*SearchDocumentsReply, error) { + if c.cfg.Common.Production { + return nil, errors.New("Not supported in production mode") + } + + docs, err := c.db.VCDatastoreColl.SearchDocuments(ctx, &db.SearchDocumentsQuery{ + AuthenticSource: req.AuthenticSource, + }) + + if err != nil { + return nil, err + } + resp := &SearchDocumentsReply{Documents: docs} + return resp, nil +} diff --git a/internal/apigw/db/methods_vc_datastore.go b/internal/apigw/db/methods_vc_datastore.go index 03f1af4c..fb2082e9 100644 --- a/internal/apigw/db/methods_vc_datastore.go +++ b/internal/apigw/db/methods_vc_datastore.go @@ -357,3 +357,32 @@ func (c *VCDatastoreColl) Replace(ctx context.Context, doc *model.CompleteDocume c.log.Info("updated document", "document_id", doc.Meta.DocumentID) return nil } + +type SearchDocumentsQuery struct { + AuthenticSource string `json:"authentic_source" bson:"authentic_source"` +} + +func (c *VCDatastoreColl) SearchDocuments(ctx context.Context, query *SearchDocumentsQuery) ([]*model.CompleteDocument, error) { + if err := helpers.Check(ctx, c.Service.cfg, query, c.Service.log); err != nil { + return nil, err + } + + filter := bson.M{} + + if query.AuthenticSource != "" { + filter["meta.authentic_source"] = bson.M{"$eq": query.AuthenticSource} + } + //TODO: add more filters + + cursor, err := c.Coll.Find(ctx, filter) + if err != nil { + return nil, err + } + + res := []*model.CompleteDocument{} + if err := cursor.All(ctx, &res); err != nil { + return nil, err + } + + return res, nil +} diff --git a/internal/apigw/httpserver/api.go b/internal/apigw/httpserver/api.go index 125e2a06..6dd0bee1 100644 --- a/internal/apigw/httpserver/api.go +++ b/internal/apigw/httpserver/api.go @@ -24,6 +24,9 @@ type Apiv1 interface { AddConsent(ctx context.Context, req *apiv1.AddConsentRequest) error GetConsent(ctx context.Context, req *apiv1.GetConsentRequest) (*model.Consent, error) + // datastore endpoints - disabled in production + SearchDocuments(ctx context.Context, req *apiv1.SearchDocumentsRequest) (*apiv1.SearchDocumentsReply, error) + // credential endpoints Revoke(ctx context.Context, req *apiv1.RevokeRequest) (*apiv1.RevokeReply, error) Credential(ctx context.Context, req *apiv1.CredentialRequest) (*apiv1_issuer.MakeSDJWTReply, error) diff --git a/internal/apigw/httpserver/endpoints.go b/internal/apigw/httpserver/endpoints.go index a798ed60..5470802f 100644 --- a/internal/apigw/httpserver/endpoints.go +++ b/internal/apigw/httpserver/endpoints.go @@ -102,6 +102,23 @@ func (s *Service) endpointGetDocument(ctx context.Context, c *gin.Context) (any, return reply, nil } +func (s *Service) endpointSearchDocuments(ctx context.Context, c *gin.Context) (any, error) { + ctx, span := s.tracer.Start(ctx, "httpserver:endpointSearchDocuments") + defer span.End() + + request := &apiv1.SearchDocumentsRequest{} + if err := s.httpHelpers.Binding.Request(ctx, c, request); err != nil { + span.SetStatus(codes.Error, err.Error()) + return nil, err + } + reply, err := s.apiv1.SearchDocuments(ctx, request) + if err != nil { + span.SetStatus(codes.Error, err.Error()) + return nil, err + } + return reply, nil +} + func (s *Service) endpointRevokeDocument(ctx context.Context, c *gin.Context) (any, error) { ctx, span := s.tracer.Start(ctx, "httpserver:endpointRevokeDocument") defer span.End() diff --git a/internal/apigw/httpserver/service.go b/internal/apigw/httpserver/service.go index 9611a680..487b29fd 100644 --- a/internal/apigw/httpserver/service.go +++ b/internal/apigw/httpserver/service.go @@ -72,6 +72,7 @@ func New(ctx context.Context, cfg *model.Cfg, apiv1 *apiv1.Client, tracer *trace s.httpHelpers.Server.RegEndpoint(ctx, rgAPIv1, http.MethodPost, "/identity/mapping", s.endpointIdentityMapping) s.httpHelpers.Server.RegEndpoint(ctx, rgAPIv1, http.MethodPost, "/document/list", s.endpointDocumentList) s.httpHelpers.Server.RegEndpoint(ctx, rgAPIv1, http.MethodPost, "/document", s.endpointGetDocument) + s.httpHelpers.Server.RegEndpoint(ctx, rgAPIv1, http.MethodPost, "/document/search", s.endpointSearchDocuments) s.httpHelpers.Server.RegEndpoint(ctx, rgAPIv1, http.MethodPost, "/consent", s.endpointAddConsent) s.httpHelpers.Server.RegEndpoint(ctx, rgAPIv1, http.MethodPost, "/consent/get", s.endpointGetConsent) s.httpHelpers.Server.RegEndpoint(ctx, rgAPIv1, http.MethodPost, "/document/revoke", s.endpointRevokeDocument) diff --git a/internal/ui/apiv1/apigw_client.go b/internal/ui/apiv1/apigw_client.go index 619cb00c..d802252d 100644 --- a/internal/ui/apiv1/apigw_client.go +++ b/internal/ui/apiv1/apigw_client.go @@ -65,8 +65,16 @@ func (c *APIGWClient) GetDocument(req *GetDocumentRequest) (any, error) { } // Notification sends POST to /api/v1/notification -func (c *APIGWClient) Notification(request *NotificationRequest) (any, error) { - reply, err := c.DoPostJSON("/api/v1/notification", request) +func (c *APIGWClient) Notification(req *NotificationRequest) (any, error) { + reply, err := c.DoPostJSON("/api/v1/notification", req) + if err != nil { + return nil, err + } + return reply, nil +} + +func (c *APIGWClient) SearchDocuments(req *apiv1_apigw.SearchDocumentsRequest) (any, error) { + reply, err := c.DoPostJSON("/api/v1/document/search", req) if err != nil { return nil, err } diff --git a/internal/ui/apiv1/handlers.go b/internal/ui/apiv1/handlers.go index ffd90c18..925c81b2 100644 --- a/internal/ui/apiv1/handlers.go +++ b/internal/ui/apiv1/handlers.go @@ -190,3 +190,11 @@ func (c *Client) DecodeCredential(ctx context.Context, req *apiv1_verifier.Decod } return reply, nil } + +func (c *Client) SearchDocuments(ctx context.Context, req *apiv1_apigw.SearchDocumentsRequest) (any, error) { + reply, err := c.apigwClient.SearchDocuments(req) + if err != nil { + return nil, err + } + return reply, nil +} diff --git a/internal/ui/httpserver/api.go b/internal/ui/httpserver/api.go index dcd20b0c..dc611d35 100644 --- a/internal/ui/httpserver/api.go +++ b/internal/ui/httpserver/api.go @@ -23,6 +23,7 @@ type Apiv1 interface { Credential(ctx context.Context, request *apiv1.CredentialRequest) (any, error) GetDocument(ctx context.Context, request *apiv1.GetDocumentRequest) (any, error) Notification(ctx context.Context, reguest *apiv1.NotificationRequest) (any, error) + SearchDocuments(ctx context.Context, request *apigw_apiv1.SearchDocumentsRequest) (any, error) // mockas HealthMockAS(ctx context.Context, request *apiv1_status.StatusRequest) (any, error) diff --git a/internal/ui/httpserver/endpoints.go b/internal/ui/httpserver/endpoints.go index ca44d391..4effc493 100644 --- a/internal/ui/httpserver/endpoints.go +++ b/internal/ui/httpserver/endpoints.go @@ -216,3 +216,16 @@ func (s *Service) endpointDecodeCredential(ctx context.Context, c *gin.Context) } return reply, nil } + +func (s *Service) endpointSearchDocuments(ctx context.Context, c *gin.Context) (any, error) { + request := &apiv1_apigw.SearchDocumentsRequest{} + if err := s.httpHelpers.Binding.Request(ctx, c, request); err != nil { + return nil, err + } + + reply, err := s.apiv1.SearchDocuments(ctx, request) + if err != nil { + return nil, err + } + return reply, nil +} diff --git a/internal/ui/httpserver/service.go b/internal/ui/httpserver/service.go index ed8d77c5..ca38dcd1 100644 --- a/internal/ui/httpserver/service.go +++ b/internal/ui/httpserver/service.go @@ -108,6 +108,7 @@ func New(ctx context.Context, cfg *model.Cfg, apiv1 *apiv1.Client, tracer *trace s.httpHelpers.Server.RegEndpoint(ctx, rgAPIGWSecure, http.MethodPost, "credential", s.endpointCredential) s.httpHelpers.Server.RegEndpoint(ctx, rgAPIGWSecure, http.MethodPost, "document", s.endpointGetDocument) s.httpHelpers.Server.RegEndpoint(ctx, rgAPIGWSecure, http.MethodPost, "notification", s.endpointNotification) + s.httpHelpers.Server.RegEndpoint(ctx, rgAPIGWSecure, http.MethodPost, "document/search", s.endpointSearchDocuments) // Run http server go func() { diff --git a/internal/ui/static/index.html b/internal/ui/static/index.html index 1d2f8cc4..e9d7bf31 100644 --- a/internal/ui/static/index.html +++ b/internal/ui/static/index.html @@ -42,6 +42,11 @@ class="navbar-item"> Create business decision + + Search documents + diff --git a/internal/ui/static/ui.js b/internal/ui/static/ui.js index b3988bb4..5207e877 100644 --- a/internal/ui/static/ui.js +++ b/internal/ui/static/ui.js @@ -54,8 +54,7 @@ function displaySecureMenyItems() { const generateUUID = () => { //UUID v4 return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) { - var r = Math.random() * 16 | 0, - v = c === 'x' ? r : (r & 0x3 | 0x8); + var r = Math.random() * 16 | 0, v = c === 'x' ? r : (r & 0x3 | 0x8); return v.toString(16); }); }; @@ -249,14 +248,11 @@ const postDocumentList = () => { const identitySchemaName = getElementById("identity-schema-name"); const documentListRequest = { - authentic_source: authenticSourceElement.value, - identity: { - authentic_source_person_id: authenticSourcePersonIdElement.value, - schema: { + authentic_source: authenticSourceElement.value, identity: { + authentic_source_person_id: authenticSourcePersonIdElement.value, schema: { name: identitySchemaName.value } - }, - document_type: documentTypeElement.value + }, document_type: documentTypeElement.value }; postAndDisplayInArticleContainerFor(path, documentListRequest, articleHeaderText); @@ -414,11 +410,10 @@ const addUploadNewMockUsingBasicEIDASattributesFormArticleToContainer = () => { const familyNameElement = createInputElement('family name', '', 'text'); const givenNameElement = createInputElement('given name', '', 'text'); const birthdateElement = createInputElement('birth date (YYYY-MM-DD)', '', 'text'); - const documentTypeSelectWithinDivElement = - createSelectElement([ - {value: 'EHIC', label: 'EHIC'}, - {value: 'PDA1', label: 'PDA1'} - ]); + const documentTypeSelectWithinDivElement = createSelectElement([{value: 'EHIC', label: 'EHIC'}, { + value: 'PDA1', + label: 'PDA1' + }]); const documentTypeDiv = documentTypeSelectWithinDivElement[0]; const documentTypeSelect = documentTypeSelectWithinDivElement[1]; @@ -437,24 +432,12 @@ const addUploadNewMockUsingBasicEIDASattributesFormArticleToContainer = () => { document_type: documentTypeSelect.value, }; - disableElements([ - familyNameElement, - givenNameElement, - birthdateElement, - documentTypeSelect, - ]); + disableElements([familyNameElement, givenNameElement, birthdateElement, documentTypeSelect,]); postAndDisplayInArticleContainerFor("/secure/mockas/mock/next", requestBody, "Uploaded business decision"); }; - return [ - familyNameElement, - givenNameElement, - birthdateElement, - documentTypeDiv, - document.createElement('br'), - createButton - ]; + return [familyNameElement, givenNameElement, birthdateElement, documentTypeDiv, document.createElement('br'), createButton]; }; const articleIdBasis = generateArticleIDBasis(); @@ -584,6 +567,7 @@ const disableElements = (elements) => { elements.forEach(el => el.disabled = true); }; + const addViewDocumentFormArticleToContainer = () => { const buildFormElements = () => { @@ -604,9 +588,7 @@ const addViewDocumentFormArticleToContainer = () => { document_type: documentTypeElement.value, }; - disableElements([ - documentIDElement, documentTypeElement, authenticSourceElement - ]); + disableElements([documentIDElement, documentTypeElement, authenticSourceElement]); postAndDisplayInArticleContainerFor("/secure/apigw/document", requestBody, "Document"); }; @@ -622,6 +604,50 @@ const addViewDocumentFormArticleToContainer = () => { document.getElementById(articleIdBasis.articleID).querySelector('input').focus(); }; +const addSearchDocumentsFormArticleToContainer = () => { + const buildFormElements = () => { + + // const documentIDElement = createInputElement('document id'); + // const documentTypeElement = createInputElement('document type (EHIC/PDA1)', 'EHIC'); + const authenticSourceElement = createInputElement('authentic source (optional)'); + + const searchButton = document.createElement('button'); + searchButton.id = generateUUID(); + searchButton.classList.add('button', 'is-link'); + searchButton.textContent = 'Search'; + searchButton.onclick = () => { + searchButton.disabled = true; + + const requestBody = { + // document_id: documentIDElement.value, + // document_type: documentTypeElement.value, + authentic_source: authenticSourceElement.value, + + }; + + disableElements([ + // documentIDElement, + // documentTypeElement, + authenticSourceElement + ]); + + postAndDisplayInArticleContainerFor("/secure/apigw/document/search", requestBody, "Documents"); + }; + + return [ + //documentIDElement, + // documentTypeElement, + authenticSourceElement, + searchButton]; + }; + + const articleIdBasis = generateArticleIDBasis(); + const articleDiv = buildArticle(articleIdBasis.articleID, "Search documents", buildFormElements()); + const articleContainer = document.getElementById('article-container'); + articleContainer.prepend(articleDiv); + + document.getElementById(articleIdBasis.articleID).querySelector('input').focus(); +}; const addViewNotificationFormArticleToContainer = () => { const buildFormElements = () => { @@ -643,9 +669,7 @@ const addViewNotificationFormArticleToContainer = () => { document_type: documentTypeElement.value, }; - disableElements([ - documentIDElement, documentTypeElement, authenticSourceElement - ]); + disableElements([documentIDElement, documentTypeElement, authenticSourceElement]); postAndDisplayInArticleContainerFor("/secure/apigw/notification", requestBody, "Notification"); }; @@ -697,11 +721,7 @@ const addCredentialFormArticleToContainer = () => { collect_id: collectIdElement.value, }; - disableElements([ - authenticSourcePersonIdElement, familyNameElement, givenNameElement, - birthdateElement, schemaNameElement, documentTypeElement, - credentialTypeElement, authenticSourceElement, collectIdElement - ]); + disableElements([authenticSourcePersonIdElement, familyNameElement, givenNameElement, birthdateElement, schemaNameElement, documentTypeElement, credentialTypeElement, authenticSourceElement, collectIdElement]); postAndDisplayInArticleContainerFor("/secure/apigw/credential", requestBody, "Credential"); }; @@ -710,11 +730,7 @@ const addCredentialFormArticleToContainer = () => { const orTextElement = document.createElement('p'); orTextElement.textContent = 'or'; - return [ - authenticSourcePersonIdElement, orTextElement, familyNameElement, givenNameElement, - birthdateElement, lineElement, collectIdElement, schemaNameElement, documentTypeElement, - credentialTypeElement, authenticSourceElement, createButton - ]; + return [authenticSourcePersonIdElement, orTextElement, familyNameElement, givenNameElement, birthdateElement, lineElement, collectIdElement, schemaNameElement, documentTypeElement, credentialTypeElement, authenticSourceElement, createButton]; }; const articleIdBasis = generateArticleIDBasis(); From f3aeaed96d8738c91b9f0efe9e6a59125beaac09 Mon Sep 17 00:00:00 2001 From: Mats Kramer Date: Mon, 2 Dec 2024 20:04:28 +0100 Subject: [PATCH 03/28] UI: concept for exact response type supported in post --- internal/ui/apiv1/apigw_client.go | 9 ++++---- internal/ui/apiv1/handlers.go | 4 ++-- internal/ui/apiv1/vc_base_client.go | 34 +++++++++++++++++++++++++++++ internal/ui/httpserver/api.go | 8 +++---- 4 files changed, 45 insertions(+), 10 deletions(-) diff --git a/internal/ui/apiv1/apigw_client.go b/internal/ui/apiv1/apigw_client.go index d802252d..a3b7bf26 100644 --- a/internal/ui/apiv1/apigw_client.go +++ b/internal/ui/apiv1/apigw_client.go @@ -20,8 +20,8 @@ func NewAPIGWClient(cfg *model.Cfg, tracer *trace.Tracer, logger *logger.Log) *A } // DocumentList sends POST to apigw /api/v1/document/list -func (c *APIGWClient) DocumentList(req *DocumentListRequest) (any, error) { - reply, err := c.DoPostJSON("/api/v1/document/list", req) +func (c *APIGWClient) DocumentList(req *DocumentListRequest) (*apiv1_apigw.DocumentListReply, error) { + reply, err := DoPostJSONGeneric[apiv1_apigw.DocumentListReply](c.VCBaseClient, "/api/v1/document/list", req) if err != nil { return nil, err } @@ -73,8 +73,9 @@ func (c *APIGWClient) Notification(req *NotificationRequest) (any, error) { return reply, nil } -func (c *APIGWClient) SearchDocuments(req *apiv1_apigw.SearchDocumentsRequest) (any, error) { - reply, err := c.DoPostJSON("/api/v1/document/search", req) +func (c *APIGWClient) SearchDocuments(req *apiv1_apigw.SearchDocumentsRequest) (*apiv1_apigw.SearchDocumentsReply, error) { + //TODO(mk): return real response type to be used in table in UI + reply, err := DoPostJSONGeneric[apiv1_apigw.SearchDocumentsReply](c.VCBaseClient, "/api/v1/document/search", req) if err != nil { return nil, err } diff --git a/internal/ui/apiv1/handlers.go b/internal/ui/apiv1/handlers.go index 925c81b2..86830157 100644 --- a/internal/ui/apiv1/handlers.go +++ b/internal/ui/apiv1/handlers.go @@ -57,7 +57,7 @@ type DocumentListRequest struct { ValidTo int64 `json:"valid_to"` } -func (c *Client) DocumentList(ctx context.Context, req *DocumentListRequest) (any, error) { +func (c *Client) DocumentList(ctx context.Context, req *DocumentListRequest) (*apiv1_apigw.DocumentListReply, error) { reply, err := c.apigwClient.DocumentList(req) if err != nil { return nil, err @@ -191,7 +191,7 @@ func (c *Client) DecodeCredential(ctx context.Context, req *apiv1_verifier.Decod return reply, nil } -func (c *Client) SearchDocuments(ctx context.Context, req *apiv1_apigw.SearchDocumentsRequest) (any, error) { +func (c *Client) SearchDocuments(ctx context.Context, req *apiv1_apigw.SearchDocumentsRequest) (*apiv1_apigw.SearchDocumentsReply, error) { reply, err := c.apigwClient.SearchDocuments(req) if err != nil { return nil, err diff --git a/internal/ui/apiv1/vc_base_client.go b/internal/ui/apiv1/vc_base_client.go index 699e9e35..29ad2ce6 100644 --- a/internal/ui/apiv1/vc_base_client.go +++ b/internal/ui/apiv1/vc_base_client.go @@ -65,6 +65,40 @@ func (c *VCBaseClient) DoPostJSON(endpoint string, reqBody any) (*map[string]any return &jsonResp, nil } +func DoPostJSONGeneric[T any](c *VCBaseClient, endpoint string, reqBody any) (*T, error) { + url := c.url(endpoint) + + reqBodyJSON, err := json.Marshal(reqBody) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", url, bytes.NewBuffer(reqBodyJSON)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + defer c.closeBody(resp) + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var jsonResp T + if err := json.Unmarshal(body, &jsonResp); err != nil { + return nil, err + } + + return &jsonResp, nil +} + func (c *VCBaseClient) DoGetJSON(endpoint string) (*map[string]any, error) { url := c.url(endpoint) diff --git a/internal/ui/httpserver/api.go b/internal/ui/httpserver/api.go index dc611d35..d9183a6d 100644 --- a/internal/ui/httpserver/api.go +++ b/internal/ui/httpserver/api.go @@ -2,7 +2,7 @@ package httpserver import ( "context" - apigw_apiv1 "vc/internal/apigw/apiv1" + apiv1_apigw "vc/internal/apigw/apiv1" "vc/internal/gen/status/apiv1_status" apiv1_mockas "vc/internal/mockas/apiv1" "vc/internal/ui/apiv1" @@ -18,12 +18,12 @@ type Apiv1 interface { // apigw HealthAPIGW(ctx context.Context, request *apiv1_status.StatusRequest) (any, error) - DocumentList(ctx context.Context, request *apiv1.DocumentListRequest) (any, error) - Upload(ctx context.Context, request *apigw_apiv1.UploadRequest) (any, error) + DocumentList(ctx context.Context, request *apiv1.DocumentListRequest) (*apiv1_apigw.DocumentListReply, error) + Upload(ctx context.Context, request *apiv1_apigw.UploadRequest) (any, error) Credential(ctx context.Context, request *apiv1.CredentialRequest) (any, error) GetDocument(ctx context.Context, request *apiv1.GetDocumentRequest) (any, error) Notification(ctx context.Context, reguest *apiv1.NotificationRequest) (any, error) - SearchDocuments(ctx context.Context, request *apigw_apiv1.SearchDocumentsRequest) (any, error) + SearchDocuments(ctx context.Context, request *apiv1_apigw.SearchDocumentsRequest) (*apiv1_apigw.SearchDocumentsReply, error) // mockas HealthMockAS(ctx context.Context, request *apiv1_status.StatusRequest) (any, error) From ed4900a95bd731d50de61ad320f0414d5eaa3e35 Mon Sep 17 00:00:00 2001 From: Mats Kramer Date: Tue, 3 Dec 2024 12:08:39 +0100 Subject: [PATCH 04/28] APIGW: search documents now has a limit for number of results --- internal/apigw/apiv1/handlers_datastore.go | 8 ++++++-- internal/apigw/db/methods_vc_datastore.go | 23 ++++++++++++++++------ internal/ui/apiv1/apigw_client.go | 1 - 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/internal/apigw/apiv1/handlers_datastore.go b/internal/apigw/apiv1/handlers_datastore.go index 6417e7f7..42833d66 100644 --- a/internal/apigw/apiv1/handlers_datastore.go +++ b/internal/apigw/apiv1/handlers_datastore.go @@ -487,6 +487,7 @@ type SearchDocumentsRequest struct { type SearchDocumentsReply struct { Documents []*model.CompleteDocument + HasMore bool `json:"has_more_results"` } func (c *Client) SearchDocuments(ctx context.Context, req *SearchDocumentsRequest) (*SearchDocumentsReply, error) { @@ -494,13 +495,16 @@ func (c *Client) SearchDocuments(ctx context.Context, req *SearchDocumentsReques return nil, errors.New("Not supported in production mode") } - docs, err := c.db.VCDatastoreColl.SearchDocuments(ctx, &db.SearchDocumentsQuery{ + docs, hasMore, err := c.db.VCDatastoreColl.SearchDocuments(ctx, &db.SearchDocumentsQuery{ AuthenticSource: req.AuthenticSource, }) if err != nil { return nil, err } - resp := &SearchDocumentsReply{Documents: docs} + resp := &SearchDocumentsReply{ + Documents: docs, + HasMore: hasMore, + } return resp, nil } diff --git a/internal/apigw/db/methods_vc_datastore.go b/internal/apigw/db/methods_vc_datastore.go index fb2082e9..d50239ff 100644 --- a/internal/apigw/db/methods_vc_datastore.go +++ b/internal/apigw/db/methods_vc_datastore.go @@ -362,9 +362,9 @@ type SearchDocumentsQuery struct { AuthenticSource string `json:"authentic_source" bson:"authentic_source"` } -func (c *VCDatastoreColl) SearchDocuments(ctx context.Context, query *SearchDocumentsQuery) ([]*model.CompleteDocument, error) { +func (c *VCDatastoreColl) SearchDocuments(ctx context.Context, query *SearchDocumentsQuery) ([]*model.CompleteDocument, bool, error) { if err := helpers.Check(ctx, c.Service.cfg, query, c.Service.log); err != nil { - return nil, err + return nil, false, err } filter := bson.M{} @@ -374,15 +374,26 @@ func (c *VCDatastoreColl) SearchDocuments(ctx context.Context, query *SearchDocu } //TODO: add more filters - cursor, err := c.Coll.Find(ctx, filter) + findOptions := options.Find() + var limit int64 = 3 + // One more than wanted to see if there are more results i db + findOptions.SetLimit(limit + 1) + + cursor, err := c.Coll.Find(ctx, filter, findOptions) if err != nil { - return nil, err + return nil, false, err } res := []*model.CompleteDocument{} if err := cursor.All(ctx, &res); err != nil { - return nil, err + return nil, false, err } - return res, nil + hasMore := len(res) > int(limit) + if hasMore { + // Remove the last entry from the result + res = res[:limit] + } + + return res, hasMore, nil } diff --git a/internal/ui/apiv1/apigw_client.go b/internal/ui/apiv1/apigw_client.go index a3b7bf26..5615ec9a 100644 --- a/internal/ui/apiv1/apigw_client.go +++ b/internal/ui/apiv1/apigw_client.go @@ -74,7 +74,6 @@ func (c *APIGWClient) Notification(req *NotificationRequest) (any, error) { } func (c *APIGWClient) SearchDocuments(req *apiv1_apigw.SearchDocumentsRequest) (*apiv1_apigw.SearchDocumentsReply, error) { - //TODO(mk): return real response type to be used in table in UI reply, err := DoPostJSONGeneric[apiv1_apigw.SearchDocumentsReply](c.VCBaseClient, "/api/v1/document/search", req) if err != nil { return nil, err From 2dc0f55ccab95013ac6a63423dd52f82b850adac Mon Sep 17 00:00:00 2001 From: Mats Kramer Date: Tue, 3 Dec 2024 19:14:24 +0100 Subject: [PATCH 05/28] APIGW: search documents now can return a dynamic projection and/or sort the result as requested. --- internal/apigw/apiv1/handlers_datastore.go | 2 +- internal/apigw/db/methods_vc_datastore.go | 27 ++++++++++++++++++---- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/internal/apigw/apiv1/handlers_datastore.go b/internal/apigw/apiv1/handlers_datastore.go index 42833d66..1c5c70b0 100644 --- a/internal/apigw/apiv1/handlers_datastore.go +++ b/internal/apigw/apiv1/handlers_datastore.go @@ -497,7 +497,7 @@ func (c *Client) SearchDocuments(ctx context.Context, req *SearchDocumentsReques docs, hasMore, err := c.db.VCDatastoreColl.SearchDocuments(ctx, &db.SearchDocumentsQuery{ AuthenticSource: req.AuthenticSource, - }) + }, nil, nil) if err != nil { return nil, err diff --git a/internal/apigw/db/methods_vc_datastore.go b/internal/apigw/db/methods_vc_datastore.go index d50239ff..38fbba90 100644 --- a/internal/apigw/db/methods_vc_datastore.go +++ b/internal/apigw/db/methods_vc_datastore.go @@ -362,7 +362,7 @@ type SearchDocumentsQuery struct { AuthenticSource string `json:"authentic_source" bson:"authentic_source"` } -func (c *VCDatastoreColl) SearchDocuments(ctx context.Context, query *SearchDocumentsQuery) ([]*model.CompleteDocument, bool, error) { +func (c *VCDatastoreColl) SearchDocuments(ctx context.Context, query *SearchDocumentsQuery, fields []string, sortFields map[string]int) ([]*model.CompleteDocument, bool, error) { if err := helpers.Check(ctx, c.Service.cfg, query, c.Service.log); err != nil { return nil, false, err } @@ -375,10 +375,29 @@ func (c *VCDatastoreColl) SearchDocuments(ctx context.Context, query *SearchDocu //TODO: add more filters findOptions := options.Find() - var limit int64 = 3 - // One more than wanted to see if there are more results i db + var limit int64 = 50 + // Set one more than wanted to see if there are more results i db findOptions.SetLimit(limit + 1) + if len(fields) > 0 { + projection := bson.M{} + for _, field := range fields { + projection[field] = 1 + } + findOptions.SetProjection(projection) + } + + sort := bson.D{} + if len(sortFields) > 0 { + for field, order := range sortFields { + // 1 for ascending, -1 for descending + sort = append(sort, bson.E{Key: field, Value: order}) + } + findOptions.SetSort(sort) + } else { + sort = append(sort, bson.E{Key: "meta.document_id", Value: 1}) + } + cursor, err := c.Coll.Find(ctx, filter, findOptions) if err != nil { return nil, false, err @@ -391,7 +410,7 @@ func (c *VCDatastoreColl) SearchDocuments(ctx context.Context, query *SearchDocu hasMore := len(res) > int(limit) if hasMore { - // Remove the last entry from the result + // Remove the last entry from the result to fit limit value res = res[:limit] } From b5c1417f4d8f28e9f3b2352fd1499bd2ee38df94 Mon Sep 17 00:00:00 2001 From: Mats Kramer Date: Wed, 4 Dec 2024 21:08:29 +0100 Subject: [PATCH 06/28] APIGW: even more advanced search documents to support known needs in both UI and portal --- internal/apigw/apiv1/handlers_datastore.go | 32 +++++++++-- internal/apigw/db/methods_vc_datastore.go | 64 +++++++++++++++++++--- internal/ui/static/ui.js | 3 +- 3 files changed, 83 insertions(+), 16 deletions(-) diff --git a/internal/apigw/apiv1/handlers_datastore.go b/internal/apigw/apiv1/handlers_datastore.go index 1c5c70b0..def6fe28 100644 --- a/internal/apigw/apiv1/handlers_datastore.go +++ b/internal/apigw/apiv1/handlers_datastore.go @@ -482,12 +482,24 @@ func (c *Client) RevokeDocument(ctx context.Context, req *RevokeDocumentRequest) } type SearchDocumentsRequest struct { - AuthenticSource string `json:"authentic_source"` + AuthenticSource string `json:"authentic_source,omitempty"` + DocumentType string `json:"document_type,omitempty"` + DocumentID string `json:"document_id,omitempty"` + CollectID string `json:"collect_id,omitempty"` + + AuthenticSourcePersonID string `json:"authentic_source_person_id,omitempty"` + FamilyName string `json:"family_name,omitempty"` + GivenName string `json:"given_name,omitempty"` + BirthDate string `json:"birth_date,omitempty"` + + Limit int64 `json:"limit,omitempty"` + Fields []string `json:"fields,omitempty"` + SortFields map[string]int `json:"sort_fields,omitempty"` } type SearchDocumentsReply struct { - Documents []*model.CompleteDocument - HasMore bool `json:"has_more_results"` + Documents []*model.CompleteDocument + HasMoreResults bool `json:"has_more_results"` } func (c *Client) SearchDocuments(ctx context.Context, req *SearchDocumentsRequest) (*SearchDocumentsReply, error) { @@ -497,14 +509,22 @@ func (c *Client) SearchDocuments(ctx context.Context, req *SearchDocumentsReques docs, hasMore, err := c.db.VCDatastoreColl.SearchDocuments(ctx, &db.SearchDocumentsQuery{ AuthenticSource: req.AuthenticSource, - }, nil, nil) + DocumentType: req.DocumentType, + DocumentID: req.DocumentID, + CollectID: req.CollectID, + + AuthenticSourcePersonID: req.AuthenticSourcePersonID, + FamilyName: req.FamilyName, + GivenName: req.GivenName, + BirthDate: req.BirthDate, + }, req.Limit, req.Fields, req.SortFields) if err != nil { return nil, err } resp := &SearchDocumentsReply{ - Documents: docs, - HasMore: hasMore, + Documents: docs, + HasMoreResults: hasMore, } return resp, nil } diff --git a/internal/apigw/db/methods_vc_datastore.go b/internal/apigw/db/methods_vc_datastore.go index 38fbba90..971852ff 100644 --- a/internal/apigw/db/methods_vc_datastore.go +++ b/internal/apigw/db/methods_vc_datastore.go @@ -358,24 +358,30 @@ func (c *VCDatastoreColl) Replace(ctx context.Context, doc *model.CompleteDocume return nil } +// TODO(mk): kolla med masv, visst fyller nedan bson ingen funktion i nedan struct när det används vid söken då bara värdet används (gäller samma för json?)? type SearchDocumentsQuery struct { - AuthenticSource string `json:"authentic_source" bson:"authentic_source"` + AuthenticSource string `json:"authentic_source,omitempty" bson:"authentic_source"` + DocumentType string `json:"document_type,omitempty" bson:"document_type"` + DocumentID string `json:"document_id,omitempty" bson:"document_id"` + CollectID string `json:"collect_id,omitempty" bson:"collect_id"` + + AuthenticSourcePersonID string `json:"authentic_source_person_id,omitempty" bson:"authentic_source_person_id"` + FamilyName string `json:"family_name,omitempty" bson:"family_name"` + GivenName string `json:"given_name,omitempty" bson:"given_name"` + BirthDate string `json:"birth_date,omitempty" bson:"birth_date"` } -func (c *VCDatastoreColl) SearchDocuments(ctx context.Context, query *SearchDocumentsQuery, fields []string, sortFields map[string]int) ([]*model.CompleteDocument, bool, error) { +func (c *VCDatastoreColl) SearchDocuments(ctx context.Context, query *SearchDocumentsQuery, limit int64, fields []string, sortFields map[string]int) ([]*model.CompleteDocument, bool, error) { if err := helpers.Check(ctx, c.Service.cfg, query, c.Service.log); err != nil { return nil, false, err } - filter := bson.M{} - - if query.AuthenticSource != "" { - filter["meta.authentic_source"] = bson.M{"$eq": query.AuthenticSource} - } - //TODO: add more filters + filter := buildSearchDocumentsFilter(query) findOptions := options.Find() - var limit int64 = 50 + if limit == 0 { + limit = 50 + } // Set one more than wanted to see if there are more results i db findOptions.SetLimit(limit + 1) @@ -416,3 +422,43 @@ func (c *VCDatastoreColl) SearchDocuments(ctx context.Context, query *SearchDocu return res, hasMore, nil } + +func buildSearchDocumentsFilter(query *SearchDocumentsQuery) bson.M { + filter := bson.M{} + + if query.AuthenticSource != "" { + filter["meta.authentic_source"] = query.AuthenticSource + } + if query.DocumentType != "" { + filter["meta.document_type"] = query.DocumentType + } + if query.DocumentID != "" { + filter["meta.document_id"] = query.DocumentID + } + if query.CollectID != "" { + filter["meta.collect.id"] = query.CollectID + } + + identityConditions := bson.M{} + if query.AuthenticSourcePersonID != "" { + identityConditions["authentic_source_person_id"] = query.AuthenticSourcePersonID + } + if query.FamilyName != "" { + identityConditions["family_name"] = query.FamilyName + } + if query.GivenName != "" { + identityConditions["given_name"] = query.GivenName + } + if query.BirthDate != "" { + identityConditions["birth_date"] = query.BirthDate + } + if len(identityConditions) > 0 { + filter["identities"] = bson.M{ + "$elemMatch": identityConditions, + } + } + + //TODO(mk): add more filters + + return filter +} diff --git a/internal/ui/static/ui.js b/internal/ui/static/ui.js index 5207e877..e4123b13 100644 --- a/internal/ui/static/ui.js +++ b/internal/ui/static/ui.js @@ -622,7 +622,8 @@ const addSearchDocumentsFormArticleToContainer = () => { // document_id: documentIDElement.value, // document_type: documentTypeElement.value, authentic_source: authenticSourceElement.value, - + limit: 3, + fields: ["meta.document_id", "meta.authentic_source", "meta.document_type", "meta.collect.id", "identities", "qr.credential_offer"], }; disableElements([ From 69207a8f7391f0b986027e06477838b4bb901860 Mon Sep 17 00:00:00 2001 From: Mats Kramer Date: Thu, 5 Dec 2024 12:48:16 +0100 Subject: [PATCH 07/28] UI: more search filters, dynamic limit and checkbox to show complete documents as raw json added to UI --- internal/apigw/db/methods_vc_datastore.go | 10 ++- internal/ui/static/index.html | 11 +-- internal/ui/static/ui.js | 93 +++++++++++++++++++---- 3 files changed, 91 insertions(+), 23 deletions(-) diff --git a/internal/apigw/db/methods_vc_datastore.go b/internal/apigw/db/methods_vc_datastore.go index 971852ff..ec94426c 100644 --- a/internal/apigw/db/methods_vc_datastore.go +++ b/internal/apigw/db/methods_vc_datastore.go @@ -414,18 +414,20 @@ func (c *VCDatastoreColl) SearchDocuments(ctx context.Context, query *SearchDocu return nil, false, err } - hasMore := len(res) > int(limit) - if hasMore { + hasMoreResults := len(res) > int(limit) + if hasMoreResults { // Remove the last entry from the result to fit limit value res = res[:limit] } - return res, hasMore, nil + return res, hasMoreResults, nil } func buildSearchDocumentsFilter(query *SearchDocumentsQuery) bson.M { filter := bson.M{} + //TODO(mk): check explain to see if any indexes are needed + if query.AuthenticSource != "" { filter["meta.authentic_source"] = query.AuthenticSource } @@ -458,7 +460,7 @@ func buildSearchDocumentsFilter(query *SearchDocumentsQuery) bson.M { } } - //TODO(mk): add more filters + //TODO(mk): add more filters? return filter } diff --git a/internal/ui/static/index.html b/internal/ui/static/index.html index e9d7bf31..7b0d6a06 100644 --- a/internal/ui/static/index.html +++ b/internal/ui/static/index.html @@ -32,6 +32,12 @@ Dev/test support