From 785cb9cfc3e7f5d2f5625c0b827a77a5da442b61 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2024 18:49:19 -0700 Subject: [PATCH 1/8] manually adding the changes from PR401 into develop since that PR was made against master instead of develop --- openapi/components/schemas/AccessMethod.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/openapi/components/schemas/AccessMethod.yaml b/openapi/components/schemas/AccessMethod.yaml index e559df19..838db8b0 100644 --- a/openapi/components/schemas/AccessMethod.yaml +++ b/openapi/components/schemas/AccessMethod.yaml @@ -29,6 +29,12 @@ properties: description: >- Name of the region in the cloud service provider that the object belongs to. example: us-east-1 + available: + type: boolean + description: >- + Availablity of file in the cloud. + This label defines if this file is immediately accessible via DRS. Any delay or requirement of thawing mechanism if the file is in offline/archival storage is classified as 0 or unavailable. + example: true authorizations: allOf: - $ref: './Authorizations.yaml' From 170a4ba9161934ec49375c001fc1d86f86faec15 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2024 20:53:31 -0700 Subject: [PATCH 2/8] adding cloud option to access method (#407) --- openapi/components/schemas/AccessMethod.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/openapi/components/schemas/AccessMethod.yaml b/openapi/components/schemas/AccessMethod.yaml index 838db8b0..c6ffd09f 100644 --- a/openapi/components/schemas/AccessMethod.yaml +++ b/openapi/components/schemas/AccessMethod.yaml @@ -24,6 +24,12 @@ properties: An arbitrary string to be passed to the `/access` method to get an `AccessURL`. This string must be unique within the scope of a single object. Note that at least one of `access_url` and `access_id` must be provided. + cloud: + type: string + description: >- + Name of the cloud service provider that the object belongs to. + If the cloud service is Amazon Web Services, Google Cloud Platform or Azure the values should be `aws`, `gcp`, or `azure` respectively. + example: aws, gcp, or azure region: type: string description: >- From aa52de7816d8bebd2446411fb59771d565301a7d Mon Sep 17 00:00:00 2001 From: Michael Lukowski Date: Tue, 10 Sep 2024 23:24:17 -0500 Subject: [PATCH 3/8] Feature/issue 396 stats endpoint (#404) * adding stats endpoint * stats endpoint * move stats into status_info * adding the two new variables to the service-info response (objectCloud and totalObjectSize). --------- Co-authored-by: Brian O'Connor --- openapi/components/schemas/DrsService.yaml | 6 ++++++ openapi/paths/service-info.yaml | 9 +++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/openapi/components/schemas/DrsService.yaml b/openapi/components/schemas/DrsService.yaml index 0e5c8d4d..49f3acaa 100644 --- a/openapi/components/schemas/DrsService.yaml +++ b/openapi/components/schemas/DrsService.yaml @@ -6,6 +6,12 @@ properties: maxBulkRequestLength: type: integer description: The max length the bullk request endpoints can handle (>= 1) before generating a 413 error e.g. how long can the arrays bulk_object_ids and bulk_object_access_ids be for this server. + objectCount: + type: integer + description: The total number of objects in this DRS service. + totalObjectSize: + type: integer + description: The total size of all objects in this DRS service in bytes. As a general best practice, file bytes are counted for each unique file and not cloud mirrors or other redundant copies. type: type: object required: diff --git a/openapi/paths/service-info.yaml b/openapi/paths/service-info.yaml index 8f2860da..e8f7ba3b 100644 --- a/openapi/paths/service-info.yaml +++ b/openapi/paths/service-info.yaml @@ -1,7 +1,7 @@ get: summary: Retrieve information about this service description: |- - Returns information about the DRS service + Returns information about the DRS service along with stats pertaning to total object count and cumulative size in bytes. Extends the [v1.0.0 GA4GH Service Info specification](https://github.com/ga4gh-discovery/ga4gh-service-info) @@ -22,9 +22,14 @@ get: ... "type": { "group": "org.ga4gh", - "artifact": "drs" + "artifact": "drs", + "version": "1.5" } ... + "stats": { + "objectCount": 774560, + "totalObjectSize": 4018437188907752 + } } ``` From c5336090effd022de7ce3d671878fa9c4fbdc63a Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2024 21:55:15 -0700 Subject: [PATCH 4/8] backporting from master to develop for PR#406 --- pages/more-background-on-compact-identifiers/openapi.yaml | 3 +++ .../tags/DRSPlusDataConnect.md | 5 +++++ 2 files changed, 8 insertions(+) create mode 100644 pages/more-background-on-compact-identifiers/tags/DRSPlusDataConnect.md diff --git a/pages/more-background-on-compact-identifiers/openapi.yaml b/pages/more-background-on-compact-identifiers/openapi.yaml index 3a0019cf..db60933d 100644 --- a/pages/more-background-on-compact-identifiers/openapi.yaml +++ b/pages/more-background-on-compact-identifiers/openapi.yaml @@ -27,3 +27,6 @@ tags: - name: Example DRS Client Compact Identifier-Based URI Resolution Process - Registering a new Compact Identifier for Your DRS Server description: $ref: ./tags/ExampleRegisterIdentifier.md + - name: Example How To Handle Extra Metadata for DRS Objects + description: + $ref: ./tags/DRSPlusDataConnect.md diff --git a/pages/more-background-on-compact-identifiers/tags/DRSPlusDataConnect.md b/pages/more-background-on-compact-identifiers/tags/DRSPlusDataConnect.md new file mode 100644 index 00000000..fd704452 --- /dev/null +++ b/pages/more-background-on-compact-identifiers/tags/DRSPlusDataConnect.md @@ -0,0 +1,5 @@ +## DRS and Data Connect + +With DRS objects it may be necessary to attach additional metadata to your objects. We believe that a change to the API of DRS to include metadata is not in the spirit of the DRS spec and in general DRS should have no knowledge of the metadata associated with the objects. We have found that a good GA4GH standard to support this is Data Connect (https://github.com/ga4gh-discovery/data-connect). The general approach would be to have a Data Connect service on your platform and to include "tables" with the ID matching your DRS ID for the same object. This means that if you have metadata associated with an object id `abcd` (ex. additional information about Compound Objects) all you need to do is request the information from the Data Connect client at `/tables/abcd/info`. There are optional functionalities of Data Connect, such as querying of tables, but we do not explore them or give any recommendations here. + +Here is an example of using Data Connect with DRS in the fasp-scripts repository (https://github.com/ga4gh/fasp-scripts/blob/master/notebooks/drs/DRS%20File%20Data.ipynb). In this notebook we can see that data connect is used to get DRS IDs from a platform. Those DRS IDs are then used to gather aditional information about the file that might be necessary for analysis. This is just one example of how DRS and Data Connect can interact with each other to gather information about data on a platform. \ No newline at end of file From 54284fc5a107edb1704a22a10037aed2219136ae Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2024 22:07:01 -0700 Subject: [PATCH 5/8] updating the validation icons and the URLs they point to --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9a17610b..9f2c9f92 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ # Data Repository Service (DRS) API -`develop` branch status: [![Build Status](https://app.travis-ci.com/ga4gh/data-repository-service-schemas.svg?branch=develop)](https://app.travis-ci.com/ga4gh/data-repository-service-schemas.svg?branch=develop) -Swagger Validator [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.1405753.svg)](https://doi.org/10.5281/zenodo.1405753) +`develop` branch status: [![Build Status](https://app.travis-ci.com/ga4gh/data-repository-service-schemas.svg?branch=develop)](https://app.travis-ci.com/github/ga4gh/data-repository-service-schemas/builds) +Swagger Validator [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.1405753.svg)](https://doi.org/10.5281/zenodo.1405753) The [Global Alliance for Genomics and Health](http://genomicsandhealth.org/) (GA4GH) is an international coalition, formed to enable the sharing of genomic and clinical data. From c9976c937007251760f29a3587d37ee95c564806 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2024 22:11:41 -0700 Subject: [PATCH 6/8] updating various version strings to 1.5.0 --- README.md | 1 + openapi/data_repository_service.openapi.yaml | 2 +- pages/more-background-on-compact-identifiers/openapi.yaml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9f2c9f92..cae575cf 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ For more information see our HTML documentation links in the table below. | **Branch** | **Reference Documentation** | Swagger Editor | | --- | --- | --- | | **develop**: the stable development branch, into which feature branches are merged | [HTML](https://ga4gh.github.io/data-repository-service-schemas/preview/develop/docs/) | [Swagger Editor](https://editor.swagger.io?url=https://ga4gh.github.io/data-repository-service-schemas/preview/develop/openapi.yaml) | +| **release 1.5.0**: The 1.5.0 release of DRS adds fields for indicating cold storage, explict cloud locations, object count and size metadata, and best practice guidance for using GA4GH Data Connect with DRS. | [HTML](https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-1.5.0/docs/) | [Swagger Editor](https://editor.swagger.io?url=https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-1.5.0/openapi.yaml) | | **release 1.4.0**: The 1.4.0 release of DRS adds bulk operations. | [HTML](https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-1.4.0/docs/) | [Swagger Editor](https://editor.swagger.io?url=https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-1.4.0/openapi.yaml) | | **release 1.3.0**: The 1.3.0 release of DRS adds the ability to use the options request method to return the auth issuers for a specific DRS object. | [HTML](https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-1.3.0/docs/) | [Swagger Editor](https://editor.swagger.io?url=https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-1.3.0/openapi.yaml) | | **release 1.2.0**: The 1.2.0 release of DRS adds the standardized `/service-info` endpoint, and 2 `POST` endpoints that are functionally equivalent to the current `GET` endpoints, but they enable the submission of large passport tokens in the request body. | [HTML](https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-1.2.0/docs/) | [Swagger Editor](https://editor.swagger.io?url=https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-1.2.0/openapi.yaml) | diff --git a/openapi/data_repository_service.openapi.yaml b/openapi/data_repository_service.openapi.yaml index 88b03af5..033f42e9 100644 --- a/openapi/data_repository_service.openapi.yaml +++ b/openapi/data_repository_service.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: title: Data Repository Service - version: 1.4.0 + version: 1.5.0 x-logo: url: 'https://www.ga4gh.org/wp-content/themes/ga4gh/dist/assets/svg/logos/logo-full-color.svg' termsOfService: 'https://www.ga4gh.org/terms-and-conditions/' diff --git a/pages/more-background-on-compact-identifiers/openapi.yaml b/pages/more-background-on-compact-identifiers/openapi.yaml index db60933d..7950cd78 100644 --- a/pages/more-background-on-compact-identifiers/openapi.yaml +++ b/pages/more-background-on-compact-identifiers/openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: title: More Background on Compact Identifiers - version: 1.4.0 + version: 1.5.0 x-logo: url: 'https://www.ga4gh.org/wp-content/themes/ga4gh/dist/assets/svg/logos/logo-full-color.svg' termsOfService: 'https://www.ga4gh.org/terms-and-conditions/' From becd4e1685453fdb50236044517dc949e14ab9d7 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2024 22:33:53 -0700 Subject: [PATCH 7/8] trying to debug build problem --- .spec-docs.json | 12 ++++++------ openapi/tags/ServiceRegistry.md | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.spec-docs.json b/.spec-docs.json index 54ec52f6..1c158cf3 100644 --- a/.spec-docs.json +++ b/.spec-docs.json @@ -4,17 +4,17 @@ "branchPathBase": "preview", "redocTheme": "ga4gh", "buildPages": [ - { - "apiSpecPath": "openapi/data_repository_service.openapi.yaml", - "htmlOutfile": "index.html", - "yamlOutfile": "openapi.yaml", - "jsonOutfile": "openapi.json" - }, { "apiSpecPath": "pages/more-background-on-compact-identifiers/openapi.yaml", "htmlOutfile": "more-background-on-compact-identifiers.html", "yamlOutfile": "more-background-on-compact-identifiers.yaml", "jsonOutfile": "more-background-on-compact-identifiers.json" + }, + { + "apiSpecPath": "openapi/data_repository_service.openapi.yaml", + "htmlOutfile": "index.html", + "yamlOutfile": "openapi.yaml", + "jsonOutfile": "openapi.json" } ] } diff --git a/openapi/tags/ServiceRegistry.md b/openapi/tags/ServiceRegistry.md index 50d8ad28..09e27669 100644 --- a/openapi/tags/ServiceRegistry.md +++ b/openapi/tags/ServiceRegistry.md @@ -17,7 +17,7 @@ Example listing of a DRS API registration from a service registry's `/services` "type": { "group": "org.ga4gh", "artifact": "drs", - "version": "1.4.0" + "version": "1.5.0" }, "description": "The Data Repository Service (DRS) API ...", "organization": { From 42a4ba88b83594567139f3c7c66e6f72562607dc Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Wed, 11 Sep 2024 21:25:16 -0700 Subject: [PATCH 8/8] did I mess this up with two descriptions? --- openapi/components/parameters/BulkObjectIdNoPassport.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/openapi/components/parameters/BulkObjectIdNoPassport.yaml b/openapi/components/parameters/BulkObjectIdNoPassport.yaml index f972c58c..f25067d2 100644 --- a/openapi/components/parameters/BulkObjectIdNoPassport.yaml +++ b/openapi/components/parameters/BulkObjectIdNoPassport.yaml @@ -3,7 +3,6 @@ description: The object that contains the DRS object IDs array properties: bulk_object_ids: type: array - description: DRS object IDs items: type: string description: An array of ObjectIDs.