From 34a53a4ba34b45915a90fa342cd39fb5841f4ff5 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Wed, 9 Oct 2024 15:09:49 -0400 Subject: [PATCH 1/4] Merge from `main` to `develop` (#483) * Versioning 5.10.1 * Rc/5.11.0 (#418) * Keycloak permission manager (#387) * enable Keycloak apiKeys * setup security config and update mvn dependencies * junit missing dependency * unit test mock Jwt Decoder * fix unit test - use a JWT decoder for testing - Remove unused clases - Remove JWT expired unit tests as validation is now implemented by Spring Security * code format * test profile * docker-compose update images * add keycloak to docker compose * fix merge conflict * fix typo curl command * update keycloak system client and apikeys * version 5.11.0 --------- Co-authored-by: Jon Eubank --- pom.xml | 2 +- score-client/pom.xml | 2 +- score-client/src/main/resources/application.yml | 4 ---- score-core/pom.xml | 2 +- score-fs/pom.xml | 2 +- score-server/pom.xml | 2 +- score-server/src/main/resources/application.yml | 4 ++-- score-test/pom.xml | 2 +- 8 files changed, 8 insertions(+), 12 deletions(-) diff --git a/pom.xml b/pom.xml index bba8cef8..622e4dff 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S bio.overture score - 5.10.1-SNAPSHOT + 5.12.0-SNAPSHOT pom ${project.artifactId} ${project.name} diff --git a/score-client/pom.xml b/score-client/pom.xml index d93ec80b..d72a8113 100644 --- a/score-client/pom.xml +++ b/score-client/pom.xml @@ -21,7 +21,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S bio.overture score - 5.10.1-SNAPSHOT + 5.12.0-SNAPSHOT ../pom.xml diff --git a/score-client/src/main/resources/application.yml b/score-client/src/main/resources/application.yml index 996265b9..2ac3a7bd 100644 --- a/score-client/src/main/resources/application.yml +++ b/score-client/src/main/resources/application.yml @@ -91,7 +91,3 @@ logging: springframework.web: DEBUG com.amazonaws.services: DEBUG ---- -############################################################################### -# Profile - "debug" -############################################################################### diff --git a/score-core/pom.xml b/score-core/pom.xml index b8db5a9b..f42bf791 100644 --- a/score-core/pom.xml +++ b/score-core/pom.xml @@ -21,7 +21,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S bio.overture score - 5.10.1-SNAPSHOT + 5.12.0-SNAPSHOT ../pom.xml diff --git a/score-fs/pom.xml b/score-fs/pom.xml index fd4fac1e..a59e03f4 100644 --- a/score-fs/pom.xml +++ b/score-fs/pom.xml @@ -21,7 +21,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S bio.overture score - 5.10.1-SNAPSHOT + 5.12.0-SNAPSHOT ../pom.xml diff --git a/score-server/pom.xml b/score-server/pom.xml index bb51e566..fb4e9b60 100644 --- a/score-server/pom.xml +++ b/score-server/pom.xml @@ -21,7 +21,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S bio.overture score - 5.10.1-SNAPSHOT + 5.12.0-SNAPSHOT ../pom.xml diff --git a/score-server/src/main/resources/application.yml b/score-server/src/main/resources/application.yml index c6f2d68a..3a7037e9 100644 --- a/score-server/src/main/resources/application.yml +++ b/score-server/src/main/resources/application.yml @@ -36,6 +36,8 @@ s3: upload.directory: upload upload.expiration: 6 data.directory: data + # COL-131: Change pre-signed URLs TTL to 1 day max + download.expiration: 1 # custom meta property with md5 hash, unused when upload state files are available (default behaviour) # customMd5Property: md5chksum @@ -59,8 +61,6 @@ bucket: object: sentinel: heliograph - # COL-131: Change pre-signed URLs TTL to 1 day max - download.expiration: 1 upload: retry.limit: 5 diff --git a/score-test/pom.xml b/score-test/pom.xml index 91a00667..db78e48a 100644 --- a/score-test/pom.xml +++ b/score-test/pom.xml @@ -21,7 +21,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S bio.overture score - 5.10.1-SNAPSHOT + 5.12.0-SNAPSHOT ../pom.xml From 12c0fb6b612713b102d558f0bb1de7675bab2ca4 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Fri, 22 Nov 2024 11:51:43 -0500 Subject: [PATCH 2/4] Add Key Features to main README --- README.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c90cbf6d..b9f3886e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Score -Score is a file transfer service designed for cloud-based projects, providing a robust API for secure file transfer and storage operations. It serves as an intermediary between object storage systems and user authorization mechanisms, using pre-signed URLs for efficient and protected data access. +Score is a file transfer service designed to enable large-file upload and download, providing a robust API for secure file transfer and storage operations. It serves as an intermediary between object storage systems and user authorization mechanisms, using pre-signed URLs for efficient and protected data access.
@@ -11,7 +11,20 @@ Score is a file transfer service designed for cloud-based projects, providing a > > *Score is part of [Overture](https://www.overture.bio/), a collection of open-source software microservices used to create platforms for researchers to organize and share genomics data.* > -> + +## Key Features + +- **Multi-cloud Support**: Compatible with AWS S3, Azure Storage, and any object storage with an S3 compliant API (Minio, Ceph, etc.) +- **High-performance Transfers**: Implements multipart uploads and downloads for optimal throughput +- **Genomic Data Handling (SamTools)**: Supports BAM/CRAM file slicing by genomic region and provides built-in samtools operations for BAM file handling +- **Data Integrity**: Ensures file integrity through MD5 checksum validation on uploads and downloads +- **Security**: Implements ACL-based security using OAuth2 with study code-scoped access +- **Metadata Integration**: Integrates with the Song metadata management system for comprehensive data tracking +- **File Bundling**: Enables efficient transfer of multiple files in a single bundle +- **Resumable Downloads**: Supports resuming downloads after network interruptions +- **FUSE Support**: Offers file system in Userspace (FUSE) support for enhanced file operations +- **Interactive API Documentation:** Built-in Swagger UI for easy API interaction and exploration + ## Repository Structure The repository is organized with the following directory structure: ``` From 9f31683ba7bc0adf5676d00e2884d6f7030d320f Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Fri, 22 Nov 2024 11:52:36 -0500 Subject: [PATCH 3/4] Update repository structure to mirror main README --- docs/overview.md | 38 +++++++++++--------------------------- 1 file changed, 11 insertions(+), 27 deletions(-) diff --git a/docs/overview.md b/docs/overview.md index ac27b442..9e56f6e0 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -1,12 +1,12 @@ # Overview -Score is a file transfer service designed for cloud-based projects, providing a robust API for secure file transfer and storage operations. It serves as an intermediary between object storage systems and user authorization mechanisms, using pre-signed URLs for efficient and protected data access. +Score is a file transfer service designed to enable large-file upload and download, providign a robust API for secure file transfer and storage operations. It serves as an intermediary between object storage systems and user authorization mechanisms, using pre-signed URLs for efficient and protected data access. ## System Architecture Score's primary function is to broker authenticated access to your object storage provider. It achieves this by: -1. Validating user access rights against an authorization system (Keycloak) +1. Validating user access rights against an authorization system (OAuth) 2. Generating time-limited pre-signed URLs for object access 3. Facilitating secure data transfer between clients and object storage @@ -14,14 +14,14 @@ Score's primary function is to broker authenticated access to your object storag As part of the larger Overture.bio software suite, Score is typically used with multiple other services including: -- **Song:** A metadata management service made to manage file metadata independently from object storage concerns. +- **Song:** A metadata management service made to manage file metadata independently from object storage concerns - **Score Client:** A command line tool to streamline interactions with Scores REST API endpoints - **Keycloak:** The authorization and authentication service used to provided OAuth2 authentication for Score ## Key Features -- **Multi-cloud Support**: Compatible with AWS S3, Azure Storage, and Google Cloud Storage +- **Multi-cloud Support**: Compatible with AWS S3, Azure Storage, and any object storage with an S3 compliant API (Minio, Ceph, etc.) - **High-performance Transfers**: Implements multipart uploads and downloads for optimal throughput - **Genomic Data Handling (SamTools)**: Supports BAM/CRAM file slicing by genomic region and provides built-in samtools operations for BAM file handling - **Data Integrity**: Ensures file integrity through MD5 checksum validation on uploads and downloads @@ -30,11 +30,11 @@ As part of the larger Overture.bio software suite, Score is typically used with - **File Bundling**: Enables efficient transfer of multiple files in a single bundle - **Resumable Downloads**: Supports resuming downloads after network interruptions - **FUSE Support**: Offers file system in Userspace (FUSE) support for enhanced file operations -- **Interactive API Documentation:** Built-in Swagger UI for easy API interaction and exploration. +- **Interactive API Documentation:** Built-in Swagger UI for easy API interaction and exploration ## Repository Structure - +The repository is organized with the following directory structure: ``` . ├── /score-client @@ -44,24 +44,8 @@ As part of the larger Overture.bio software suite, Score is typically used with └── /score-test ``` -[Click here to view the Score respository on GitHub ](https://github.com/overture-stack/score) - -#### Score-client - -[Explaination] - -#### Score-core - -[Explaination] - -#### Score-fs - -[Explaination] - -#### Score-server - -[Explaination] - -#### Score-test - -[Explaination] +- **score-client:** Command line app for uploading and downloading files, published as a [docker container](https://github.com/overture-stack/score/pkgs/container/score) and availabe as an executable jar from [github releases](https://github.com/overture-stack/score/releases) +- **score-core:** Core library containing shared utilities and data models used by all other packages +- **score-fs:** File system operations module for managing local files +- **score-server:** Main server application that handles object storage and transfers, published as a [docker container](https://github.com/overture-stack/score/pkgs/container/score-server) +- **score-test:** Integration and end-to-end test suite for all packages From 299846a1ad65d1b75bc9f301030f0cf75358f1b2 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Fri, 22 Nov 2024 11:52:59 -0500 Subject: [PATCH 4/4] Formatting and minor edits --- docs/setup.md | 44 +++++++++---------- .../src/main/resources/application.yml | 13 +++--- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/docs/setup.md b/docs/setup.md index 483c1b5b..9d9cc752 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -23,9 +23,9 @@ We'll use our Conductor service, a flexible Docker Compose setup, to spin up Sco 2. Run the appropriate start command for your operating system: - | Operating System | Command | - |------------------|---------| - | Unix/macOS | `make scoreDev` | + | Operating System | Command | + | ---------------- | --------------------- | + | Unix/macOS | `make scoreDev` | | Windows | `./make.bat scoreDev` |
@@ -35,16 +35,16 @@ We'll use our Conductor service, a flexible Docker Compose setup, to spin up Sco ![ScoreDev](./assets/scoreDev.svg 'Score Dev Environment') - | Service | Port | Description | Purpose in Score Development | - |---------|------|-------------|------------------------------| - | Conductor | `9204` | Orchestrates deployments and environment setups | Manages the overall development environment | - | Keycloak-db | - | Database for Keycloak (no exposed port) | Stores Keycloak data for authentication | - | Keycloak | `8180` | Authorization and authentication service | Provides OAuth2 authentication for Score | - | Song-db | `5433` | Database for Song | Stores metadata managed by Song | - | Song | `8080` | Metadata management service | Manages metadata for files stored by Score | - | Minio | `9000` | Object storage provider | Simulates S3-compatible storage for Score | + | Service | Port | Description | Purpose in Score Development | + | ----------- | ------ | ----------------------------------------------- | ------------------------------------------- | + | Conductor | `9204` | Orchestrates deployments and environment setups | Manages the overall development environment | + | Keycloak-db | - | Database for Keycloak (no exposed port) | Stores Keycloak data for authentication | + | Keycloak | `8180` | Authorization and authentication service | Provides OAuth2 authentication for Score | + | Song-db | `5433` | Database for Song | Stores metadata managed by Song | + | Song | `8080` | Metadata management service | Manages metadata for files stored by Score | + | Minio | `9000` | Object storage provider | Simulates S3-compatible storage for Score | - - Ensure all ports are free on your system before starting the environment. + - Ensure these ports are free on your system before starting the environment. - You may need to adjust the ports in the `docker-compose.yml` file if you have conflicts with existing services. For more information, see our [Conductor documentation linked here](/docs/other-software/Conductor) @@ -99,16 +99,16 @@ We'll use our Conductor service, a flexible Docker Compose setup, to spin up Sco **Click here for a summary of the Score-server spring profiles** **Score Profiles** - | Profile | Description | - |---------|-------------| - | `default` | Common settings for all environments. Includes server, S3, bucket, object, upload, and authentication configurations. | - | `ssl` | Enables SSL configuration for using a self-signed certificate in production deployments. | - | `azure` | Configuration for Azure blob storage. Includes Azure-specific settings and bucket policies. | - | `s3` | Configuration for Amazon S3 or S3-compatible storage. Includes endpoint, access key, and secret key settings. | - | `prod` | Production environment configuration. Enables secure S3 connections and sets the metadata URL. | - | `secure` | Security configuration for OAuth2 and JWT. Includes settings for resource server, authentication server, and scope definitions. | - | `dev` | Development environment configuration. Uses non-secure S3 connections, local endpoints, and disables upload cleaning. | - | `benchmark` | Configuration for benchmarking purposes. Includes SSL settings and a non-secure S3 endpoint. | + | Profile | Description | + | ----------- | ------------------------------------------------------------------------------------------------------------------------------- | + | `default` | Common settings for all environments. Includes server, S3, bucket, object, upload, and authentication configurations. | + | `ssl` | Enables SSL configuration for using a self-signed certificate in production deployments. | + | `azure` | Configuration for Azure blob storage. Includes Azure-specific settings and bucket policies. | + | `s3` | Configuration for Amazon S3 or S3-compatible storage. Includes endpoint, access key, and secret key settings. | + | `prod` | Production environment configuration. Enables secure S3 connections and sets the metadata URL. | + | `secure` | Security configuration for OAuth2 and JWT. Includes settings for resource server, authentication server, and scope definitions. | + | `dev` | Development environment configuration. Uses non-secure S3 connections, local endpoints, and disables upload cleaning. | + | `benchmark` | Configuration for benchmarking purposes. Includes SSL settings and a non-secure S3 endpoint. |
diff --git a/score-server/src/main/resources/application.yml b/score-server/src/main/resources/application.yml index 50f56562..8cb84841 100644 --- a/score-server/src/main/resources/application.yml +++ b/score-server/src/main/resources/application.yml @@ -19,7 +19,8 @@ ############################################################################### # Application Configurations for the Score Server # -# This file contains various configuration profiles for the Score Server application. +# This file contains placeholder configuration values for the various +# application profiles used by Score. # Please update the values according to your specific environment. ############################################################################### @@ -35,7 +36,7 @@ server: mime-types: application/json # Allowed origins for CORS (Cross-Origin Resource Sharing), multiple values can be added, separated by commas cors: - allowedOrigins: http://localhost:8081 # Update this with your frontend application URL(s) + allowedOrigins: http://localhost:8081 # If fetching files from a web application, update this with your frontend application domain s3: # S3 connection settings secured: true # Use HTTPS for S3 connections @@ -55,8 +56,8 @@ metadata: useLegacyMode: false # Set to true to use legacy metadata handling bucket: name: - object: oicr.icgc # Bucket name for object storage (update as needed) - state: oicr.icgc # Bucket name for state storage (update as needed) + object: score.data # Bucket name for object storage (update as needed) + state: score.data # Bucket name for state storage (update as needed) size: pool: 0 # Size of the connection pool key: 2 # Size of the key used for encryption @@ -146,7 +147,7 @@ spring: on-profile: azure azure: endpointProtocol: https - accountName: oicricgc # Replace with your Azure account name + accountName: # Add your Azure account name accountKey: # Add your Azure account key here bucket: name: @@ -191,7 +192,7 @@ spring: s3: secured: true metadata: - url: https://meta.icgc.org # Update with your actual metadata server URL + url: https://song.example.com # Update with your actual metadata server URL ---