Merge pull request #759 from umccr/feat/htsget

feat(htsget): add htsget support

config/config.ts

@@ -65,6 +65,7 @@ import {
 import { getOraDecompressionManagerStackProps } from './stacks/oraDecompressionPipelineManager';
 import { getPgDDProps } from './stacks/pgDD';
 import { getDataMigrateStackProps } from './stacks/dataMigrate';
+import { getHtsgetProps } from './stacks/htsget';
 
 interface EnvironmentConfig {
   name: string;
@@ -133,6 +134,7 @@ export const getEnvironmentConfig = (stage: AppStage): EnvironmentConfig | null
       stackyMcStackFaceProps: getGlueStackProps(stage),
       fmAnnotatorProps: getFmAnnotatorProps(),
       dataMigrateProps: getDataMigrateStackProps(stage),
+      htsgetProps: getHtsgetProps(stage),
       pgDDProps: getPgDDProps(stage),
     },
   };

config/stacks/htsget.ts

@@ -0,0 +1,21 @@
+import {
+  AppStage,
+  cognitoApiGatewayConfig,
+  corsAllowOrigins,
+  logsApiGatewayConfig,
+  vpcProps,
+} from '../constants';
+import { HtsgetStackConfigurableProps } from '../../lib/workload/stateless/stacks/htsget/stack';
+
+export const getHtsgetProps = (stage: AppStage): HtsgetStackConfigurableProps => {
+  return {
+    vpcProps,
+    apiGatewayCognitoProps: {
+      ...cognitoApiGatewayConfig,
+      corsAllowOrigins: corsAllowOrigins[stage],
+      apiGwLogsConfig: logsApiGatewayConfig[stage],
+      apiName: 'Htsget',
+      customDomainNamePrefix: 'htsget-file',
+    },
+  };
+};

lib/workload/stateless/stacks/filemanager/deploy/stack.ts

@@ -46,6 +46,7 @@ export class Filemanager extends Stack {
   private readonly securityGroup: ISecurityGroup;
   private readonly queue: IQueue;
   readonly domainName: string;
+  readonly role: Role;
 
   constructor(scope: Construct, id: string, props: FilemanagerProps) {
     super(scope, id, props);
@@ -64,7 +65,7 @@ export class Filemanager extends Stack {
       props.databaseClusterEndpointHostParameter
     );
 
-    const role = this.createRole(props.fileManagerRoleName);
+    this.role = this.createRole(props.fileManagerRoleName);
     if (props?.migrateDatabase) {
       const migrateFunction = new MigrateFunction(this, 'MigrateFunction', {
         vpc: this.vpc,
@@ -91,10 +92,10 @@ export class Filemanager extends Stack {
       )
     );
 
-    this.createIngestFunction(props, role);
-    this.createInventoryFunction(props, role);
+    this.createIngestFunction(props);
+    this.createInventoryFunction(props);
 
-    this.domainName = this.createApiFunction(props, role);
+    this.domainName = this.createApiFunction(props);
   }
 
   private createRole(name: string) {
@@ -107,42 +108,42 @@ export class Filemanager extends Stack {
   /**
    * Lambda function definitions and surrounding infrastructure.
    */
-  private createIngestFunction(props: FilemanagerProps, role: Role) {
+  private createIngestFunction(props: FilemanagerProps) {
     return new IngestFunction(this, 'IngestFunction', {
       vpc: this.vpc,
       host: this.host,
       securityGroup: this.securityGroup,
       eventSources: [this.queue],
       buckets: props.eventSourceBuckets,
-      role,
+      role: this.role,
       ...props,
     });
   }
 
   /**
    * Create the inventory function.
    */
-  private createInventoryFunction(props: FilemanagerProps, role: Role) {
+  private createInventoryFunction(props: FilemanagerProps) {
     return new InventoryFunction(this, 'InventoryFunction', {
       vpc: this.vpc,
       host: this.host,
       securityGroup: this.securityGroup,
       port: props.port,
       buckets: props.inventorySourceBuckets,
-      role,
+      role: this.role,
     });
   }
 
   /**
    * Query function and API Gateway fronting the function. Returns the configured domain name.
    */
-  private createApiFunction(props: FilemanagerProps, role: Role): string {
+  private createApiFunction(props: FilemanagerProps): string {
     let apiLambda = new ApiFunction(this, 'ApiFunction', {
       vpc: this.vpc,
       host: this.host,
       securityGroup: this.securityGroup,
       buckets: [...props.eventSourceBuckets, ...props.inventorySourceBuckets],
-      role,
+      role: this.role,
       ...props,
     });
 

lib/workload/stateless/stacks/filemanager/docs/API_GUIDE.md

@@ -235,7 +235,28 @@ There is also no way to POST an attribute linking rule, which can be used to upd
 as they are received by filemanager. See [ATTRIBUTE_LINKING.md][attribute-linking] for a discussion on some approaches
 for this. The likely solution will involve merging the above wildcard matching logic with attribute rules.
 
+## Htsget
+
+Htsget support is enabled under the `htsget.file` subdomain, see [here] for more details. For each current object
+returned by the filemanager, it can be reached via htsget by combining the key and the bucket in the query path:
+
+```sh
+curl -H "Authorization: Bearer $TOKEN" "https://htsget-file.dev.umccr.org/reads/<filemanager_bucket>/<filemanager_key>" | jq
+```
+
+For example, fetching a current object:
+
+```sh
+export RESULT=$(curl -H "Authorization: Bearer $TOKEN" "https://file.dev.umccr.org/api/v1/s3?bucket=umccr-temp-dev&key=*analysis*.bam&rowsPerPage=1&currentState=true" | jq)
+export BUCKET=$(echo $RESULT | jq -r '.results[] | .bucket')
+export KEY=$(echo $RESULT | jq -r '.results[] | .key')
+
+# Note that you must remove the file extension.
+curl -H "Authorization: Bearer $TOKEN" "https://htsget-file.dev.umccr.org/reads/${BUCKET}/${KEY%.*}" | jq
+```
+
 [json-patch]: https://jsonpatch.com/
 [qs]: https://github.com/ljharb/qs
 [s3-events]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/EventNotifications.html
-[attribute-linking]: ATTRIBUTE_LINKING.md
+[attribute-linking]: ATTRIBUTE_LINKING.md
+[here]: ../../htsget/README.md

lib/workload/stateless/stacks/htsget/README.md

@@ -0,0 +1,18 @@
+# htsget stack
+
+This stack deploys [htsget-rs] to access files. Any files accessible by filemanager are also accessible using htsget.
+
+The deployed instance of the htsget-rs can be reached using stage at `https://htsget-file.<stage>.umccr.org`
+and the orcabus API token. To retrieve the token, run:
+
+```sh
+export TOKEN=$(aws secretsmanager get-secret-value --secret-id orcabus/token-service-jwt --output json --query SecretString | jq -r 'fromjson | .id_token')
+```
+
+Then, the API can be queried:
+
+```sh
+curl -H "Authorization: Bearer $TOKEN" "https://htsget-file.dev.umccr.org/reads/service-info" | jq
+```
+
+[htsget-rs]: https://github.com/umccr/htsget-rs

lib/workload/stateless/stacks/htsget/deploy.toml

@@ -0,0 +1,24 @@
+# TODO this will eventually be removed for props-only configuration.
+
+ticket_server_cors_allow_headers = "All"
+ticket_server_cors_allow_origins = "Mirror"
+ticket_server_cors_allow_methods = "All"
+ticket_server_cors_allow_credentials = true
+ticket_server_cors_max_age = 300
+
+data_server_enabled = false
+
+name = "orcabus-htsget-rs"
+version = "0.1.0"
+organization_name = "UMCCR"
+organization_url = "https://umccr.org/"
+contact_url = "https://umccr.org/"
+documentation_url = "https://github.com/umccr/htsget-rs"
+
+# The role should prevent any access to other files, although it should probably
+# be set here as well.
+[[resolvers]]
+regex = '^(?P<bucket>.*?)/(?P<key>.*)$'
+substitution_string = '$key'
+storage.backend = 'S3'
+

lib/workload/stateless/stacks/htsget/stack.ts

@@ -0,0 +1,54 @@
+import { Construct } from 'constructs';
+import { Stack, StackProps } from 'aws-cdk-lib';
+import { Role } from 'aws-cdk-lib/aws-iam';
+import { IVpc, Vpc, VpcLookupOptions } from 'aws-cdk-lib/aws-ec2';
+import { ApiGatewayConstruct, ApiGatewayConstructProps } from '../../../components/api-gateway';
+import path from 'path';
+import { HtsgetLambdaConstruct } from 'htsget-lambda';
+
+/**
+ * Configurable props for the htsget stack.
+ */
+export type HtsgetStackConfigurableProps = {
+  /**
+   * Props to lookup vpc.
+   */
+  vpcProps: VpcLookupOptions;
+  /**
+   * API gateway construct props.
+   */
+  apiGatewayCognitoProps: ApiGatewayConstructProps;
+};
+
+/**
+ * Props for the data migrate stack.
+ */
+export type HtsgetStackProps = HtsgetStackConfigurableProps & {
+  /**
+   * The role to use.
+   */
+  role: Role;
+};
+
+/**
+ * Deploys htsget-rs with access to filemanager data.
+ */
+export class HtsgetStack extends Stack {
+  private readonly vpc: IVpc;
+  private readonly apiGateway: ApiGatewayConstruct;
+
+  constructor(scope: Construct, id: string, props: StackProps & HtsgetStackProps) {
+    super(scope, id, props);
+
+    this.vpc = Vpc.fromLookup(this, 'MainVpc', props.vpcProps);
+    this.apiGateway = new ApiGatewayConstruct(this, 'ApiGateway', props.apiGatewayCognitoProps);
+
+    const configPath = path.join(__dirname, 'deploy.toml');
+    new HtsgetLambdaConstruct(this, 'Htsget', {
+      config: configPath,
+      vpc: this.vpc,
+      role: props.role,
+      httpApi: this.apiGateway.httpApi,
+    });
+  }
+}

lib/workload/stateless/statelessStackCollectionClass.ts

@@ -81,6 +81,7 @@ import {
 } from './stacks/ora-decompression-manager/deploy';
 import { PgDDStack, PgDDStackProps } from './stacks/pg-dd/deploy/stack';
 import { DataMigrateStack, DataMigrateStackProps } from './stacks/data-migrate/deploy/stack';
+import { HtsgetStack, HtsgetStackConfigurableProps } from './stacks/htsget/stack';
 
 export interface StatelessStackCollectionProps {
   metadataManagerStackProps: MetadataManagerStackProps;
@@ -107,6 +108,7 @@ export interface StatelessStackCollectionProps {
   stackyMcStackFaceProps: GlueStackProps;
   fmAnnotatorProps: FMAnnotatorConfigurableProps;
   dataMigrateProps: DataMigrateStackProps;
+  htsgetProps: HtsgetStackConfigurableProps;
   pgDDProps?: PgDDStackProps;
 }
 
@@ -136,6 +138,7 @@ export class StatelessStackCollection {
   readonly stackyMcStackFaceStack: Stack;
   readonly fmAnnotator: Stack;
   readonly dataMigrate: Stack;
+  readonly htsgetStack: Stack;
   readonly pgDDStack: Stack;
 
   constructor(
@@ -319,6 +322,11 @@ export class StatelessStackCollection {
       ...this.createTemplateProps(env, 'DataMigrateStack'),
       ...statelessConfiguration.dataMigrateProps,
     });
+    this.htsgetStack = new HtsgetStack(scope, 'HtsgetStack', {
+      ...this.createTemplateProps(env, 'HtsgetStack'),
+      ...statelessConfiguration.htsgetProps,
+      role: fileManagerStack.role,
+    });
 
     if (statelessConfiguration.pgDDProps) {
       this.pgDDStack = new PgDDStack(scope, 'PgDDStack', {

package.json

@@ -33,6 +33,7 @@
     "constructs": "^10.4.2",
     "core-js-pure": "^3.38.1",
     "dotenv": "^16.4.5",
+    "htsget-lambda": "^0.6.2",
     "source-map-support": "^0.5.21",
     "sqs-dlq-monitoring": "^1.2.16"
   },