diff --git a/client/src/api/landings.ts b/client/src/api/landings.ts new file mode 100644 index 000000000000..03bbfc01d840 --- /dev/null +++ b/client/src/api/landings.ts @@ -0,0 +1,4 @@ +import { type components } from "@/api/schema"; + +export type ClaimLandingPayload = components["schemas"]["ClaimLandingPayload"]; +export type WorkflowLandingRequest = components["schemas"]["WorkflowLandingRequest"]; diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 01a0046d0a32..49f8c021d5f2 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -4937,6 +4937,57 @@ export interface paths { patch?: never; trace?: never; }; + "/api/workflow_landings": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Create Landing */ + post: operations["create_landing_api_workflow_landings_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/workflow_landings/{uuid}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Landing */ + get: operations["get_landing_api_workflow_landings__uuid__get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/workflow_landings/{uuid}/claim": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Claim Landing */ + post: operations["claim_landing_api_workflow_landings__uuid__claim_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/workflows": { parameters: { query?: never; @@ -6026,18 +6077,18 @@ export interface components { source: "admin" | "galaxy"; /** Type */ type: - | ( - | "faster" - | "slower" - | "short_term" - | "backed_up" - | "not_backed_up" - | "more_secure" - | "less_secure" - | "more_stable" - | "less_stable" - ) - | ("cloud" | "quota" | "no_quota" | "restricted" | "user_defined"); + | ( + | "faster" + | "slower" + | "short_term" + | "backed_up" + | "not_backed_up" + | "more_secure" + | "less_secure" + | "more_stable" + | "less_stable" + ) + | ("cloud" | "quota" | "no_quota" | "restricted" | "user_defined"); }; /** BasicRoleModel */ BasicRoleModel: { @@ -6396,6 +6447,11 @@ export interface components { */ type: string; }; + /** ClaimLandingPayload */ + ClaimLandingPayload: { + /** Client Secret */ + client_secret?: string | null; + }; /** CleanableItemsSummary */ CleanableItemsSummary: { /** @@ -6456,6 +6512,12 @@ export interface components { CompositeDataElement: { /** Md5 */ MD5?: string | null; + /** Sha-1 */ + "SHA-1"?: string | null; + /** Sha-256 */ + "SHA-256"?: string | null; + /** Sha-512 */ + "SHA-512"?: string | null; /** * Auto Decompress * @description Decompress compressed data before sniffing? @@ -6486,6 +6548,8 @@ export interface components { */ ext: string; extra_files?: components["schemas"]["ExtraFiles"] | null; + /** Hashes */ + hashes?: components["schemas"]["FetchDatasetHash"][] | null; /** Info */ info?: string | null; /** Name */ @@ -6604,8 +6668,8 @@ export interface components { input: components["schemas"]["InputReferenceByOrderIndex"] | components["schemas"]["InputReferenceByLabel"]; /** Output */ output: - | components["schemas"]["OutputReferenceByOrderIndex"] - | components["schemas"]["OutputReferenceByLabel"]; + | components["schemas"]["OutputReferenceByOrderIndex"] + | components["schemas"]["OutputReferenceByLabel"]; }; /** ContentsObject */ ContentsObject: { @@ -7038,6 +7102,20 @@ export interface components { * @enum {string} */ CreateType: "file" | "folder" | "collection"; + /** CreateWorkflowLandingRequestPayload */ + CreateWorkflowLandingRequestPayload: { + /** Client Secret */ + client_secret?: string | null; + /** Request State */ + request_state?: Record | null; + /** Workflow Id */ + workflow_id: string; + /** + * Workflow Target Type + * @enum {string} + */ + workflow_target_type: "stored_workflow" | "workflow"; + }; /** CreatedEntryResponse */ CreatedEntryResponse: { /** @@ -7523,10 +7601,10 @@ export interface components { * @description The element's specific data depending on the value of `element_type`. */ object?: - | components["schemas"]["HDAObject"] - | components["schemas"]["HDADetailed"] - | components["schemas"]["DCObject"] - | null; + | components["schemas"]["HDAObject"] + | components["schemas"]["HDADetailed"] + | components["schemas"]["DCObject"] + | null; }; /** * DCEType @@ -7585,9 +7663,9 @@ export interface components { auto_decompress: boolean; /** Destination */ destination: - | components["schemas"]["HdaDestination"] - | components["schemas"]["LibraryFolderDestination"] - | components["schemas"]["LibraryDestination"]; + | components["schemas"]["HdaDestination"] + | components["schemas"]["LibraryFolderDestination"] + | components["schemas"]["LibraryDestination"]; elements_from: components["schemas"]["ElementsFromType"]; /** Ftp Path */ ftp_path?: string | null; @@ -7609,20 +7687,20 @@ export interface components { auto_decompress: boolean; /** Destination */ destination: - | components["schemas"]["HdaDestination"] - | components["schemas"]["LibraryFolderDestination"] - | components["schemas"]["LibraryDestination"]; + | components["schemas"]["HdaDestination"] + | components["schemas"]["LibraryFolderDestination"] + | components["schemas"]["LibraryDestination"]; /** Elements */ elements: ( | ( - | components["schemas"]["FileDataElement"] - | components["schemas"]["PastedDataElement"] - | components["schemas"]["UrlDataElement"] - | components["schemas"]["PathDataElement"] - | components["schemas"]["ServerDirElement"] - | components["schemas"]["FtpImportElement"] - | components["schemas"]["CompositeDataElement"] - ) + | components["schemas"]["FileDataElement"] + | components["schemas"]["PastedDataElement"] + | components["schemas"]["UrlDataElement"] + | components["schemas"]["PathDataElement"] + | components["schemas"]["ServerDirElement"] + | components["schemas"]["FtpImportElement"] + | components["schemas"]["CompositeDataElement"] + ) | components["schemas"]["NestedElement"] )[]; }; @@ -7827,18 +7905,18 @@ export interface components { * @enum {string} */ DatasetState: - | "new" - | "upload" - | "queued" - | "running" - | "ok" - | "empty" - | "error" - | "paused" - | "setting_metadata" - | "failed_metadata" - | "deferred" - | "discarded"; + | "new" + | "upload" + | "queued" + | "running" + | "ok" + | "empty" + | "error" + | "paused" + | "setting_metadata" + | "failed_metadata" + | "deferred" + | "discarded"; /** DatasetStorageDetails */ DatasetStorageDetails: { /** @@ -8272,8 +8350,8 @@ export interface components { input: components["schemas"]["InputReferenceByOrderIndex"] | components["schemas"]["InputReferenceByLabel"]; /** Output */ output: - | components["schemas"]["OutputReferenceByOrderIndex"] - | components["schemas"]["OutputReferenceByLabel"]; + | components["schemas"]["OutputReferenceByOrderIndex"] + | components["schemas"]["OutputReferenceByLabel"]; }; /** * DisplayApp @@ -8652,8 +8730,8 @@ export interface components { object_type: components["schemas"]["ExportObjectType"]; /** Payload */ payload: - | components["schemas"]["WriteStoreToPayload"] - | components["schemas"]["ShortTermStoreExportPayload"]; + | components["schemas"]["WriteStoreToPayload"] + | components["schemas"]["ShortTermStoreExportPayload"]; /** User Id */ user_id?: string | null; }; @@ -8799,10 +8877,26 @@ export interface components { } & { [key: string]: unknown; }; + /** FetchDatasetHash */ + FetchDatasetHash: { + /** + * Hash Function + * @enum {string} + */ + hash_function: "MD5" | "SHA-1" | "SHA-256" | "SHA-512"; + /** Hash Value */ + hash_value: string; + }; /** FileDataElement */ FileDataElement: { /** Md5 */ MD5?: string | null; + /** Sha-1 */ + "SHA-1"?: string | null; + /** Sha-256 */ + "SHA-256"?: string | null; + /** Sha-512 */ + "SHA-512"?: string | null; /** * Auto Decompress * @description Decompress compressed data before sniffing? @@ -8832,6 +8926,8 @@ export interface components { */ ext: string; extra_files?: components["schemas"]["ExtraFiles"] | null; + /** Hashes */ + hashes?: components["schemas"]["FetchDatasetHash"][] | null; /** Info */ info?: string | null; /** Name */ @@ -8945,13 +9041,13 @@ export interface components { type: "ftp" | "posix" | "s3fs" | "azure" | "onedata"; /** Variables */ variables?: - | ( - | components["schemas"]["TemplateVariableString"] - | components["schemas"]["TemplateVariableInteger"] - | components["schemas"]["TemplateVariablePathComponent"] - | components["schemas"]["TemplateVariableBoolean"] - )[] - | null; + | ( + | components["schemas"]["TemplateVariableString"] + | components["schemas"]["TemplateVariableInteger"] + | components["schemas"]["TemplateVariablePathComponent"] + | components["schemas"]["TemplateVariableBoolean"] + )[] + | null; /** * Version * @default 0 @@ -9097,6 +9193,12 @@ export interface components { FtpImportElement: { /** Md5 */ MD5?: string | null; + /** Sha-1 */ + "SHA-1"?: string | null; + /** Sha-256 */ + "SHA-256"?: string | null; + /** Sha-512 */ + "SHA-512"?: string | null; /** * Auto Decompress * @description Decompress compressed data before sniffing? @@ -9128,6 +9230,8 @@ export interface components { extra_files?: components["schemas"]["ExtraFiles"] | null; /** Ftp Path */ ftp_path: string; + /** Hashes */ + hashes?: components["schemas"]["FetchDatasetHash"][] | null; /** Info */ info?: string | null; /** Name */ @@ -10612,14 +10716,14 @@ export interface components { /** Elements */ elements: ( | ( - | components["schemas"]["FileDataElement"] - | components["schemas"]["PastedDataElement"] - | components["schemas"]["UrlDataElement"] - | components["schemas"]["PathDataElement"] - | components["schemas"]["ServerDirElement"] - | components["schemas"]["FtpImportElement"] - | components["schemas"]["CompositeDataElement"] - ) + | components["schemas"]["FileDataElement"] + | components["schemas"]["PastedDataElement"] + | components["schemas"]["UrlDataElement"] + | components["schemas"]["PathDataElement"] + | components["schemas"]["ServerDirElement"] + | components["schemas"]["FtpImportElement"] + | components["schemas"]["CompositeDataElement"] + ) | components["schemas"]["NestedElement"] )[]; /** Name */ @@ -10918,10 +11022,10 @@ export interface components { operation: components["schemas"]["HistoryContentItemOperation"]; /** Params */ params?: - | components["schemas"]["ChangeDatatypeOperationParams"] - | components["schemas"]["ChangeDbkeyOperationParams"] - | components["schemas"]["TagOperationParams"] - | null; + | components["schemas"]["ChangeDatatypeOperationParams"] + | components["schemas"]["ChangeDbkeyOperationParams"] + | components["schemas"]["TagOperationParams"] + | null; }; /** HistoryContentBulkOperationResult */ HistoryContentBulkOperationResult: { @@ -10948,15 +11052,15 @@ export interface components { * @enum {string} */ HistoryContentItemOperation: - | "hide" - | "unhide" - | "delete" - | "undelete" - | "purge" - | "change_datatype" - | "change_dbkey" - | "add_tags" - | "remove_tags"; + | "hide" + | "unhide" + | "delete" + | "undelete" + | "purge" + | "change_datatype" + | "change_dbkey" + | "add_tags" + | "remove_tags"; /** * HistoryContentSource * @enum {string} @@ -11273,8 +11377,8 @@ export interface components { ImportToolDataBundle: { /** Source */ source: - | components["schemas"]["ImportToolDataBundleDatasetSource"] - | components["schemas"]["ImportToolDataBundleUriSource"]; + | components["schemas"]["ImportToolDataBundleDatasetSource"] + | components["schemas"]["ImportToolDataBundleUriSource"]; }; /** ImportToolDataBundleDatasetSource */ ImportToolDataBundleDatasetSource: { @@ -11794,17 +11898,17 @@ export interface components { }; }; InvocationMessageResponseUnion: - | components["schemas"]["InvocationCancellationReviewFailedResponse"] - | components["schemas"]["InvocationCancellationHistoryDeletedResponse"] - | components["schemas"]["InvocationCancellationUserRequestResponse"] - | components["schemas"]["InvocationFailureDatasetFailedResponse"] - | components["schemas"]["InvocationFailureCollectionFailedResponse"] - | components["schemas"]["InvocationFailureJobFailedResponse"] - | components["schemas"]["InvocationFailureOutputNotFoundResponse"] - | components["schemas"]["InvocationFailureExpressionEvaluationFailedResponse"] - | components["schemas"]["InvocationFailureWhenNotBooleanResponse"] - | components["schemas"]["InvocationUnexpectedFailureResponse"] - | components["schemas"]["InvocationEvaluationWarningWorkflowOutputNotFoundResponse"]; + | components["schemas"]["InvocationCancellationReviewFailedResponse"] + | components["schemas"]["InvocationCancellationHistoryDeletedResponse"] + | components["schemas"]["InvocationCancellationUserRequestResponse"] + | components["schemas"]["InvocationFailureDatasetFailedResponse"] + | components["schemas"]["InvocationFailureCollectionFailedResponse"] + | components["schemas"]["InvocationFailureJobFailedResponse"] + | components["schemas"]["InvocationFailureOutputNotFoundResponse"] + | components["schemas"]["InvocationFailureExpressionEvaluationFailedResponse"] + | components["schemas"]["InvocationFailureWhenNotBooleanResponse"] + | components["schemas"]["InvocationUnexpectedFailureResponse"] + | components["schemas"]["InvocationEvaluationWarningWorkflowOutputNotFoundResponse"]; /** InvocationOutput */ InvocationOutput: { /** @@ -11953,7 +12057,14 @@ export interface components { * InvocationState * @enum {string} */ - InvocationState: "new" | "ready" | "scheduled" | "cancelled" | "cancelling" | "failed"; + InvocationState: + | "new" + | "requires_materialization" + | "ready" + | "scheduled" + | "cancelled" + | "cancelling" + | "failed"; /** * InvocationStep * @description Information about workflow invocation step @@ -12733,11 +12844,11 @@ export interface components { * @description The values of the job parameter */ value?: - | (components["schemas"]["EncodedJobParameterHistoryItem"] | null)[] - | number - | boolean - | string - | null; + | (components["schemas"]["EncodedJobParameterHistoryItem"] | null)[] + | number + | boolean + | string + | null; }; /** * JobSourceType @@ -12750,21 +12861,21 @@ export interface components { * @enum {string} */ JobState: - | "new" - | "resubmitted" - | "upload" - | "waiting" - | "queued" - | "running" - | "ok" - | "error" - | "failed" - | "paused" - | "deleting" - | "deleted" - | "stop" - | "stopped" - | "skipped"; + | "new" + | "resubmitted" + | "upload" + | "waiting" + | "queued" + | "running" + | "ok" + | "error" + | "failed" + | "paused" + | "deleting" + | "deleted" + | "stop" + | "stopped" + | "skipped"; /** JobStateSummary */ JobStateSummary: { /** @@ -12887,6 +12998,11 @@ export interface components { */ value: string; }; + /** + * LandingRequestState + * @enum {string} + */ + LandingRequestState: "unclaimed" | "claimed"; /** LegacyLibraryPermissionsPayload */ LegacyLibraryPermissionsPayload: { /** @@ -13879,6 +13995,21 @@ export interface components { * @description The source of the content. Can be other history element to be copied or library elements. */ source: components["schemas"]["DatasetSourceType"]; + /** + * Validate hashes + * @description Set to true to enable dataset validation during materialization. + * @default false + */ + validate_hashes: boolean; + }; + /** MaterializeDatasetOptions */ + MaterializeDatasetOptions: { + /** + * Validate hashes + * @description Set to true to enable dataset validation during materialization. + * @default false + */ + validate_hashes: boolean; }; /** MessageExceptionModel */ MessageExceptionModel: { @@ -13954,6 +14085,12 @@ export interface components { NestedElement: { /** Md5 */ MD5?: string | null; + /** Sha-1 */ + "SHA-1"?: string | null; + /** Sha-256 */ + "SHA-256"?: string | null; + /** Sha-512 */ + "SHA-512"?: string | null; /** * Auto Decompress * @description Decompress compressed data before sniffing? @@ -13979,14 +14116,14 @@ export interface components { /** Elements */ elements: ( | ( - | components["schemas"]["FileDataElement"] - | components["schemas"]["PastedDataElement"] - | components["schemas"]["UrlDataElement"] - | components["schemas"]["PathDataElement"] - | components["schemas"]["ServerDirElement"] - | components["schemas"]["FtpImportElement"] - | components["schemas"]["CompositeDataElement"] - ) + | components["schemas"]["FileDataElement"] + | components["schemas"]["PastedDataElement"] + | components["schemas"]["UrlDataElement"] + | components["schemas"]["PathDataElement"] + | components["schemas"]["ServerDirElement"] + | components["schemas"]["FtpImportElement"] + | components["schemas"]["CompositeDataElement"] + ) | components["schemas"]["NestedElement"] )[]; elements_from?: components["schemas"]["ElementsFromType"] | null; @@ -13996,6 +14133,8 @@ export interface components { */ ext: string; extra_files?: components["schemas"]["ExtraFiles"] | null; + /** Hashes */ + hashes?: components["schemas"]["FetchDatasetHash"][] | null; /** Info */ info?: string | null; /** Name */ @@ -14122,16 +14261,16 @@ export interface components { * @description The category of the notification. Represents the type of the notification. E.g. 'message' or 'new_shared_item'. */ category: - | components["schemas"]["MandatoryNotificationCategory"] - | components["schemas"]["PersonalNotificationCategory"]; + | components["schemas"]["MandatoryNotificationCategory"] + | components["schemas"]["PersonalNotificationCategory"]; /** * Content * @description The content of the notification. The structure depends on the category. */ content: - | components["schemas"]["MessageNotificationContent"] - | components["schemas"]["NewSharedItemNotificationContent"] - | components["schemas"]["BroadcastNotificationContent"]; + | components["schemas"]["MessageNotificationContent"] + | components["schemas"]["NewSharedItemNotificationContent"] + | components["schemas"]["BroadcastNotificationContent"]; /** * Expiration time * @description The time when the notification should expire. By default it will expire after 6 months. Expired notifications will be permanently deleted. @@ -14210,16 +14349,16 @@ export interface components { * @description The category of the notification. Represents the type of the notification. E.g. 'message' or 'new_shared_item'. */ category: - | components["schemas"]["MandatoryNotificationCategory"] - | components["schemas"]["PersonalNotificationCategory"]; + | components["schemas"]["MandatoryNotificationCategory"] + | components["schemas"]["PersonalNotificationCategory"]; /** * Content * @description The content of the notification. The structure depends on the category. */ content: - | components["schemas"]["MessageNotificationContent"] - | components["schemas"]["NewSharedItemNotificationContent"] - | components["schemas"]["BroadcastNotificationContent"]; + | components["schemas"]["MessageNotificationContent"] + | components["schemas"]["NewSharedItemNotificationContent"] + | components["schemas"]["BroadcastNotificationContent"]; /** * Create time * Format: date-time @@ -14369,13 +14508,13 @@ export interface components { type: "aws_s3" | "azure_blob" | "boto3" | "disk" | "generic_s3" | "onedata"; /** Variables */ variables?: - | ( - | components["schemas"]["TemplateVariableString"] - | components["schemas"]["TemplateVariableInteger"] - | components["schemas"]["TemplateVariablePathComponent"] - | components["schemas"]["TemplateVariableBoolean"] - )[] - | null; + | ( + | components["schemas"]["TemplateVariableString"] + | components["schemas"]["TemplateVariableInteger"] + | components["schemas"]["TemplateVariablePathComponent"] + | components["schemas"]["TemplateVariableBoolean"] + )[] + | null; /** * Version * @default 0 @@ -14598,6 +14737,12 @@ export interface components { PastedDataElement: { /** Md5 */ MD5?: string | null; + /** Sha-1 */ + "SHA-1"?: string | null; + /** Sha-256 */ + "SHA-256"?: string | null; + /** Sha-512 */ + "SHA-512"?: string | null; /** * Auto Decompress * @description Decompress compressed data before sniffing? @@ -14627,6 +14772,8 @@ export interface components { */ ext: string; extra_files?: components["schemas"]["ExtraFiles"] | null; + /** Hashes */ + hashes?: components["schemas"]["FetchDatasetHash"][] | null; /** Info */ info?: string | null; /** Name */ @@ -14658,6 +14805,12 @@ export interface components { PathDataElement: { /** Md5 */ MD5?: string | null; + /** Sha-1 */ + "SHA-1"?: string | null; + /** Sha-256 */ + "SHA-256"?: string | null; + /** Sha-512 */ + "SHA-512"?: string | null; /** * Auto Decompress * @description Decompress compressed data before sniffing? @@ -14687,6 +14840,8 @@ export interface components { */ ext: string; extra_files?: components["schemas"]["ExtraFiles"] | null; + /** Hashes */ + hashes?: components["schemas"]["FetchDatasetHash"][] | null; /** Info */ info?: string | null; /** Link Data Only */ @@ -15018,26 +15173,26 @@ export interface components { RefactorActionExecution: { /** Action */ action: - | components["schemas"]["AddInputAction"] - | components["schemas"]["AddStepAction"] - | components["schemas"]["ConnectAction"] - | components["schemas"]["DisconnectAction"] - | components["schemas"]["ExtractInputAction"] - | components["schemas"]["ExtractUntypedParameter"] - | components["schemas"]["FileDefaultsAction"] - | components["schemas"]["FillStepDefaultsAction"] - | components["schemas"]["UpdateAnnotationAction"] - | components["schemas"]["UpdateCreatorAction"] - | components["schemas"]["UpdateNameAction"] - | components["schemas"]["UpdateLicenseAction"] - | components["schemas"]["UpdateOutputLabelAction"] - | components["schemas"]["UpdateReportAction"] - | components["schemas"]["UpdateStepLabelAction"] - | components["schemas"]["UpdateStepPositionAction"] - | components["schemas"]["UpgradeSubworkflowAction"] - | components["schemas"]["UpgradeToolAction"] - | components["schemas"]["UpgradeAllStepsAction"] - | components["schemas"]["RemoveUnlabeledWorkflowOutputs"]; + | components["schemas"]["AddInputAction"] + | components["schemas"]["AddStepAction"] + | components["schemas"]["ConnectAction"] + | components["schemas"]["DisconnectAction"] + | components["schemas"]["ExtractInputAction"] + | components["schemas"]["ExtractUntypedParameter"] + | components["schemas"]["FileDefaultsAction"] + | components["schemas"]["FillStepDefaultsAction"] + | components["schemas"]["UpdateAnnotationAction"] + | components["schemas"]["UpdateCreatorAction"] + | components["schemas"]["UpdateNameAction"] + | components["schemas"]["UpdateLicenseAction"] + | components["schemas"]["UpdateOutputLabelAction"] + | components["schemas"]["UpdateReportAction"] + | components["schemas"]["UpdateStepLabelAction"] + | components["schemas"]["UpdateStepPositionAction"] + | components["schemas"]["UpgradeSubworkflowAction"] + | components["schemas"]["UpgradeToolAction"] + | components["schemas"]["UpgradeAllStepsAction"] + | components["schemas"]["RemoveUnlabeledWorkflowOutputs"]; /** Messages */ messages: components["schemas"]["RefactorActionExecutionMessage"][]; }; @@ -15108,10 +15263,10 @@ export interface components { * @enum {string} */ RefactorActionExecutionMessageTypeEnum: - | "tool_version_change" - | "tool_state_adjustment" - | "connection_drop_forced" - | "workflow_output_drop_forced"; + | "tool_version_change" + | "tool_state_adjustment" + | "connection_drop_forced" + | "workflow_output_drop_forced"; /** RefactorRequest */ RefactorRequest: { /** Actions */ @@ -15278,14 +15433,14 @@ export interface components { * @enum {string} */ RequestDataType: - | "state" - | "converted_datasets_state" - | "data" - | "features" - | "raw_data" - | "track_config" - | "genome_data" - | "in_use_state"; + | "state" + | "converted_datasets_state" + | "data" + | "features" + | "raw_data" + | "track_config" + | "genome_data" + | "in_use_state"; /** * Requirement * @description Available types of job sources (model classes) that produce dataset collections. @@ -15379,6 +15534,12 @@ export interface components { ServerDirElement: { /** Md5 */ MD5?: string | null; + /** Sha-1 */ + "SHA-1"?: string | null; + /** Sha-256 */ + "SHA-256"?: string | null; + /** Sha-512 */ + "SHA-512"?: string | null; /** * Auto Decompress * @description Decompress compressed data before sniffing? @@ -15408,6 +15569,8 @@ export interface components { */ ext: string; extra_files?: components["schemas"]["ExtraFiles"] | null; + /** Hashes */ + hashes?: components["schemas"]["FetchDatasetHash"][] | null; /** Info */ info?: string | null; /** Link Data Only */ @@ -16038,8 +16201,8 @@ export interface components { * @description Additional information about the creator (or multiple creators) of this workflow. */ creator?: - | (components["schemas"]["Person"] | components["schemas"]["galaxy__schema__schema__Organization"])[] - | null; + | (components["schemas"]["Person"] | components["schemas"]["galaxy__schema__schema__Organization"])[] + | null; /** * Deleted * @description Whether this item is marked as deleted. @@ -16132,12 +16295,12 @@ export interface components { */ steps: { [key: string]: - | components["schemas"]["InputDataStep"] - | components["schemas"]["InputDataCollectionStep"] - | components["schemas"]["InputParameterStep"] - | components["schemas"]["PauseStep"] - | components["schemas"]["ToolStep"] - | components["schemas"]["SubworkflowStep"]; + | components["schemas"]["InputDataStep"] + | components["schemas"]["InputDataCollectionStep"] + | components["schemas"]["InputParameterStep"] + | components["schemas"]["PauseStep"] + | components["schemas"]["ToolStep"] + | components["schemas"]["SubworkflowStep"]; }; tags: components["schemas"]["TagCollection"]; /** @@ -16256,13 +16419,13 @@ export interface components { * @enum {string} */ TaggableItemClass: - | "History" - | "HistoryDatasetAssociation" - | "HistoryDatasetCollectionAssociation" - | "LibraryDatasetDatasetAssociation" - | "Page" - | "StoredWorkflow" - | "Visualization"; + | "History" + | "HistoryDatasetAssociation" + | "HistoryDatasetCollectionAssociation" + | "LibraryDatasetDatasetAssociation" + | "Page" + | "StoredWorkflow" + | "Visualization"; /** * TaskState * @description Enum representing the possible states of a task. @@ -16870,8 +17033,8 @@ export interface components { action_type: "update_output_label"; /** Output */ output: - | components["schemas"]["OutputReferenceByOrderIndex"] - | components["schemas"]["OutputReferenceByLabel"]; + | components["schemas"]["OutputReferenceByOrderIndex"] + | components["schemas"]["OutputReferenceByLabel"]; /** Output Label */ output_label: string; }; @@ -17028,6 +17191,12 @@ export interface components { UrlDataElement: { /** Md5 */ MD5?: string | null; + /** Sha-1 */ + "SHA-1"?: string | null; + /** Sha-256 */ + "SHA-256"?: string | null; + /** Sha-512 */ + "SHA-512"?: string | null; /** * Auto Decompress * @description Decompress compressed data before sniffing? @@ -17057,6 +17226,8 @@ export interface components { */ ext: string; extra_files?: components["schemas"]["ExtraFiles"] | null; + /** Hashes */ + hashes?: components["schemas"]["FetchDatasetHash"][] | null; /** Info */ info?: string | null; /** Name */ @@ -17285,8 +17456,8 @@ export interface components { * @description The content of the notification. The structure depends on the category. */ content: - | components["schemas"]["MessageNotificationContent"] - | components["schemas"]["NewSharedItemNotificationContent"]; + | components["schemas"]["MessageNotificationContent"] + | components["schemas"]["NewSharedItemNotificationContent"]; /** * Create time * Format: date-time @@ -17924,8 +18095,8 @@ export interface components { }; /** WorkflowInvocationResponse */ WorkflowInvocationResponse: - | components["schemas"]["WorkflowInvocationElementView"] - | components["schemas"]["WorkflowInvocationCollectionView"]; + | components["schemas"]["WorkflowInvocationElementView"] + | components["schemas"]["WorkflowInvocationCollectionView"]; /** WorkflowInvocationStateSummary */ WorkflowInvocationStateSummary: { /** @@ -17952,6 +18123,25 @@ export interface components { [key: string]: number; }; }; + /** WorkflowLandingRequest */ + WorkflowLandingRequest: { + /** Request State */ + request_state: Record; + state: components["schemas"]["LandingRequestState"]; + /** + * UUID + * Format: uuid4 + * @description Universal unique identifier for this dataset. + */ + uuid: string; + /** Workflow Id */ + workflow_id: string; + /** + * Workflow Target Type + * @enum {string} + */ + workflow_target_type: "stored_workflow" | "workflow"; + }; /** WriteInvocationStoreToPayload */ WriteInvocationStoreToPayload: { /** @@ -18597,9 +18787,9 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["HDCACustom"] - | components["schemas"]["HDCADetailed"] - | components["schemas"]["HDCASummary"]; + | components["schemas"]["HDCACustom"] + | components["schemas"]["HDCADetailed"] + | components["schemas"]["HDCASummary"]; }; }; /** @description Request Error */ @@ -19054,13 +19244,13 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["HDACustom"] - | components["schemas"]["HDADetailed"] - | components["schemas"]["HDASummary"] - | components["schemas"]["HDAInaccessible"] - | components["schemas"]["HDCACustom"] - | components["schemas"]["HDCADetailed"] - | components["schemas"]["HDCASummary"]; + | components["schemas"]["HDACustom"] + | components["schemas"]["HDADetailed"] + | components["schemas"]["HDASummary"] + | components["schemas"]["HDAInaccessible"] + | components["schemas"]["HDCACustom"] + | components["schemas"]["HDCADetailed"] + | components["schemas"]["HDCASummary"]; }; }; /** @description Request Error */ @@ -19278,10 +19468,10 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["HDACustom"] - | components["schemas"]["HDADetailed"] - | components["schemas"]["HDASummary"] - | components["schemas"]["HDAInaccessible"]; + | components["schemas"]["HDACustom"] + | components["schemas"]["HDADetailed"] + | components["schemas"]["HDASummary"] + | components["schemas"]["HDAInaccessible"]; }; }; /** @description Request Error */ @@ -19648,9 +19838,9 @@ export interface operations { requestBody: { content: { "application/json": - | components["schemas"]["UpdateDatasetPermissionsPayload"] - | components["schemas"]["UpdateDatasetPermissionsPayloadAliasB"] - | components["schemas"]["UpdateDatasetPermissionsPayloadAliasC"]; + | components["schemas"]["UpdateDatasetPermissionsPayload"] + | components["schemas"]["UpdateDatasetPermissionsPayloadAliasB"] + | components["schemas"]["UpdateDatasetPermissionsPayloadAliasC"]; }; }; responses: { @@ -20608,9 +20798,9 @@ export interface operations { requestBody: { content: { "application/json": - | components["schemas"]["UpdateInstanceSecretPayload"] - | components["schemas"]["UpgradeInstancePayload"] - | components["schemas"]["UpdateInstancePayload"]; + | components["schemas"]["UpdateInstanceSecretPayload"] + | components["schemas"]["UpgradeInstancePayload"] + | components["schemas"]["UpdateInstancePayload"]; }; }; responses: { @@ -21097,8 +21287,8 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["LibraryFolderCurrentPermissions"] - | components["schemas"]["LibraryAvailablePermissions"]; + | components["schemas"]["LibraryFolderCurrentPermissions"] + | components["schemas"]["LibraryAvailablePermissions"]; }; }; /** @description Request Error */ @@ -21296,8 +21486,8 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["ListUriResponse"] - | components["schemas"]["ListJstreeResponse"]; + | components["schemas"]["ListUriResponse"] + | components["schemas"]["ListJstreeResponse"]; }; }; /** @description Request Error */ @@ -22514,10 +22704,10 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["JobImportHistoryResponse"] - | components["schemas"]["CustomHistoryView"] - | components["schemas"]["HistoryDetailed"] - | components["schemas"]["HistorySummary"]; + | components["schemas"]["JobImportHistoryResponse"] + | components["schemas"]["CustomHistoryView"] + | components["schemas"]["HistoryDetailed"] + | components["schemas"]["HistorySummary"]; }; }; /** @description Request Error */ @@ -22839,9 +23029,9 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["CustomHistoryView"] - | components["schemas"]["HistoryDetailed"] - | components["schemas"]["HistorySummary"]; + | components["schemas"]["CustomHistoryView"] + | components["schemas"]["HistoryDetailed"] + | components["schemas"]["HistorySummary"]; }; }; /** @description Request Error */ @@ -22892,9 +23082,9 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["CustomHistoryView"] - | components["schemas"]["HistoryDetailed"] - | components["schemas"]["HistorySummary"]; + | components["schemas"]["CustomHistoryView"] + | components["schemas"]["HistoryDetailed"] + | components["schemas"]["HistorySummary"]; }; }; /** @description Request Error */ @@ -22986,9 +23176,9 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["CustomHistoryView"] - | components["schemas"]["HistoryDetailed"] - | components["schemas"]["HistorySummary"]; + | components["schemas"]["CustomHistoryView"] + | components["schemas"]["HistoryDetailed"] + | components["schemas"]["HistorySummary"]; }; }; /** @description Request Error */ @@ -23158,9 +23348,9 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["CustomHistoryView"] - | components["schemas"]["HistoryDetailed"] - | components["schemas"]["HistorySummary"]; + | components["schemas"]["CustomHistoryView"] + | components["schemas"]["HistoryDetailed"] + | components["schemas"]["HistorySummary"]; }; }; /** @description Request Error */ @@ -23214,9 +23404,9 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["CustomHistoryView"] - | components["schemas"]["HistoryDetailed"] - | components["schemas"]["HistorySummary"]; + | components["schemas"]["CustomHistoryView"] + | components["schemas"]["HistoryDetailed"] + | components["schemas"]["HistorySummary"]; }; }; /** @description Request Error */ @@ -23271,9 +23461,9 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["CustomHistoryView"] - | components["schemas"]["HistoryDetailed"] - | components["schemas"]["HistorySummary"]; + | components["schemas"]["CustomHistoryView"] + | components["schemas"]["HistoryDetailed"] + | components["schemas"]["HistorySummary"]; }; }; /** @description Request Error */ @@ -23322,9 +23512,9 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["CustomArchivedHistoryView"] - | components["schemas"]["ArchivedHistoryDetailed"] - | components["schemas"]["ArchivedHistorySummary"]; + | components["schemas"]["CustomArchivedHistoryView"] + | components["schemas"]["ArchivedHistoryDetailed"] + | components["schemas"]["ArchivedHistorySummary"]; }; }; /** @description Request Error */ @@ -23372,9 +23562,9 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["CustomHistoryView"] - | components["schemas"]["HistoryDetailed"] - | components["schemas"]["HistorySummary"]; + | components["schemas"]["CustomHistoryView"] + | components["schemas"]["HistoryDetailed"] + | components["schemas"]["HistorySummary"]; }; }; /** @description Request Error */ @@ -23618,6 +23808,14 @@ export interface operations { }; content: { "application/json": + | components["schemas"]["HDACustom"] + | components["schemas"]["HDADetailed"] + | components["schemas"]["HDASummary"] + | components["schemas"]["HDAInaccessible"] + | components["schemas"]["HDCACustom"] + | components["schemas"]["HDCADetailed"] + | components["schemas"]["HDCASummary"] + | ( | components["schemas"]["HDACustom"] | components["schemas"]["HDADetailed"] | components["schemas"]["HDASummary"] @@ -23625,15 +23823,7 @@ export interface operations { | components["schemas"]["HDCACustom"] | components["schemas"]["HDCADetailed"] | components["schemas"]["HDCASummary"] - | ( - | components["schemas"]["HDACustom"] - | components["schemas"]["HDADetailed"] - | components["schemas"]["HDASummary"] - | components["schemas"]["HDAInaccessible"] - | components["schemas"]["HDCACustom"] - | components["schemas"]["HDCADetailed"] - | components["schemas"]["HDCASummary"] - )[]; + )[]; }; }; /** @description Request Error */ @@ -23891,7 +24081,11 @@ export interface operations { }; cookie?: never; }; - requestBody?: never; + requestBody?: { + content: { + "application/json": components["schemas"]["MaterializeDatasetOptions"] | null; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -23940,9 +24134,9 @@ export interface operations { requestBody: { content: { "application/json": - | components["schemas"]["UpdateDatasetPermissionsPayload"] - | components["schemas"]["UpdateDatasetPermissionsPayloadAliasB"] - | components["schemas"]["UpdateDatasetPermissionsPayloadAliasC"]; + | components["schemas"]["UpdateDatasetPermissionsPayload"] + | components["schemas"]["UpdateDatasetPermissionsPayloadAliasB"] + | components["schemas"]["UpdateDatasetPermissionsPayloadAliasC"]; }; }; responses: { @@ -24447,13 +24641,13 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["HDACustom"] - | components["schemas"]["HDADetailed"] - | components["schemas"]["HDASummary"] - | components["schemas"]["HDAInaccessible"] - | components["schemas"]["HDCACustom"] - | components["schemas"]["HDCADetailed"] - | components["schemas"]["HDCASummary"]; + | components["schemas"]["HDACustom"] + | components["schemas"]["HDADetailed"] + | components["schemas"]["HDASummary"] + | components["schemas"]["HDAInaccessible"] + | components["schemas"]["HDCACustom"] + | components["schemas"]["HDCADetailed"] + | components["schemas"]["HDCASummary"]; }; }; /** @description Request Error */ @@ -24511,13 +24705,13 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["HDACustom"] - | components["schemas"]["HDADetailed"] - | components["schemas"]["HDASummary"] - | components["schemas"]["HDAInaccessible"] - | components["schemas"]["HDCACustom"] - | components["schemas"]["HDCADetailed"] - | components["schemas"]["HDCASummary"]; + | components["schemas"]["HDACustom"] + | components["schemas"]["HDADetailed"] + | components["schemas"]["HDASummary"] + | components["schemas"]["HDAInaccessible"] + | components["schemas"]["HDCACustom"] + | components["schemas"]["HDCADetailed"] + | components["schemas"]["HDCASummary"]; }; }; /** @description Request Error */ @@ -24793,6 +24987,14 @@ export interface operations { }; content: { "application/json": + | components["schemas"]["HDACustom"] + | components["schemas"]["HDADetailed"] + | components["schemas"]["HDASummary"] + | components["schemas"]["HDAInaccessible"] + | components["schemas"]["HDCACustom"] + | components["schemas"]["HDCADetailed"] + | components["schemas"]["HDCASummary"] + | ( | components["schemas"]["HDACustom"] | components["schemas"]["HDADetailed"] | components["schemas"]["HDASummary"] @@ -24800,15 +25002,7 @@ export interface operations { | components["schemas"]["HDCACustom"] | components["schemas"]["HDCADetailed"] | components["schemas"]["HDCASummary"] - | ( - | components["schemas"]["HDACustom"] - | components["schemas"]["HDADetailed"] - | components["schemas"]["HDASummary"] - | components["schemas"]["HDAInaccessible"] - | components["schemas"]["HDCACustom"] - | components["schemas"]["HDCADetailed"] - | components["schemas"]["HDCASummary"] - )[]; + )[]; }; }; /** @description Request Error */ @@ -24864,13 +25058,13 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["HDACustom"] - | components["schemas"]["HDADetailed"] - | components["schemas"]["HDASummary"] - | components["schemas"]["HDAInaccessible"] - | components["schemas"]["HDCACustom"] - | components["schemas"]["HDCADetailed"] - | components["schemas"]["HDCASummary"]; + | components["schemas"]["HDACustom"] + | components["schemas"]["HDADetailed"] + | components["schemas"]["HDASummary"] + | components["schemas"]["HDAInaccessible"] + | components["schemas"]["HDCACustom"] + | components["schemas"]["HDCADetailed"] + | components["schemas"]["HDCASummary"]; }; }; /** @description Request Error */ @@ -24928,13 +25122,13 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["HDACustom"] - | components["schemas"]["HDADetailed"] - | components["schemas"]["HDASummary"] - | components["schemas"]["HDAInaccessible"] - | components["schemas"]["HDCACustom"] - | components["schemas"]["HDCADetailed"] - | components["schemas"]["HDCASummary"]; + | components["schemas"]["HDACustom"] + | components["schemas"]["HDADetailed"] + | components["schemas"]["HDASummary"] + | components["schemas"]["HDAInaccessible"] + | components["schemas"]["HDCACustom"] + | components["schemas"]["HDCADetailed"] + | components["schemas"]["HDCASummary"]; }; }; /** @description Request Error */ @@ -25064,9 +25258,9 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["JobStateSummary"] - | components["schemas"]["ImplicitCollectionJobsStateSummary"] - | components["schemas"]["WorkflowInvocationStateSummary"]; + | components["schemas"]["JobStateSummary"] + | components["schemas"]["ImplicitCollectionJobsStateSummary"] + | components["schemas"]["WorkflowInvocationStateSummary"]; }; }; /** @description Request Error */ @@ -25464,8 +25658,8 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["JobExportHistoryArchiveModel"] - | components["schemas"]["JobIdResponse"]; + | components["schemas"]["JobExportHistoryArchiveModel"] + | components["schemas"]["JobIdResponse"]; }; }; /** @description The exported archive file is not ready yet. */ @@ -27088,8 +27282,8 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["ShowFullJobResponse"] - | components["schemas"]["EncodedJobDetails"]; + | components["schemas"]["ShowFullJobResponse"] + | components["schemas"]["EncodedJobDetails"]; }; }; /** @description Request Error */ @@ -27927,8 +28121,8 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["LibraryCurrentPermissions"] - | components["schemas"]["LibraryAvailablePermissions"]; + | components["schemas"]["LibraryCurrentPermissions"] + | components["schemas"]["LibraryAvailablePermissions"]; }; }; /** @description Request Error */ @@ -27970,8 +28164,8 @@ export interface operations { requestBody: { content: { "application/json": - | components["schemas"]["LibraryPermissionsPayload"] - | components["schemas"]["LegacyLibraryPermissionsPayload"]; + | components["schemas"]["LibraryPermissionsPayload"] + | components["schemas"]["LegacyLibraryPermissionsPayload"]; }; }; responses: { @@ -27982,8 +28176,8 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["LibraryLegacySummary"] - | components["schemas"]["LibraryCurrentPermissions"]; + | components["schemas"]["LibraryLegacySummary"] + | components["schemas"]["LibraryCurrentPermissions"]; }; }; /** @description Request Error */ @@ -28074,10 +28268,10 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["LibraryContentsCreateFolderListResponse"] - | components["schemas"]["LibraryContentsCreateFileListResponse"] - | components["schemas"]["LibraryContentsCreateDatasetCollectionResponse"] - | components["schemas"]["LibraryContentsCreateDatasetResponse"]; + | components["schemas"]["LibraryContentsCreateFolderListResponse"] + | components["schemas"]["LibraryContentsCreateFileListResponse"] + | components["schemas"]["LibraryContentsCreateDatasetCollectionResponse"] + | components["schemas"]["LibraryContentsCreateDatasetResponse"]; }; }; /** @description Request Error */ @@ -28123,8 +28317,8 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["LibraryContentsShowFolderResponse"] - | components["schemas"]["LibraryContentsShowDatasetResponse"]; + | components["schemas"]["LibraryContentsShowFolderResponse"] + | components["schemas"]["LibraryContentsShowDatasetResponse"]; }; }; /** @description Request Error */ @@ -28477,8 +28671,8 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["NotificationCreatedResponse"] - | components["schemas"]["AsyncTaskResultSummary"]; + | components["schemas"]["NotificationCreatedResponse"] + | components["schemas"]["AsyncTaskResultSummary"]; }; }; /** @description Request Error */ @@ -29174,9 +29368,9 @@ export interface operations { requestBody: { content: { "application/json": - | components["schemas"]["UpdateInstanceSecretPayload"] - | components["schemas"]["UpgradeInstancePayload"] - | components["schemas"]["UpdateInstancePayload"]; + | components["schemas"]["UpdateInstanceSecretPayload"] + | components["schemas"]["UpgradeInstancePayload"] + | components["schemas"]["UpdateInstancePayload"]; }; }; responses: { @@ -30500,8 +30694,8 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["ListUriResponse"] - | components["schemas"]["ListJstreeResponse"]; + | components["schemas"]["ListUriResponse"] + | components["schemas"]["ListJstreeResponse"]; }; }; /** @description Request Error */ @@ -32078,8 +32272,8 @@ export interface operations { requestBody: { content: { "application/json": - | components["schemas"]["UserCreationPayload"] - | components["schemas"]["RemoteUserCreationPayload"]; + | components["schemas"]["UserCreationPayload"] + | components["schemas"]["RemoteUserCreationPayload"]; }; }; responses: { @@ -32233,8 +32427,8 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["DetailedUserModel"] - | components["schemas"]["AnonUserModel"]; + | components["schemas"]["DetailedUserModel"] + | components["schemas"]["AnonUserModel"]; }; }; /** @description Request Error */ @@ -32374,8 +32568,8 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["DetailedUserModel"] - | components["schemas"]["AnonUserModel"]; + | components["schemas"]["DetailedUserModel"] + | components["schemas"]["AnonUserModel"]; }; }; /** @description Request Error */ @@ -33912,6 +34106,143 @@ export interface operations { }; }; }; + create_landing_api_workflow_landings_post: { + parameters: { + query?: never; + header?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + "run-as"?: string | null; + }; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CreateWorkflowLandingRequestPayload"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["WorkflowLandingRequest"]; + }; + }; + /** @description Request Error */ + "4XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + /** @description Server Error */ + "5XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + }; + }; + get_landing_api_workflow_landings__uuid__get: { + parameters: { + query?: never; + header?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + "run-as"?: string | null; + }; + path: { + /** @description The UUID used to identify a persisted landing request. */ + uuid: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["WorkflowLandingRequest"]; + }; + }; + /** @description Request Error */ + "4XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + /** @description Server Error */ + "5XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + }; + }; + claim_landing_api_workflow_landings__uuid__claim_post: { + parameters: { + query?: never; + header?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + "run-as"?: string | null; + }; + path: { + /** @description The UUID used to identify a persisted landing request. */ + uuid: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ClaimLandingPayload"] | null; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["WorkflowLandingRequest"]; + }; + }; + /** @description Request Error */ + "4XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + /** @description Server Error */ + "5XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + }; + }; index_api_workflows_get: { parameters: { query?: { @@ -34391,8 +34722,8 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["WorkflowInvocationResponse"] - | components["schemas"]["WorkflowInvocationResponse"][]; + | components["schemas"]["WorkflowInvocationResponse"] + | components["schemas"]["WorkflowInvocationResponse"][]; }; }; /** @description Request Error */ @@ -35449,8 +35780,8 @@ export interface operations { }; content: { "application/json": - | components["schemas"]["WorkflowInvocationResponse"] - | components["schemas"]["WorkflowInvocationResponse"][]; + | components["schemas"]["WorkflowInvocationResponse"] + | components["schemas"]["WorkflowInvocationResponse"][]; }; }; /** @description Request Error */ diff --git a/client/src/components/Form/Elements/FormData/FormDataUri.vue b/client/src/components/Form/Elements/FormData/FormDataUri.vue new file mode 100644 index 000000000000..0c888fbc4fce --- /dev/null +++ b/client/src/components/Form/Elements/FormData/FormDataUri.vue @@ -0,0 +1,66 @@ + + + + diff --git a/client/src/components/Form/FormElement.vue b/client/src/components/Form/FormElement.vue index 72af07dd7f8d..ed81622799e1 100644 --- a/client/src/components/Form/FormElement.vue +++ b/client/src/components/Form/FormElement.vue @@ -14,6 +14,7 @@ import type { FormParameterAttributes, FormParameterTypes, FormParameterValue } import FormBoolean from "./Elements/FormBoolean.vue"; import FormColor from "./Elements/FormColor.vue"; import FormData from "./Elements/FormData/FormData.vue"; +import FormDataUri from "./Elements/FormData/FormDataUri.vue"; import FormDataDialog from "./Elements/FormDataDialog.vue"; import FormDirectory from "./Elements/FormDirectory.vue"; import FormDrilldown from "./Elements/FormDrilldown/FormDrilldown.vue"; @@ -130,6 +131,14 @@ const elementId = computed(() => `form-element-${props.id}`); const hasAlert = computed(() => alerts.value.length > 0); const showPreview = computed(() => (collapsed.value && attrs.value["collapsible_preview"]) || props.disabled); const showField = computed(() => !collapsed.value && !props.disabled); +const isUriDataField = computed(() => { + const dataField = props.type == "data"; + if (dataField && props.value && "src" in props.value) { + const src = props.value.src; + return src == "url"; + } + return false; +}); const previewText = computed(() => attrs.value["text_value"]); const helpText = computed(() => { @@ -285,6 +294,12 @@ function onAlert(value: string | undefined) { :options="attrs.options" :optional="attrs.optional" :multiple="attrs.multiple" /> + + + diff --git a/client/src/components/Landing/WorkflowLanding.vue b/client/src/components/Landing/WorkflowLanding.vue new file mode 100644 index 000000000000..b5f7e2126809 --- /dev/null +++ b/client/src/components/Landing/WorkflowLanding.vue @@ -0,0 +1,61 @@ + + + diff --git a/client/src/components/Markdown/MarkdownVisualization.test.js b/client/src/components/Markdown/MarkdownVisualization.test.js index 55ab54943181..da48cf495b37 100644 --- a/client/src/components/Markdown/MarkdownVisualization.test.js +++ b/client/src/components/Markdown/MarkdownVisualization.test.js @@ -9,7 +9,7 @@ describe("Markdown/MarkdownVisualization", () => { argumentName: "name", argumentPayload: { settings: [{}, {}], - groups: [{}], + tracks: [{}], }, history: "history_id", labels: [], diff --git a/client/src/components/Markdown/MarkdownVisualization.vue b/client/src/components/Markdown/MarkdownVisualization.vue index 614cb1725fab..02fd69bb6439 100644 --- a/client/src/components/Markdown/MarkdownVisualization.vue +++ b/client/src/components/Markdown/MarkdownVisualization.vue @@ -73,14 +73,14 @@ export default { if (this.argumentPayload.settings && this.argumentPayload.settings.length > 0) { settings = this.argumentPayload.settings.slice(); } - if (this.argumentPayload.groups && this.argumentPayload.groups.length > 0) { + if (this.argumentPayload.tracks && this.argumentPayload.tracks.length > 0) { settings = settings || []; settings.push({ type: "repeat", title: "Columns", - name: "groups", + name: "tracks", min: 1, - inputs: this.argumentPayload.groups.map((x) => { + inputs: this.argumentPayload.tracks.map((x) => { if (x.type == "data_column") { x.is_workflow = true; } diff --git a/client/src/components/Workflow/Run/WorkflowRun.vue b/client/src/components/Workflow/Run/WorkflowRun.vue index 4190257e3cdd..5e8d9dea8bb6 100644 --- a/client/src/components/Workflow/Run/WorkflowRun.vue +++ b/client/src/components/Workflow/Run/WorkflowRun.vue @@ -30,6 +30,7 @@ interface Props { preferSimpleForm?: boolean; simpleFormTargetHistory?: string; simpleFormUseJobCache?: boolean; + requestState?: Record; } const props = withDefaults(defineProps(), { @@ -37,6 +38,7 @@ const props = withDefaults(defineProps(), { preferSimpleForm: false, simpleFormTargetHistory: "current", simpleFormUseJobCache: false, + requestState: undefined, }); const loading = ref(true); @@ -200,6 +202,7 @@ defineExpose({ :target-history="simpleFormTargetHistory" :use-job-cache="simpleFormUseJobCache" :can-mutate-current-history="canRunOnHistory" + :request-state="requestState" @submissionSuccess="handleInvocations" @submissionError="handleSubmissionError" @showAdvanced="showAdvanced" /> diff --git a/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue b/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue index e7e45723ff16..feb1bb1b2ac2 100644 --- a/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue +++ b/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue @@ -99,6 +99,10 @@ export default { type: Boolean, required: true, }, + requestState: { + type: Object, + required: false, + }, }, setup() { const { config, isConfigLoaded } = useConfig(true); @@ -135,6 +139,7 @@ export default { if (isWorkflowInput(step.step_type)) { const stepName = new String(step.step_index); const stepLabel = step.step_label || new String(step.step_index + 1); + const stepType = step.step_type; const help = step.annotation; const longFormInput = step.inputs[0]; const stepAsInput = Object.assign({}, longFormInput, { @@ -142,10 +147,14 @@ export default { help: help, label: stepLabel, }); + if (this.requestState && this.requestState[stepLabel]) { + const value = this.requestState[stepLabel]; + stepAsInput.value = value; + } // disable collection mapping... stepAsInput.flavor = "module"; inputs.push(stepAsInput); - this.inputTypes[stepName] = step.step_type; + this.inputTypes[stepName] = stepType; } }); return inputs; diff --git a/client/src/entry/analysis/router.js b/client/src/entry/analysis/router.js index 0128829043b9..18024224e9d3 100644 --- a/client/src/entry/analysis/router.js +++ b/client/src/entry/analysis/router.js @@ -17,6 +17,8 @@ import HistoryImport from "components/HistoryImport"; import InteractiveTools from "components/InteractiveTools/InteractiveTools"; import JobDetails from "components/JobInformation/JobDetails"; import CarbonEmissionsCalculations from "components/JobMetrics/CarbonEmissions/CarbonEmissionsCalculations"; +import ToolLanding from "components/Landing/ToolLanding"; +import WorkflowLanding from "components/Landing/WorkflowLanding"; import PageDisplay from "components/PageDisplay/PageDisplay"; import PageEditor from "components/PageEditor/PageEditor"; import ToolSuccess from "components/Tool/ToolSuccess"; @@ -494,6 +496,16 @@ export function getRouter(Galaxy) { path: "tools/json", component: ToolsJson, }, + { + path: "tool_landings/:uuid", + component: ToolLanding, + props: true, + }, + { + path: "workflow_landings/:uuid", + component: WorkflowLanding, + props: true, + }, { path: "user", component: UserPreferences, diff --git a/client/src/mvc/visualization/chart/components/model.js b/client/src/mvc/visualization/chart/components/model.js index 458663122b1f..814bbafe6386 100644 --- a/client/src/mvc/visualization/chart/components/model.js +++ b/client/src/mvc/visualization/chart/components/model.js @@ -2,7 +2,7 @@ import Backbone from "backbone"; import { Visualization } from "mvc/visualization/visualization-model"; import Utils from "utils/utils"; -const MATCH_GROUP = /^groups_([0-9]+)\|([\w]+)/; +const MATCH_GROUP = /^(groups|tracks)_([0-9]+)\|([\w]+)/; export default Backbone.Model.extend({ defaults: { diff --git a/client/src/mvc/visualization/chart/views/editor.js b/client/src/mvc/visualization/chart/views/editor.js index 3c9c21bb3f32..3b0a3f026285 100644 --- a/client/src/mvc/visualization/chart/views/editor.js +++ b/client/src/mvc/visualization/chart/views/editor.js @@ -36,7 +36,7 @@ export default Backbone.View.extend({ ) .append(new Settings(this.app).$el), }); - if (this.chart.plugin.groups) { + if (this.chart.plugin.tracks) { this.tabs.add({ id: "groups", icon: "fa-database", diff --git a/client/src/mvc/visualization/chart/views/groups.js b/client/src/mvc/visualization/chart/views/groups.js index fa3f68f4f8be..b262c9968fbe 100644 --- a/client/src/mvc/visualization/chart/views/groups.js +++ b/client/src/mvc/visualization/chart/views/groups.js @@ -25,7 +25,7 @@ var GroupView = Backbone.View.extend({ }, render: function () { var self = this; - var inputs = Utils.clone(this.chart.plugin.groups) || []; + var inputs = Utils.clone(this.chart.plugin.tracks) || []; var dataset_id = this.chart.get("dataset_id"); if (dataset_id) { this.chart.state("wait", "Loading metadata..."); @@ -127,7 +127,7 @@ export default Backbone.View.extend({ }); }, render: function () { - if (_.size(this.chart.plugin.groups) > 0) { + if (_.size(this.chart.plugin.tracks) > 0) { this.repeat.$el.show(); } else { this.repeat.$el.hide(); diff --git a/client/src/nls/es/locale.js b/client/src/nls/es/locale.js index df68712e1d18..fbfaf079c891 100644 --- a/client/src/nls/es/locale.js +++ b/client/src/nls/es/locale.js @@ -608,7 +608,7 @@ define({ "Agregar o modificar la configuración que permite que Galaxy acceda a tus recursos en la nube", "Manage Toolbox Filters": "Manejar filtros de la caja de herramientas", - "Manage Custom Builds": "Administrar compilaciones personalizadas", + "Manage Custom Builds": "Administrar construcciones personalizadas", "Enable notifications": "Habilitar notificaciones", "Allow push and tab notifcations on job completion. To disable, revoke the site notification privilege in your browser.": diff --git a/doc/source/dev/tool_state_state_classes.plantuml.svg b/doc/source/dev/tool_state_state_classes.plantuml.svg index de86dbcd3787..28c7da1c9092 100644 --- a/doc/source/dev/tool_state_state_classes.plantuml.svg +++ b/doc/source/dev/tool_state_state_classes.plantuml.svg @@ -1,21 +1,17 @@ -galaxy.tool_util.parameters.stateToolStatestate_representation: strinput_state: Dict[str, Any]validate(input_models: ToolParameterBundle)_to_base_model(input_models: ToolParameterBundle): Optional[Type[BaseModel]]RequestToolStatestate_representation = "request"_to_base_model(input_models: ToolParameterBundle): Type[BaseModel]Object references of the form{src: "hda", id: <encoded_id>}.Allow mapping/reduce constructs.RequestInternalToolStatestate_representation = "request_internal"_to_base_model(input_models: ToolParameterBundle): Type[BaseModel]Object references of the form{src: "hda", id: <decoded_id>}.Allow mapping/reduce constructs.JobInternalToolStatestate_representation = "job_internal"_to_base_model(input_models: ToolParameterBundle): Type[BaseModel]Object references of the form{src: "hda", id: <decoded_id>}.Mapping constructs expanded out.(Defaults are inserted?)TestCaseToolStatestate_representation = "test_case"_to_base_model(input_models: ToolParameterBundle): Type[BaseModel]Object references of the form file name and URIs.Mapping constructs not allowed. WorkflowStepToolStatestate_representation = "workflow_step"_to_base_model(input_models: ToolParameterBundle): Type[BaseModel]Nearly everything optional except conditional discriminators. WorkflowStepLinkedToolStatestate_representation = "workflow_step_linked"_to_base_model(input_models: ToolParameterBundle): Type[BaseModel]Expect pre-process ``in`` dictionaries and bring in representationof links and defaults and validate them in model. decodeexpandpreprocess_links_and_defaultsgalaxy.tool_util.parameters.stateToolStatestate_representation: strinput_state: Dict[str, Any]validate(parameters: ToolParameterBundle)_to_base_model(parameters: ToolParameterBundle): Optional[Type[BaseModel]]RequestToolStatestate_representation = "request"_to_base_model(parameters: ToolParameterBundle): Type[BaseModel]Object references of the form{src: "hda", id: <encoded_id>}.Allow mapping/reduce constructs.RequestInternalToolStatestate_representation = "request_internal"_to_base_model(parameters: ToolParameterBundle): Type[BaseModel]Object references of the form{src: "hda", id: <decoded_id>}.Allow mapping/reduce constructs. Allows URI src dicts.RequestInternalDereferencedToolStatestate_representation = "request_internal"_to_base_model(parameters: ToolParameterBundle): Type[BaseModel]Object references of the form{src: "hda", id: <decoded_id>}.Allow mapping/reduce constructs. No URI src dicts - all converted to HDAs.JobInternalToolStatestate_representation = "job_internal"_to_base_model(parameters: ToolParameterBundle): Type[BaseModel]Object references of the form{src: "hda", id: <decoded_id>}.Mapping constructs expanded out.(Defaults are inserted?)decodedereferenceexpand \ No newline at end of file diff --git a/doc/source/dev/tool_state_task.plantuml.txt b/doc/source/dev/tool_state_task.plantuml.txt new file mode 100644 index 000000000000..64286f2f5ed5 --- /dev/null +++ b/doc/source/dev/tool_state_task.plantuml.txt @@ -0,0 +1,25 @@ +@startuml +'!include plantuml_options.txt +queue TaskQueue as queue +participant "queue_jobs Task" as task +participant "JobSubmitter.queue_jobs" as queue_jobs +participant "JobSubmitter.dereference" as dereference +participant "materialize Task" as materialize_task +participant "Tool.handle_input_async" as handle_input +participant "expand_meta_parameters_async" as expand +participant "ToolAction.execute" as tool_action + +queue -> task : +task -> queue_jobs : QueueJobs pydantic model +queue_jobs -> dereference : RequestInternalToolState +dereference -> queue_jobs : RequestInternalDereferencedToolState +queue_jobs -> materialize_task : HDA (with state deferred) +materialize_task -> queue_jobs : return when state is okay +queue_jobs -> handle_input : RequestInternalDereferencedToolState +handle_input -> expand : RequestInternalDereferencedToolState +expand -> handle_input : JobInternalToolState[] +loop over expanded job tool states + handle_input -> tool_action : + tool_action -> handle_input : A Galaxy Job +end +@enduml diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 2f67d6ec3b1b..fcbd1a1b7cb0 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -34,6 +34,7 @@ GalaxyTaskBeforeStartUserRateLimitPostgres, GalaxyTaskBeforeStartUserRateLimitStandard, ) +from galaxy.config import GalaxyAppConfiguration from galaxy.config_watchers import ConfigWatchers from galaxy.datatypes.registry import Registry from galaxy.files import ( @@ -206,7 +207,7 @@ def shutdown(self): class SentryClientMixin: - config: config.GalaxyAppConfiguration + config: GalaxyAppConfiguration application_stack: ApplicationStack def configure_sentry_client(self): @@ -263,7 +264,7 @@ class MinimalGalaxyApplication(BasicSharedApp, HaltableContainer, SentryClientMi """Encapsulates the state of a minimal Galaxy application""" model: GalaxyModelMapping - config: config.GalaxyAppConfiguration + config: GalaxyAppConfiguration tool_cache: ToolCache job_config: jobs.JobConfiguration toolbox_search: ToolBoxSearch @@ -287,7 +288,7 @@ def __init__(self, fsmon=False, **kwargs) -> None: self.name = "galaxy" self.is_webapp = False # Read config file and check for errors - self.config = self._register_singleton(config.GalaxyAppConfiguration, config.GalaxyAppConfiguration(**kwargs)) + self.config = self._register_singleton(GalaxyAppConfiguration, GalaxyAppConfiguration(**kwargs)) self.config.check() config_file = kwargs.get("global_conf", {}).get("__file__", None) if config_file: diff --git a/lib/galaxy/app_unittest_utils/tools_support.py b/lib/galaxy/app_unittest_utils/tools_support.py index 9af461601a83..f2f880da0704 100644 --- a/lib/galaxy/app_unittest_utils/tools_support.py +++ b/lib/galaxy/app_unittest_utils/tools_support.py @@ -20,6 +20,7 @@ from galaxy.tool_util.parser import get_tool_source from galaxy.tools import create_tool_from_source from galaxy.util.bunch import Bunch +from galaxy.util.path import StrPath datatypes_registry = galaxy.datatypes.registry.Registry() datatypes_registry.load_datatypes() @@ -83,10 +84,10 @@ def _init_tool( tool_id="test_tool", extra_file_contents=None, extra_file_path=None, - tool_path=None, + tool_path: Optional[StrPath] = None, ): if tool_path is None: - self.tool_file = os.path.join(self.test_directory, filename) + self.tool_file: StrPath = os.path.join(self.test_directory, filename) contents_template = string.Template(tool_contents) tool_contents = contents_template.safe_substitute(dict(version=version, profile=profile, tool_id=tool_id)) self.__write_tool(tool_contents) @@ -96,7 +97,7 @@ def _init_tool( self.tool_file = tool_path return self.__setup_tool() - def _init_tool_for_path(self, tool_file): + def _init_tool_for_path(self, tool_file: StrPath): self.tool_file = tool_file return self.__setup_tool() diff --git a/lib/galaxy/celery/base_task.py b/lib/galaxy/celery/base_task.py index 94bb8d75956f..c4e986cc94b6 100644 --- a/lib/galaxy/celery/base_task.py +++ b/lib/galaxy/celery/base_task.py @@ -80,7 +80,7 @@ class GalaxyTaskBeforeStartUserRateLimitPostgres(GalaxyTaskBeforeStartUserRateLi We take advantage of efficiencies in its dialect. """ - def calculate_task_start_time( # type: ignore + def calculate_task_start_time( self, user_id: int, sa_session: galaxy_scoped_session, task_interval_secs: float, now: datetime.datetime ) -> datetime.datetime: with transaction(sa_session): diff --git a/lib/galaxy/celery/tasks.py b/lib/galaxy/celery/tasks.py index 3b2e4c6272a7..1bfe6b34f39f 100644 --- a/lib/galaxy/celery/tasks.py +++ b/lib/galaxy/celery/tasks.py @@ -75,10 +75,8 @@ def setup_data_table_manager(app): @lru_cache -def cached_create_tool_from_representation(app, raw_tool_source): - return create_tool_from_representation( - app=app, raw_tool_source=raw_tool_source, tool_dir="", tool_source_class="XmlToolSource" - ) +def cached_create_tool_from_representation(app: MinimalManagerApp, raw_tool_source: str): + return create_tool_from_representation(app=app, raw_tool_source=raw_tool_source, tool_source_class="XmlToolSource") @galaxy_task(action="recalculate a user's disk usage") @@ -305,7 +303,7 @@ def _fetch_data(setup_return): working_directory = Path(tool_job_working_directory) / "working" datatypes_registry = DatatypesRegistry() datatypes_registry.load_datatypes( - galaxy_directory, + galaxy_directory(), config=Path(tool_job_working_directory) / "metadata" / "registry.xml", use_build_sites=False, use_converters=False, diff --git a/lib/galaxy/config/schemas/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml index 5098d923d8ca..afcb6ec9c3ed 100644 --- a/lib/galaxy/config/schemas/config_schema.yml +++ b/lib/galaxy/config/schemas/config_schema.yml @@ -3738,6 +3738,21 @@ mapping: Optional configuration file similar to `job_config_file` to specify which Galaxy processes should schedule workflows. + workflow_scheduling_separate_materialization_iteration: + type: bool + default: false + required: false + desc: | + Workflows launched with URI/URL inputs that are not marked as 'deferred' + are "materialized" (or undeferred) by the workflow scheduler. This might be + a lengthy process. Setting this to 'True' will place the invocation back in + the queue after materialization before scheduling the workflow so it is less + likely to starve other workflow scheduling. Ideally, Galaxy would allow more + fine grain control of handlers but until then, this provides a way to tip the + balance between "doing more work" and "being more fair". The default here is + pretty arbitrary - it has been to False to optimize Galaxy for automated, + single user applications where "fairness" is mostly irrelevant. + cache_user_job_count: type: bool default: false diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index 05f5adbad6bf..0024adea6155 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -207,6 +207,7 @@ class Data(metaclass=DataMeta): edam_data = "data_0006" edam_format = "format_1915" file_ext = "data" + is_subclass = False # Data is not chunkable by default. CHUNKABLE = False diff --git a/lib/galaxy/datatypes/display_applications/configs/biom/biom_simple.xml b/lib/galaxy/datatypes/display_applications/configs/biom/biom_simple.xml index 787a4b3ef4dc..4c5d4e10f159 100644 --- a/lib/galaxy/datatypes/display_applications/configs/biom/biom_simple.xml +++ b/lib/galaxy/datatypes/display_applications/configs/biom/biom_simple.xml @@ -2,6 +2,6 @@ ${ url % { 'biom_file_url_qp': $biom_file.qp } } - + diff --git a/lib/galaxy/datatypes/display_applications/configs/qiime/qiime2/q2view.xml b/lib/galaxy/datatypes/display_applications/configs/qiime/qiime2/q2view.xml index 1035cbb679a0..5cbba046610c 100644 --- a/lib/galaxy/datatypes/display_applications/configs/qiime/qiime2/q2view.xml +++ b/lib/galaxy/datatypes/display_applications/configs/qiime/qiime2/q2view.xml @@ -2,6 +2,6 @@ ${ url % { 'q2view_file_url_qp': $q2view_file.qp } } - + diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 9e7bf7d561a7..ce883da59009 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -6,15 +6,16 @@ import logging import os import pkgutil -from pathlib import Path from string import Template from typing import ( + Any, cast, Dict, Iterable, List, Optional, Tuple, + Type, TYPE_CHECKING, Union, ) @@ -24,8 +25,12 @@ import galaxy.util from galaxy.datatypes.protocols import DatasetProtocol from galaxy.tool_util.edam_util import load_edam_tree -from galaxy.util import RW_R__R__ +from galaxy.util import ( + Element, + RW_R__R__, +) from galaxy.util.bunch import Bunch +from galaxy.util.path import StrPath from . import ( binary, coverage, @@ -43,6 +48,7 @@ if TYPE_CHECKING: from galaxy.datatypes.data import Data + from galaxy.tool_util.toolbox.base import AbstractToolBox class ConfigurationError(Exception): @@ -65,7 +71,7 @@ def __init__(self, config=None): self.log.addHandler(logging.NullHandler()) self.config = config self.edam = edam - self.datatypes_by_extension = {} + self.datatypes_by_extension: Dict[str, Data] = {} self.datatypes_by_suffix_inferences = {} self.mimetypes_by_extension = {} self.datatype_converters = {} @@ -75,7 +81,7 @@ def __init__(self, config=None): self.converter_deps = {} self.available_tracks = [] self.set_external_metadata_tool = None - self.sniff_order = [] + self.sniff_order: List[Data] = [] self.upload_file_formats = [] # Datatype elements defined in local datatypes_conf.xml that contain display applications. self.display_app_containers = [] @@ -92,7 +98,7 @@ def __init__(self, config=None): self.inherit_display_application_by_class = [] self.datatype_elems = [] self.datatype_info_dicts = [] - self.sniffer_elems = [] + self.sniffer_elems: List[Element] = [] self._registry_xml_string = None self._edam_formats_mapping = None self._edam_data_mapping = None @@ -104,13 +110,13 @@ def __init__(self, config=None): def load_datatypes( self, - root_dir=None, - config=None, - override=True, - use_converters=True, - use_display_applications=True, - use_build_sites=True, - ): + root_dir: Optional[StrPath] = None, + config: Optional[Union[Element, StrPath]] = None, + override: bool = True, + use_converters: bool = True, + use_display_applications: bool = True, + use_build_sites: bool = True, + ) -> None: """ Parse a datatypes XML file located at root_dir/config (if processing the Galaxy distributed config) or contained within an installed Tool Shed repository. @@ -127,8 +133,8 @@ def __import_module(full_path: str, datatype_module: str): return module if root_dir and config: - compressed_sniffers = {} - if isinstance(config, (str, Path)): + compressed_sniffers: Dict[Type[Data], List[Data]] = {} + if isinstance(config, (str, os.PathLike)): # Parse datatypes_conf.xml tree = galaxy.util.parse_xml(config) root = tree.getroot() @@ -137,6 +143,7 @@ def __import_module(full_path: str, datatype_module: str): else: root = config registration = root.find("registration") + assert registration is not None # Set default paths defined in local datatypes_conf.xml. if use_converters: if not self.converters_path: @@ -167,7 +174,6 @@ def __import_module(full_path: str, datatype_module: str): for elem in registration.findall("datatype"): # Keep a status of the process steps to enable stopping the process of handling the datatype if necessary. - ok = True extension = self.get_extension(elem) dtype = elem.get("type", None) type_extension = elem.get("type_extension", None) @@ -199,7 +205,9 @@ def __import_module(full_path: str, datatype_module: str): if override or extension not in self.datatypes_by_extension: can_process_datatype = True if can_process_datatype: + datatype_class: Optional[Type[Data]] = None if dtype is not None: + ok = True try: fields = dtype.split(":") datatype_module = fields[0] @@ -208,21 +216,18 @@ def __import_module(full_path: str, datatype_module: str): self.log.exception("Error parsing datatype definition for dtype %s", str(dtype)) ok = False if ok: - datatype_class = None - if datatype_class is None: - try: - # The datatype class name must be contained in one of the datatype modules in the Galaxy distribution. - fields = datatype_module.split(".")[1:] - module = __import__(datatype_module) - for mod in fields: - module = getattr(module, mod) - datatype_class = getattr(module, datatype_class_name) - self.log.debug( - f"Retrieved datatype module {str(datatype_module)}:{datatype_class_name} from the datatype registry for extension {extension}." - ) - except Exception: - self.log.exception("Error importing datatype module %s", str(datatype_module)) - ok = False + try: + # The datatype class name must be contained in one of the datatype modules in the Galaxy distribution. + fields = datatype_module.split(".")[1:] + module = __import__(datatype_module) + for mod in fields: + module = getattr(module, mod) + datatype_class = getattr(module, datatype_class_name) + self.log.debug( + f"Retrieved datatype module {str(datatype_module)}:{datatype_class_name} from the datatype registry for extension {extension}." + ) + except Exception: + self.log.exception("Error importing datatype module %s", str(datatype_module)) elif type_extension is not None: try: datatype_class = self.datatypes_by_extension[type_extension].__class__ @@ -233,8 +238,7 @@ def __import_module(full_path: str, datatype_module: str): self.log.exception( "Error determining datatype_class for type_extension %s", str(type_extension) ) - ok = False - if ok: + if datatype_class: # A new tool shed repository that contains custom datatypes is being installed, and since installation is # occurring after the datatypes registry has been initialized at server startup, its contents cannot be # overridden by new introduced conflicting data types unless the value of override is True. @@ -262,7 +266,7 @@ def __import_module(full_path: str, datatype_module: str): for upload_warning_el in upload_warning_els: if upload_warning_template is not None: raise NotImplementedError("Multiple upload_warnings not implemented") - upload_warning_template = Template(upload_warning_el.text) + upload_warning_template = Template(upload_warning_el.text or "") datatype_instance = datatype_class() self.datatypes_by_extension[extension] = datatype_instance if mimetype is None: @@ -282,9 +286,9 @@ def __import_module(full_path: str, datatype_module: str): # compressed files in the future (e.g. maybe some day faz will be a compressed fasta # or something along those lines) for infer_from in elem.findall("infer_from"): - suffix = infer_from.get("suffix", None) + suffix = infer_from.get("suffix") if suffix is None: - raise Exception("Failed to parse infer_from datatype element") + raise ConfigurationError("Failed to parse infer_from datatype element") infer_from_suffixes.append(suffix) self.datatypes_by_suffix_inferences[suffix] = datatype_instance for converter in elem.findall("converter"): @@ -300,9 +304,11 @@ def __import_module(full_path: str, datatype_module: str): self.converters.append((converter_config, extension, target_datatype)) # Add composite files. for composite_file in elem.findall("composite_file"): - name = composite_file.get("name", None) + name = composite_file.get("name") if name is None: - self.log.warning(f"You must provide a name for your composite_file ({composite_file}).") + raise ConfigurationError( + f"You must provide a name for your composite_file ({composite_file})." + ) optional = composite_file.get("optional", False) mimetype = composite_file.get("mimetype", None) self.datatypes_by_extension[extension].add_composite_file( @@ -321,8 +327,8 @@ def __import_module(full_path: str, datatype_module: str): composite_files = datatype_instance.get_composite_files() if composite_files: _composite_files = [] - for name, composite_file in composite_files.items(): - _composite_file = composite_file.dict() + for name, composite_file_bunch in composite_files.items(): + _composite_file = composite_file_bunch.dict() _composite_file["name"] = name _composite_files.append(_composite_file) datatype_info_dict["composite_files"] = _composite_files @@ -332,16 +338,18 @@ def __import_module(full_path: str, datatype_module: str): compressed_extension = f"{extension}.{auto_compressed_type}" upper_compressed_type = auto_compressed_type[0].upper() + auto_compressed_type[1:] auto_compressed_type_name = datatype_class_name + upper_compressed_type - attributes = {} + attributes: Dict[str, Any] = {} if auto_compressed_type == "gz": - dynamic_parent = binary.GzDynamicCompressedArchive + dynamic_parent: Type[binary.DynamicCompressedArchive] = ( + binary.GzDynamicCompressedArchive + ) elif auto_compressed_type == "bz2": dynamic_parent = binary.Bz2DynamicCompressedArchive else: - raise Exception(f"Unknown auto compression type [{auto_compressed_type}]") + raise ConfigurationError(f"Unknown auto compression type [{auto_compressed_type}]") attributes["file_ext"] = compressed_extension attributes["uncompressed_datatype_instance"] = datatype_instance - compressed_datatype_class = type( + compressed_datatype_class: Type[Data] = type( auto_compressed_type_name, ( datatype_class, @@ -411,7 +419,7 @@ def __import_module(full_path: str, datatype_module: str): self._load_build_sites(root) self.set_default_values() - def append_to_sniff_order(): + def append_to_sniff_order() -> None: sniff_order_classes = {type(_) for _ in self.sniff_order} for datatype in self.datatypes_by_extension.values(): # Add a datatype only if it is not already in sniff_order, it @@ -482,7 +490,12 @@ def get_legacy_sites_by_build(self, site_type, build): def get_display_sites(self, site_type): return self.display_sites.get(site_type, []) - def load_datatype_sniffers(self, root, override=False, compressed_sniffers=None): + def load_datatype_sniffers( + self, + root: Element, + override: bool = False, + compressed_sniffers: Optional[Dict[Type["Data"], List["Data"]]] = None, + ) -> None: """ Process the sniffers element from a parsed a datatypes XML file located at root_dir/config (if processing the Galaxy distributed config) or contained within an installed Tool Shed repository. @@ -514,17 +527,15 @@ def load_datatype_sniffers(self, root, override=False, compressed_sniffers=None) ok = False if ok: try: - aclass = getattr(module, datatype_class_name)() + aclass = getattr(module, datatype_class_name) except Exception: - self.log.exception( - "Error calling method %s from class %s", str(datatype_class_name), str(module) - ) + self.log.exception("Error getting class %s from module %s", datatype_class_name, module) ok = False if ok: # We are loading new sniffer, so see if we have a conflicting sniffer already loaded. conflict = False - for conflict_loc, sniffer_class in enumerate(self.sniff_order): - if sniffer_class.__class__ == aclass.__class__: + for conflict_loc, sniffer_datatype_instance in enumerate(self.sniff_order): + if sniffer_datatype_instance.__class__ == aclass: # We have a conflicting sniffer, so replace the one previously loaded. conflict = True if override: @@ -532,16 +543,15 @@ def load_datatype_sniffers(self, root, override=False, compressed_sniffers=None) self.log.debug(f"Removed conflicting sniffer for datatype '{dtype}'") break if not conflict or override: - if compressed_sniffers and aclass.__class__ in compressed_sniffers: - for compressed_sniffer in compressed_sniffers[aclass.__class__]: + if compressed_sniffers and aclass in compressed_sniffers: + for compressed_sniffer in compressed_sniffers[aclass]: self.sniff_order.append(compressed_sniffer) - self.sniff_order.append(aclass) + self.sniff_order.append(aclass()) self.log.debug(f"Loaded sniffer for datatype '{dtype}'") # Processing the new sniffer elem is now complete, so make sure the element defining it is loaded if necessary. - sniffer_class = elem.get("type", None) - if sniffer_class is not None: - if sniffer_class not in sniffer_elem_classes: - self.sniffer_elems.append(elem) + sniffer_class = elem.get("type") + if sniffer_class is not None and sniffer_class not in sniffer_elem_classes: + self.sniffer_elems.append(elem) def get_datatype_from_filename(self, name): max_extension_parts = 3 @@ -614,7 +624,7 @@ def change_datatype(self, data, ext): data.init_meta(copy_from=data) return data - def load_datatype_converters(self, toolbox, use_cached=False): + def load_datatype_converters(self, toolbox: "AbstractToolBox", use_cached: bool = False): """ Add datatype converters from self.converters to the calling app's toolbox. """ diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 0affcde217da..310ee9d21c7f 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -17,8 +17,10 @@ Callable, Dict, IO, + Iterable, NamedTuple, Optional, + TYPE_CHECKING, Union, ) @@ -44,6 +46,8 @@ pass import magic # isort:skip +if TYPE_CHECKING: + from .data import Data log = logging.getLogger(__name__) @@ -689,7 +693,7 @@ def _get_file_prefix(filename_or_file_prefix: Union[str, FilePrefix], auto_decom return filename_or_file_prefix -def run_sniffers_raw(file_prefix: FilePrefix, sniff_order): +def run_sniffers_raw(file_prefix: FilePrefix, sniff_order: Iterable["Data"]): """Run through sniffers specified by sniff_order, return None of None match.""" fname = file_prefix.filename file_ext = None @@ -718,15 +722,16 @@ def run_sniffers_raw(file_prefix: FilePrefix, sniff_order): continue try: if hasattr(datatype, "sniff_prefix"): - if file_prefix.compressed_format and getattr(datatype, "compressed_format", None): + datatype_compressed_format = getattr(datatype, "compressed_format", None) + if file_prefix.compressed_format and datatype_compressed_format: # Compare the compressed format detected # to the expected. - if file_prefix.compressed_format != datatype.compressed_format: + if file_prefix.compressed_format != datatype_compressed_format: continue if datatype.sniff_prefix(file_prefix): file_ext = datatype.file_ext break - elif datatype.sniff(fname): + elif hasattr(datatype, "sniff") and datatype.sniff(fname): file_ext = datatype.file_ext break except Exception: diff --git a/lib/galaxy/exceptions/__init__.py b/lib/galaxy/exceptions/__init__.py index f8e8c956ac98..749b80a57433 100644 --- a/lib/galaxy/exceptions/__init__.py +++ b/lib/galaxy/exceptions/__init__.py @@ -224,6 +224,11 @@ class UserActivationRequiredException(MessageException): err_code = error_codes_by_name["USER_ACTIVATION_REQUIRED"] +class ItemAlreadyClaimedException(MessageException): + status_code = 403 + err_code = error_codes_by_name["ITEM_IS_CLAIMED"] + + class ObjectNotFound(MessageException): """Accessed object was not found""" diff --git a/lib/galaxy/exceptions/error_codes.json b/lib/galaxy/exceptions/error_codes.json index 1c57fbc7bbad..cfaca436a65c 100644 --- a/lib/galaxy/exceptions/error_codes.json +++ b/lib/galaxy/exceptions/error_codes.json @@ -144,6 +144,11 @@ "code": 403007, "message": "Action requires account activation." }, + { + "name": "ITEM_IS_CLAIMED", + "code": 403008, + "message": "This item has already been claimed and cannot be re-claimed." + }, { "name": "USER_REQUIRED", "code": 403008, diff --git a/lib/galaxy/files/sources/util.py b/lib/galaxy/files/sources/util.py index 3704aaa7a71b..a3af04f6bff4 100644 --- a/lib/galaxy/files/sources/util.py +++ b/lib/galaxy/files/sources/util.py @@ -1,10 +1,8 @@ import time -from os import PathLike from typing import ( List, Optional, Tuple, - Union, ) from galaxy import exceptions @@ -20,8 +18,7 @@ requests, ) from galaxy.util.config_parsers import IpAllowedListEntryT - -TargetPathT = Union[str, PathLike] +from galaxy.util.path import StrPath def _not_implemented(drs_uri: str, desc: str) -> NotImplementedError: @@ -79,7 +76,7 @@ def _get_access_info(obj_url: str, access_method: dict, headers=None) -> Tuple[s def fetch_drs_to_file( drs_uri: str, - target_path: TargetPathT, + target_path: StrPath, user_context: Optional[FileSourcesUserContext], force_http=False, retry_options: Optional[RetryOptions] = None, diff --git a/lib/galaxy/jobs/handler.py b/lib/galaxy/jobs/handler.py index 0213a797aab4..e66065aab2e4 100644 --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -130,7 +130,13 @@ def setup_query(self): if self.grab_model is model.Job: grab_condition = self.grab_model.state == self.grab_model.states.NEW elif self.grab_model is model.WorkflowInvocation: - grab_condition = self.grab_model.state.in_((self.grab_model.states.NEW, self.grab_model.states.CANCELLING)) + grab_condition = self.grab_model.state.in_( + ( + self.grab_model.states.NEW, + self.grab_model.states.REQUIRES_MATERIALIZATION, + self.grab_model.states.CANCELLING, + ) + ) else: raise NotImplementedError(f"Grabbing {self.grab_model.__name__} not implemented") subq = ( diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py index 511c431ac8e5..fd2a13804345 100644 --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -10,12 +10,18 @@ import threading import time import traceback -import typing import uuid from queue import ( Empty, Queue, ) +from typing import ( + Any, + Dict, + Optional, + TYPE_CHECKING, + Union, +) from sqlalchemy import select from sqlalchemy.orm import object_session @@ -58,7 +64,7 @@ from galaxy.util.monitors import Monitors from .state_handler_factory import build_state_handlers -if typing.TYPE_CHECKING: +if TYPE_CHECKING: from galaxy.app import GalaxyManagerApplication from galaxy.jobs import ( JobDestination, @@ -183,7 +189,7 @@ def run_next(self): # Prevent fail_job cycle in the work_queue self.work_queue.put((self.fail_job, job_state)) - def _ensure_db_session(self, arg: typing.Union["JobWrapper", "JobState"]) -> None: + def _ensure_db_session(self, arg: Union["JobWrapper", "JobState"]) -> None: """Ensure Job object belongs to current session.""" try: job_wrapper = arg.job_wrapper # type: ignore[union-attr] @@ -264,7 +270,7 @@ def url_to_destination(self, url: str): """ return galaxy.jobs.JobDestination(runner=url.split(":")[0]) - def parse_destination_params(self, params: typing.Dict[str, typing.Any]): + def parse_destination_params(self, params: Dict[str, Any]): """Parse the JobDestination ``params`` dict and return the runner's native representation of those params.""" raise NotImplementedError() @@ -347,8 +353,8 @@ def build_command_line( def get_work_dir_outputs( self, job_wrapper: "MinimalJobWrapper", - job_working_directory: typing.Optional[str] = None, - tool_working_directory: typing.Optional[str] = None, + job_working_directory: Optional[str] = None, + tool_working_directory: Optional[str] = None, ): """ Returns list of pairs (source_file, destination) describing path @@ -527,10 +533,10 @@ def write_executable_script(self, path: str, contents: str, job_io: DescribesScr def _find_container( self, job_wrapper: "MinimalJobWrapper", - compute_working_directory: typing.Optional[str] = None, - compute_tool_directory: typing.Optional[str] = None, - compute_job_directory: typing.Optional[str] = None, - compute_tmp_directory: typing.Optional[str] = None, + compute_working_directory: Optional[str] = None, + compute_tool_directory: Optional[str] = None, + compute_job_directory: Optional[str] = None, + compute_tmp_directory: Optional[str] = None, ): job_directory_type = "galaxy" if compute_working_directory is None else "pulsar" if not compute_working_directory: @@ -542,7 +548,7 @@ def _find_container( tool = job_wrapper.tool assert tool if not compute_tool_directory: - compute_tool_directory = tool.tool_dir + compute_tool_directory = str(tool.tool_dir) if tool.tool_dir is not None else None if not compute_tmp_directory: compute_tmp_directory = job_wrapper.tmp_directory() @@ -600,7 +606,7 @@ def fail_job(self, job_state: "JobState", exception=False, message="Job failed", fail_message, tool_stdout=tool_stdout, tool_stderr=tool_stderr, exception=exception ) - def mark_as_resubmitted(self, job_state: "JobState", info: typing.Optional[str] = None): + def mark_as_resubmitted(self, job_state: "JobState", info: Optional[str] = None): job_state.job_wrapper.mark_as_resubmitted(info=info) if not self.app.config.track_jobs_in_database: job_state.job_wrapper.change_state(model.Job.states.QUEUED) diff --git a/lib/galaxy/managers/context.py b/lib/galaxy/managers/context.py index f578fe79b8f2..89a9c32c8902 100644 --- a/lib/galaxy/managers/context.py +++ b/lib/galaxy/managers/context.py @@ -39,10 +39,13 @@ import string from json import dumps from typing import ( + Any, Callable, cast, + Dict, List, Optional, + Tuple, ) from sqlalchemy import select @@ -206,6 +209,13 @@ class ProvidesUserContext(ProvidesAppContext): galaxy_session: Optional[GalaxySession] = None _tag_handler: Optional[GalaxyTagHandlerSession] = None + _short_term_cache: Dict[Tuple[str, ...], Any] + + def set_cache_value(self, args: Tuple[str, ...], value: Any): + self._short_term_cache[args] = value + + def get_cache_value(self, args: Tuple[str, ...], default: Any = None) -> Any: + return self._short_term_cache.get(args, default) @property def tag_handler(self): diff --git a/lib/galaxy/managers/hdas.py b/lib/galaxy/managers/hdas.py index 86f9b8c98c02..faef0b0c1ac3 100644 --- a/lib/galaxy/managers/hdas.py +++ b/lib/galaxy/managers/hdas.py @@ -44,6 +44,7 @@ taggable, users, ) +from galaxy.managers.context import ProvidesHistoryContext from galaxy.model import ( Job, JobStateHistory, @@ -51,6 +52,7 @@ ) from galaxy.model.base import transaction from galaxy.model.deferred import materializer_factory +from galaxy.model.dereference import dereference_to_model from galaxy.schema.schema import DatasetSourceType from galaxy.schema.storage_cleaner import ( CleanableItemsSummary, @@ -68,6 +70,7 @@ MinimalManagerApp, StructuredApp, ) +from galaxy.tool_util.parameters import DataRequestUri from galaxy.util.compression_utils import get_fileobj log = logging.getLogger(__name__) @@ -173,7 +176,7 @@ def create( session.commit() return hda - def materialize(self, request: MaterializeDatasetInstanceTaskRequest) -> None: + def materialize(self, request: MaterializeDatasetInstanceTaskRequest, in_place: bool = False) -> bool: request_user: RequestUser = request.user materializer = materializer_factory( True, # attached... @@ -187,11 +190,17 @@ def materialize(self, request: MaterializeDatasetInstanceTaskRequest) -> None: else: dataset_instance = self.ldda_manager.get_accessible(request.content, user) history = self.app.history_manager.by_id(request.history_id) - new_hda = materializer.ensure_materialized(dataset_instance, target_history=history) - history.add_dataset(new_hda, set_hid=True) + new_hda = materializer.ensure_materialized( + dataset_instance, target_history=history, validate_hashes=request.validate_hashes, in_place=in_place + ) + if not in_place: + history.add_dataset(new_hda, set_hid=True) + else: + new_hda.set_total_size() session = self.session() with transaction(session): session.commit() + return new_hda.is_ok def copy( self, item: Any, history=None, hide_copy: bool = False, flush: bool = True, **kwargs: Any @@ -342,6 +351,18 @@ def _set_permissions(self, trans, hda, role_ids_dict): raise exceptions.RequestParameterInvalidException(error) +def dereference_input( + trans: ProvidesHistoryContext, data_request: DataRequestUri, history: Optional[model.History] = None +) -> model.HistoryDatasetAssociation: + target_history = history or trans.history + hda = dereference_to_model(trans.sa_session, trans.user, target_history, data_request) + permissions = trans.app.security_agent.history_get_default_permissions(target_history) + trans.app.security_agent.set_all_dataset_permissions(hda.dataset, permissions, new=True, flush=False) + with transaction(trans.sa_session): + trans.sa_session.commit() + return hda + + class HDAStorageCleanerManager(base.StorageCleanerManager): def __init__(self, hda_manager: HDAManager, dataset_manager: datasets.DatasetManager): self.hda_manager = hda_manager diff --git a/lib/galaxy/managers/landing.py b/lib/galaxy/managers/landing.py new file mode 100644 index 000000000000..2138c0c81ca2 --- /dev/null +++ b/lib/galaxy/managers/landing.py @@ -0,0 +1,172 @@ +from typing import ( + Optional, + Union, +) +from uuid import uuid4 + +from pydantic import UUID4 +from sqlalchemy import select + +from galaxy.exceptions import ( + InconsistentDatabase, + InsufficientPermissionsException, + ItemAlreadyClaimedException, + ObjectNotFound, + RequestParameterMissingException, +) +from galaxy.model import ( + ToolLandingRequest as ToolLandingRequestModel, + WorkflowLandingRequest as WorkflowLandingRequestModel, +) +from galaxy.model.base import transaction +from galaxy.model.scoped_session import galaxy_scoped_session +from galaxy.schema.schema import ( + ClaimLandingPayload, + CreateToolLandingRequestPayload, + CreateWorkflowLandingRequestPayload, + LandingRequestState, + ToolLandingRequest, + WorkflowLandingRequest, +) +from galaxy.security.idencoding import IdEncodingHelper +from galaxy.util import safe_str_cmp +from .context import ProvidesUserContext + +LandingRequestModel = Union[ToolLandingRequestModel, WorkflowLandingRequestModel] + + +class LandingRequestManager: + + def __init__(self, sa_session: galaxy_scoped_session, security: IdEncodingHelper): + self.sa_session = sa_session + self.security = security + + def create_tool_landing_request(self, payload: CreateToolLandingRequestPayload) -> ToolLandingRequest: + model = ToolLandingRequestModel() + model.tool_id = payload.tool_id + model.tool_version = payload.tool_version + model.request_state = payload.request_state + model.uuid = uuid4() + model.client_secret = payload.client_secret + self._save(model) + return self._tool_response(model) + + def create_workflow_landing_request(self, payload: CreateWorkflowLandingRequestPayload) -> WorkflowLandingRequest: + model = WorkflowLandingRequestModel() + if payload.workflow_target_type == "stored_workflow": + model.stored_workflow_id = self.security.decode_id(payload.workflow_id) + else: + model.workflow_id = self.security.decode_id(payload.workflow_id) + model.request_state = payload.request_state + model.uuid = uuid4() + model.client_secret = payload.client_secret + self._save(model) + return self._workflow_response(model) + + def claim_tool_landing_request( + self, trans: ProvidesUserContext, uuid: UUID4, claim: Optional[ClaimLandingPayload] + ) -> ToolLandingRequest: + request = self._get_tool_landing_request(uuid) + self._check_can_claim(trans, request, claim) + request.user_id = trans.user.id + self._save(request) + return self._tool_response(request) + + def claim_workflow_landing_request( + self, trans: ProvidesUserContext, uuid: UUID4, claim: Optional[ClaimLandingPayload] + ) -> WorkflowLandingRequest: + request = self._get_workflow_landing_request(uuid) + self._check_can_claim(trans, request, claim) + request.user_id = trans.user.id + self._save(request) + return self._workflow_response(request) + + def get_tool_landing_request(self, trans: ProvidesUserContext, uuid: UUID4) -> ToolLandingRequest: + request = self._get_claimed_tool_landing_request(trans, uuid) + return self._tool_response(request) + + def get_workflow_landing_request(self, trans: ProvidesUserContext, uuid: UUID4) -> WorkflowLandingRequest: + request = self._get_claimed_workflow_landing_request(trans, uuid) + return self._workflow_response(request) + + def _check_can_claim( + self, trans: ProvidesUserContext, request: LandingRequestModel, claim: Optional[ClaimLandingPayload] + ): + if request.client_secret is not None: + if claim is None or not claim.client_secret: + raise RequestParameterMissingException() + if not safe_str_cmp(request.client_secret, claim.client_secret): + raise InsufficientPermissionsException() + if request.user_id is not None: + raise ItemAlreadyClaimedException() + + def _get_tool_landing_request(self, uuid: UUID4) -> ToolLandingRequestModel: + request = self.sa_session.scalars( + select(ToolLandingRequestModel).where(ToolLandingRequestModel.uuid == str(uuid)) + ).one_or_none() + if request is None: + raise ObjectNotFound() + return request + + def _get_workflow_landing_request(self, uuid: UUID4) -> WorkflowLandingRequestModel: + request = self.sa_session.scalars( + select(WorkflowLandingRequestModel).where(WorkflowLandingRequestModel.uuid == str(uuid)) + ).one_or_none() + if request is None: + raise ObjectNotFound() + return request + + def _get_claimed_tool_landing_request(self, trans: ProvidesUserContext, uuid: UUID4) -> ToolLandingRequestModel: + request = self._get_tool_landing_request(uuid) + self._check_ownership(trans, request) + return request + + def _get_claimed_workflow_landing_request( + self, trans: ProvidesUserContext, uuid: UUID4 + ) -> WorkflowLandingRequestModel: + request = self._get_workflow_landing_request(uuid) + self._check_ownership(trans, request) + return request + + def _tool_response(self, model: ToolLandingRequestModel) -> ToolLandingRequest: + response_model = ToolLandingRequest( + tool_id=model.tool_id, + tool_version=model.tool_version, + request_state=model.request_state, + uuid=model.uuid, + state=self._state(model), + ) + return response_model + + def _workflow_response(self, model: WorkflowLandingRequestModel) -> WorkflowLandingRequest: + workflow_id: Optional[int] + if model.stored_workflow_id is not None: + workflow_id = model.stored_workflow_id + target_type = "stored_workflow" + else: + workflow_id = model.workflow_id + target_type = "workflow" + if workflow_id is None: + raise InconsistentDatabase() + assert workflow_id + response_model = WorkflowLandingRequest( + workflow_id=self.security.encode_id(workflow_id), + workflow_target_type=target_type, + request_state=model.request_state, + uuid=model.uuid, + state=self._state(model), + ) + return response_model + + def _check_ownership(self, trans: ProvidesUserContext, model: LandingRequestModel): + if model.user_id != trans.user.id: + raise InsufficientPermissionsException() + + def _state(self, model: LandingRequestModel) -> LandingRequestState: + return LandingRequestState.UNCLAIMED if model.user_id is None else LandingRequestState.CLAIMED + + def _save(self, model: LandingRequestModel): + sa_session = self.sa_session + sa_session.add(model) + with transaction(sa_session): + sa_session.commit() diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 36aee6b45441..9967c08fe667 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -18,6 +18,7 @@ import string from collections import defaultdict from collections.abc import Callable +from dataclasses import dataclass from datetime import ( datetime, timedelta, @@ -176,6 +177,7 @@ DatasetValidatedState, InvocationsStateCounts, JobState, + ToolRequestState, ) from galaxy.schema.workflow.comments import WorkflowCommentModel from galaxy.security import get_permitted_actions @@ -212,6 +214,7 @@ WorkflowMappingField, ) from galaxy.util.hash_util import ( + HashFunctionNameEnum, md5_hash_str, new_insecure_hash, ) @@ -1328,6 +1331,30 @@ def __init__(self, user, token=None): self.expiration_time = now() + timedelta(hours=24) +class ToolSource(Base, Dictifiable, RepresentById): + __tablename__ = "tool_source" + + id: Mapped[int] = mapped_column(primary_key=True) + hash: Mapped[Optional[str]] = mapped_column(Unicode(255)) + source: Mapped[dict] = mapped_column(JSONType) + + +class ToolRequest(Base, Dictifiable, RepresentById): + __tablename__ = "tool_request" + + states: TypeAlias = ToolRequestState + + id: Mapped[int] = mapped_column(primary_key=True) + tool_source_id: Mapped[int] = mapped_column(ForeignKey("tool_source.id"), index=True) + history_id: Mapped[Optional[int]] = mapped_column(ForeignKey("history.id"), index=True) + request: Mapped[dict] = mapped_column(JSONType) + state: Mapped[Optional[str]] = mapped_column(TrimmedString(32), index=True) + state_message: Mapped[Optional[str]] = mapped_column(JSONType, index=True) + + tool_source: Mapped["ToolSource"] = relationship() + history: Mapped[Optional["History"]] = relationship(back_populates="tool_requests") + + class DynamicTool(Base, Dictifiable, RepresentById): __tablename__ = "dynamic_tool" @@ -1454,7 +1481,9 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable): handler: Mapped[Optional[str]] = mapped_column(TrimmedString(255), index=True) preferred_object_store_id: Mapped[Optional[str]] = mapped_column(String(255)) object_store_id_overrides: Mapped[Optional[STR_TO_STR_DICT]] = mapped_column(JSONType) + tool_request_id: Mapped[Optional[int]] = mapped_column(ForeignKey("tool_request.id"), index=True) + tool_request: Mapped[Optional["ToolRequest"]] = relationship() user: Mapped[Optional["User"]] = relationship() galaxy_session: Mapped[Optional["GalaxySession"]] = relationship() history: Mapped[Optional["History"]] = relationship(back_populates="jobs") @@ -3177,6 +3206,7 @@ class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, Serializable ) user: Mapped[Optional["User"]] = relationship(back_populates="histories") jobs: Mapped[List["Job"]] = relationship(back_populates="history", cascade_backrefs=False) + tool_requests: Mapped[List["ToolRequest"]] = relationship(back_populates="history") update_time = column_property( select(func.max(HistoryAudit.update_time)).where(HistoryAudit.history_id == id).scalar_subquery(), @@ -4464,7 +4494,17 @@ def copy(self) -> "DatasetSource": return new_source -class DatasetSourceHash(Base, Serializable): +class HasHashFunctionName: + hash_function: Mapped[Optional[str]] + + @property + def hash_func_name(self) -> HashFunctionNameEnum: + as_str = self.hash_function + assert as_str + return HashFunctionNameEnum(self.hash_function) + + +class DatasetSourceHash(Base, Serializable, HasHashFunctionName): __tablename__ = "dataset_source_hash" id: Mapped[int] = mapped_column(primary_key=True) @@ -4489,7 +4529,7 @@ def copy(self) -> "DatasetSourceHash": return new_hash -class DatasetHash(Base, Dictifiable, Serializable): +class DatasetHash(Base, Dictifiable, Serializable, HasHashFunctionName): __tablename__ = "dataset_hash" id: Mapped[int] = mapped_column(primary_key=True) @@ -4518,6 +4558,15 @@ def copy(self) -> "DatasetHash": new_hash.extra_files_path = self.extra_files_path return new_hash + @property + def hash_func_name(self) -> HashFunctionNameEnum: + as_str = self.hash_function + assert as_str + return HashFunctionNameEnum(self.hash_function) + + +DescribesHash = Union[DatasetSourceHash, DatasetHash] + def datatype_for_extension(extension, datatypes_registry=None) -> "Data": if extension is not None: @@ -8518,6 +8567,18 @@ class StoredWorkflowMenuEntry(Base, RepresentById): ) +@dataclass +class InputWithRequest: + input: Any + request: Dict[str, Any] + + +@dataclass +class InputToMaterialize: + hda: "HistoryDatasetAssociation" + input_dataset: "WorkflowRequestToInputDatasetAssociation" + + class WorkflowInvocation(Base, UsesCreateAndUpdateTime, Dictifiable, Serializable): __tablename__ = "workflow_invocation" @@ -8749,6 +8810,7 @@ def poll_active_workflow_ids(engine, scheduler=None, handler=None): and_conditions = [ or_( WorkflowInvocation.state == WorkflowInvocation.states.NEW, + WorkflowInvocation.state == WorkflowInvocation.states.REQUIRES_MATERIALIZATION, WorkflowInvocation.state == WorkflowInvocation.states.READY, WorkflowInvocation.state == WorkflowInvocation.states.CANCELLING, ), @@ -8840,6 +8902,21 @@ def input_associations(self): inputs.append(input_dataset_collection_assoc) return inputs + def inputs_requiring_materialization(self) -> List[InputToMaterialize]: + hdas_to_materialize: List[InputToMaterialize] = [] + for input_dataset_assoc in self.input_datasets: + request = input_dataset_assoc.request + if request: + deferred = request.get("deferred", False) + if not deferred: + hdas_to_materialize.append( + InputToMaterialize( + input_dataset_assoc.dataset, + input_dataset_assoc, + ) + ) + return hdas_to_materialize + def _serialize(self, id_encoder, serialization_options): invocation_attrs = dict_for(self) invocation_attrs["state"] = self.state @@ -9002,20 +9079,28 @@ def attach_step(request_to_content): else: request_to_content.workflow_step = step + request: Optional[Dict[str, Any]] = None + if isinstance(content, InputWithRequest): + request = content.request + content = content.input + history_content_type = getattr(content, "history_content_type", None) if history_content_type == "dataset": request_to_content = WorkflowRequestToInputDatasetAssociation() request_to_content.dataset = content + request_to_content.request = request attach_step(request_to_content) self.input_datasets.append(request_to_content) elif history_content_type == "dataset_collection": request_to_content = WorkflowRequestToInputDatasetCollectionAssociation() request_to_content.dataset_collection = content + request_to_content.request = request attach_step(request_to_content) self.input_dataset_collections.append(request_to_content) else: request_to_content = WorkflowRequestInputStepParameter() request_to_content.parameter_value = content + request_to_content.request = request attach_step(request_to_content) self.input_step_parameters.append(request_to_content) @@ -9439,6 +9524,7 @@ class WorkflowRequestToInputDatasetAssociation(Base, Dictifiable, Serializable): workflow_invocation_id: Mapped[Optional[int]] = mapped_column(ForeignKey("workflow_invocation.id"), index=True) workflow_step_id: Mapped[Optional[int]] = mapped_column(ForeignKey("workflow_step.id")) dataset_id: Mapped[Optional[int]] = mapped_column(ForeignKey("history_dataset_association.id"), index=True) + request: Mapped[Optional[Dict]] = mapped_column(JSONType) workflow_step: Mapped[Optional["WorkflowStep"]] = relationship() dataset: Mapped[Optional["HistoryDatasetAssociation"]] = relationship() @@ -9474,6 +9560,7 @@ class WorkflowRequestToInputDatasetCollectionAssociation(Base, Dictifiable, Seri workflow_invocation: Mapped[Optional["WorkflowInvocation"]] = relationship( back_populates="input_dataset_collections" ) + request: Mapped[Optional[Dict]] = mapped_column(JSONType) history_content_type = "dataset_collection" dict_collection_visible_keys = ["id", "workflow_invocation_id", "workflow_step_id", "dataset_collection_id", "name"] @@ -9497,6 +9584,7 @@ class WorkflowRequestInputStepParameter(Base, Dictifiable, Serializable): workflow_invocation_id: Mapped[Optional[int]] = mapped_column(ForeignKey("workflow_invocation.id"), index=True) workflow_step_id: Mapped[Optional[int]] = mapped_column(ForeignKey("workflow_step.id")) parameter_value: Mapped[Optional[bytes]] = mapped_column(MutableJSONType) + request: Mapped[Optional[Dict]] = mapped_column(JSONType) workflow_step: Mapped[Optional["WorkflowStep"]] = relationship() workflow_invocation: Mapped[Optional["WorkflowInvocation"]] = relationship(back_populates="input_step_parameters") @@ -9520,7 +9608,6 @@ class WorkflowInvocationOutputDatasetAssociation(Base, Dictifiable, Serializable workflow_step_id: Mapped[Optional[int]] = mapped_column(ForeignKey("workflow_step.id"), index=True) dataset_id: Mapped[Optional[int]] = mapped_column(ForeignKey("history_dataset_association.id"), index=True) workflow_output_id: Mapped[Optional[int]] = mapped_column(ForeignKey("workflow_output.id"), index=True) - workflow_invocation: Mapped[Optional["WorkflowInvocation"]] = relationship(back_populates="output_datasets") workflow_step: Mapped[Optional["WorkflowStep"]] = relationship() dataset: Mapped[Optional["HistoryDatasetAssociation"]] = relationship() @@ -11162,6 +11249,43 @@ def file_source_configuration( raise ValueError("No template sent to file_source_configuration") +# TODO: add link from tool_request to this +class ToolLandingRequest(Base): + __tablename__ = "tool_landing_request" + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[Optional[int]] = mapped_column(ForeignKey("galaxy_user.id"), index=True) + create_time: Mapped[datetime] = mapped_column(default=now, nullable=True) + update_time: Mapped[Optional[datetime]] = mapped_column(index=True, default=now, onupdate=now, nullable=True) + uuid: Mapped[Union[UUID, str]] = mapped_column(UUIDType(), index=True) + tool_id: Mapped[str] = mapped_column(String(255)) + tool_version: Mapped[Optional[str]] = mapped_column(String(255), default=None) + request_state: Mapped[Optional[Dict]] = mapped_column(JSONType) + client_secret: Mapped[Optional[str]] = mapped_column(String(255), default=None) + + user: Mapped[Optional["User"]] = relationship() + + +# TODO: add link from workflow_invocation to this +class WorkflowLandingRequest(Base): + __tablename__ = "workflow_landing_request" + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[Optional[int]] = mapped_column(ForeignKey("galaxy_user.id"), index=True) + workflow_id: Mapped[Optional[int]] = mapped_column(ForeignKey("stored_workflow.id"), nullable=True) + stored_workflow_id: Mapped[Optional[int]] = mapped_column(ForeignKey("workflow.id"), nullable=True) + + create_time: Mapped[datetime] = mapped_column(default=now, nullable=True) + update_time: Mapped[Optional[datetime]] = mapped_column(index=True, default=now, onupdate=now, nullable=True) + uuid: Mapped[Union[UUID, str]] = mapped_column(UUIDType(), index=True) + request_state: Mapped[Optional[Dict]] = mapped_column(JSONType) + client_secret: Mapped[Optional[str]] = mapped_column(String(255), default=None) + + user: Mapped[Optional["User"]] = relationship() + stored_workflow: Mapped[Optional["StoredWorkflow"]] = relationship() + workflow: Mapped[Optional["Workflow"]] = relationship() + + class UserAction(Base, RepresentById): __tablename__ = "user_action" diff --git a/lib/galaxy/model/deferred.py b/lib/galaxy/model/deferred.py index a241d1418c1c..f9b5f1414d3c 100644 --- a/lib/galaxy/model/deferred.py +++ b/lib/galaxy/model/deferred.py @@ -4,6 +4,7 @@ import shutil from typing import ( cast, + List, NamedTuple, Optional, Union, @@ -25,7 +26,9 @@ Dataset, DatasetCollection, DatasetCollectionElement, + DatasetHash, DatasetSource, + DescribesHash, History, HistoryDatasetAssociation, HistoryDatasetCollectionAssociation, @@ -36,6 +39,7 @@ ObjectStore, ObjectStorePopulator, ) +from galaxy.util.hash_util import verify_hash log = logging.getLogger(__name__) @@ -95,6 +99,8 @@ def ensure_materialized( self, dataset_instance: Union[HistoryDatasetAssociation, LibraryDatasetDatasetAssociation], target_history: Optional[History] = None, + validate_hashes: bool = False, + in_place: bool = False, ) -> HistoryDatasetAssociation: """Create a new detached dataset instance from the supplied instance. @@ -106,15 +112,21 @@ def ensure_materialized( if dataset.state != Dataset.states.DEFERRED and isinstance(dataset_instance, HistoryDatasetAssociation): return dataset_instance - materialized_dataset = Dataset() - materialized_dataset.state = Dataset.states.OK - materialized_dataset.deleted = False - materialized_dataset.purged = False - materialized_dataset.sources = [s.copy() for s in dataset.sources] - materialized_dataset.hashes = [h.copy() for h in dataset.hashes] - + materialized_dataset_hashes = [h.copy() for h in dataset.hashes] + if in_place: + materialized_dataset = dataset_instance.dataset + materialized_dataset.state = Dataset.states.OK + else: + materialized_dataset = Dataset() + materialized_dataset.state = Dataset.states.OK + materialized_dataset.deleted = False + materialized_dataset.purged = False + materialized_dataset.sources = [s.copy() for s in dataset.sources] + materialized_dataset.hashes = materialized_dataset_hashes target_source = self._find_closest_dataset_source(dataset) transient_paths = None + + exception_materializing: Optional[Exception] = None if attached: object_store_populator = self._object_store_populator assert object_store_populator @@ -130,17 +142,28 @@ def ensure_materialized( with transaction(sa_session): sa_session.commit() object_store_populator.set_dataset_object_store_id(materialized_dataset) - path = self._stream_source(target_source, datatype=dataset_instance.datatype) - object_store.update_from_file(materialized_dataset, file_name=path) + try: + path = self._stream_source( + target_source, dataset_instance.datatype, validate_hashes, materialized_dataset_hashes + ) + object_store.update_from_file(materialized_dataset, file_name=path) + materialized_dataset.set_size() + except Exception as e: + exception_materializing = e else: transient_path_mapper = self._transient_path_mapper assert transient_path_mapper transient_paths = transient_path_mapper.transient_paths_for(dataset) # TODO: optimize this by streaming right to this path... # TODO: take into acount transform and ensure we are and are not modifying the file as appropriate. - path = self._stream_source(target_source, datatype=dataset_instance.datatype) - shutil.move(path, transient_paths.external_filename) - materialized_dataset.external_filename = transient_paths.external_filename + try: + path = self._stream_source( + target_source, dataset_instance.datatype, validate_hashes, materialized_dataset_hashes + ) + shutil.move(path, transient_paths.external_filename) + materialized_dataset.external_filename = transient_paths.external_filename + except Exception as e: + exception_materializing = e history = target_history if history is None and isinstance(dataset_instance, HistoryDatasetAssociation): @@ -148,19 +171,31 @@ def ensure_materialized( history = dataset_instance.history except DetachedInstanceError: history = None - materialized_dataset_instance = HistoryDatasetAssociation( - create_dataset=False, # is the default but lets make this really clear... - history=history, - ) + + materialized_dataset_instance: HistoryDatasetAssociation + if not in_place: + materialized_dataset_instance = HistoryDatasetAssociation( + create_dataset=False, # is the default but lets make this really clear... + history=history, + ) + else: + assert isinstance(dataset_instance, HistoryDatasetAssociation) + materialized_dataset_instance = cast(HistoryDatasetAssociation, dataset_instance) + if exception_materializing is not None: + materialized_dataset.state = Dataset.states.ERROR + materialized_dataset_instance.info = ( + f"Failed to materialize deferred dataset with exception: {exception_materializing}" + ) if attached: sa_session = self._sa_session if sa_session is None: sa_session = object_session(dataset_instance) assert sa_session sa_session.add(materialized_dataset_instance) - materialized_dataset_instance.copy_from( - dataset_instance, new_dataset=materialized_dataset, include_tags=attached, include_metadata=True - ) + if not in_place: + materialized_dataset_instance.copy_from( + dataset_instance, new_dataset=materialized_dataset, include_tags=attached, include_metadata=True + ) require_metadata_regeneration = ( materialized_dataset_instance.has_metadata_files or materialized_dataset_instance.metadata_deferred ) @@ -176,11 +211,17 @@ def ensure_materialized( materialized_dataset_instance.metadata_deferred = False return materialized_dataset_instance - def _stream_source(self, target_source: DatasetSource, datatype) -> str: + def _stream_source( + self, target_source: DatasetSource, datatype, validate_hashes: bool, dataset_hashes: List[DatasetHash] + ) -> str: source_uri = target_source.source_uri if source_uri is None: raise Exception("Cannot stream from dataset source without specified source_uri") path = stream_url_to_file(source_uri, file_sources=self._file_sources) + if validate_hashes and target_source.hashes: + for source_hash in target_source.hashes: + _validate_hash(path, source_hash, "downloaded file") + transform = target_source.transform or [] to_posix_lines = False spaces_to_tabs = False @@ -202,6 +243,11 @@ def _stream_source(self, target_source: DatasetSource, datatype) -> str: path = convert_result.converted_path if datatype_groom: datatype.groom_dataset_content(path) + + if validate_hashes and dataset_hashes: + for dataset_hash in dataset_hashes: + _validate_hash(path, dataset_hash, "dataset contents") + return path def _find_closest_dataset_source(self, dataset: Dataset) -> DatasetSource: @@ -298,3 +344,9 @@ def materializer_factory( file_sources=file_sources, sa_session=sa_session, ) + + +def _validate_hash(path: str, describes_hash: DescribesHash, what: str) -> None: + hash_value = describes_hash.hash_value + if hash_value is not None: + verify_hash(path, hash_func_name=describes_hash.hash_func_name, hash_value=hash_value) diff --git a/lib/galaxy/model/dereference.py b/lib/galaxy/model/dereference.py new file mode 100644 index 000000000000..ef84cea67b3d --- /dev/null +++ b/lib/galaxy/model/dereference.py @@ -0,0 +1,46 @@ +import os.path +from typing import List + +from galaxy.model import ( + DatasetSource, + DatasetSourceHash, + HistoryDatasetAssociation, + TransformAction, +) +from galaxy.tool_util.parameters import DataRequestUri + + +def dereference_to_model(sa_session, user, history, data_request_uri: DataRequestUri) -> HistoryDatasetAssociation: + name = data_request_uri.name or os.path.basename(data_request_uri.url) + dbkey = data_request_uri.dbkey or "?" + hda = HistoryDatasetAssociation( + name=name, + extension=data_request_uri.ext, + dbkey=dbkey, # TODO + history=history, + create_dataset=True, + sa_session=sa_session, + ) + hda.state = hda.states.DEFERRED + dataset_source = DatasetSource() + dataset_source.source_uri = data_request_uri.url + hashes = [] + for dataset_hash in data_request_uri.hashes or []: + hash_object = DatasetSourceHash() + hash_object.hash_function = dataset_hash.hash_function + hash_object.hash_value = dataset_hash.hash_value + hashes.append(hash_object) + dataset_source.hashes = hashes + hda.dataset.sources = [dataset_source] + transform: List[TransformAction] = [] + if data_request_uri.space_to_tab: + transform.append({"action": "space_to_tab"}) + elif data_request_uri.to_posix_lines: + transform.append({"action": "to_posix_lines"}) + if len(transform) > 0: + dataset_source.transform = transform + + sa_session.add(hda) + sa_session.add(dataset_source) + history.add_dataset(hda, genome_build=dbkey, quota=False) + return hda diff --git a/lib/galaxy/model/migrations/alembic/versions_gxy/7ffd33d5d144_implement_structured_tool_state.py b/lib/galaxy/model/migrations/alembic/versions_gxy/7ffd33d5d144_implement_structured_tool_state.py new file mode 100644 index 000000000000..027338dac279 --- /dev/null +++ b/lib/galaxy/model/migrations/alembic/versions_gxy/7ffd33d5d144_implement_structured_tool_state.py @@ -0,0 +1,178 @@ +"""implement structured tool state + +Revision ID: 7ffd33d5d144 +Revises: 25b092f7938b +Create Date: 2022-11-09 15:53:11.451185 + +""" + +from sqlalchemy import ( + Column, + DateTime, + Integer, + String, +) + +from galaxy.model.custom_types import ( + JSONType, + UUIDType, +) +from galaxy.model.database_object_names import build_index_name +from galaxy.model.migrations.util import ( + add_column, + create_foreign_key, + create_index, + create_table, + drop_column, + drop_index, + drop_table, + transaction, +) + +# revision identifiers, used by Alembic. +revision = "7ffd33d5d144" +down_revision = "25b092f7938b" +branch_labels = None +depends_on = None + +job_table_name = "job" +tool_source_table_name = "tool_source" +tool_request_table_name = "tool_request" +request_column_name = "tool_request_id" +job_request_index_name = build_index_name(job_table_name, request_column_name) +workflow_request_to_input_dataset_table_name = "workflow_request_to_input_dataset" +workflow_request_to_input_collection_table_name = "workflow_request_to_input_collection_dataset" +workflow_request_to_input_parameter_table_name = "workflow_request_input_step_parameter" +workflow_input_request_column_name = "request" +workflow_landing_request_table_name = "workflow_landing_request" +tool_landing_request_table_name = "tool_landing_request" + + +def upgrade(): + with transaction(): + create_table( + tool_landing_request_table_name, + Column("id", Integer, primary_key=True), + Column("user_id", Integer, index=True, nullable=True), + Column("create_time", DateTime), + Column("update_time", DateTime, nullable=True), + Column("request_state", JSONType), + Column("client_secret", String(255), nullable=True), + Column("uuid", UUIDType, nullable=False, index=True), + ) + + create_table( + workflow_landing_request_table_name, + Column("id", Integer, primary_key=True), + Column("user_id", Integer, index=True, nullable=True), + Column("stored_workflow_id", Integer, index=True), + Column("workflow_id", Integer, index=True), + Column("create_time", DateTime), + Column("update_time", DateTime, nullable=True), + Column("request_state", JSONType), + Column("client_secret", String(255), nullable=True), + Column("uuid", UUIDType, nullable=False, index=True), + ) + + create_foreign_key( + "foreign_key_user_id", + tool_landing_request_table_name, + "galaxy_user", + ["user_id"], + ["id"], + ) + create_foreign_key( + "foreign_key_user_id", + workflow_landing_request_table_name, + "galaxy_user", + ["user_id"], + ["id"], + ) + create_foreign_key( + "foreign_key_stored_workflow_id", + workflow_landing_request_table_name, + "stored_workflow", + ["stored_workflow_id"], + ["id"], + ) + create_foreign_key( + "foreign_key_workflow_id", + workflow_landing_request_table_name, + "workflow", + ["workflow_id"], + ["id"], + ) + + create_table( + tool_source_table_name, + Column("id", Integer, primary_key=True), + Column("hash", String(255), index=True), + Column("source", JSONType), + ) + create_table( + tool_request_table_name, + Column("id", Integer, primary_key=True), + Column("request", JSONType), + Column("state", String(32)), + Column("state_message", JSONType), + Column("tool_source_id", Integer, index=True), + Column("history_id", Integer, index=True), + ) + + create_foreign_key( + "foreign_key_tool_source_id", + tool_request_table_name, + tool_source_table_name, + ["tool_source_id"], + ["id"], + ) + + create_foreign_key( + "foreign_key_history_id", + tool_request_table_name, + "history", + ["history_id"], + ["id"], + ) + + add_column( + job_table_name, + Column(request_column_name, Integer, default=None), + ) + + create_foreign_key( + "foreign_key_tool_request_id", + job_table_name, + tool_request_table_name, + ["tool_request_id"], + ["id"], + ) + + create_index(job_request_index_name, job_table_name, [request_column_name]) + + add_column( + workflow_request_to_input_dataset_table_name, + Column(workflow_input_request_column_name, JSONType, default=None), + ) + add_column( + workflow_request_to_input_collection_table_name, + Column(workflow_input_request_column_name, JSONType, default=None), + ) + add_column( + workflow_request_to_input_parameter_table_name, + Column(workflow_input_request_column_name, JSONType, default=None), + ) + + +def downgrade(): + with transaction(): + drop_column(workflow_request_to_input_dataset_table_name, workflow_input_request_column_name) + drop_column(workflow_request_to_input_collection_table_name, workflow_input_request_column_name) + drop_column(workflow_request_to_input_parameter_table_name, workflow_input_request_column_name) + drop_index(job_request_index_name, job_table_name) + drop_column(job_table_name, request_column_name) + drop_table(tool_request_table_name) + drop_table(tool_source_table_name) + + drop_table(tool_landing_request_table_name) + drop_table(workflow_landing_request_table_name) diff --git a/lib/galaxy/model/unittest_utils/store_fixtures.py b/lib/galaxy/model/unittest_utils/store_fixtures.py index d779e79d698e..c05f0660fcf1 100644 --- a/lib/galaxy/model/unittest_utils/store_fixtures.py +++ b/lib/galaxy/model/unittest_utils/store_fixtures.py @@ -11,7 +11,7 @@ TEST_SOURCE_URI = "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/2.bed" TEST_SOURCE_URI_BAM = "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bam" TEST_HASH_FUNCTION = "MD5" -TEST_HASH_VALUE = "moocowpretendthisisahas" +TEST_HASH_VALUE = "f568c29421792b1b1df4474dafae01f1" TEST_HISTORY_NAME = "My history in a model store" TEST_EXTENSION = "bed" TEST_LIBRARY_NAME = "My cool library" diff --git a/lib/galaxy/schema/fetch_data.py b/lib/galaxy/schema/fetch_data.py index 2603b1471ea5..b415ffebf64e 100644 --- a/lib/galaxy/schema/fetch_data.py +++ b/lib/galaxy/schema/fetch_data.py @@ -101,6 +101,13 @@ class ExtraFiles(FetchBaseModel): ) +class FetchDatasetHash(Model): + hash_function: Literal["MD5", "SHA-1", "SHA-256", "SHA-512"] + hash_value: str + + model_config = ConfigDict(extra="forbid") + + class BaseDataElement(FetchBaseModel): name: Optional[CoercedStringType] = None dbkey: str = Field("?") @@ -116,6 +123,10 @@ class BaseDataElement(FetchBaseModel): items_from: Optional[ElementsFromType] = Field(None, alias="elements_from") collection_type: Optional[str] = None MD5: Optional[str] = None + SHA1: Optional[str] = Field(None, alias="SHA-1") + SHA256: Optional[str] = Field(None, alias="SHA-256") + SHA512: Optional[str] = Field(None, alias="SHA-512") + hashes: Optional[List[FetchDatasetHash]] = None description: Optional[str] = None model_config = ConfigDict(extra="forbid") diff --git a/lib/galaxy/schema/invocation.py b/lib/galaxy/schema/invocation.py index 4d5ce80548e8..1040aac4d1b4 100644 --- a/lib/galaxy/schema/invocation.py +++ b/lib/galaxy/schema/invocation.py @@ -281,6 +281,7 @@ class InvocationMessageResponseModel(RootModel): class InvocationState(str, Enum): NEW = "new" # Brand new workflow invocation... maybe this should be same as READY + REQUIRES_MATERIALIZATION = "requires_materialization" # an otherwise NEW or READY workflow that requires inputs to be materialized (undeferred) READY = "ready" # Workflow ready for another iteration of scheduling. SCHEDULED = "scheduled" # Workflow has been scheduled. CANCELLED = "cancelled" diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index 8b9c1ecc0bdd..a2cb9ef72272 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -3338,7 +3338,7 @@ class HDACustom(HDADetailed): # TODO: Fix this workaround for partial_model not supporting UUID fields for some reason. # The error otherwise is: `PydanticUserError: 'UuidVersion' cannot annotate 'nullable'.` # Also ignoring mypy complaints about the type redefinition. - uuid: Optional[UUID4] # type: ignore + uuid: Optional[UUID4] # type: ignore[assignment] # Add fields that are not part of any view here visualizations: Annotated[ @@ -3681,7 +3681,15 @@ class PageSummaryBase(Model): ) -class MaterializeDatasetInstanceAPIRequest(Model): +class MaterializeDatasetOptions(Model): + validate_hashes: bool = Field( + False, + title="Validate hashes", + description="Set to true to enable dataset validation during materialization.", + ) + + +class MaterializeDatasetInstanceAPIRequest(MaterializeDatasetOptions): source: DatasetSourceType = Field( title="Source", description="The source of the content. Can be other history element to be copied or library elements.", @@ -3737,6 +3745,22 @@ class AsyncTaskResultSummary(Model): ) +ToolRequestIdField = Field(title="ID", description="Encoded ID of the role") + + +class ToolRequestState(str, Enum): + NEW = "new" + SUBMITTED = "submitted" + FAILED = "failed" + + +class ToolRequestModel(Model): + id: EncodedDatabaseIdField = ToolRequestIdField + request: Dict[str, Any] + state: ToolRequestState + state_message: Optional[str] + + class AsyncFile(Model): storage_request_id: UUID task: AsyncTaskResultSummary @@ -3816,6 +3840,49 @@ class PageSummaryList(RootModel): ) +class LandingRequestState(str, Enum): + UNCLAIMED = "unclaimed" + CLAIMED = "claimed" + + +ToolLandingRequestIdField = Field(title="ID", description="Encoded ID of the tool landing request") +WorkflowLandingRequestIdField = Field(title="ID", description="Encoded ID of the workflow landing request") + + +class CreateToolLandingRequestPayload(Model): + tool_id: str + tool_version: Optional[str] = None + request_state: Optional[Dict[str, Any]] = None + client_secret: Optional[str] = None + + +class CreateWorkflowLandingRequestPayload(Model): + workflow_id: str + workflow_target_type: Literal["stored_workflow", "workflow"] + request_state: Optional[Dict[str, Any]] = None + client_secret: Optional[str] = None + + +class ClaimLandingPayload(Model): + client_secret: Optional[str] = None + + +class ToolLandingRequest(Model): + uuid: UuidField + tool_id: str + tool_version: Optional[str] = None + request_state: Optional[Dict[str, Any]] = None + state: LandingRequestState + + +class WorkflowLandingRequest(Model): + uuid: UuidField + workflow_id: str + workflow_target_type: Literal["stored_workflow", "workflow"] + request_state: Dict[str, Any] + state: LandingRequestState + + class MessageExceptionModel(BaseModel): err_msg: str err_code: int diff --git a/lib/galaxy/schema/tasks.py b/lib/galaxy/schema/tasks.py index 022d82666aed..d83640f52ffa 100644 --- a/lib/galaxy/schema/tasks.py +++ b/lib/galaxy/schema/tasks.py @@ -104,10 +104,15 @@ class MaterializeDatasetInstanceTaskRequest(Model): title="Content", description=( "Depending on the `source` it can be:\n" - "- The encoded id of the source library dataset\n" - "- The encoded id of the HDA\n" + "- The decoded id of the source library dataset\n" + "- The decoded id of the HDA\n" ), ) + validate_hashes: bool = Field( + False, + title="Validate hashes", + description="Set to true to enable dataset validation during materialization.", + ) class ComputeDatasetHashTaskRequest(Model): diff --git a/lib/galaxy/tool_util/client/landing.py b/lib/galaxy/tool_util/client/landing.py new file mode 100644 index 000000000000..425559a87ffa --- /dev/null +++ b/lib/galaxy/tool_util/client/landing.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python + +# . .venv/bin/activate; PYTHONPATH=lib python lib/galaxy/tool_util/client/landing.py -g http://localhost:8081 simple_workflow + +import argparse +import random +import string +import sys +from dataclasses import dataclass +from typing import Optional + +import requests +import yaml + +from galaxy.util.resources import resource_string + +DESCRIPTION = """ +A small utility for demoing creation of tool and workflow landing endpoints. + +This allows an external developer to create tool and workflow forms with +pre-populated parameters and handing them off with URLs to their clients/users. +""" +RANDOM_SECRET_LENGTH = 10 + + +def load_default_catalog(): + catalog_yaml = resource_string("galaxy.tool_util.client", "landing_catalog.sample.yml") + return yaml.safe_load(catalog_yaml) + + +@dataclass +class Request: + template_id: str + catalog: str + client_secret: Optional[str] + galaxy_url: str + + +@dataclass +class Response: + landing_url: str + + +def generate_claim_url(request: Request) -> Response: + template_id = request.template_id + catalog_path = request.catalog + galaxy_url = request.galaxy_url + client_secret = request.client_secret + if client_secret == "__GEN__": + client_secret = "".join( + random.choice(string.ascii_lowercase + string.digits) for _ in range(RANDOM_SECRET_LENGTH) + ) + if catalog_path: + with open(catalog_path) as f: + catalog = yaml.safe_load(f) + else: + catalog = load_default_catalog() + template = catalog[template_id] + template_type = "tool" if "tool_id" in template else "workflow" + if client_secret: + template["client_secret"] = client_secret + + landing_request_url = f"{galaxy_url}/api/{template_type}_landings" + raw_response = requests.post( + landing_request_url, + json=template, + ) + raw_response.raise_for_status() + response = raw_response.json() + url = f"{galaxy_url}/{template_type}_landings/{response['uuid']}" + if client_secret: + url = url + f"?secret={client_secret}" + return Response(url) + + +def arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=DESCRIPTION) + parser.add_argument("template_id") + parser.add_argument( + "-g", + "--galaxy-url", + dest="galaxy_url", + default="https://usegalaxy.org/", + help="Galxy target for the landing request", + ) + parser.add_argument( + "-c", + "--catalog", + dest="catalog", + default=None, + help="YAML catalog to load landing request templates from.", + ) + parser.add_argument( + "-s", + "--secret", + dest="secret", + default=None, + help="An optional client secret to verify the request against, set to __GEN__ to generate one at random for this request.", + ) + return parser + + +def main(argv=None) -> None: + if argv is None: + argv = sys.argv[1:] + + args = arg_parser().parse_args(argv) + request = Request( + args.template_id, + args.catalog, + args.secret, + args.galaxy_url, + ) + response = generate_claim_url(request) + print(f"Your customized form is located at {response.landing_url}") + + +if __name__ == "__main__": + main() diff --git a/lib/galaxy/tool_util/client/landing_library.catalog.yml b/lib/galaxy/tool_util/client/landing_library.catalog.yml new file mode 100644 index 000000000000..71a1c6e03f62 --- /dev/null +++ b/lib/galaxy/tool_util/client/landing_library.catalog.yml @@ -0,0 +1,21 @@ +# Catalog of artifacts that the landing script can generate landing URLs for against +# the target Galaxy. +simple_workflow: + # Encoded ID for a workflow to target - if workflow_target_type is stored_workflow this + # will always be the latest version of that workflow and stored workflow ID should be provided. + # If instead workflow_target_type is workflow - this is a particular version of a workflow and + # the workflow ID should be provided. + workflow_id: f2db41e1fa331b3e + workflow_target_type: stored_workflow + # request state provides prepopulated parameters for this workflow or stored workflow when the + # user navigates to the landing URL. + request_state: + myinput: + src: url + url: https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bed + ext: txt +int_workflow: + workflow_id: f597429621d6eb2b + workflow_target_type: stored_workflow + request_state: + int_input: 8 diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_update_singularity_containers.py b/lib/galaxy/tool_util/deps/mulled/mulled_update_singularity_containers.py index deb62f0b24b8..b1fd62feed80 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_update_singularity_containers.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_update_singularity_containers.py @@ -1,7 +1,6 @@ #!/usr/bin/env python import argparse -import os import os.path import subprocess import tempfile @@ -11,10 +10,10 @@ Any, Dict, List, - Union, ) from galaxy.util import unicodify +from galaxy.util.path import StrPath from .get_tests import ( hashed_test_search, import_test_to_command_list, @@ -46,7 +45,7 @@ def docker_to_singularity(container, installation, filepath, no_sudo=False): def singularity_container_test( - tests: Dict[str, Dict[str, Any]], installation: str, filepath: Union[str, os.PathLike] + tests: Dict[str, Dict[str, Any]], installation: str, filepath: StrPath ) -> Dict[str, List]: """ Run tests, record if they pass or fail diff --git a/lib/galaxy/tool_util/models.py b/lib/galaxy/tool_util/models.py index 4f1ea35670c6..eb76e1d711ef 100644 --- a/lib/galaxy/tool_util/models.py +++ b/lib/galaxy/tool_util/models.py @@ -13,12 +13,16 @@ ) from pydantic import ( + AfterValidator, AnyUrl, BaseModel, ConfigDict, + Field, RootModel, ) from typing_extensions import ( + Annotated, + Literal, NotRequired, TypedDict, ) @@ -113,7 +117,7 @@ class BaseTestOutputModel(StrictModel): class TestDataOutputAssertions(BaseTestOutputModel): - pass + class_: Optional[Literal["File"]] = Field("File", alias="class") class TestCollectionCollectionElementAssertions(StrictModel): @@ -131,14 +135,29 @@ class TestCollectionDatasetElementAssertions(BaseTestOutputModel): TestCollectionCollectionElementAssertions.model_rebuild() +def _check_collection_type(v: str) -> str: + if len(v) == 0: + raise ValueError("Invalid empty collection_type specified.") + collection_levels = v.split(":") + for collection_level in collection_levels: + if collection_level not in ["list", "paired"]: + raise ValueError(f"Invalid collection_type specified [{v}]") + return v + + +CollectionType = Annotated[Optional[str], AfterValidator(_check_collection_type)] + + class CollectionAttributes(StrictModel): - collection_type: Optional[str] = None + collection_type: CollectionType = None class TestCollectionOutputAssertions(StrictModel): + class_: Optional[Literal["Collection"]] = Field("Collection", alias="class") elements: Optional[Dict[str, TestCollectionElementAssertion]] = None element_tests: Optional[Dict[str, "TestCollectionElementAssertion"]] = None attributes: Optional[CollectionAttributes] = None + collection_type: CollectionType = None TestOutputLiteral = Union[bool, int, float, str] diff --git a/lib/galaxy/tool_util/parameters/__init__.py b/lib/galaxy/tool_util/parameters/__init__.py index ac6ca0dbbf3c..4a54ba0a8373 100644 --- a/lib/galaxy/tool_util/parameters/__init__.py +++ b/lib/galaxy/tool_util/parameters/__init__.py @@ -1,7 +1,10 @@ from .case import test_case_state from .convert import ( decode, + dereference, encode, + encode_test, + fill_static_defaults, ) from .factory import ( from_input_source, @@ -26,7 +29,12 @@ CwlStringParameterModel, CwlUnionParameterModel, DataCollectionParameterModel, + DataCollectionRequest, DataParameterModel, + DataRequest, + DataRequestHda, + DataRequestInternalHda, + DataRequestUri, FloatParameterModel, HiddenParameterModel, IntegerParameterModel, @@ -42,6 +50,7 @@ validate_against_model, validate_internal_job, validate_internal_request, + validate_internal_request_dereferenced, validate_request, validate_test_case, validate_workflow_step, @@ -49,6 +58,7 @@ ) from .state import ( JobInternalToolState, + RequestInternalDereferencedToolState, RequestInternalToolState, RequestToolState, TestCaseToolState, @@ -75,6 +85,11 @@ "JobInternalToolState", "ToolParameterBundle", "ToolParameterBundleModel", + "DataRequest", + "DataRequestInternalHda", + "DataRequestHda", + "DataRequestUri", + "DataCollectionRequest", "ToolParameterModel", "IntegerParameterModel", "BooleanParameterModel", @@ -101,6 +116,7 @@ "validate_against_model", "validate_internal_job", "validate_internal_request", + "validate_internal_request_dereferenced", "validate_request", "validate_test_case", "validate_workflow_step", @@ -113,6 +129,7 @@ "test_case_state", "RequestToolState", "RequestInternalToolState", + "RequestInternalDereferencedToolState", "flat_state_path", "keys_starting_with", "visit_input_values", @@ -120,6 +137,9 @@ "VISITOR_NO_REPLACEMENT", "decode", "encode", + "encode_test", + "fill_static_defaults", + "dereference", "WorkflowStepToolState", "WorkflowStepLinkedToolState", ) diff --git a/lib/galaxy/tool_util/parameters/case.py b/lib/galaxy/tool_util/parameters/case.py index cd8aa4f71e04..d1ff72b67d24 100644 --- a/lib/galaxy/tool_util/parameters/case.py +++ b/lib/galaxy/tool_util/parameters/case.py @@ -43,6 +43,7 @@ ) INTEGER_STR_PATTERN = compile(r"^(\d+)$") +INTEGERS_STR_PATTERN = compile(r"^(\d+)(\s*,\s*(\d+))*$") COLUMN_NAME_STR_PATTERN = compile(r"^c(\d+): .*$") # In an effort to squeeze all the ambiguity out of test cases - at some point Marius and John # agree tools should be using value_json for typed inputs to parameters but John has come around on @@ -111,22 +112,31 @@ def legacy_from_string(parameter: ToolParameterT, value: Optional[Any], warnings "Likely using deprected truevalue/falsevalue in tool parameter - switch to 'true' or 'false'" ) elif isinstance(parameter, (DataColumnParameterModel,)): - integer_match = INTEGER_STR_PATTERN.match(value_str) - if integer_match: - if WARN_ON_UNTYPED_XML_STRINGS: + if parameter.multiple: + integers_match = INTEGER_STR_PATTERN.match(value_str) + if integers_match: + if WARN_ON_UNTYPED_XML_STRINGS: + warnings.append( + f"Implicitly converted {parameter.name} to a column index integer from a string value, please use 'value_json' to define this test input parameter value instead." + ) + result_value = [int(v.strip()) for v in value_str.split(",")] + else: + integer_match = INTEGER_STR_PATTERN.match(value_str) + if integer_match: + if WARN_ON_UNTYPED_XML_STRINGS: + warnings.append( + f"Implicitly converted {parameter.name} to a column index integer from a string value, please use 'value_json' to define this test input parameter value instead." + ) + result_value = int(value_str) + elif Version(profile) < Version("24.2"): + # allow this for older tools but new tools will just require the integer index warnings.append( - f"Implicitly converted {parameter.name} to a column index integer from a string value, please use 'value_json' to define this test input parameter value instead." + f"Using column names as test case values is deprecated, please adjust {parameter.name} to just use an integer column index." ) - result_value = int(value_str) - elif Version(profile) < Version("24.2"): - # allow this for older tools but new tools will just require the integer index - warnings.append( - f"Using column names as test case values is deprecated, please adjust {parameter.name} to just use an integer column index." - ) - column_name_value_match = COLUMN_NAME_STR_PATTERN.match(value_str) - if column_name_value_match: - column_part = column_name_value_match.group(1) - result_value = int(column_part) + column_name_value_match = COLUMN_NAME_STR_PATTERN.match(value_str) + if column_name_value_match: + column_part = column_name_value_match.group(1) + result_value = int(column_part) return result_value @@ -268,8 +278,8 @@ def _merge_into_state( for test_input_value in test_input_values: instance_test_input = test_input.copy() instance_test_input["value"] = test_input_value - input_value = xml_data_input_to_json(test_input) - input_value_list.append(input_value) + input_value_json = xml_data_input_to_json(instance_test_input) + input_value_list.append(input_value_json) input_value = input_value_list else: input_value = xml_data_input_to_json(test_input) @@ -286,11 +296,14 @@ def _select_which_when( conditional: ConditionalParameterModel, state: dict, inputs: ToolSourceTestInputs, prefix: str ) -> ConditionalWhen: test_parameter = conditional.test_parameter + is_boolean = test_parameter.parameter_type == "gx_boolean" test_parameter_name = test_parameter.name test_parameter_flat_path = flat_state_path(test_parameter_name, prefix) test_input = _input_for(test_parameter_flat_path, inputs) explicit_test_value = test_input["value"] if test_input else None + if is_boolean and isinstance(explicit_test_value, str): + explicit_test_value = asbool(explicit_test_value) test_value = validate_explicit_conditional_test_value(test_parameter_name, explicit_test_value) for when in conditional.whens: if test_value is None and when.is_default_when: diff --git a/lib/galaxy/tool_util/parameters/convert.py b/lib/galaxy/tool_util/parameters/convert.py index 14caed47e92c..9484cc6c7e71 100644 --- a/lib/galaxy/tool_util/parameters/convert.py +++ b/lib/galaxy/tool_util/parameters/convert.py @@ -1,24 +1,58 @@ """Utilities for converting between request states. """ +import logging from typing import ( Any, Callable, + cast, + Dict, + List, + Optional, + Union, ) +from galaxy.tool_util.parser.interface import ( + JsonTestCollectionDefDict, + JsonTestDatasetDefDict, +) from .models import ( + BooleanParameterModel, + ConditionalParameterModel, + ConditionalWhen, + DataCollectionRequest, + DataColumnParameterModel, + DataParameterModel, + DataRequestHda, + DataRequestInternalHda, + DataRequestUri, + DiscriminatorType, + DrillDownParameterModel, + FloatParameterModel, + GenomeBuildParameterModel, + HiddenParameterModel, + IntegerParameterModel, + RepeatParameterModel, + SectionParameterModel, + SelectParameterModel, ToolParameterBundle, ToolParameterT, ) from .state import ( + JobInternalToolState, + RequestInternalDereferencedToolState, RequestInternalToolState, RequestToolState, + TestCaseToolState, ) from .visitor import ( + validate_explicit_conditional_test_value, visit_input_values, VISITOR_NO_REPLACEMENT, ) +log = logging.getLogger(__name__) + def decode( external_state: RequestToolState, input_models: ToolParameterBundle, decode_id: Callable[[str], int] @@ -27,13 +61,30 @@ def decode( external_state.validate(input_models) + def decode_src_dict(src_dict: dict): + if "id" in src_dict: + decoded_dict = src_dict.copy() + decoded_dict["id"] = decode_id(src_dict["id"]) + return decoded_dict + else: + return src_dict + def decode_callback(parameter: ToolParameterT, value: Any): if parameter.parameter_type == "gx_data": + if value is None: + return VISITOR_NO_REPLACEMENT + data_parameter = cast(DataParameterModel, parameter) + if data_parameter.multiple: + assert isinstance(value, list), str(value) + return list(map(decode_src_dict, value)) + else: + assert isinstance(value, dict), str(value) + return decode_src_dict(value) + elif parameter.parameter_type == "gx_data_collection": + if value is None: + return VISITOR_NO_REPLACEMENT assert isinstance(value, dict), str(value) - assert "id" in value - decoded_dict = value.copy() - decoded_dict["id"] = decode_id(value["id"]) - return decoded_dict + return decode_src_dict(value) else: return VISITOR_NO_REPLACEMENT @@ -53,13 +104,26 @@ def encode( ) -> RequestToolState: """Prepare an external representation of tool state (request) for storing in the database (request_internal).""" + def encode_src_dict(src_dict: dict): + if "id" in src_dict: + encoded_dict = src_dict.copy() + encoded_dict["id"] = encode_id(src_dict["id"]) + return encoded_dict + else: + return src_dict + def encode_callback(parameter: ToolParameterT, value: Any): if parameter.parameter_type == "gx_data": + data_parameter = cast(DataParameterModel, parameter) + if data_parameter.multiple: + assert isinstance(value, list), str(value) + return list(map(encode_src_dict, value)) + else: + assert isinstance(value, dict), str(value) + return encode_src_dict(value) + elif parameter.parameter_type == "gx_data_collection": assert isinstance(value, dict), str(value) - assert "id" in value - encoded_dict = value.copy() - encoded_dict["id"] = encode_id(value["id"]) - return encoded_dict + return encode_src_dict(value) else: return VISITOR_NO_REPLACEMENT @@ -71,3 +135,226 @@ def encode_callback(parameter: ToolParameterT, value: Any): request_state = RequestToolState(request_state_dict) request_state.validate(input_models) return request_state + + +DereferenceCallable = Callable[[DataRequestUri], DataRequestInternalHda] + + +def dereference( + internal_state: RequestInternalToolState, input_models: ToolParameterBundle, dereference: DereferenceCallable +) -> RequestInternalDereferencedToolState: + + def derefrence_dict(src_dict: dict): + src = src_dict.get("src") + if src == "url": + data_request_uri: DataRequestUri = DataRequestUri.model_validate(src_dict) + data_request_hda: DataRequestInternalHda = dereference(data_request_uri) + return data_request_hda.model_dump() + else: + return src_dict + + def dereference_callback(parameter: ToolParameterT, value: Any): + if parameter.parameter_type == "gx_data": + if value is None: + return VISITOR_NO_REPLACEMENT + data_parameter = cast(DataParameterModel, parameter) + if data_parameter.multiple: + assert isinstance(value, list), str(value) + return list(map(derefrence_dict, value)) + else: + assert isinstance(value, dict), str(value) + return derefrence_dict(value) + else: + return VISITOR_NO_REPLACEMENT + + request_state_dict = visit_input_values( + input_models, + internal_state, + dereference_callback, + ) + request_state = RequestInternalDereferencedToolState(request_state_dict) + request_state.validate(input_models) + return request_state + + +# interfaces for adapting test data dictionaries to tool request dictionaries +# e.g. {class: File, path: foo.bed} => {src: hda, id: ab1235cdfea3} +AdaptDatasets = Callable[[JsonTestDatasetDefDict], DataRequestHda] +AdaptCollections = Callable[[JsonTestCollectionDefDict], DataCollectionRequest] + + +def encode_test( + test_case_state: TestCaseToolState, + input_models: ToolParameterBundle, + adapt_datasets: AdaptDatasets, + adapt_collections: AdaptCollections, +): + + def encode_callback(parameter: ToolParameterT, value: Any): + if parameter.parameter_type == "gx_data": + data_parameter = cast(DataParameterModel, parameter) + if value is not None: + if data_parameter.multiple: + assert isinstance(value, list), str(value) + test_datasets = cast(List[JsonTestDatasetDefDict], value) + return [d.model_dump() for d in map(adapt_datasets, test_datasets)] + else: + assert isinstance(value, dict), str(value) + test_dataset = cast(JsonTestDatasetDefDict, value) + return adapt_datasets(test_dataset).model_dump() + elif parameter.parameter_type == "gx_data_collection": + # data_collection_parameter = cast(DataCollectionParameterModel, parameter) + if value is not None: + assert isinstance(value, dict), str(value) + test_collection = cast(JsonTestCollectionDefDict, value) + return adapt_collections(test_collection).model_dump() + elif parameter.parameter_type == "gx_select": + select_parameter = cast(SelectParameterModel, parameter) + if select_parameter.multiple and value is not None: + return [v.strip() for v in value.split(",")] + else: + return VISITOR_NO_REPLACEMENT + elif parameter.parameter_type == "gx_drill_down": + drilldown = cast(DrillDownParameterModel, parameter) + if drilldown.multiple and value is not None: + return [v.strip() for v in value.split(",")] + else: + return VISITOR_NO_REPLACEMENT + elif parameter.parameter_type == "gx_data_column": + data_column = cast(DataColumnParameterModel, parameter) + is_multiple = data_column.multiple + if is_multiple and value is not None and isinstance(value, (str,)): + return [int(v.strip()) for v in value.split(",")] + else: + return VISITOR_NO_REPLACEMENT + + return VISITOR_NO_REPLACEMENT + + request_state_dict = visit_input_values( + input_models, + test_case_state, + encode_callback, + ) + request_state = RequestToolState(request_state_dict) + request_state.validate(input_models) + return request_state + + +def fill_static_defaults( + tool_state: Dict[str, Any], input_models: ToolParameterBundle, profile: float, partial: bool = True +) -> Dict[str, Any]: + """If additional defaults might stem from Galaxy runtime, partial should be true. + + Setting partial to True, prevents runtime validation. + """ + _fill_defaults(tool_state, input_models) + + if not partial: + internal_state = JobInternalToolState(tool_state) + internal_state.validate(input_models) + return tool_state + + +def _fill_defaults(tool_state: Dict[str, Any], input_models: ToolParameterBundle) -> None: + for parameter in input_models.parameters: + _fill_default_for(tool_state, parameter) + + +def _fill_default_for(tool_state: Dict[str, Any], parameter: ToolParameterT) -> None: + parameter_name = parameter.name + parameter_type = parameter.parameter_type + if parameter_type == "gx_boolean": + boolean = cast(BooleanParameterModel, parameter) + if parameter_name not in tool_state: + # even optional parameters default to false if not in the body of the request :_( + # see test_tools.py -> expression_null_handling_boolean or test cases for gx_boolean_optional.xml + tool_state[parameter_name] = boolean.value or False + + if parameter_type in ["gx_integer", "gx_float", "gx_hidden"]: + has_value_parameter = cast( + Union[ + IntegerParameterModel, + FloatParameterModel, + HiddenParameterModel, + ], + parameter, + ) + if parameter_name not in tool_state: + tool_state[parameter_name] = has_value_parameter.value + elif parameter_type == "gx_genomebuild": + genomebuild = cast(GenomeBuildParameterModel, parameter) + if parameter_name not in tool_state and genomebuild.optional: + tool_state[parameter_name] = None + elif parameter_type == "gx_select": + select = cast(SelectParameterModel, parameter) + # don't fill in dynamic parameters - wait for runtime to specify the default + if select.dynamic_options: + return + + if parameter_name not in tool_state: + if not select.multiple: + tool_state[parameter_name] = select.default_value + else: + tool_state[parameter_name] = None + elif parameter_type == "gx_drill_down": + if parameter_name not in tool_state: + drilldown = cast(DrillDownParameterModel, parameter) + if drilldown.multiple: + options = drilldown.default_options + if options is not None: + tool_state[parameter_name] = options + else: + option = drilldown.default_option + if option is not None: + tool_state[parameter_name] = option + elif parameter_type in ["gx_conditional"]: + if parameter_name not in tool_state: + tool_state[parameter_name] = {} + + raw_conditional_state = tool_state[parameter_name] + assert isinstance(raw_conditional_state, dict) + conditional_state = cast(Dict[str, Any], raw_conditional_state) + + conditional = cast(ConditionalParameterModel, parameter) + test_parameter = conditional.test_parameter + test_parameter_name = test_parameter.name + + explicit_test_value: Optional[DiscriminatorType] = ( + conditional_state[test_parameter_name] if test_parameter_name in conditional_state else None + ) + test_value = validate_explicit_conditional_test_value(test_parameter_name, explicit_test_value) + when = _select_which_when(conditional, test_value, conditional_state) + test_parameter = conditional.test_parameter + _fill_default_for(conditional_state, test_parameter) + _fill_defaults(conditional_state, when) + elif parameter_type in ["gx_repeat"]: + if parameter_name not in tool_state: + tool_state[parameter_name] = [] + repeat_instances = cast(List[Dict[str, Any]], tool_state[parameter_name]) + repeat = cast(RepeatParameterModel, parameter) + if repeat.min: + while len(repeat_instances) < repeat.min: + repeat_instances.append({}) + + for instance_state in tool_state[parameter_name]: + _fill_defaults(instance_state, repeat) + elif parameter_type in ["gx_section"]: + if parameter_name not in tool_state: + tool_state[parameter_name] = {} + section_state = cast(Dict[str, Any], tool_state[parameter_name]) + section = cast(SectionParameterModel, parameter) + _fill_defaults(section_state, section) + + +def _select_which_when( + conditional: ConditionalParameterModel, test_value: Optional[DiscriminatorType], conditional_state: Dict[str, Any] +) -> ConditionalWhen: + for when in conditional.whens: + if test_value is None and when.is_default_when: + return when + elif test_value == when.discriminator: + return when + else: + raise Exception( + f"Invalid conditional test value ({test_value}) for parameter ({conditional.test_parameter.name})" + ) diff --git a/lib/galaxy/tool_util/parameters/factory.py b/lib/galaxy/tool_util/parameters/factory.py index a636b13a17e6..e200b67aeb0e 100644 --- a/lib/galaxy/tool_util/parameters/factory.py +++ b/lib/galaxy/tool_util/parameters/factory.py @@ -14,6 +14,7 @@ PagesSource, ToolSource, ) +from galaxy.tool_util.parser.util import parse_profile_version from galaxy.util import string_as_bool from .models import ( BaseUrlParameterModel, @@ -64,7 +65,7 @@ def get_color_value(input_source: InputSource) -> str: return input_source.get("value", "#000000") -def _from_input_source_galaxy(input_source: InputSource) -> ToolParameterT: +def _from_input_source_galaxy(input_source: InputSource, profile: float) -> ToolParameterT: input_type = input_source.parse_input_type() if input_type == "param": param_type = input_source.get("type") @@ -84,11 +85,11 @@ def _from_input_source_galaxy(input_source: InputSource) -> ToolParameterT: return IntegerParameterModel(name=input_source.parse_name(), optional=optional, value=int_value) elif param_type == "boolean": nullable = input_source.parse_optional() - checked = input_source.get_bool("checked", None if nullable else False) + value = input_source.get_bool_or_none("checked", None if nullable else False) return BooleanParameterModel( name=input_source.parse_name(), optional=nullable, - value=checked, + value=value, ) elif param_type == "text": optional = input_source.parse_optional() @@ -213,7 +214,8 @@ def _from_input_source_galaxy(input_source: InputSource) -> ToolParameterT: elif input_type == "conditional": test_param_input_source = input_source.parse_test_input_source() test_parameter = cast( - Union[BooleanParameterModel, SelectParameterModel], _from_input_source_galaxy(test_param_input_source) + Union[BooleanParameterModel, SelectParameterModel], + _from_input_source_galaxy(test_param_input_source, profile), ) whens = [] default_test_value = cond_test_parameter_default_value(test_parameter) @@ -224,12 +226,14 @@ def _from_input_source_galaxy(input_source: InputSource) -> ToolParameterT: else: typed_value = value - tool_parameter_models = input_models_for_page(case_inputs_sources) + tool_parameter_models = input_models_for_page(case_inputs_sources, profile) is_default_when = False if typed_value == default_test_value: is_default_when = True whens.append( - ConditionalWhen(discriminator=value, parameters=tool_parameter_models, is_default_when=is_default_when) + ConditionalWhen( + discriminator=typed_value, parameters=tool_parameter_models, is_default_when=is_default_when + ) ) return ConditionalParameterModel( name=input_source.parse_name(), @@ -241,7 +245,7 @@ def _from_input_source_galaxy(input_source: InputSource) -> ToolParameterT: # title = input_source.get("title") # help = input_source.get("help", None) instance_sources = input_source.parse_nested_inputs_source() - instance_tool_parameter_models = input_models_for_page(instance_sources) + instance_tool_parameter_models = input_models_for_page(instance_sources, profile) min_raw = input_source.get("min", None) max_raw = input_source.get("max", None) min = int(min_raw) if min_raw is not None else None @@ -255,7 +259,7 @@ def _from_input_source_galaxy(input_source: InputSource) -> ToolParameterT: elif input_type == "section": name = input_source.get("name") instance_sources = input_source.parse_nested_inputs_source() - instance_tool_parameter_models = input_models_for_page(instance_sources) + instance_tool_parameter_models = input_models_for_page(instance_sources, profile) return SectionParameterModel( name=name, parameters=instance_tool_parameter_models, @@ -328,34 +332,35 @@ def tool_parameter_bundle_from_json(json: Dict[str, Any]) -> ToolParameterBundle def input_models_for_tool_source(tool_source: ToolSource) -> ToolParameterBundleModel: pages = tool_source.parse_input_pages() - return ToolParameterBundleModel(parameters=input_models_for_pages(pages)) + profile = parse_profile_version(tool_source) + return ToolParameterBundleModel(parameters=input_models_for_pages(pages, profile)) -def input_models_for_pages(pages: PagesSource) -> List[ToolParameterT]: +def input_models_for_pages(pages: PagesSource, profile: float) -> List[ToolParameterT]: input_models = [] if pages.inputs_defined: for page_source in pages.page_sources: - input_models.extend(input_models_for_page(page_source)) + input_models.extend(input_models_for_page(page_source, profile)) return input_models -def input_models_for_page(page_source: PageSource) -> List[ToolParameterT]: +def input_models_for_page(page_source: PageSource, profile: float) -> List[ToolParameterT]: input_models = [] for input_source in page_source.parse_input_sources(): input_type = input_source.parse_input_type() if input_type == "display": # not a real input... just skip this. Should this be handled in the parser layer better? continue - tool_parameter_model = from_input_source(input_source) + tool_parameter_model = from_input_source(input_source, profile) input_models.append(tool_parameter_model) return input_models -def from_input_source(input_source: InputSource) -> ToolParameterT: +def from_input_source(input_source: InputSource, profile: float) -> ToolParameterT: tool_parameter: ToolParameterT if isinstance(input_source, CwlInputSource): tool_parameter = _from_input_source_cwl(input_source) else: - tool_parameter = _from_input_source_galaxy(input_source) + tool_parameter = _from_input_source_galaxy(input_source, profile) return tool_parameter diff --git a/lib/galaxy/tool_util/parameters/models.py b/lib/galaxy/tool_util/parameters/models.py index 30d6b5a5a483..11957c37099a 100644 --- a/lib/galaxy/tool_util/parameters/models.py +++ b/lib/galaxy/tool_util/parameters/models.py @@ -61,7 +61,13 @@ # + request_internal: This is a pydantic model to validate what Galaxy expects to find in the database, # in particular dataset and collection references should be decoded integers. StateRepresentationT = Literal[ - "request", "request_internal", "job_internal", "test_case_xml", "workflow_step", "workflow_step_linked" + "request", + "request_internal", + "request_internal_dereferenced", + "job_internal", + "test_case_xml", + "workflow_step", + "workflow_step_linked", ] @@ -182,7 +188,10 @@ def pydantic_template(self, state_representation: StateRepresentationT) -> Dynam py_type = self.py_type if state_representation == "workflow_step_linked": py_type = allow_connected_value(py_type) - return dynamic_model_information_from_py_type(self, py_type) + requires_value = self.request_requires_value + if state_representation == "job_internal": + requires_value = True + return dynamic_model_information_from_py_type(self, py_type, requires_value=requires_value) @property def request_requires_value(self) -> bool: @@ -204,7 +213,10 @@ def pydantic_template(self, state_representation: StateRepresentationT) -> Dynam py_type = self.py_type if state_representation == "workflow_step_linked": py_type = allow_connected_value(py_type) - return dynamic_model_information_from_py_type(self, py_type) + requires_value = self.request_requires_value + if state_representation == "job_internal": + requires_value = True + return dynamic_model_information_from_py_type(self, py_type, requires_value=requires_value) @property def request_requires_value(self) -> bool: @@ -239,49 +251,109 @@ def request_requires_value(self) -> bool: TestCaseDataSrcT = Literal["File"] -class DataRequest(StrictModel): - src: DataSrcT +class DataRequestHda(StrictModel): + src: Literal["hda"] = "hda" id: StrictStr -class BatchDataInstance(StrictModel): - src: MultiDataSrcT +class DataRequestLdda(StrictModel): + src: Literal["ldda"] = "ldda" id: StrictStr -class MultiDataInstance(StrictModel): +class DataRequestHdca(StrictModel): + src: Literal["hdca"] = "hdca" + id: StrictStr + + +class DatasetHash(StrictModel): + hash_function: Literal["MD5", "SHA-1", "SHA-256", "SHA-512"] + hash_value: StrictStr + + +class DataRequestUri(StrictModel): + # calling it url instead of uri to match data fetch schema... + src: Literal["url"] = "url" + url: StrictStr + name: Optional[StrictStr] = None + ext: StrictStr + dbkey: StrictStr = "?" + deferred: StrictBool = False + created_from_basename: Optional[StrictStr] = None + info: Optional[StrictStr] = None + hashes: Optional[List[DatasetHash]] = None + space_to_tab: bool = False + to_posix_lines: bool = False + # to implement: + # tags + + +DataRequest: Type = cast( + Type, Annotated[union_type([DataRequestHda, DataRequestLdda, DataRequestUri]), Field(discriminator="src")] +) + + +class BatchDataInstance(StrictModel): src: MultiDataSrcT id: StrictStr -MultiDataRequest: Type = union_type([MultiDataInstance, List[MultiDataInstance]]) +MultiDataInstance: Type = cast( + Type, + Annotated[ + union_type([DataRequestHda, DataRequestLdda, DataRequestHdca, DataRequestUri]), Field(discriminator="src") + ], +) +MultiDataRequest: Type = cast(Type, union_type([MultiDataInstance, list_type(MultiDataInstance)])) -class DataRequestInternal(StrictModel): - src: DataSrcT +class DataRequestInternalHda(StrictModel): + src: Literal["hda"] = "hda" id: StrictInt -class BatchDataInstanceInternal(StrictModel): - src: MultiDataSrcT +class DataRequestInternalLdda(StrictModel): + src: Literal["ldda"] = "ldda" id: StrictInt -class MultiDataInstanceInternal(StrictModel): - src: MultiDataSrcT +class DataRequestInternalHdca(StrictModel): + src: Literal["hdca"] = "hdca" id: StrictInt -class DataTestCaseValue(StrictModel): - src: TestCaseDataSrcT - path: str +DataRequestInternal: Type = cast( + Type, Annotated[Union[DataRequestInternalHda, DataRequestInternalLdda, DataRequestUri], Field(discriminator="src")] +) +DataRequestInternalDereferenced: Type = cast( + Type, Annotated[Union[DataRequestInternalHda, DataRequestInternalLdda], Field(discriminator="src")] +) +DataJobInternal = DataRequestInternalDereferenced + +class BatchDataInstanceInternal(StrictModel): + src: MultiDataSrcT + id: StrictInt -class MultipleDataTestCaseValue(RootModel): - root: List[DataTestCaseValue] +MultiDataInstanceInternal: Type = cast( + Type, + Annotated[ + Union[DataRequestInternalHda, DataRequestInternalLdda, DataRequestInternalHdca, DataRequestUri], + Field(discriminator="src"), + ], +) +MultiDataInstanceInternalDereferenced: Type = cast( + Type, + Annotated[ + Union[DataRequestInternalHda, DataRequestInternalLdda, DataRequestInternalHdca], Field(discriminator="src") + ], +) -MultiDataRequestInternal: Type = union_type([MultiDataInstanceInternal, List[MultiDataInstanceInternal]]) +MultiDataRequestInternal: Type = union_type([MultiDataInstanceInternal, list_type(MultiDataInstanceInternal)]) +MultiDataRequestInternalDereferenced: Type = union_type( + [MultiDataInstanceInternalDereferenced, list_type(MultiDataInstanceInternalDereferenced)] +) class DataParameterModel(BaseGalaxyToolParameterModelDefinition): @@ -309,6 +381,15 @@ def py_type_internal(self) -> Type: base_model = DataRequestInternal return optional_if_needed(base_model, self.optional) + @property + def py_type_internal_dereferenced(self) -> Type: + base_model: Type + if self.multiple: + base_model = MultiDataRequestInternalDereferenced + else: + base_model = DataRequestInternalDereferenced + return optional_if_needed(base_model, self.optional) + @property def py_type_test_case(self) -> Type: base_model: Type @@ -325,8 +406,13 @@ def pydantic_template(self, state_representation: StateRepresentationT) -> Dynam return allow_batching( dynamic_model_information_from_py_type(self, self.py_type_internal), BatchDataInstanceInternal ) + elif state_representation == "request_internal_dereferenced": + return allow_batching( + dynamic_model_information_from_py_type(self, self.py_type_internal_dereferenced), + BatchDataInstanceInternal, + ) elif state_representation == "job_internal": - return dynamic_model_information_from_py_type(self, self.py_type_internal) + return dynamic_model_information_from_py_type(self, self.py_type_internal_dereferenced, requires_value=True) elif state_representation == "test_case_xml": return dynamic_model_information_from_py_type(self, self.py_type_test_case) elif state_representation == "workflow_step": @@ -366,8 +452,10 @@ def py_type_internal(self) -> Type: def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: if state_representation == "request": return allow_batching(dynamic_model_information_from_py_type(self, self.py_type)) - elif state_representation == "request_internal": + elif state_representation in ["request_internal", "request_internal_dereferenced"]: return allow_batching(dynamic_model_information_from_py_type(self, self.py_type_internal)) + elif state_representation == "job_internal": + return dynamic_model_information_from_py_type(self, self.py_type_internal, requires_value=True) elif state_representation == "workflow_step": return dynamic_model_information_from_py_type(self, type(None), requires_value=False) elif state_representation == "workflow_step_linked": @@ -401,6 +489,8 @@ def pydantic_template(self, state_representation: StateRepresentationT) -> Dynam # allow it to be linked in so force allow optional... py_type = optional(py_type) requires_value = False + elif state_representation == "job_internal": + requires_value = True return dynamic_model_information_from_py_type(self, py_type, requires_value=requires_value) @property @@ -487,7 +577,10 @@ def pydantic_template(self, state_representation: StateRepresentationT) -> Dynam py_type = self.py_type if state_representation == "workflow_step_linked": py_type = allow_connected_value(py_type) - return dynamic_model_information_from_py_type(self, py_type) + requires_value = self.request_requires_value + if state_representation == "job_internal": + requires_value = True + return dynamic_model_information_from_py_type(self, py_type, requires_value=requires_value) @property def request_requires_value(self) -> bool: @@ -568,7 +661,7 @@ def py_type_if_required(self, allow_connections: bool = False) -> Type: @property def py_type(self) -> Type: - return optional_if_needed(self.py_type_if_required(), self.optional) + return optional_if_needed(self.py_type_if_required(), self.optional or self.multiple) @property def py_type_workflow_step(self) -> Type: @@ -580,7 +673,9 @@ def pydantic_template(self, state_representation: StateRepresentationT) -> Dynam return dynamic_model_information_from_py_type(self, self.py_type_workflow_step, requires_value=False) elif state_representation == "workflow_step_linked": py_type = self.py_type_if_required(allow_connections=True) - return dynamic_model_information_from_py_type(self, optional_if_needed(py_type, self.optional)) + return dynamic_model_information_from_py_type( + self, optional_if_needed(py_type, self.optional or self.multiple) + ) elif state_representation == "test_case_xml": # in a YAML test case representation this can be string, in XML we are still expecting a comma separated string py_type = self.py_type_if_required(allow_connections=False) @@ -592,7 +687,10 @@ def pydantic_template(self, state_representation: StateRepresentationT) -> Dynam self, optional_if_needed(py_type, self.optional), validators=validators ) else: - return dynamic_model_information_from_py_type(self, self.py_type) + requires_value = self.request_requires_value + if state_representation == "job_internal": + requires_value = True + return dynamic_model_information_from_py_type(self, self.py_type, requires_value=requires_value) @property def has_selected_static_option(self): @@ -614,8 +712,13 @@ def default_value(self) -> Optional[str]: def request_requires_value(self) -> bool: # API will allow an empty value and just grab the first static option # see API Tests -> test_tools.py -> test_select_first_by_default - # so only require a value in the multiple case if optional is False - return self.multiple and not self.optional + # If it is multiple - it will also always just allow null regardless of + # optional - see test_select_multiple_null_handling + return False + + @property + def dynamic_options(self) -> bool: + return self.options is None class GenomeBuildParameterModel(BaseGalaxyToolParameterModelDefinition): @@ -630,7 +733,10 @@ def py_type(self) -> Type: return optional_if_needed(py_type, self.optional) def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: - return dynamic_model_information_from_py_type(self, self.py_type) + requires_value = self.request_requires_value + if state_representation == "job_internal": + requires_value = True + return dynamic_model_information_from_py_type(self, self.py_type, requires_value=requires_value) @property def request_requires_value(self) -> bool: @@ -641,11 +747,13 @@ def request_requires_value(self) -> bool: DrillDownHierarchyT = Literal["recurse", "exact"] -def drill_down_possible_values(options: List[DrillDownOptionsDict], multiple: bool) -> List[str]: +def drill_down_possible_values( + options: List[DrillDownOptionsDict], multiple: bool, hierarchy: DrillDownHierarchyT +) -> List[str]: possible_values = [] def add_value(option: str, is_leaf: bool): - if not multiple and not is_leaf: + if not multiple and not is_leaf and hierarchy == "recurse": return possible_values.append(option) @@ -673,7 +781,8 @@ class DrillDownParameterModel(BaseGalaxyToolParameterModelDefinition): def py_type(self) -> Type: if self.options is not None: literal_options: List[Type] = [ - cast_as_type(Literal[o]) for o in drill_down_possible_values(self.options, self.multiple) + cast_as_type(Literal[o]) + for o in drill_down_possible_values(self.options, self.multiple, self.hierarchy) ] py_type = union_type(literal_options) else: @@ -690,10 +799,12 @@ def py_type_test_case_xml(self) -> Type: return optional_if_needed(base_model, not self.request_requires_value) def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: - if state_representation == "test_case_xml": - return dynamic_model_information_from_py_type(self, self.py_type_test_case_xml) - else: - return dynamic_model_information_from_py_type(self, self.py_type) + py_type = self.py_type_test_case_xml if state_representation == "test_case_xml" else self.py_type + requires_value = self.request_requires_value + if state_representation == "job_internal": + requires_value = True + + return dynamic_model_information_from_py_type(self, py_type, requires_value=requires_value) @property def request_requires_value(self) -> bool: @@ -706,6 +817,24 @@ def request_requires_value(self) -> bool: # do we need to default to assuming they're not required? return False + @property + def default_option(self) -> Optional[str]: + options = self.options + if options: + selected_options = selected_drill_down_options(options) + if len(selected_options) > 0: + return selected_options[0] + return None + + @property + def default_options(self) -> Optional[List[str]]: + options = self.options + if options: + selected_options = selected_drill_down_options(options) + return selected_options + + return None + def any_drill_down_options_selected(options: List[DrillDownOptionsDict]) -> bool: for option in options: @@ -719,6 +848,19 @@ def any_drill_down_options_selected(options: List[DrillDownOptionsDict]) -> bool return False +def selected_drill_down_options(options: List[DrillDownOptionsDict]) -> List[str]: + selected_options: List[str] = [] + for option in options: + selected = option.get("selected") + value = option.get("value") + if selected and value: + selected_options.append(value) + child_options = option.get("options", []) + selected_options.extend(selected_drill_down_options(child_options)) + + return selected_options + + class DataColumnParameterModel(BaseGalaxyToolParameterModelDefinition): parameter_type: Literal["gx_data_column"] = "gx_data_column" multiple: bool @@ -749,7 +891,10 @@ def pydantic_template(self, state_representation: StateRepresentationT) -> Dynam validators = {} return dynamic_model_information_from_py_type(self, self.py_type, validators=validators) else: - return dynamic_model_information_from_py_type(self, self.py_type) + requires_value = self.request_requires_value + if state_representation == "job_internal": + requires_value = True + return dynamic_model_information_from_py_type(self, self.py_type, requires_value=requires_value) @property def request_requires_value(self) -> bool: @@ -768,7 +913,10 @@ def py_type(self) -> Type: return optional_if_needed(py_type, self.optional) def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: - return dynamic_model_information_from_py_type(self, self.py_type) + requires_value = self.request_requires_value + if state_representation == "job_internal": + requires_value = True + return dynamic_model_information_from_py_type(self, self.py_type, requires_value=requires_value) @property def request_requires_value(self) -> bool: @@ -819,10 +967,14 @@ class ConditionalParameterModel(BaseGalaxyToolParameterModelDefinition): whens: List[ConditionalWhen] def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + is_boolean = isinstance(self.test_parameter, BooleanParameterModel) test_param_name = self.test_parameter.name test_info = self.test_parameter.pydantic_template(state_representation) extra_validators = test_info.validators - test_parameter_requires_value = self.test_parameter.request_requires_value + if state_representation == "job_internal": + test_parameter_requires_value = True + else: + test_parameter_requires_value = self.test_parameter.request_requires_value when_types: List[Type[BaseModel]] = [] default_type = None for when in self.whens: @@ -832,7 +984,7 @@ def pydantic_template(self, state_representation: StateRepresentationT) -> Dynam initialize_test = ... else: initialize_test = None - + tag = str(discriminator) if not is_boolean else str(discriminator).lower() extra_kwd = {test_param_name: (Union[str, bool], initialize_test)} when_types.append( cast( @@ -845,22 +997,29 @@ def pydantic_template(self, state_representation: StateRepresentationT) -> Dynam extra_kwd=extra_kwd, extra_validators=extra_validators, ), - Tag(str(discriminator)), + Tag(tag), ], ) ) - if when.is_default_when: - extra_kwd = {} - default_type = create_field_model( - parameters, - f"When_{test_param_name}___absent", - state_representation, - extra_kwd=extra_kwd, - extra_validators={}, - ) - when_types.append(cast(Type[BaseModel], Annotated[default_type, Tag("__absent__")])) - - def model_x_discriminator(v: Any) -> str: + # job_internal requires parameters are filled in - so don't allow the absent branch + # here that most other state representations allow + if state_representation != "job_internal": + if when.is_default_when: + extra_kwd = {} + default_type = create_field_model( + parameters, + f"When_{test_param_name}___absent", + state_representation, + extra_kwd=extra_kwd, + extra_validators={}, + ) + when_types.append(cast(Type[BaseModel], Annotated[default_type, Tag("__absent__")])) + + def model_x_discriminator(v: Any) -> Optional[str]: + # returning None causes a validation error, this is what we would want if + # if the conditional state is not a dictionary. + if not isinstance(v, dict): + return None if test_param_name not in v: return "__absent__" else: @@ -872,17 +1031,29 @@ def model_x_discriminator(v: Any) -> str: else: return str(test_param_val) - cond_type = union_type(when_types) + py_type: Type - class ConditionalType(RootModel): - root: cond_type = Field(..., discriminator=Discriminator(model_x_discriminator)) # type: ignore[valid-type] + if len(when_types) > 1: + cond_type = union_type(when_types) - if default_type is not None: - initialize_cond = None - else: - initialize_cond = ... + class ConditionalType(RootModel): + root: cond_type = Field(..., discriminator=Discriminator(model_x_discriminator)) # type: ignore[valid-type] - py_type = ConditionalType + if default_type is not None: + initialize_cond = None + else: + initialize_cond = ... + + py_type = ConditionalType + + else: + py_type = when_types[0] + # a better check here would be if any of the parameters below this have a required value, + # in the case of job_internal though this is correct + if state_representation == "job_internal": + initialize_cond = ... + else: + initialize_cond = None return DynamicModelInformation( self.name, @@ -906,9 +1077,12 @@ def pydantic_template(self, state_representation: StateRepresentationT) -> Dynam instance_class: Type[BaseModel] = create_field_model( self.parameters, f"Repeat_{self.name}", state_representation ) + requires_value = self.request_requires_value + if state_representation == "job_internal": + requires_value = True initialize_repeat: Any - if self.request_requires_value: + if requires_value: initialize_repeat = ... else: initialize_repeat = None @@ -942,7 +1116,10 @@ def pydantic_template(self, state_representation: StateRepresentationT) -> Dynam instance_class: Type[BaseModel] = create_field_model( self.parameters, f"Section_{self.name}", state_representation ) - if self.request_requires_value: + requires_value = self.request_requires_value + if state_representation == "job_internal": + requires_value = True + if requires_value: initialize_section = ... else: initialize_section = None @@ -1199,6 +1376,12 @@ def create_request_internal_model(tool: ToolParameterBundle, name: str = "Dynami return create_field_model(tool.parameters, name, "request_internal") +def create_request_internal_dereferenced_model( + tool: ToolParameterBundle, name: str = "DynamicModelForTool" +) -> Type[BaseModel]: + return create_field_model(tool.parameters, name, "request_internal_dereferenced") + + def create_job_internal_model(tool: ToolParameterBundle, name: str = "DynamicModelForTool") -> Type[BaseModel]: return create_field_model(tool.parameters, name, "job_internal") @@ -1259,6 +1442,11 @@ def validate_internal_request(tool: ToolParameterBundle, request: Dict[str, Any] validate_against_model(pydantic_model, request) +def validate_internal_request_dereferenced(tool: ToolParameterBundle, request: Dict[str, Any]) -> None: + pydantic_model = create_request_internal_dereferenced_model(tool) + validate_against_model(pydantic_model, request) + + def validate_internal_job(tool: ToolParameterBundle, request: Dict[str, Any]) -> None: pydantic_model = create_job_internal_model(tool) validate_against_model(pydantic_model, request) diff --git a/lib/galaxy/tool_util/parameters/state.py b/lib/galaxy/tool_util/parameters/state.py index 94eb48b5de52..af15cf23ac7b 100644 --- a/lib/galaxy/tool_util/parameters/state.py +++ b/lib/galaxy/tool_util/parameters/state.py @@ -15,6 +15,7 @@ from .models import ( create_job_internal_model, + create_request_internal_dereferenced_model, create_request_internal_model, create_request_model, create_test_case_model, @@ -83,6 +84,14 @@ def _parameter_model_for(cls, parameters: ToolParameterBundle) -> Type[BaseModel return create_request_internal_model(parameters) +class RequestInternalDereferencedToolState(ToolState): + state_representation: Literal["request_internal_dereferenced"] = "request_internal_dereferenced" + + @classmethod + def _parameter_model_for(cls, parameters: ToolParameterBundle) -> Type[BaseModel]: + return create_request_internal_dereferenced_model(parameters) + + class JobInternalToolState(ToolState): state_representation: Literal["job_internal"] = "job_internal" diff --git a/lib/galaxy/tool_util/parameters/visitor.py b/lib/galaxy/tool_util/parameters/visitor.py index 5b8e059f2895..35d4bc176a3e 100644 --- a/lib/galaxy/tool_util/parameters/visitor.py +++ b/lib/galaxy/tool_util/parameters/visitor.py @@ -12,14 +12,27 @@ from typing_extensions import Protocol from .models import ( + ConditionalParameterModel, + ConditionalWhen, + RepeatParameterModel, + SectionParameterModel, simple_input_models, ToolParameterBundle, ToolParameterT, ) from .state import ToolState -VISITOR_NO_REPLACEMENT = object() -VISITOR_UNDEFINED = object() + +class VisitorNoReplacement: + pass + + +class VisitorUndefined: + pass + + +VISITOR_NO_REPLACEMENT = VisitorNoReplacement() +VISITOR_UNDEFINED = VisitorUndefined() class Callback(Protocol): @@ -47,20 +60,79 @@ def _visit_input_values( callback: Callback, no_replacement_value=VISITOR_NO_REPLACEMENT, ) -> Dict[str, Any]: - new_input_values = {} + + def _callback(name: str, old_values: Dict[str, Any], new_values: Dict[str, Any]): + input_value = old_values.get(name, VISITOR_UNDEFINED) + if input_value is VISITOR_UNDEFINED: + return + replacement = callback(model, input_value) + if replacement != no_replacement_value: + new_values[name] = replacement + else: + new_values[name] = input_value + + new_input_values: Dict[str, Any] = {} for model in input_models: name = model.name + parameter_type = model.parameter_type input_value = input_values.get(name, VISITOR_UNDEFINED) - replacement = callback(model, input_value) - if replacement != no_replacement_value: - new_input_values[name] = replacement - elif replacement is VISITOR_UNDEFINED: - pass + if input_value is VISITOR_UNDEFINED: + continue + + if parameter_type == "gx_repeat": + repeat_parameter = cast(RepeatParameterModel, model) + repeat_parameters = repeat_parameter.parameters + repeat_values = cast(list, input_value) + new_repeat_values = [] + for repeat_instance_values in repeat_values: + new_repeat_values.append( + _visit_input_values( + repeat_parameters, repeat_instance_values, callback, no_replacement_value=no_replacement_value + ) + ) + new_input_values[name] = new_repeat_values + elif parameter_type == "gx_section": + section_parameter = cast(SectionParameterModel, model) + section_parameters = section_parameter.parameters + section_values = cast(dict, input_value) + new_section_values = _visit_input_values( + section_parameters, section_values, callback, no_replacement_value=no_replacement_value + ) + new_input_values[name] = new_section_values + elif parameter_type == "gx_conditional": + conditional_parameter = cast(ConditionalParameterModel, model) + test_parameter = conditional_parameter.test_parameter + test_parameter_name = test_parameter.name + + conditional_values = cast(dict, input_value) + when: ConditionalWhen = _select_which_when(conditional_parameter, conditional_values) + new_conditional_values = _visit_input_values( + when.parameters, conditional_values, callback, no_replacement_value=no_replacement_value + ) + if test_parameter_name in conditional_values: + _callback(test_parameter_name, conditional_values, new_conditional_values) + new_input_values[name] = new_conditional_values else: - new_input_values[name] = input_value + _callback(name, input_values, new_input_values) return new_input_values +def _select_which_when(conditional: ConditionalParameterModel, state: dict) -> ConditionalWhen: + test_parameter = conditional.test_parameter + test_parameter_name = test_parameter.name + explicit_test_value = state.get(test_parameter_name) + test_value = validate_explicit_conditional_test_value(test_parameter_name, explicit_test_value) + for when in conditional.whens: + print(when.discriminator) + print(type(when.discriminator)) + if test_value is None and when.is_default_when: + return when + elif test_value == when.discriminator: + return when + else: + raise Exception(f"Invalid conditional test value ({explicit_test_value}) for parameter ({test_parameter_name})") + + def flat_state_path(has_name: Union[str, ToolParameterT], prefix: Optional[str] = None) -> str: """Given a parameter name or model and an optional prefix, give 'flat' name for parameter in tree.""" if hasattr(has_name, "name"): diff --git a/lib/galaxy/tool_util/parser/factory.py b/lib/galaxy/tool_util/parser/factory.py index 4dda4589e596..f36647dfd40d 100644 --- a/lib/galaxy/tool_util/parser/factory.py +++ b/lib/galaxy/tool_util/parser/factory.py @@ -1,13 +1,11 @@ """Constructors for concrete tool and input source objects.""" import logging -from pathlib import PurePath from typing import ( Callable, Dict, List, Optional, - Union, ) from yaml import safe_load @@ -17,6 +15,7 @@ ElementTree, parse_xml_string_to_etree, ) +from galaxy.util.path import StrPath from galaxy.util.yaml_util import ordered_load from .cwl import ( CwlToolSource, @@ -61,7 +60,7 @@ def build_yaml_tool_source(yaml_string: str) -> YamlToolSource: def get_tool_source( - config_file: Optional[Union[str, PurePath]] = None, + config_file: Optional[StrPath] = None, xml_tree: Optional[ElementTree] = None, enable_beta_formats: bool = True, tool_location_fetcher: Optional[ToolLocationFetcher] = None, @@ -87,8 +86,7 @@ def get_tool_source( tool_location_fetcher = ToolLocationFetcher() assert config_file - if isinstance(config_file, PurePath): - config_file = str(config_file) + config_file = str(config_file) config_file = tool_location_fetcher.to_tool_path(config_file) if not enable_beta_formats: diff --git a/lib/galaxy/tool_util/parser/interface.py b/lib/galaxy/tool_util/parser/interface.py index c137955dbeb1..a52e39e0200b 100644 --- a/lib/galaxy/tool_util/parser/interface.py +++ b/lib/galaxy/tool_util/parser/interface.py @@ -155,7 +155,7 @@ def parse_id(self) -> Optional[str]: def parse_version(self) -> Optional[str]: """Parse a version describing the abstract tool.""" - def parse_tool_module(self): + def parse_tool_module(self) -> Optional[Tuple[str, str]]: """Load Tool class from a custom module. (Optional). If not None, return pair containing module and class (as strings). @@ -169,7 +169,7 @@ def parse_action_module(self): """ return None - def parse_tool_type(self): + def parse_tool_type(self) -> Optional[str]: """Load simple tool type string (e.g. 'data_source', 'default').""" return None @@ -478,6 +478,12 @@ def get_bool(self, key, default): keys to be supported depend on the parameter type. """ + @abstractmethod + def get_bool_or_none(self, key, default): + """Return simple named properties as boolean or none for this input source. + keys to be supported depend on the parameter type. + """ + def parse_label(self): return self.get("label") @@ -655,12 +661,14 @@ class XmlTestCollectionDefDict(TypedDict): ) -def xml_data_input_to_json(xml_input: ToolSourceTestInput) -> "JsonTestDatasetDefDict": +def xml_data_input_to_json(xml_input: ToolSourceTestInput) -> Optional["JsonTestDatasetDefDict"]: attributes = xml_input["attributes"] + value = xml_input["value"] + if value is None and attributes.get("location") is None: + return None as_dict: JsonTestDatasetDefDict = { "class": "File", } - value = xml_input["value"] if value: as_dict["path"] = value _copy_if_exists(attributes, as_dict, "location") @@ -705,10 +713,16 @@ def to_element(xml_element_dict: "TestCollectionDefElementInternal") -> "JsonTes identifier=identifier, **element_object._test_format_to_dict() ) else: - as_dict = JsonTestCollectionDefDatasetElementDict( - identifier=identifier, - **xml_data_input_to_json(cast(ToolSourceTestInput, element_object)), + input_as_dict: Optional[JsonTestDatasetDefDict] = xml_data_input_to_json( + cast(ToolSourceTestInput, element_object) ) + if input_as_dict is not None: + as_dict = JsonTestCollectionDefDatasetElementDict( + identifier=identifier, + **input_as_dict, + ) + else: + raise Exception("Invalid empty test element...") return as_dict test_format_dict = BaseJsonTestCollectionDefCollectionElementDict( @@ -870,6 +884,19 @@ def from_dict(as_dict): element_tests=as_dict["element_tests"], ) + @staticmethod + def from_yaml_test_format(as_dict): + if "attributes" not in as_dict: + as_dict["attributes"] = {} + attributes = as_dict["attributes"] + # setup preferred name "elements" in accordance with work in https://github.com/galaxyproject/planemo/pull/1417 + # TODO: test this works recursively... + if "elements" in as_dict and "element_tests" not in as_dict: + as_dict["element_tests"] = as_dict["elements"] + if "collection_type" in as_dict: + attributes["type"] = as_dict["collection_type"] + return TestCollectionOutputDef.from_dict(as_dict) + def to_dict(self): return dict(name=self.name, attributes=self.attrib, element_tests=self.element_tests) diff --git a/lib/galaxy/tool_util/parser/xml.py b/lib/galaxy/tool_util/parser/xml.py index a89754553f84..c5f39568447b 100644 --- a/lib/galaxy/tool_util/parser/xml.py +++ b/lib/galaxy/tool_util/parser/xml.py @@ -30,6 +30,7 @@ Element, ElementTree, string_as_bool, + string_as_bool_or_none, XML, xml_text, xml_to_string, @@ -190,8 +191,7 @@ def parse_action_module(self): def parse_tool_type(self): root = self.root - if root.get("tool_type", None) is not None: - return root.get("tool_type") + return root.get("tool_type") def parse_name(self): return self.root.get("name") or self.parse_id() @@ -1328,6 +1328,9 @@ def get(self, key, value=None): def get_bool(self, key, default): return string_as_bool(self.get(key, default)) + def get_bool_or_none(self, key, default): + return string_as_bool_or_none(self.get(key, default)) + def parse_label(self): return xml_text(self.input_elem, "label") diff --git a/lib/galaxy/tool_util/parser/yaml.py b/lib/galaxy/tool_util/parser/yaml.py index c2cde0752f3a..c2e6352384c0 100644 --- a/lib/galaxy/tool_util/parser/yaml.py +++ b/lib/galaxy/tool_util/parser/yaml.py @@ -338,6 +338,9 @@ def get(self, key, default=None): def get_bool(self, key, default): return self.input_dict.get(key, default) + def get_bool_or_none(self, key, default): + return self.input_dict.get(key, default) + def parse_input_type(self): input_type = self.input_dict["type"] if input_type == "repeat": diff --git a/lib/galaxy/tool_util/verify/codegen.py b/lib/galaxy/tool_util/verify/codegen.py index 2e93d29ccbf0..18132fb078e8 100644 --- a/lib/galaxy/tool_util/verify/codegen.py +++ b/lib/galaxy/tool_util/verify/codegen.py @@ -3,8 +3,6 @@ # how to use this function... # PYTHONPATH=lib python lib/galaxy/tool_util/verify/codegen.py -from __future__ import annotations - import argparse import inspect import os @@ -34,7 +32,7 @@ Children = Literal["allowed", "required", "forbidden"] -DESCRIPTION = """This script synchronizes dynamic code aritfacts against models in Galaxy. +DESCRIPTION = """This script synchronizes dynamic code artifacts against models in Galaxy. Right now this just synchronizes Galaxy's XSD file against documentation in Galaxy's assertion modules but in the future it will also build Pydantic models for these functions. diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 29800c43b795..c085ac512699 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -176,6 +176,7 @@ safe_loads, swap_inf_nan, ) +from galaxy.util.path import StrPath from galaxy.util.rules_dsl import RuleSet from galaxy.util.template import ( fill_template, @@ -370,15 +371,16 @@ class ToolNotFoundException(Exception): pass -def create_tool_from_source(app, tool_source, config_file=None, **kwds): +def create_tool_from_source(app, tool_source: ToolSource, config_file: Optional[StrPath] = None, **kwds): # Allow specifying a different tool subclass to instantiate if (tool_module := tool_source.parse_tool_module()) is not None: module, cls = tool_module mod = __import__(module, globals(), locals(), [cls]) ToolClass = getattr(mod, cls) - elif tool_source.parse_tool_type(): - tool_type = tool_source.parse_tool_type() + elif tool_type := tool_source.parse_tool_type(): ToolClass = tool_types.get(tool_type) + if not ToolClass: + raise ValueError(f"Unrecognized tool type: {tool_type}") else: # Normal tool root = getattr(tool_source, "root", None) @@ -388,7 +390,7 @@ def create_tool_from_source(app, tool_source, config_file=None, **kwds): def create_tool_from_representation( - app, raw_tool_source: str, tool_dir: str, tool_source_class="XmlToolSource" + app, raw_tool_source: str, tool_dir: Optional[StrPath] = None, tool_source_class="XmlToolSource" ) -> "Tool": tool_source = get_tool_source(tool_source_class=tool_source_class, raw_tool_source=raw_tool_source) return create_tool_from_source(app, tool_source=tool_source, tool_dir=tool_dir) @@ -560,7 +562,7 @@ def get_cache_region(self, tool_cache_data_dir): self.cache_regions[tool_cache_data_dir] = ToolDocumentCache(cache_dir=tool_cache_data_dir) return self.cache_regions[tool_cache_data_dir] - def create_tool(self, config_file, tool_cache_data_dir=None, **kwds): + def create_tool(self, config_file: str, tool_cache_data_dir=None, **kwds): cache = self.get_cache_region(tool_cache_data_dir) if config_file.endswith(".xml") and cache and not cache.disabled: tool_document = cache.get(config_file) @@ -617,7 +619,6 @@ def get_tool_components(self, tool_id, tool_version=None, get_loaded_tools_by_li Retrieve all loaded versions of a tool from the toolbox and return a select list enabling selection of a different version, the list of the tool's loaded versions, and the specified tool. """ - toolbox = self tool_version_select_field = None tools = [] tool = None @@ -629,11 +630,11 @@ def get_tool_components(self, tool_id, tool_version=None, get_loaded_tools_by_li # Some data sources send back redirects ending with `/`, this takes care of that case tool_id = tool_id[:-1] if get_loaded_tools_by_lineage: - tools = toolbox.get_loaded_tools_by_lineage(tool_id) + tools = self.get_loaded_tools_by_lineage(tool_id) else: - tools = toolbox.get_tool(tool_id, tool_version=tool_version, get_all_versions=True) + tools = self.get_tool(tool_id, tool_version=tool_version, get_all_versions=True) if tools: - tool = toolbox.get_tool(tool_id, tool_version=tool_version, get_all_versions=False) + tool = self.get_tool(tool_id, tool_version=tool_version, get_all_versions=False) if len(tools) > 1: tool_version_select_field = self.__build_tool_version_select_field(tools, tool.id, set_selected) break @@ -775,24 +776,22 @@ class Tool(UsesDictVisibleKeys): def __init__( self, - config_file, + config_file: Optional[StrPath], tool_source: ToolSource, app: "UniverseApplication", - guid=None, + guid: Optional[str] = None, repository_id=None, tool_shed_repository=None, - allow_code_files=True, - dynamic=False, - tool_dir=None, + allow_code_files: bool = True, + dynamic: bool = False, + tool_dir: Optional[StrPath] = None, ): """Load a tool from the config named by `config_file`""" + self.config_file = config_file # Determine the full path of the directory where the tool config is if config_file is not None: - self.config_file = config_file - self.tool_dir = tool_dir or os.path.dirname(config_file) - else: - self.config_file = None - self.tool_dir = tool_dir + tool_dir = tool_dir or os.path.dirname(config_file) + self.tool_dir = tool_dir self.app = app self.repository_id = repository_id @@ -1032,7 +1031,7 @@ def allow_user_access(self, user, attempting_access=True): return False return True - def parse(self, tool_source: ToolSource, guid=None, dynamic=False): + def parse(self, tool_source: ToolSource, guid: Optional[str] = None, dynamic: bool = False) -> None: """ Read tool configuration from the element `root` and fill in `self`. """ @@ -1130,6 +1129,7 @@ def parse(self, tool_source: ToolSource, guid=None, dynamic=False): version_cmd_interpreter = tool_source.parse_version_command_interpreter() if version_cmd_interpreter: executable = self.version_string_cmd.split()[0] + assert self.tool_dir is not None abs_executable = os.path.abspath(os.path.join(self.tool_dir, executable)) command_line = self.version_string_cmd.replace(executable, abs_executable, 1) self.version_string_cmd = f"{version_cmd_interpreter} {command_line}" @@ -1249,7 +1249,7 @@ def parse(self, tool_source: ToolSource, guid=None, dynamic=False): self._is_workflow_compatible = self.check_workflow_compatible(self.tool_source) - def __parse_legacy_features(self, tool_source): + def __parse_legacy_features(self, tool_source: ToolSource): self.code_namespace: Dict[str, str] = {} self.hook_map: Dict[str, str] = {} self.uihints: Dict[str, str] = {} @@ -1268,6 +1268,7 @@ def __parse_legacy_features(self, tool_source): # map hook to function self.hook_map[key] = value file_name = code_elem.get("file") + assert self.tool_dir is not None code_path = os.path.join(self.tool_dir, file_name) if self._allow_code_files: with open(code_path) as f: @@ -1349,9 +1350,8 @@ def tests(self): @property def _repository_dir(self): """If tool shed installed tool, the base directory of the repository installed.""" - repository_base_dir = None - if getattr(self, "tool_shed", None): + assert self.tool_dir is not None tool_dir = Path(self.tool_dir) for repo_dir in itertools.chain([tool_dir], tool_dir.parents): if repo_dir.name == self.repository_name and repo_dir.parent.name == self.installed_changeset_revision: @@ -1359,7 +1359,7 @@ def _repository_dir(self): else: log.error(f"Problem finding repository dir for tool '{self.id}'") - return repository_base_dir + return None def test_data_path(self, filename): test_data = None @@ -2402,16 +2402,16 @@ def discover_outputs( return collected def to_archive(self): - tool = self tarball_files = [] temp_files = [] - with open(os.path.abspath(tool.config_file)) as fh1: + assert self.config_file + with open(os.path.abspath(self.config_file)) as fh1: tool_xml = fh1.read() # Retrieve tool help images and rewrite the tool's xml into a temporary file with the path # modified to be relative to the repository root. image_found = False - if tool.help is not None: - tool_help = tool.help._source + if self.help is not None: + tool_help = self.help._source # Check each line of the rendered tool help for an image tag that points to a location under static/ for help_line in tool_help.split("\n"): image_regex = re.compile(r'img alt="[^"]+" src="\${static_path}/([^"]+)"') @@ -2429,25 +2429,25 @@ def to_archive(self): with tempfile.NamedTemporaryFile(mode="w", suffix=".xml", delete=False) as fh2: new_tool_config = fh2.name fh2.write(tool_xml) - tool_tup = (new_tool_config, os.path.split(tool.config_file)[-1]) + tool_tup = (new_tool_config, os.path.split(self.config_file)[-1]) temp_files.append(new_tool_config) else: - tool_tup = (os.path.abspath(tool.config_file), os.path.split(tool.config_file)[-1]) + tool_tup = (os.path.abspath(self.config_file), os.path.split(self.config_file)[-1]) tarball_files.append(tool_tup) # TODO: This feels hacky. - tool_command = tool.command.strip().split()[0] - tool_path = os.path.dirname(os.path.abspath(tool.config_file)) + tool_command = self.command.strip().split()[0] + tool_path = os.path.dirname(os.path.abspath(self.config_file)) # Add the tool XML to the tuple that will be used to populate the tarball. if os.path.exists(os.path.join(tool_path, tool_command)): tarball_files.append((os.path.join(tool_path, tool_command), tool_command)) # Find and add macros and code files. - for external_file in tool.get_externally_referenced_paths(os.path.abspath(tool.config_file)): + for external_file in self.get_externally_referenced_paths(os.path.abspath(self.config_file)): external_file_abspath = os.path.abspath(os.path.join(tool_path, external_file)) tarball_files.append((external_file_abspath, external_file)) if os.path.exists(os.path.join(tool_path, "Dockerfile")): tarball_files.append((os.path.join(tool_path, "Dockerfile"), "Dockerfile")) # Find tests, and check them for test data. - if (tests := tool.tests) is not None: + if (tests := self.tests) is not None: for test in tests: # Add input file tuples to the list. for input in test.inputs: @@ -2463,7 +2463,7 @@ def to_archive(self): if os.path.exists(output_filepath): td_tup = (output_filepath, os.path.join("test-data", filename)) tarball_files.append(td_tup) - for param in tool.input_params: + for param in self.input_params: # Check for tool data table definitions. param_options = getattr(param, "options", None) if param_options is not None: @@ -3790,6 +3790,17 @@ def element_is_valid(element: model.DatasetCollectionElement): return False +class FilterNullTool(FilterDatasetsTool): + tool_type = "filter_null" + require_dataset_ok = True + + @staticmethod + def element_is_valid(element: model.DatasetCollectionElement): + element_object = element.element_object + assert isinstance(element_object, model.DatasetInstance) + return element_object.extension == "expression.json" and element_object.blurb == "skipped" + + class FlattenTool(DatabaseOperationTool): tool_type = "flatten_collection" require_terminal_states = False @@ -4157,7 +4168,6 @@ def produce_outputs(self, trans, out_data, output_collections, incoming, history # Populate tool_type to ToolClass mappings -tool_types = {} TOOL_CLASSES: List[Type[Tool]] = [ Tool, SetMetadataTool, @@ -4177,9 +4187,7 @@ def produce_outputs(self, trans, out_data, output_collections, incoming, history ExtractDatasetCollectionTool, DataDestinationTool, ] -for tool_class in TOOL_CLASSES: - tool_types[tool_class.tool_type] = tool_class - +tool_types = {tool_class.tool_type: tool_class for tool_class in TOOL_CLASSES} # ---- Utility classes to be factored out ----------------------------------- diff --git a/lib/galaxy/tools/data_fetch.py b/lib/galaxy/tools/data_fetch.py index a6786c725dc9..fe1419aa1434 100644 --- a/lib/galaxy/tools/data_fetch.py +++ b/lib/galaxy/tools/data_fetch.py @@ -34,7 +34,7 @@ from galaxy.util.compression_utils import CompressedFile from galaxy.util.hash_util import ( HASH_NAMES, - memory_bound_hexdigest, + verify_hash, ) DESCRIPTION = """Data Import Script""" @@ -250,6 +250,10 @@ def _resolve_item_with_primary(item): if url: sources.append(source_dict) hashes = item.get("hashes", []) + for hash_function in HASH_NAMES: + hash_value = item.get(hash_function) + if hash_value: + hashes.append({"hash_function": hash_function, "hash_value": hash_value}) for hash_dict in hashes: hash_function = hash_dict.get("hash_function") hash_value = hash_dict.get("hash_value") @@ -511,11 +515,7 @@ def _has_src_to_path(upload_config, item, is_dataset=False) -> Tuple[str, str]: def _handle_hash_validation(upload_config, hash_function, hash_value, path): if upload_config.validate_hashes: - calculated_hash_value = memory_bound_hexdigest(hash_func_name=hash_function, path=path) - if calculated_hash_value != hash_value: - raise Exception( - f"Failed to validate upload with [{hash_function}] - expected [{hash_value}] got [{calculated_hash_value}]" - ) + verify_hash(path, hash_func_name=hash_function, hash_value=hash_value, what="upload") def _arg_parser(): diff --git a/lib/galaxy/tools/execute.py b/lib/galaxy/tools/execute.py index d6e65f592a6e..31369c948052 100644 --- a/lib/galaxy/tools/execute.py +++ b/lib/galaxy/tools/execute.py @@ -285,7 +285,7 @@ def __init__( self.collection_info = collection_info self.completed_jobs = completed_jobs - self._on_text = None + self._on_text: Optional[str] = None # Populated as we go... self.failed_jobs = 0 @@ -322,7 +322,7 @@ def record_error(self, error): self.execution_errors.append(error) @property - def on_text(self): + def on_text(self) -> Optional[str]: collection_info = self.collection_info if self._on_text is None and collection_info is not None: collection_names = ["collection %d" % c.hid for c in collection_info.collections.values()] diff --git a/lib/galaxy/tools/execution_helpers.py b/lib/galaxy/tools/execution_helpers.py index 66ae3c853681..76413a5f6370 100644 --- a/lib/galaxy/tools/execution_helpers.py +++ b/lib/galaxy/tools/execution_helpers.py @@ -5,6 +5,7 @@ """ import logging +from typing import Collection log = logging.getLogger(__name__) @@ -47,7 +48,7 @@ def filter_output(tool, output, incoming): return False -def on_text_for_names(input_names): +def on_text_for_names(input_names: Collection[str]) -> str: # input_names may contain duplicates... this is because the first value in # multiple input dataset parameters will appear twice once as param_name # and once as param_name1. diff --git a/lib/galaxy/tools/flatten_collection.xml b/lib/galaxy/tools/flatten_collection.xml index 67a96252dfeb..a9cb4f04f075 100644 --- a/lib/galaxy/tools/flatten_collection.xml +++ b/lib/galaxy/tools/flatten_collection.xml @@ -1,7 +1,7 @@ + tool_type="flatten_collection"> >> print(p.name) _name >>> assert sorted(p.to_dict(trans).items()) == [('argument', None), ('falsevalue', '_falsevalue'), ('help', ''), ('hidden', False), ('is_dynamic', False), ('label', ''), ('model_class', 'BooleanToolParameter'), ('name', '_name'), ('optional', False), ('refresh_on_change', False), ('truevalue', '_truevalue'), ('type', 'boolean'), ('value', True)] - >>> print(p.from_json('true')) + >>> print(p.from_json('true', trans)) True >>> print(p.to_param_dict_string(True)) _truevalue - >>> print(p.from_json('false')) + >>> print(p.from_json('false', trans)) False >>> print(p.to_param_dict_string(False)) _falsevalue @@ -615,7 +615,7 @@ def __init__(self, tool, input_source): self.optional = input_source.get_bool("optional", False) self.checked = boolean_is_checked(input_source) - def from_json(self, value, trans=None, other_values=None): + def from_json(self, value, trans, other_values=None): return self.to_python(value) def to_python(self, value, app=None): @@ -666,7 +666,7 @@ class FileToolParameter(ToolParameter): def __init__(self, tool, input_source): super().__init__(tool, input_source) - def from_json(self, value, trans=None, other_values=None): + def from_json(self, value, trans, other_values=None): # Middleware or proxies may encode files in special ways (TODO: this # should be pluggable) if isinstance(value, FilesPayload): @@ -765,7 +765,7 @@ def to_param_dict_string(self, value, other_values=None): else: return lst[0] - def from_json(self, value, trans=None, other_values=None): + def from_json(self, value, trans, other_values=None): return self.to_python(value, trans.app, validate=True) def to_json(self, value, app, use_security): @@ -885,7 +885,7 @@ def __init__(self, tool, input_source): def get_initial_value(self, trans, other_values): return self._get_value(trans) - def from_json(self, value=None, trans=None, other_values=None): + def from_json(self, value, trans, other_values=None): return self._get_value(trans) def _get_value(self, trans): @@ -1004,7 +1004,7 @@ def get_legal_names(self, trans, other_values): """ return {n: v for n, v, _ in self.get_options(trans, other_values)} - def from_json(self, value, trans=None, other_values=None): + def from_json(self, value, trans, other_values=None): return self._select_from_json(value, trans, other_values=other_values, require_legal_value=True) def _select_from_json(self, value, trans, other_values=None, require_legal_value=True): @@ -1284,7 +1284,7 @@ def __init__(self, tool, input_source): self.default_value = input_source.get("value", None) self.is_dynamic = True - def from_json(self, value, trans=None, other_values=None): + def from_json(self, value, trans, other_values=None): other_values = other_values or {} if self.multiple: tag_list = [] @@ -1412,7 +1412,7 @@ def to_json(self, value, app, use_security): return value.strip() return value - def from_json(self, value, trans=None, other_values=None): + def from_json(self, value, trans, other_values=None): """ Label convention prepends column number with a 'c', but tool uses the integer. This removes the 'c' when entered into a workflow. @@ -1447,10 +1447,9 @@ def from_json(self, value, trans=None, other_values=None): @staticmethod def _strip_c(column): - if isinstance(column, str): - column = column.strip() - if column.startswith("c") and len(column) > 1 and all(c.isdigit() for c in column[1:]): - column = column.lower()[1:] + column = str(column).strip() + if column.startswith("c") and len(column) > 1 and all(c.isdigit() for c in column[1:]): + column = column.lower()[1:] return column def get_column_list(self, trans, other_values): @@ -1701,7 +1700,7 @@ def recurse_options(legal_values, options): recurse_options(legal_values, self.get_options(trans=trans, other_values=other_values)) return legal_values - def from_json(self, value, trans=None, other_values=None): + def from_json(self, value, trans, other_values=None): other_values = other_values or {} legal_values = self.get_legal_values(trans, other_values, value) if not legal_values and trans.workflow_building_mode: @@ -2102,7 +2101,7 @@ def __init__(self, tool, input_source, trans=None): ) self.conversions.append((name, conv_extension, [conv_type])) - def from_json(self, value, trans=None, other_values=None): + def from_json(self, value, trans, other_values=None): session = trans.sa_session other_values = other_values or {} @@ -2459,7 +2458,7 @@ def match_multirun_collections(self, trans, history, dataset_collection_matcher) if match: yield history_dataset_collection, match.implicit_conversion - def from_json(self, value, trans=None, other_values=None): + def from_json(self, value, trans, other_values=None): session = trans.sa_session other_values = other_values or {} diff --git a/lib/galaxy/tools/parameters/dynamic_options.py b/lib/galaxy/tools/parameters/dynamic_options.py index b8cd20dab37d..e354c93eabb5 100644 --- a/lib/galaxy/tools/parameters/dynamic_options.py +++ b/lib/galaxy/tools/parameters/dynamic_options.py @@ -286,14 +286,12 @@ def get_dependency_name(self): return self.ref_name def filter_options(self, options, trans, other_values): - if trans is not None and trans.workflow_building_mode: - return [] ref = other_values.get(self.ref_name, None) - if ref is None: + if ref is None or is_runtime_value(ref): ref = [] # - for HDCAs the list of contained HDAs is extracted - # - single values are transformed in a single eleent list + # - single values are transformed in a single element list # - remaining cases are already lists (select and data parameters with multiple=true) if isinstance(ref, HistoryDatasetCollectionAssociation): ref = ref.to_hda_representative(multiple=True) @@ -835,6 +833,9 @@ def get_field_by_name_for_value(self, field_name, value, trans, other_values): return rval def get_options(self, trans, other_values): + + rval = [] + def to_triple(values): if len(values) == 2: return [str(values[0]), str(values[1]), False] @@ -877,8 +878,7 @@ def to_triple(values): data = [] # We only support the very specific ["name", "value", "selected"] format for now. - return [to_triple(d) for d in data] - rval = [] + rval = [to_triple(d) for d in data] if ( self.file_fields is not None or self.tool_data_table is not None diff --git a/lib/galaxy/tools/remote_tool_eval.py b/lib/galaxy/tools/remote_tool_eval.py index b07c6cf6a875..6f2e273ffbe4 100644 --- a/lib/galaxy/tools/remote_tool_eval.py +++ b/lib/galaxy/tools/remote_tool_eval.py @@ -72,7 +72,7 @@ def __init__( self.security = None # type: ignore[assignment] -def main(TMPDIR, WORKING_DIRECTORY, IMPORT_STORE_DIRECTORY): +def main(TMPDIR, WORKING_DIRECTORY, IMPORT_STORE_DIRECTORY) -> None: metadata_params = get_metadata_params(WORKING_DIRECTORY) datatypes_config = metadata_params["datatypes_config"] if not os.path.exists(datatypes_config): diff --git a/lib/galaxy/tours/_impl.py b/lib/galaxy/tours/_impl.py index 60ab97bfc80a..ab6f280f45e5 100644 --- a/lib/galaxy/tours/_impl.py +++ b/lib/galaxy/tours/_impl.py @@ -4,10 +4,7 @@ import logging import os -from typing import ( - List, - Union, -) +from typing import List import yaml from pydantic import parse_obj_as @@ -15,6 +12,7 @@ from galaxy.exceptions import ObjectNotFound from galaxy.navigation.data import load_root_component from galaxy.util import config_directories_from_setting +from galaxy.util.path import StrPath from ._interface import ToursRegistry from ._schema import TourList @@ -61,12 +59,12 @@ def load_tour_steps(contents_dict, warn=None, resolve_components=True): step["title"] = title_default -def get_tour_id_from_path(tour_path: Union[str, os.PathLike]) -> str: +def get_tour_id_from_path(tour_path: StrPath) -> str: filename = os.path.basename(tour_path) return os.path.splitext(filename)[0] -def load_tour_from_path(tour_path: Union[str, os.PathLike], warn=None, resolve_components=True) -> dict: +def load_tour_from_path(tour_path: StrPath, warn=None, resolve_components=True) -> dict: with open(tour_path) as f: tour = yaml.safe_load(f) load_tour_steps(tour, warn=warn, resolve_components=resolve_components) @@ -80,7 +78,7 @@ def is_yaml(filename: str) -> bool: return False -def tour_paths(target_path: Union[str, os.PathLike]) -> List[str]: +def tour_paths(target_path: StrPath) -> List[str]: paths = [] if os.path.isdir(target_path): for filename in os.listdir(target_path): diff --git a/lib/galaxy/util/hash_util.py b/lib/galaxy/util/hash_util.py index 100adc23bcc4..efb5d0ce7e1b 100644 --- a/lib/galaxy/util/hash_util.py +++ b/lib/galaxy/util/hash_util.py @@ -6,7 +6,6 @@ import hashlib import hmac import logging -import os from enum import Enum from typing import ( Any, @@ -19,6 +18,7 @@ ) from . import smart_str +from .path import StrPath log = logging.getLogger(__name__) @@ -82,7 +82,7 @@ def memory_bound_hexdigest( file.close() -def md5_hash_file(path: Union[str, os.PathLike]) -> Optional[str]: +def md5_hash_file(path: StrPath) -> Optional[str]: """ Return a md5 hashdigest for a file or None if path could not be read. """ @@ -153,6 +153,14 @@ def parse_checksum_hash(checksum: str) -> Tuple[HashFunctionNameEnum, str]: return HashFunctionNameEnum(hash_name), hash_value +def verify_hash(path: str, hash_func_name: HashFunctionNameEnum, hash_value: str, what: str = "path"): + calculated_hash_value = memory_bound_hexdigest(hash_func_name=hash_func_name, path=path) + if calculated_hash_value != hash_value: + raise Exception( + f"Failed to validate {what} with [{hash_func_name}] - expected [{hash_value}] got [{calculated_hash_value}]" + ) + + __all__ = ( "md5", "hashlib", diff --git a/lib/galaxy/util/plugin_config.py b/lib/galaxy/util/plugin_config.py index 3b4bc405ebd4..50852627e3e7 100644 --- a/lib/galaxy/util/plugin_config.py +++ b/lib/galaxy/util/plugin_config.py @@ -1,4 +1,3 @@ -from pathlib import Path from types import ModuleType from typing import ( Any, @@ -16,9 +15,9 @@ import yaml from galaxy.util import parse_xml +from galaxy.util.path import StrPath from galaxy.util.submodules import import_submodules -PathT = Union[str, Path] PluginDictConfigT = Dict[str, Any] PluginConfigsT = Union[PluginDictConfigT, List[PluginDictConfigT]] @@ -132,7 +131,7 @@ def __load_plugins_from_dicts( return plugins -def plugin_source_from_path(path: PathT) -> PluginConfigSource: +def plugin_source_from_path(path: StrPath) -> PluginConfigSource: filename = str(path) if ( filename.endswith(".yaml") @@ -149,7 +148,7 @@ def plugin_source_from_dict(as_dict: PluginConfigsT) -> PluginConfigSource: return PluginConfigSource("dict", as_dict) -def __read_yaml(path: PathT): +def __read_yaml(path: StrPath): if yaml is None: raise ImportError("Attempting to read YAML configuration file - but PyYAML dependency unavailable.") diff --git a/lib/galaxy/visualization/plugins/config_parser.py b/lib/galaxy/visualization/plugins/config_parser.py index 160f52acef2c..2e784abda7bc 100644 --- a/lib/galaxy/visualization/plugins/config_parser.py +++ b/lib/galaxy/visualization/plugins/config_parser.py @@ -151,9 +151,9 @@ def parse_visualization(self, xml_tree): if (specs_section := xml_tree.find("specs")) is not None: returned["specs"] = DictParser(specs_section) - # load group specifiers - if (groups_section := xml_tree.find("groups")) is not None: - returned["groups"] = ListParser(groups_section) + # load tracks specifiers (allow 'groups' section for backward compatibility) + if (tracks_section := xml_tree.find("tracks") or xml_tree.find("groups")) is not None: + returned["tracks"] = ListParser(tracks_section) # load settings specifiers if (settings_section := xml_tree.find("settings")) is not None: diff --git a/lib/galaxy/visualization/plugins/plugin.py b/lib/galaxy/visualization/plugins/plugin.py index c53f7977834a..e3bdf0e03d3c 100644 --- a/lib/galaxy/visualization/plugins/plugin.py +++ b/lib/galaxy/visualization/plugins/plugin.py @@ -125,8 +125,8 @@ def to_dict(self): "embeddable": self.config.get("embeddable"), "entry_point": self.config.get("entry_point"), "settings": self.config.get("settings"), - "groups": self.config.get("groups"), "specs": self.config.get("specs"), + "tracks": self.config.get("tracks"), "href": self._get_url(), } diff --git a/lib/galaxy/webapps/base/api.py b/lib/galaxy/webapps/base/api.py index b6d12d834014..9df5b838eea2 100644 --- a/lib/galaxy/webapps/base/api.py +++ b/lib/galaxy/webapps/base/api.py @@ -1,8 +1,16 @@ import os import stat -import typing import uuid from logging import getLogger +from typing import ( + Any, + Dict, + Mapping, + Optional, + Tuple, + TYPE_CHECKING, + Union, +) import anyio from fastapi import ( @@ -26,10 +34,11 @@ from galaxy.exceptions import MessageException from galaxy.exceptions.utils import api_error_to_dict +from galaxy.util.path import StrPath from galaxy.web.framework.base import walk_controller_modules from galaxy.web.framework.decorators import validation_error_to_message_exception -if typing.TYPE_CHECKING: +if TYPE_CHECKING: from starlette.background import BackgroundTask from starlette.types import ( Receive, @@ -43,7 +52,7 @@ # Copied from https://github.com/tiangolo/fastapi/issues/1240#issuecomment-1055396884 -def _get_range_header(range_header: str, file_size: int) -> typing.Tuple[int, int]: +def _get_range_header(range_header: str, file_size: int) -> Tuple[int, int]: def _invalid_range(): return HTTPException( status.HTTP_416_REQUESTED_RANGE_NOT_SATISFIABLE, @@ -67,19 +76,19 @@ class GalaxyFileResponse(FileResponse): Augments starlette FileResponse with x-accel-redirect/x-sendfile and byte-range handling. """ - nginx_x_accel_redirect_base: typing.Optional[str] = None - apache_xsendfile: typing.Optional[bool] = None + nginx_x_accel_redirect_base: Optional[str] = None + apache_xsendfile: Optional[bool] = None def __init__( self, - path: typing.Union[str, "os.PathLike[str]"], + path: StrPath, status_code: int = 200, - headers: typing.Optional[typing.Mapping[str, str]] = None, - media_type: typing.Optional[str] = None, - background: typing.Optional["BackgroundTask"] = None, - filename: typing.Optional[str] = None, - stat_result: typing.Optional[os.stat_result] = None, - method: typing.Optional[str] = None, + headers: Optional[Mapping[str, str]] = None, + media_type: Optional[str] = None, + background: Optional["BackgroundTask"] = None, + filename: Optional[str] = None, + stat_result: Optional[os.stat_result] = None, + method: Optional[str] = None, content_disposition_type: str = "attachment", ) -> None: super().__init__( @@ -184,8 +193,8 @@ def get_error_response_for_request(request: Request, exc: MessageException) -> J else: content = error_dict - retry_after: typing.Optional[int] = getattr(exc, "retry_after", None) - headers: typing.Dict[str, str] = {} + retry_after: Optional[int] = getattr(exc, "retry_after", None) + headers: Dict[str, str] = {} if retry_after: headers["Retry-After"] = str(retry_after) return JSONResponse(status_code=status_code, content=content, headers=headers) @@ -237,7 +246,7 @@ def add_request_id_middleware(app: FastAPI): def include_all_package_routers(app: FastAPI, package_name: str): - responses: typing.Dict[typing.Union[int, str], typing.Dict[str, typing.Any]] = { + responses: Dict[Union[int, str], Dict[str, Any]] = { "4XX": { "description": "Request Error", "model": MessageExceptionModel, diff --git a/lib/galaxy/webapps/base/webapp.py b/lib/galaxy/webapps/base/webapp.py index 822e5b7ec236..75a14f04e301 100644 --- a/lib/galaxy/webapps/base/webapp.py +++ b/lib/galaxy/webapps/base/webapp.py @@ -14,6 +14,7 @@ Any, Dict, Optional, + Tuple, ) from urllib.parse import urlparse @@ -325,6 +326,7 @@ def __init__( self.galaxy_session = None self.error_message = None self.host = self.request.host + self._short_term_cache: Dict[Tuple[str, ...], Any] = {} # set any cross origin resource sharing headers if configured to do so self.set_cors_headers() diff --git a/lib/galaxy/webapps/galaxy/api/__init__.py b/lib/galaxy/webapps/galaxy/api/__init__.py index 3b96292f955d..260f90cebced 100644 --- a/lib/galaxy/webapps/galaxy/api/__init__.py +++ b/lib/galaxy/webapps/galaxy/api/__init__.py @@ -26,6 +26,7 @@ APIRouter, Form, Header, + Path, Query, Request, Response, @@ -41,7 +42,10 @@ HTTPAuthorizationCredentials, HTTPBearer, ) -from pydantic import ValidationError +from pydantic import ( + UUID4, + ValidationError, +) from pydantic.main import BaseModel from routes import ( Mapper, @@ -618,3 +622,10 @@ def search_query_param(model_name: str, tags: list, free_text_fields: list) -> O title="Search query.", description=description, ) + + +LandingUuidPathParam: UUID4 = Path( + ..., + title="Landing UUID", + description="The UUID used to identify a persisted landing request.", +) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index 95641f244c85..f5653a6f7dae 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -7,6 +7,7 @@ List, Literal, Optional, + Set, Union, ) @@ -53,6 +54,7 @@ HistoryContentType, MaterializeDatasetInstanceAPIRequest, MaterializeDatasetInstanceRequest, + MaterializeDatasetOptions, StoreExportPayload, UpdateHistoryContentsBatchPayload, UpdateHistoryContentsPayload, @@ -335,11 +337,10 @@ def parse_content_types(types: Union[List[str], str]) -> List[HistoryContentType def parse_dataset_details(details: Optional[str]): """Parses the different values that the `dataset_details` parameter can have from a string.""" - dataset_details = None if details is not None and details != "all": - dataset_details = set(util.listify(details)) + dataset_details: Union[None, Set[str], str] = set(util.listify(details)) else: # either None or 'all' - dataset_details = details # type: ignore + dataset_details = details return dataset_details @@ -1072,12 +1073,17 @@ def materialize_dataset( history_id: HistoryIDPathParam, id: HistoryItemIDPathParam, trans: ProvidesHistoryContext = DependsOnTrans, + materialize_api_payload: Optional[MaterializeDatasetOptions] = Body(None), ) -> AsyncTaskResultSummary: + validate_hashes: bool = ( + materialize_api_payload.validate_hashes if materialize_api_payload is not None else False + ) # values are already validated, use model_construct materialize_request = MaterializeDatasetInstanceRequest.model_construct( history_id=history_id, source=DatasetSourceType.hda, content=id, + validate_hashes=validate_hashes, ) rval = self.service.materialize(trans, materialize_request) return rval diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index 05a475d430e6..002e705c8d98 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -42,6 +42,7 @@ ProvidesHistoryContext, ProvidesUserContext, ) +from galaxy.managers.landing import LandingRequestManager from galaxy.managers.workflows import ( MissingToolsException, RefactorRequest, @@ -68,12 +69,15 @@ from galaxy.schema.schema import ( AsyncFile, AsyncTaskResultSummary, + ClaimLandingPayload, + CreateWorkflowLandingRequestPayload, InvocationSortByEnum, InvocationsStateCounts, SetSlugPayload, ShareWithPayload, ShareWithStatus, SharingStatus, + WorkflowLandingRequest, WorkflowSortByEnum, ) from galaxy.schema.workflows import ( @@ -102,6 +106,7 @@ depends, DependsOnTrans, IndexQueryTag, + LandingUuidPathParam, Router, search_query_param, ) @@ -909,6 +914,7 @@ def __get_stored_workflow(self, trans, workflow_id, **kwd): @router.cbv class FastAPIWorkflows: service: WorkflowsService = depends(WorkflowsService) + landing_manager: LandingRequestManager = depends(LandingRequestManager) @router.get( "/api/workflows", @@ -1159,6 +1165,39 @@ def show_workflow( ) -> StoredWorkflowDetailed: return self.service.show_workflow(trans, workflow_id, instance, legacy, version) + @router.post("/api/workflow_landings", public=True) + def create_landing( + self, + trans: ProvidesUserContext = DependsOnTrans, + workflow_landing_request: CreateWorkflowLandingRequestPayload = Body(...), + ) -> WorkflowLandingRequest: + try: + return self.landing_manager.create_workflow_landing_request(workflow_landing_request) + except Exception: + log.exception("Problem...") + raise + + @router.post("/api/workflow_landings/{uuid}/claim") + def claim_landing( + self, + trans: ProvidesUserContext = DependsOnTrans, + uuid: UUID4 = LandingUuidPathParam, + payload: Optional[ClaimLandingPayload] = Body(...), + ) -> WorkflowLandingRequest: + try: + return self.landing_manager.claim_workflow_landing_request(trans, uuid, payload) + except Exception: + log.exception("claiim problem...") + raise + + @router.get("/api/workflow_landings/{uuid}") + def get_landing( + self, + trans: ProvidesUserContext = DependsOnTrans, + uuid: UUID4 = LandingUuidPathParam, + ) -> WorkflowLandingRequest: + return self.landing_manager.get_workflow_landing_request(trans, uuid) + StepDetailQueryParam = Annotated[ bool, diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index ef56cd174fdb..e1897420f9a5 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -227,6 +227,8 @@ def app_pair(global_conf, load_app_kwds=None, wsgi_preflight=True, **kwargs): webapp.add_client_route("/login/start") webapp.add_client_route("/tools/list") webapp.add_client_route("/tools/json") + webapp.add_client_route("/tool_landings/{uuid}") + webapp.add_client_route("/workflow_landings/{uuid}") webapp.add_client_route("/tours") webapp.add_client_route("/tours/{tour_id}") webapp.add_client_route("/user") diff --git a/lib/galaxy/webapps/galaxy/services/history_contents.py b/lib/galaxy/webapps/galaxy/services/history_contents.py index b1edebd3e897..42913b7f2b42 100644 --- a/lib/galaxy/webapps/galaxy/services/history_contents.py +++ b/lib/galaxy/webapps/galaxy/services/history_contents.py @@ -573,6 +573,7 @@ def materialize( history_id=request.history_id, source=request.source, content=request.content, + validate_hashes=request.validate_hashes, user=trans.async_request_user, ) results = materialize_task.delay(request=task_request) diff --git a/lib/galaxy/work/context.py b/lib/galaxy/work/context.py index 81db0f7e1c6c..cded013f2e74 100644 --- a/lib/galaxy/work/context.py +++ b/lib/galaxy/work/context.py @@ -49,12 +49,6 @@ def __init__( self.workflow_building_mode = workflow_building_mode self.galaxy_session = galaxy_session - def set_cache_value(self, args: Tuple[str, ...], value: Any): - self._short_term_cache[args] = value - - def get_cache_value(self, args: Tuple[str, ...], default: Any = None) -> Any: - return self._short_term_cache.get(args, default) - @property def app(self): return self._app diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index de1b65223309..448d07ecbe90 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -1423,7 +1423,7 @@ def restrict_options(self, step, connections: Iterable[WorkflowStepConnection], def callback(input, prefixed_name, context, **kwargs): if prefixed_name == connection.input_name and hasattr(input, "get_options"): # noqa: B023 - static_options.append(input.get_options(self.trans, {})) + static_options.append(input.get_options(self.trans, context)) visit_input_values(tool_inputs, module.state.inputs, callback) elif isinstance(module, SubWorkflowModule): diff --git a/lib/galaxy/workflow/run.py b/lib/galaxy/workflow/run.py index 5ebb586fd56f..5619a4547b8f 100644 --- a/lib/galaxy/workflow/run.py +++ b/lib/galaxy/workflow/run.py @@ -144,7 +144,12 @@ def queue_invoke( ) workflow_invocation = workflow_run_config_to_request(trans, workflow_run_config, workflow) workflow_invocation.workflow = workflow - return trans.app.workflow_scheduling_manager.queue(workflow_invocation, request_params, flush=flush) + initial_state = model.WorkflowInvocation.states.NEW + if workflow_run_config.requires_materialization: + initial_state = model.WorkflowInvocation.states.REQUIRES_MATERIALIZATION + return trans.app.workflow_scheduling_manager.queue( + workflow_invocation, request_params, flush=flush, initial_state=initial_state + ) class WorkflowInvoker: diff --git a/lib/galaxy/workflow/run_request.py b/lib/galaxy/workflow/run_request.py index 516c85d07eb1..24b81fff849d 100644 --- a/lib/galaxy/workflow/run_request.py +++ b/lib/galaxy/workflow/run_request.py @@ -10,11 +10,13 @@ ) from galaxy import exceptions +from galaxy.managers.hdas import dereference_input from galaxy.model import ( EffectiveOutput, History, HistoryDatasetAssociation, HistoryDatasetCollectionAssociation, + InputWithRequest, LibraryDataset, LibraryDatasetDatasetAssociation, WorkflowInvocation, @@ -25,6 +27,7 @@ ensure_object_added_to_session, transaction, ) +from galaxy.tool_util.parameters import DataRequestUri from galaxy.tools.parameters.meta import expand_workflow_inputs from galaxy.workflow.resources import get_resource_mapper_function @@ -57,6 +60,9 @@ class WorkflowRunConfig: :param inputs: Map from step ids to dict's containing HDA for these steps. :type inputs: dict + :param requires_materialization: True if an input requires materialization before + the workflow is scheduled. + :param inputs_by: How inputs maps to inputs (datasets/collections) to workflows steps - by unencoded database id ('step_id'), index in workflow 'step_index' (independent of database), or by input name for @@ -78,6 +84,7 @@ def __init__( copy_inputs_to_history: bool = False, use_cached_job: bool = False, resource_params: Optional[Dict[int, Any]] = None, + requires_materialization: bool = False, preferred_object_store_id: Optional[str] = None, preferred_outputs_object_store_id: Optional[str] = None, preferred_intermediate_object_store_id: Optional[str] = None, @@ -91,6 +98,7 @@ def __init__( self.resource_params = resource_params or {} self.allow_tool_state_corrections = allow_tool_state_corrections self.use_cached_job = use_cached_job + self.requires_materialization = requires_materialization self.preferred_object_store_id = preferred_object_store_id self.preferred_outputs_object_store_id = preferred_outputs_object_store_id self.preferred_intermediate_object_store_id = preferred_intermediate_object_store_id @@ -310,7 +318,7 @@ def build_workflow_run_configs( legacy = payload.get("legacy", False) already_normalized = payload.get("parameters_normalized", False) raw_parameters = payload.get("parameters", {}) - + requires_materialization: bool = False run_configs = [] unexpanded_param_map = _normalize_step_parameters( workflow.steps, raw_parameters, legacy=legacy, already_normalized=already_normalized @@ -368,16 +376,22 @@ def build_workflow_run_configs( raise exceptions.RequestParameterInvalidException( f"Not input source type defined for input '{input_dict}'." ) - if "id" not in input_dict: - raise exceptions.RequestParameterInvalidException(f"Not input id defined for input '{input_dict}'.") + input_source = input_dict["src"] + if "id" not in input_dict and input_source != "url": + raise exceptions.RequestParameterInvalidException(f"No input id defined for input '{input_dict}'.") + elif input_source == "url" and not input_dict.get("url"): + raise exceptions.RequestParameterInvalidException( + f"Supplied 'url' is empty or absent for input '{input_dict}'." + ) if "content" in input_dict: raise exceptions.RequestParameterInvalidException( f"Input cannot specify explicit 'content' attribute {input_dict}'." ) - input_source = input_dict["src"] - input_id = input_dict["id"] + input_id = input_dict.get("id") try: + added_to_history = False if input_source == "ldda": + assert input_id ldda = trans.sa_session.get(LibraryDatasetDatasetAssociation, trans.security.decode_id(input_id)) assert ldda assert trans.user_is_admin or trans.app.security_agent.can_access_dataset( @@ -385,6 +399,7 @@ def build_workflow_run_configs( ) content = ldda.to_history_dataset_association(history, add_to_history=add_to_history) elif input_source == "ld": + assert input_id library_dataset = trans.sa_session.get(LibraryDataset, trans.security.decode_id(input_id)) assert library_dataset ldda = library_dataset.library_dataset_dataset_association @@ -394,6 +409,7 @@ def build_workflow_run_configs( ) content = ldda.to_history_dataset_association(history, add_to_history=add_to_history) elif input_source == "hda": + assert input_id # Get dataset handle, add to dict and history if necessary content = trans.sa_session.get(HistoryDatasetAssociation, trans.security.decode_id(input_id)) assert trans.user_is_admin or trans.app.security_agent.can_access_dataset( @@ -401,11 +417,21 @@ def build_workflow_run_configs( ) elif input_source == "hdca": content = app.dataset_collection_manager.get_dataset_collection_instance(trans, "history", input_id) + elif input_source == "url": + data_request = DataRequestUri.model_validate(input_dict) + hda: HistoryDatasetAssociation = dereference_input(trans, data_request, history) + added_to_history = True + content = InputWithRequest( + input=hda, + request=data_request.model_dump(mode="json"), + ) + if not data_request.deferred: + requires_materialization = True else: raise exceptions.RequestParameterInvalidException( f"Unknown workflow input source '{input_source}' specified." ) - if add_to_history and content.history != history: + if not added_to_history and add_to_history and content.history != history: if isinstance(content, HistoryDatasetCollectionAssociation): content = content.copy(element_destination=history, flush=False) else: @@ -474,6 +500,7 @@ def build_workflow_run_configs( allow_tool_state_corrections=allow_tool_state_corrections, use_cached_job=use_cached_job, resource_params=resource_params, + requires_materialization=requires_materialization, preferred_object_store_id=preferred_object_store_id, preferred_outputs_object_store_id=preferred_outputs_object_store_id, preferred_intermediate_object_store_id=preferred_intermediate_object_store_id, diff --git a/lib/galaxy/workflow/scheduling_manager.py b/lib/galaxy/workflow/scheduling_manager.py index 8d31130bad21..918ca431a0dd 100644 --- a/lib/galaxy/workflow/scheduling_manager.py +++ b/lib/galaxy/workflow/scheduling_manager.py @@ -1,11 +1,22 @@ import os from functools import partial +from typing import Optional import galaxy.workflow.schedulers from galaxy import model from galaxy.exceptions import HandlerAssignmentError from galaxy.jobs.handler import InvocationGrabber from galaxy.model.base import transaction +from galaxy.schema.invocation import ( + FailureReason, + InvocationFailureDatasetFailed, + InvocationState, + InvocationUnexpectedFailure, +) +from galaxy.schema.tasks import ( + MaterializeDatasetInstanceTaskRequest, + RequestUser, +) from galaxy.util import plugin_config from galaxy.util.custom_logging import get_logger from galaxy.util.monitors import Monitors @@ -154,8 +165,9 @@ def shutdown(self): if exception: raise exception - def queue(self, workflow_invocation, request_params, flush=True): - workflow_invocation.set_state(model.WorkflowInvocation.states.NEW) + def queue(self, workflow_invocation, request_params, flush=True, initial_state: Optional[InvocationState] = None): + initial_state = initial_state or model.WorkflowInvocation.states.NEW + workflow_invocation.set_state(initial_state) workflow_invocation.scheduler = request_params.get("scheduler", None) or self.default_scheduler_id sa_session = self.app.model.context sa_session.add(workflow_invocation) @@ -326,9 +338,55 @@ def __schedule(self, workflow_scheduler_id, workflow_scheduler): if not self.monitor_running: return + def __attempt_materialize(self, workflow_invocation, session) -> bool: + try: + inputs_to_materialize = workflow_invocation.inputs_requiring_materialization() + for input_to_materialize in inputs_to_materialize: + hda = input_to_materialize.hda + user = RequestUser(user_id=workflow_invocation.history.user_id) + task_request = MaterializeDatasetInstanceTaskRequest( + user=user, + history_id=workflow_invocation.history.id, + source="hda", + content=hda.id, + validate_hashes=True, + ) + materialized_okay = self.app.hda_manager.materialize(task_request, in_place=True) + if not materialized_okay: + workflow_invocation.fail() + workflow_invocation.add_message( + InvocationFailureDatasetFailed( + workflow_step_id=input_to_materialize.input_dataset.workflow_step.id, + reason=FailureReason.dataset_failed, + hda_id=hda.id, + ) + ) + session.add(workflow_invocation) + session.commit() + return False + + # place back into ready and let it proceed normally on next iteration? + workflow_invocation.set_state(model.WorkflowInvocation.states.READY) + session.add(workflow_invocation) + session.commit() + return True + except Exception as e: + log.exception(f"Failed to materialize dataset for workflow {workflow_invocation.id} - {e}") + workflow_invocation.fail() + failure = InvocationUnexpectedFailure(reason=FailureReason.unexpected_failure, details=str(e)) + workflow_invocation.add_message(failure) + session.add(workflow_invocation) + session.commit() + return False + def __attempt_schedule(self, invocation_id, workflow_scheduler): with self.app.model.context() as session: workflow_invocation = session.get(model.WorkflowInvocation, invocation_id) + if workflow_invocation.state == workflow_invocation.states.REQUIRES_MATERIALIZATION: + if not self.__attempt_materialize(workflow_invocation, session): + return None + if self.app.config.workflow_scheduling_separate_materialization_iteration: + return None try: if workflow_invocation.state == workflow_invocation.states.CANCELLING: workflow_invocation.cancel_invocation_steps() diff --git a/lib/galaxy_test/api/test_landing.py b/lib/galaxy_test/api/test_landing.py new file mode 100644 index 000000000000..84057f92097e --- /dev/null +++ b/lib/galaxy_test/api/test_landing.py @@ -0,0 +1,52 @@ +from base64 import b64encode +from typing import ( + Any, + Dict, +) + +from galaxy.schema.schema import CreateWorkflowLandingRequestPayload +from galaxy_test.base.populators import ( + DatasetPopulator, + skip_without_tool, + WorkflowPopulator, +) +from ._framework import ApiTestCase + + +class TestLandingApi(ApiTestCase): + dataset_populator: DatasetPopulator + workflow_populator: WorkflowPopulator + + def setUp(self): + super().setUp() + self.dataset_populator = DatasetPopulator(self.galaxy_interactor) + self.workflow_populator = WorkflowPopulator(self.galaxy_interactor) + + @skip_without_tool("cat1") + def test_workflow_landing(self): + workflow_id = self.workflow_populator.simple_workflow("test_landing") + workflow_target_type = "stored_workflow" + request_state = _workflow_request_state() + request = CreateWorkflowLandingRequestPayload( + workflow_id=workflow_id, + workflow_target_type=workflow_target_type, + request_state=request_state, + ) + response = self.dataset_populator.create_workflow_landing(request) + assert response.workflow_id == workflow_id + assert response.workflow_target_type == workflow_target_type + + response = self.dataset_populator.claim_workflow_landing(response.uuid) + assert response.workflow_id == workflow_id + assert response.workflow_target_type == workflow_target_type + + +def _workflow_request_state() -> Dict[str, Any]: + deferred = False + input_b64_1 = b64encode(b"1 2 3").decode("utf-8") + input_b64_2 = b64encode(b"4 5 6").decode("utf-8") + inputs = { + "WorkflowInput1": {"src": "url", "url": f"base64://{input_b64_1}", "ext": "txt", "deferred": deferred}, + "WorkflowInput2": {"src": "url", "url": f"base64://{input_b64_2}", "ext": "txt", "deferred": deferred}, + } + return inputs diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py index 8a29c39aaa2b..921d706b79cf 100644 --- a/lib/galaxy_test/api/test_tools.py +++ b/lib/galaxy_test/api/test_tools.py @@ -1002,6 +1002,70 @@ def test_optional_repeats_with_mins_filled_id(self): assert "false" in output1_content assert "length: 2" in output1_content + def test_data_column_defaults(self): + for input_format in ["legacy", "21.01"]: + tabular_contents = "1\t2\t3\t\n4\t5\t6\n" + with self.dataset_populator.test_history(require_new=True) as history_id: + hda = dataset_to_param( + self.dataset_populator.new_dataset(history_id, content=tabular_contents, file_type="tabular") + ) + details = self.dataset_populator.get_history_dataset_details(history_id, dataset=hda, assert_ok=True) + inputs = {"ref_parameter": hda} + response = self._run( + "gx_data_column", history_id, inputs, assert_ok=False, input_format=input_format + ).json() + output = response["outputs"] + details = self.dataset_populator.get_history_dataset_details( + history_id, dataset=output[0], assert_ok=False + ) + assert details["state"] == "ok" + + bed1_contents = open(self.get_filename("1.bed")).read() + hda = dataset_to_param( + self.dataset_populator.new_dataset(history_id, content=bed1_contents, file_type="bed") + ) + details = self.dataset_populator.get_history_dataset_details(history_id, dataset=hda, assert_ok=True) + inputs = {"ref_parameter": hda} + response = self._run( + "gx_data_column", history_id, inputs, assert_ok=False, input_format=input_format + ).json() + output = response["outputs"] + details = self.dataset_populator.get_history_dataset_details( + history_id, dataset=output[0], assert_ok=False + ) + assert details["state"] == "ok" + + with self.dataset_populator.test_history(require_new=False) as history_id: + response = self._run("gx_data_column_multiple", history_id, inputs, assert_ok=False).json() + assert "err_msg" in response, str(response) + assert "parameter 'parameter': an invalid option" in response["err_msg"] + + with self.dataset_populator.test_history(require_new=True) as history_id: + response = self._run("gx_data_column_optional", history_id, inputs, assert_ok=True) + output = response["outputs"] + content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output[0]) + assert "parameter: None" in content + + response = self._run("gx_data_column_with_default", history_id, inputs, assert_ok=True) + output = response["outputs"] + content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output[0]) + assert "parameter: 2" in content + + response = self._run("gx_data_column_with_default_legacy", history_id, inputs, assert_ok=True) + output = response["outputs"] + content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output[0]) + assert "parameter: 3" in content + + response = self._run("gx_data_column_accept_default", history_id, inputs, assert_ok=True) + output = response["outputs"] + content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output[0]) + assert "parameter: 1" in content + + response = self._run("gx_data_column_multiple_with_default", history_id, inputs, assert_ok=True) + output = response["outputs"] + content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output[0]) + assert "parameter: 1,2" in content + @skip_without_tool("library_data") def test_library_data_param(self): with self.dataset_populator.test_history(require_new=False) as history_id: diff --git a/lib/galaxy_test/api/test_tools_upload.py b/lib/galaxy_test/api/test_tools_upload.py index 64d7e97365b9..ed32bc92caf0 100644 --- a/lib/galaxy_test/api/test_tools_upload.py +++ b/lib/galaxy_test/api/test_tools_upload.py @@ -1,6 +1,7 @@ import json import os import urllib.parse +from base64 import b64encode import pytest from tusclient import client @@ -25,6 +26,9 @@ ) from ._framework import ApiTestCase +B64_FOR_1_2_3 = b64encode(b"1 2 3").decode("utf-8") +URI_FOR_1_2_3 = f"base64://{B64_FOR_1_2_3}" + class TestToolsUpload(ApiTestCase): dataset_populator: DatasetPopulator @@ -927,6 +931,63 @@ def test_upload_and_validate_valid(self): terminal_validated_state = self.dataset_populator.validate_dataset_and_wait(history_id, dataset_id) assert terminal_validated_state == "ok", terminal_validated_state + def test_upload_and_validate_hash_valid(self): + with self.dataset_populator.test_history() as history_id: + destination = {"type": "hdas"} + targets = [ + { + "destination": destination, + "items": [ + { + "src": "url", + "url": URI_FOR_1_2_3, + "hashes": [ + {"hash_function": "SHA-1", "hash_value": "65e9d53484d28eef5447bc06fe2d754d1090975a"} + ], + }, + ], + } + ] + payload = { + "history_id": history_id, + "targets": targets, + "validate_hashes": True, + } + fetch_response = self.dataset_populator.fetch(payload) + self._assert_status_code_is(fetch_response, 200) + # history ok implies the dataset upload work + self.dataset_populator.wait_for_history(history_id, assert_ok=True) + + def test_upload_and_validate_hash_invalid(self): + with self.dataset_populator.test_history() as history_id: + destination = {"type": "hdas"} + targets = [ + { + "destination": destination, + "items": [ + { + "src": "url", + "url": URI_FOR_1_2_3, + "hashes": [{"hash_function": "SHA-1", "hash_value": "invalidhash"}], + }, + ], + } + ] + payload = { + "history_id": history_id, + "targets": targets, + "validate_hashes": True, + } + fetch_response = self.dataset_populator.fetch(payload, assert_ok=True, wait=False) + self._assert_status_code_is(fetch_response, 200) + outputs = fetch_response.json()["outputs"] + new_dataset = outputs[0] + self.dataset_populator.wait_for_history(history_id, assert_ok=False) + dataset_details = self.dataset_populator.get_history_dataset_details( + history_id, dataset=new_dataset, assert_ok=False + ) + assert dataset_details["state"] == "error" + def _velvet_upload(self, history_id, extra_inputs): payload = self.dataset_populator.upload_payload( history_id, diff --git a/lib/galaxy_test/api/test_workflow_build_module.py b/lib/galaxy_test/api/test_workflow_build_module.py new file mode 100644 index 000000000000..546f6192e397 --- /dev/null +++ b/lib/galaxy_test/api/test_workflow_build_module.py @@ -0,0 +1,19 @@ +from galaxy_test.base.populators import ( + skip_without_tool, + WorkflowPopulator, +) +from ._framework import ApiTestCase + + +class TestBuildWorkflowModule(ApiTestCase): + + def setUp(self): + super().setUp() + self.workflow_populator = WorkflowPopulator(self.galaxy_interactor) + + @skip_without_tool("select_from_url") + def test_build_module_filter_dynamic_select(self): + # Verify that filtering on parameters that depend on parameter and validators works + # fine in workflow building mode. + module = self.workflow_populator.build_module(step_type="tool", content_id="select_from_url") + assert not module["errors"], module["errors"] diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py index b6a5ec037bea..8408837df347 100644 --- a/lib/galaxy_test/api/test_workflows.py +++ b/lib/galaxy_test/api/test_workflows.py @@ -1533,11 +1533,144 @@ def test_run_workflow_by_name(self): def test_run_workflow(self): self.__run_cat_workflow(inputs_by="step_id") - def __run_cat_workflow(self, inputs_by): + @skip_without_tool("cat1") + def test_run_workflow_by_deferred_url(self): + with self.dataset_populator.test_history() as history_id: + self.__run_cat_workflow(inputs_by="deferred_url", history_id=history_id) + # it did an upload of the inputs anyway - so this is a 3 is a bit of a hack... + # TODO fix this. + input_dataset_details = self.dataset_populator.get_history_dataset_details(history_id, hid=3) + assert input_dataset_details["state"] == "deferred" + + @skip_without_tool("cat1") + def test_run_workflow_by_url(self): + with self.dataset_populator.test_history() as history_id: + self.__run_cat_workflow(inputs_by="url", history_id=history_id) + input_dataset_details = self.dataset_populator.get_history_dataset_details( + history_id, hid=3, assert_ok=False + ) + assert input_dataset_details["state"] == "ok" + + @skip_without_tool("cat1") + def test_run_workflow_with_valid_url_hashes(self): + with self.dataset_populator.test_history() as history_id: + workflow = self.workflow_populator.load_workflow(name="test_for_run_invalid_url_hashes") + workflow_id = self.workflow_populator.create_workflow(workflow) + input_b64_1 = base64.b64encode(b"1 2 3").decode("utf-8") + input_b64_2 = base64.b64encode(b"4 5 6").decode("utf-8") + deferred = False + hashes_1 = [{"hash_function": "MD5", "hash_value": "5ba48b6e5a7c4d4930fda256f411e55b"}] + hashes_2 = [{"hash_function": "MD5", "hash_value": "ad0f811416f7ed2deb9122007d649fb0"}] + inputs = { + "WorkflowInput1": { + "src": "url", + "url": f"base64://{input_b64_1}", + "ext": "txt", + "deferred": deferred, + "hashes": hashes_1, + }, + "WorkflowInput2": { + "src": "url", + "url": f"base64://{input_b64_2}", + "ext": "txt", + "deferred": deferred, + "hashes": hashes_2, + }, + } + workflow_request = dict( + history=f"hist_id={history_id}", + ) + workflow_request["inputs"] = json.dumps(inputs) + workflow_request["inputs_by"] = "name" + invocation_id = self.workflow_populator.invoke_workflow_and_wait( + workflow_id, request=workflow_request + ).json()["id"] + invocation = self._invocation_details(workflow_id, invocation_id) + assert invocation["state"] == "scheduled", invocation + invocation_jobs = self.workflow_populator.get_invocation_jobs(invocation_id) + for job in invocation_jobs: + assert job["state"] == "ok" + + @skip_without_tool("cat1") + def test_run_workflow_with_invalid_url_hashes(self): + with self.dataset_populator.test_history() as history_id: + workflow = self.workflow_populator.load_workflow(name="test_for_run_invalid_url_hashes") + workflow_id = self.workflow_populator.create_workflow(workflow) + input_b64_1 = base64.b64encode(b"1 2 3").decode("utf-8") + input_b64_2 = base64.b64encode(b"4 5 6").decode("utf-8") + deferred = False + hashes = [{"hash_function": "MD5", "hash_value": "abadmd5sumhash"}] + inputs = { + "WorkflowInput1": { + "src": "url", + "url": f"base64://{input_b64_1}", + "ext": "txt", + "deferred": deferred, + "hashes": hashes, + }, + "WorkflowInput2": { + "src": "url", + "url": f"base64://{input_b64_2}", + "ext": "txt", + "deferred": deferred, + "hashes": hashes, + }, + } + workflow_request = dict( + history=f"hist_id={history_id}", + ) + workflow_request["inputs"] = json.dumps(inputs) + workflow_request["inputs_by"] = "name" + invocation_id = self.workflow_populator.invoke_workflow_and_wait( + workflow_id, request=workflow_request, assert_ok=False + ).json()["id"] + invocation_details = self._invocation_details(workflow_id, invocation_id) + assert invocation_details["state"] == "failed" + assert len(invocation_details["messages"]) == 1 + message = invocation_details["messages"][0] + assert message["reason"] == "dataset_failed" + + @skip_without_tool("cat1") + def test_run_workflow_with_invalid_url(self): + with self.dataset_populator.test_history() as history_id: + workflow = self.workflow_populator.load_workflow(name="test_for_run_invalid_url") + workflow_id = self.workflow_populator.create_workflow(workflow) + deferred = False + inputs = { + "WorkflowInput1": { + "src": "url", + "url": "gxfiles://thisurl/doesnt/work", + "ext": "txt", + "deferred": deferred, + }, + "WorkflowInput2": { + "src": "url", + "url": "gxfiles://thisurl/doesnt/work", + "ext": "txt", + "deferred": deferred, + }, + } + workflow_request = dict( + history=f"hist_id={history_id}", + ) + workflow_request["inputs"] = json.dumps(inputs) + workflow_request["inputs_by"] = "name" + invocation_id = self.workflow_populator.invoke_workflow_and_wait( + workflow_id, request=workflow_request, assert_ok=False + ).json()["id"] + invocation_details = self._invocation_details(workflow_id, invocation_id) + assert invocation_details["state"] == "failed" + assert len(invocation_details["messages"]) == 1 + message = invocation_details["messages"][0] + assert message["reason"] == "dataset_failed" + + def __run_cat_workflow(self, inputs_by, history_id: Optional[str] = None): workflow = self.workflow_populator.load_workflow(name="test_for_run") workflow["steps"]["0"]["uuid"] = str(uuid4()) workflow["steps"]["1"]["uuid"] = str(uuid4()) - workflow_request, _, workflow_id = self._setup_workflow_run(workflow, inputs_by=inputs_by) + workflow_request, _, workflow_id = self._setup_workflow_run( + workflow, inputs_by=inputs_by, history_id=history_id + ) invocation_id = self.workflow_populator.invoke_workflow_and_wait(workflow_id, request=workflow_request).json()[ "id" ] diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index a0087a5cba4d..0ee28827d414 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -77,10 +77,17 @@ ImporterGalaxyInterface, ) from gxformat2.yaml import ordered_load +from pydantic import UUID4 from requests import Response from rocrate.rocrate import ROCrate from typing_extensions import Literal +from galaxy.schema.schema import ( + CreateToolLandingRequestPayload, + CreateWorkflowLandingRequestPayload, + ToolLandingRequest, + WorkflowLandingRequest, +) from galaxy.tool_util.client.staging import InteractorStaging from galaxy.tool_util.cwl.util import ( download_output, @@ -101,6 +108,7 @@ galaxy_root_path, UNKNOWN, ) +from galaxy.util.path import StrPath from galaxy.util.resources import resource_string from galaxy.util.unittest_utils import skip_if_site_down from galaxy_test.base.decorators import ( @@ -369,7 +377,9 @@ class BasePopulator(metaclass=ABCMeta): galaxy_interactor: ApiTestInteractor @abstractmethod - def _post(self, route, data=None, files=None, headers=None, admin=False, json: bool = False) -> Response: + def _post( + self, route, data=None, files=None, headers=None, admin=False, json: bool = False, anon: bool = False + ) -> Response: """POST data to target Galaxy instance on specified route.""" @abstractmethod @@ -758,6 +768,34 @@ def _wait_for_purge(): wait_on(_wait_for_purge, "dataset to become purged", timeout=2) return self._get(dataset_url) + def create_tool_landing(self, payload: CreateToolLandingRequestPayload) -> ToolLandingRequest: + create_url = "tool_landings" + json = payload.model_dump(mode="json") + create_response = self._post(create_url, json, json=True, anon=True) + api_asserts.assert_status_code_is(create_response, 200) + create_response.raise_for_status() + return ToolLandingRequest.model_validate(create_response.json()) + + def create_workflow_landing(self, payload: CreateWorkflowLandingRequestPayload) -> WorkflowLandingRequest: + create_url = "workflow_landings" + json = payload.model_dump(mode="json") + create_response = self._post(create_url, json, json=True, anon=True) + api_asserts.assert_status_code_is(create_response, 200) + create_response.raise_for_status() + return WorkflowLandingRequest.model_validate(create_response.json()) + + def claim_tool_landing(self, uuid: UUID4) -> ToolLandingRequest: + url = f"tool_landings/{uuid}/claim" + claim_response = self._post(url, {"client_secret": "foobar"}, json=True) + api_asserts.assert_status_code_is(claim_response, 200) + return ToolLandingRequest.model_validate(claim_response.json()) + + def claim_workflow_landing(self, uuid: UUID4) -> WorkflowLandingRequest: + url = f"workflow_landings/{uuid}/claim" + claim_response = self._post(url, {"client_secret": "foobar"}, json=True) + api_asserts.assert_status_code_is(claim_response, 200) + return WorkflowLandingRequest.model_validate(claim_response.json()) + def create_tool_from_path(self, tool_path: str) -> Dict[str, Any]: tool_directory = os.path.dirname(os.path.abspath(tool_path)) payload = dict( @@ -944,7 +982,9 @@ def tools_post(self, payload: dict, url="tools") -> Response: tool_response = self._post(url, data=payload) return tool_response - def materialize_dataset_instance(self, history_id: str, id: str, source: str = "hda"): + def materialize_dataset_instance( + self, history_id: str, id: str, source: str = "hda", validate_hashes: bool = False + ): payload: Dict[str, Any] if source == "ldda": url = f"histories/{history_id}/materialize" @@ -955,6 +995,8 @@ def materialize_dataset_instance(self, history_id: str, id: str, source: str = " else: url = f"histories/{history_id}/contents/datasets/{id}/materialize" payload = {} + if validate_hashes: + payload["validate_hashes"] = True create_response = self._post(url, payload, json=True) api_asserts.assert_status_code_is_ok(create_response) create_response_json = create_response.json() @@ -1664,8 +1706,10 @@ class GalaxyInteractorHttpMixin: def _api_key(self): return self.galaxy_interactor.api_key - def _post(self, route, data=None, files=None, headers=None, admin=False, json: bool = False) -> Response: - return self.galaxy_interactor.post(route, data, files=files, admin=admin, headers=headers, json=json) + def _post( + self, route, data=None, files=None, headers=None, admin=False, json: bool = False, anon: bool = False + ) -> Response: + return self.galaxy_interactor.post(route, data, files=files, admin=admin, headers=headers, json=json, anon=anon) def _put(self, route, data=None, headers=None, admin=False, json: bool = False): return self.galaxy_interactor.put(route, data, headers=headers, admin=admin, json=json) @@ -1701,7 +1745,7 @@ def _test_history( # Things gxformat2 knows how to upload as workflows -YamlContentT = Union[str, os.PathLike, dict] +YamlContentT = Union[StrPath, dict] class BaseWorkflowPopulator(BasePopulator): @@ -1956,6 +2000,7 @@ def invoke_workflow_and_wait( history_id: Optional[str] = None, inputs: Optional[dict] = None, request: Optional[dict] = None, + assert_ok: bool = True, ) -> Response: invoke_return = self.invoke_workflow(workflow_id, history_id=history_id, inputs=inputs, request=request) invoke_return.raise_for_status() @@ -1968,7 +2013,7 @@ def invoke_workflow_and_wait( if history_id.startswith("hist_id="): history_id = history_id[len("hist_id=") :] assert history_id - self.wait_for_workflow(workflow_id, invocation_id, history_id, assert_ok=True) + self.wait_for_workflow(workflow_id, invocation_id, history_id, assert_ok=assert_ok) return invoke_return def workflow_report_json(self, workflow_id: str, invocation_id: str) -> dict: @@ -2155,23 +2200,33 @@ def setup_workflow_run( workflow_id = self.create_workflow(workflow) if not history_id: history_id = self.dataset_populator.new_history() - hda1 = self.dataset_populator.new_dataset(history_id, content="1 2 3", wait=True) - hda2 = self.dataset_populator.new_dataset(history_id, content="4 5 6", wait=True) + hda1: Optional[Dict[str, Any]] = None + hda2: Optional[Dict[str, Any]] = None + label_map: Optional[Dict[str, Any]] = None + if inputs_by != "url": + hda1 = self.dataset_populator.new_dataset(history_id, content="1 2 3", wait=True) + hda2 = self.dataset_populator.new_dataset(history_id, content="4 5 6", wait=True) + label_map = {"WorkflowInput1": ds_entry(hda1), "WorkflowInput2": ds_entry(hda2)} workflow_request = dict( history=f"hist_id={history_id}", ) - label_map = {"WorkflowInput1": ds_entry(hda1), "WorkflowInput2": ds_entry(hda2)} if inputs_by == "step_id": + assert label_map ds_map = self.build_ds_map(workflow_id, label_map) workflow_request["ds_map"] = ds_map elif inputs_by == "step_index": + assert hda1 + assert hda2 index_map = {"0": ds_entry(hda1), "1": ds_entry(hda2)} workflow_request["inputs"] = json.dumps(index_map) workflow_request["inputs_by"] = "step_index" elif inputs_by == "name": + assert label_map workflow_request["inputs"] = json.dumps(label_map) workflow_request["inputs_by"] = "name" elif inputs_by in ["step_uuid", "uuid_implicitly"]: + assert hda1 + assert hda2 assert workflow, f"Must specify workflow for this inputs_by {inputs_by} parameter value" uuid_map = { workflow["steps"]["0"]["uuid"]: ds_entry(hda1), @@ -2180,6 +2235,16 @@ def setup_workflow_run( workflow_request["inputs"] = json.dumps(uuid_map) if inputs_by == "step_uuid": workflow_request["inputs_by"] = "step_uuid" + elif inputs_by in ["url", "deferred_url"]: + input_b64_1 = base64.b64encode(b"1 2 3").decode("utf-8") + input_b64_2 = base64.b64encode(b"4 5 6").decode("utf-8") + deferred = inputs_by == "deferred_url" + inputs = { + "WorkflowInput1": {"src": "url", "url": f"base64://{input_b64_1}", "ext": "txt", "deferred": deferred}, + "WorkflowInput2": {"src": "url", "url": f"base64://{input_b64_2}", "ext": "txt", "deferred": deferred}, + } + workflow_request["inputs"] = json.dumps(inputs) + workflow_request["inputs_by"] = "name" return workflow_request, history_id, workflow_id @@ -2298,6 +2363,12 @@ def import_tool(self, tool) -> Dict[str, Any]: assert upload_response.status_code == 200, upload_response return upload_response.json() + def build_module(self, step_type: str, content_id: Optional[str] = None, inputs: Optional[Dict[str, Any]] = None): + payload = {"inputs": inputs or {}, "type": step_type, "content_id": content_id} + response = self._post("workflows/build_module", data=payload, json=True) + assert response.status_code == 200, response + return response.json() + def _import_tool_response(self, tool) -> Response: using_requirement("admin") tool_str = json.dumps(tool, indent=4) @@ -3239,6 +3310,7 @@ def get_state(): "queued", "new", "ready", + "requires_materialization", "stop", "stopped", "setting_metadata", @@ -3292,11 +3364,14 @@ def _api_url(self): def _get(self, route, data=None, headers=None, admin=False) -> Response: return self._gi.make_get_request(self._url(route), params=data) - def _post(self, route, data=None, files=None, headers=None, admin=False, json: bool = False) -> Response: + def _post( + self, route, data=None, files=None, headers=None, admin=False, json: bool = False, anon: bool = False + ) -> Response: if headers is None: headers = {} headers = headers.copy() - headers["x-api-key"] = self._gi.key + if not anon: + headers["x-api-key"] = self._gi.key return requests.post(self._url(route), data=data, headers=headers, timeout=DEFAULT_SOCKET_TIMEOUT) def _put(self, route, data=None, headers=None, admin=False, json: bool = False): diff --git a/lib/galaxy_test/selenium/framework.py b/lib/galaxy_test/selenium/framework.py index 310c88c88053..9c300dc7b0de 100644 --- a/lib/galaxy_test/selenium/framework.py +++ b/lib/galaxy_test/selenium/framework.py @@ -744,12 +744,14 @@ def _get(self, route, data=None, headers=None, admin=False) -> Response: response = requests.get(full_url, params=data, cookies=cookies, headers=headers, timeout=DEFAULT_SOCKET_TIMEOUT) return response - def _post(self, route, data=None, files=None, headers=None, admin=False, json: bool = False) -> Response: + def _post( + self, route, data=None, files=None, headers=None, admin=False, json: bool = False, anon: bool = False + ) -> Response: full_url = self.selenium_context.build_url(f"api/{route}", for_selenium=False) cookies = None if admin: full_url = f"{full_url}?key={self._mixin_admin_api_key}" - else: + elif not anon: cookies = self.selenium_context.selenium_to_requests_cookies() request_kwd = prepare_request_params(data=data, files=files, as_json=json, headers=headers, cookies=cookies) response = requests.post(full_url, timeout=DEFAULT_SOCKET_TIMEOUT, **request_kwd) diff --git a/lib/galaxy_test/workflow/empty_collection_sort.gxwf-tests.yml b/lib/galaxy_test/workflow/empty_collection_sort.gxwf-tests.yml index ad83bb676b67..d5ac9d9b15b4 100644 --- a/lib/galaxy_test/workflow/empty_collection_sort.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/empty_collection_sort.gxwf-tests.yml @@ -9,6 +9,8 @@ filter_file: i1 outputs: output: + class: Collection + collection_type: list elements: i1: asserts: diff --git a/lib/galaxy_test/workflow/flatten_collection.gxwf-tests.yml b/lib/galaxy_test/workflow/flatten_collection.gxwf-tests.yml index bfd0a6a02435..7af6cff256bc 100644 --- a/lib/galaxy_test/workflow/flatten_collection.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/flatten_collection.gxwf-tests.yml @@ -3,7 +3,8 @@ job: {} outputs: out: - attributes: {collection_type: 'list'} + class: Collection + collection_type: list elements: 'oe1-ie1': asserts: diff --git a/lib/galaxy_test/workflow/flatten_collection_over_execution.gxwf-tests.yml b/lib/galaxy_test/workflow/flatten_collection_over_execution.gxwf-tests.yml index 5c54212815e9..85d986adbb10 100644 --- a/lib/galaxy_test/workflow/flatten_collection_over_execution.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/flatten_collection_over_execution.gxwf-tests.yml @@ -8,6 +8,8 @@ content: "0 mycoolline\n1 mysecondline\n" outputs: out: + class: Collection + collection_type: list elements: 'samp1-0': asserts: diff --git a/lib/galaxy_test/workflow/integer_into_data_column.gxwf-tests.yml b/lib/galaxy_test/workflow/integer_into_data_column.gxwf-tests.yml index 890acb144ee1..eae87aa6f314 100644 --- a/lib/galaxy_test/workflow/integer_into_data_column.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/integer_into_data_column.gxwf-tests.yml @@ -10,6 +10,7 @@ type: raw outputs: output: + class: File asserts: - that: has_line line: "parameter: 2" diff --git a/lib/galaxy_test/workflow/map_over_expression.gxwf-tests.yml b/lib/galaxy_test/workflow/map_over_expression.gxwf-tests.yml index 0357ca6c53ca..7a52cb858c28 100644 --- a/lib/galaxy_test/workflow/map_over_expression.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/map_over_expression.gxwf-tests.yml @@ -10,7 +10,8 @@ content: B outputs: out1: - attributes: { collection_type: list } + class: Collection + collection_type: list elements: A: asserts: diff --git a/lib/galaxy_test/workflow/multi_select_mapping.gxwf-tests.yml b/lib/galaxy_test/workflow/multi_select_mapping.gxwf-tests.yml index 065849a34082..b9857cc30642 100644 --- a/lib/galaxy_test/workflow/multi_select_mapping.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/multi_select_mapping.gxwf-tests.yml @@ -16,6 +16,8 @@ ext: 'expression.json' outputs: output: + class: Collection + collection_type: list elements: the_example_2: asserts: diff --git a/lib/galaxy_test/workflow/multiple_integer_into_data_column.gxwf-tests.yml b/lib/galaxy_test/workflow/multiple_integer_into_data_column.gxwf-tests.yml index c960ea73e8ff..365bf7f6de1f 100644 --- a/lib/galaxy_test/workflow/multiple_integer_into_data_column.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/multiple_integer_into_data_column.gxwf-tests.yml @@ -10,6 +10,7 @@ type: raw outputs: output: + class: File asserts: - that: has_text text: "col 1,2" diff --git a/lib/galaxy_test/workflow/multiple_text.gxwf-tests.yml b/lib/galaxy_test/workflow/multiple_text.gxwf-tests.yml index 15fd0dba0254..09a7dd89e557 100644 --- a/lib/galaxy_test/workflow/multiple_text.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/multiple_text.gxwf-tests.yml @@ -9,6 +9,7 @@ type: raw outputs: output: + class: File asserts: - that: has_line line: '--ex1,ex2,--ex3' diff --git a/lib/galaxy_test/workflow/multiple_versions.gxwf-tests.yml b/lib/galaxy_test/workflow/multiple_versions.gxwf-tests.yml index 1d1db5dba980..bbe2eee671e1 100644 --- a/lib/galaxy_test/workflow/multiple_versions.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/multiple_versions.gxwf-tests.yml @@ -6,12 +6,14 @@ type: raw outputs: output_1: + class: File asserts: - that: has_text text: 'Version 0.1' - that: not_has_text text: 'Version 0.2' output_2: + class: File asserts: - that: has_text text: 'Version 0.2' diff --git a/lib/galaxy_test/workflow/rename_based_on_input_collection.gxwf-tests.yml b/lib/galaxy_test/workflow/rename_based_on_input_collection.gxwf-tests.yml index a6b357635316..9f76add4f574 100644 --- a/lib/galaxy_test/workflow/rename_based_on_input_collection.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/rename_based_on_input_collection.gxwf-tests.yml @@ -18,5 +18,6 @@ type: File outputs: output: + class: File metadata: name: 'the_dataset_pair suffix' diff --git a/lib/galaxy_test/workflow/replacement_parameters_legacy.gxwf-tests.yml b/lib/galaxy_test/workflow/replacement_parameters_legacy.gxwf-tests.yml index a3120f277072..b800042dc1a4 100644 --- a/lib/galaxy_test/workflow/replacement_parameters_legacy.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/replacement_parameters_legacy.gxwf-tests.yml @@ -5,8 +5,10 @@ replaceme: moocow outputs: out1: + class: File metadata: name: 'moocow name' out2: + class: File metadata: name: 'moocow name 2' diff --git a/lib/galaxy_test/workflow/replacement_parameters_nested.gxwf-tests.yml b/lib/galaxy_test/workflow/replacement_parameters_nested.gxwf-tests.yml index c5152f0e4add..f9acae8274f1 100644 --- a/lib/galaxy_test/workflow/replacement_parameters_nested.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/replacement_parameters_nested.gxwf-tests.yml @@ -6,8 +6,10 @@ type: raw outputs: out1: + class: File metadata: name: 'moocow name' out2: + class: File metadata: name: 'moocow name 2' diff --git a/lib/galaxy_test/workflow/replacement_parameters_text.gxwf-tests.yml b/lib/galaxy_test/workflow/replacement_parameters_text.gxwf-tests.yml index b76353ae9325..fd8dc8eb7bfc 100644 --- a/lib/galaxy_test/workflow/replacement_parameters_text.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/replacement_parameters_text.gxwf-tests.yml @@ -6,8 +6,10 @@ type: raw outputs: out1: + class: File metadata: name: 'moocow name' out2: + class: File metadata: name: 'moocow name 2' diff --git a/lib/galaxy_test/workflow/tests.py b/lib/galaxy_test/workflow/tests.py index a850490740b6..d879a1f310ae 100644 --- a/lib/galaxy_test/workflow/tests.py +++ b/lib/galaxy_test/workflow/tests.py @@ -96,10 +96,7 @@ def verify_dataset(dataset: dict, test_properties: OutputChecks): if is_collection_test: assert isinstance(test_properties, dict) test_properties["name"] = output_name - # setup preferred name "elements" in accordance with work in https://github.com/galaxyproject/planemo/pull/1417 - test_properties["element_tests"] = test_properties["elements"] - output_def = TestCollectionOutputDef.from_dict(test_properties) - + output_def = TestCollectionOutputDef.from_yaml_test_format(test_properties) invocation_details = self.workflow_populator.get_invocation(run_summary.invocation_id, step_details=True) assert output_name in invocation_details["output_collections"] test_output = invocation_details["output_collections"][output_name] diff --git a/lib/tool_shed/webapp/app.py b/lib/tool_shed/webapp/app.py index 6778d9028463..1a10551ed0f7 100644 --- a/lib/tool_shed/webapp/app.py +++ b/lib/tool_shed/webapp/app.py @@ -1,10 +1,7 @@ import logging import sys import time -from typing import ( - Any, - Optional, -) +from typing import Optional from sqlalchemy.orm.scoping import scoped_session @@ -54,7 +51,7 @@ def __init__(self, **kwd) -> None: # will be overwritten when building WSGI app self.is_webapp = False # Read the tool_shed.ini configuration file and check for errors. - self.config: Any = config.Configuration(**kwd) + self.config = config.Configuration(**kwd) self.config.check() configure_logging(self.config) self.application_stack = application_stack_instance() diff --git a/lib/tool_shed/webapp/frontend/.eslintignore b/lib/tool_shed/webapp/frontend/.eslintignore index b22c816bfd5e..a220c535ece5 100644 --- a/lib/tool_shed/webapp/frontend/.eslintignore +++ b/lib/tool_shed/webapp/frontend/.eslintignore @@ -3,5 +3,5 @@ node_modules # don't lint build output (make sure it's set to your correct build folder name) dist -# Ignore codegen aritfacts +# Ignore codegen artifacts src/gql/*.ts diff --git a/mypy.ini b/mypy.ini index 0ec6ae269986..70250f4d521f 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,4 +1,5 @@ [mypy] +enable_error_code = ignore-without-code plugins = pydantic.mypy show_error_codes = True ignore_missing_imports = True diff --git a/test/functional/tools/parameters/gx_boolean.xml b/test/functional/tools/parameters/gx_boolean.xml index e42c9c9b6af6..d89ec53fe3ff 100644 --- a/test/functional/tools/parameters/gx_boolean.xml +++ b/test/functional/tools/parameters/gx_boolean.xml @@ -9,6 +9,13 @@ echo '$parameter' >> '$output' + + + + + + + diff --git a/test/functional/tools/parameters/gx_boolean_checked.xml b/test/functional/tools/parameters/gx_boolean_checked.xml new file mode 100644 index 000000000000..de33a26e6f56 --- /dev/null +++ b/test/functional/tools/parameters/gx_boolean_checked.xml @@ -0,0 +1,36 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_boolean_optional.xml b/test/functional/tools/parameters/gx_boolean_optional.xml index dc667b614a1a..57a6d6f3f1be 100644 --- a/test/functional/tools/parameters/gx_boolean_optional.xml +++ b/test/functional/tools/parameters/gx_boolean_optional.xml @@ -34,7 +34,6 @@ cat '$inputs' >> $inputs_json; - diff --git a/test/functional/tools/parameters/gx_boolean_optional_checked.xml b/test/functional/tools/parameters/gx_boolean_optional_checked.xml new file mode 100644 index 000000000000..f1c27f98ce46 --- /dev/null +++ b/test/functional/tools/parameters/gx_boolean_optional_checked.xml @@ -0,0 +1,58 @@ + + > '$output'; +cat '$inputs' >> $inputs_json; + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_data_column_accept_default.xml b/test/functional/tools/parameters/gx_data_column_accept_default.xml new file mode 100644 index 000000000000..5b189f4ebda0 --- /dev/null +++ b/test/functional/tools/parameters/gx_data_column_accept_default.xml @@ -0,0 +1,21 @@ + + + macros.xml + + > '$output' + ]]> + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_data_column_multiple.xml b/test/functional/tools/parameters/gx_data_column_multiple.xml index ed37a530d19b..bd6b30a2a545 100644 --- a/test/functional/tools/parameters/gx_data_column_multiple.xml +++ b/test/functional/tools/parameters/gx_data_column_multiple.xml @@ -28,9 +28,6 @@ echo 'parameter: $parameter' >> '$output' - - - diff --git a/test/functional/tools/parameters/gx_data_column_multiple_accept_default.xml b/test/functional/tools/parameters/gx_data_column_multiple_accept_default.xml new file mode 100644 index 000000000000..401d416668d2 --- /dev/null +++ b/test/functional/tools/parameters/gx_data_column_multiple_accept_default.xml @@ -0,0 +1,21 @@ + + + macros.xml + + > '$output' + ]]> + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/data_column_multiple_optional.xml b/test/functional/tools/parameters/gx_data_column_multiple_optional.xml similarity index 100% rename from test/functional/tools/parameters/data_column_multiple_optional.xml rename to test/functional/tools/parameters/gx_data_column_multiple_optional.xml diff --git a/test/functional/tools/parameters/gx_data_column_multiple_optional_with_default.xml b/test/functional/tools/parameters/gx_data_column_multiple_optional_with_default.xml new file mode 100644 index 000000000000..0700e38a23a9 --- /dev/null +++ b/test/functional/tools/parameters/gx_data_column_multiple_optional_with_default.xml @@ -0,0 +1,21 @@ + + + macros.xml + + > '$output' + ]]> + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_data_column_multiple_with_default.xml b/test/functional/tools/parameters/gx_data_column_multiple_with_default.xml new file mode 100644 index 000000000000..fca81373d544 --- /dev/null +++ b/test/functional/tools/parameters/gx_data_column_multiple_with_default.xml @@ -0,0 +1,21 @@ + + + macros.xml + + > '$output' + ]]> + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_data_column_optional_accept_default.xml b/test/functional/tools/parameters/gx_data_column_optional_accept_default.xml new file mode 100644 index 000000000000..be893d34337f --- /dev/null +++ b/test/functional/tools/parameters/gx_data_column_optional_accept_default.xml @@ -0,0 +1,21 @@ + + + macros.xml + + > '$output' + ]]> + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_data_column_with_default.xml b/test/functional/tools/parameters/gx_data_column_with_default.xml new file mode 100644 index 000000000000..126c4dfc8ec1 --- /dev/null +++ b/test/functional/tools/parameters/gx_data_column_with_default.xml @@ -0,0 +1,22 @@ + + + macros.xml + + > '$output' + ]]> + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_data_column_with_default_legacy.xml b/test/functional/tools/parameters/gx_data_column_with_default_legacy.xml new file mode 100644 index 000000000000..b77b546add50 --- /dev/null +++ b/test/functional/tools/parameters/gx_data_column_with_default_legacy.xml @@ -0,0 +1,22 @@ + + + macros.xml + + > '$output' + ]]> + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_select_optional.xml b/test/functional/tools/parameters/gx_select_optional.xml index 5f6b63813dd3..03fc045dba2a 100644 --- a/test/functional/tools/parameters/gx_select_optional.xml +++ b/test/functional/tools/parameters/gx_select_optional.xml @@ -23,5 +23,12 @@ echo '$parameter' >> '$output' + + + + + + + diff --git a/test/functional/tools/select_from_url.xml b/test/functional/tools/select_from_url.xml index 60a712e74631..488055677e85 100644 --- a/test/functional/tools/select_from_url.xml +++ b/test/functional/tools/select_from_url.xml @@ -10,6 +10,13 @@ echo '$url_param_value_header_and_body' > '$param_value_header_and_body' + + + + + + + > '$output1' && echo select_mult_opt $select_mult_opt >> '$output1' && diff --git a/test/integration/test_materialize_dataset_instance_tasks.py b/test/integration/test_materialize_dataset_instance_tasks.py index 7bd606cd390a..5d3d209e7221 100644 --- a/test/integration/test_materialize_dataset_instance_tasks.py +++ b/test/integration/test_materialize_dataset_instance_tasks.py @@ -78,6 +78,26 @@ def test_materialize_gxfiles_uri(self, history_id: str): assert new_hda_details["state"] == "ok" assert not new_hda_details["deleted"] + @pytest.mark.require_new_history + def test_materialize_hash_failure(self, history_id: str): + store_dict = deferred_hda_model_store_dict(source_uri="gxfiles://testdatafiles/2.bed") + store_dict["datasets"][0]["file_metadata"]["hashes"][0]["hash_value"] = "invalidhash" + as_list = self.dataset_populator.create_contents_from_store(history_id, store_dict=store_dict) + assert len(as_list) == 1 + deferred_hda = as_list[0] + assert deferred_hda["model_class"] == "HistoryDatasetAssociation" + assert deferred_hda["state"] == "deferred" + assert not deferred_hda["deleted"] + + self.dataset_populator.materialize_dataset_instance(history_id, deferred_hda["id"], validate_hashes=True) + self.dataset_populator.wait_on_history_length(history_id, 2) + new_hda_details = self.dataset_populator.get_history_dataset_details( + history_id, hid=2, assert_ok=False, wait=False + ) + assert new_hda_details["model_class"] == "HistoryDatasetAssociation" + assert new_hda_details["state"] == "error" + assert not new_hda_details["deleted"] + @pytest.mark.require_new_history def test_materialize_history_dataset_bam(self, history_id: str): as_list = self.dataset_populator.create_contents_from_store( diff --git a/test/unit/app/managers/test_landing.py b/test/unit/app/managers/test_landing.py new file mode 100644 index 000000000000..211e1f8010e5 --- /dev/null +++ b/test/unit/app/managers/test_landing.py @@ -0,0 +1,176 @@ +from uuid import uuid4 + +from galaxy.exceptions import ( + InsufficientPermissionsException, + ItemAlreadyClaimedException, + ObjectNotFound, +) +from galaxy.managers.landing import LandingRequestManager +from galaxy.model import ( + StoredWorkflow, + Workflow, +) +from galaxy.model.base import transaction +from galaxy.schema.schema import ( + ClaimLandingPayload, + CreateToolLandingRequestPayload, + CreateWorkflowLandingRequestPayload, + LandingRequestState, + ToolLandingRequest, + WorkflowLandingRequest, +) +from .base import BaseTestCase + +TEST_TOOL_ID = "cat1" +TEST_TOOL_VERSION = "1.0.0" +TEST_STATE = { + "input1": { + "src": "url", + "url": "https://raw.githubusercontent.com/galaxyproject/planemo/7be1bf5b3971a43eaa73f483125bfb8cabf1c440/tests/data/hello.txt", + "ext": "txt", + }, +} +CLIENT_SECRET = "mycoolsecret" + + +class TestLanding(BaseTestCase): + + def setUp(self): + super().setUp() + self.landing_manager = LandingRequestManager(self.trans.sa_session, self.app.security) + + def test_tool_landing_requests_typical_flow(self): + landing_request: ToolLandingRequest = self.landing_manager.create_tool_landing_request(self._tool_request) + assert landing_request.state == LandingRequestState.UNCLAIMED + assert landing_request.uuid is not None + uuid = landing_request.uuid + claim_payload = ClaimLandingPayload(client_secret=CLIENT_SECRET) + landing_request = self.landing_manager.claim_tool_landing_request(self.trans, uuid, claim_payload) + assert landing_request.state == LandingRequestState.CLAIMED + assert landing_request.uuid == uuid + landing_request = self.landing_manager.get_tool_landing_request(self.trans, uuid) + assert landing_request.state == LandingRequestState.CLAIMED + assert landing_request.uuid == uuid + + def test_tool_landing_requests_requires_matching_client_secret(self): + landing_request: ToolLandingRequest = self.landing_manager.create_tool_landing_request(self._tool_request) + uuid = landing_request.uuid + claim_payload = ClaimLandingPayload(client_secret="wrongsecret") + exception = None + try: + self.landing_manager.claim_tool_landing_request(self.trans, uuid, claim_payload) + except InsufficientPermissionsException as e: + exception = e + assert exception is not None + + def test_tool_landing_requests_get_requires_claim(self): + landing_request: ToolLandingRequest = self.landing_manager.create_tool_landing_request(self._tool_request) + uuid = landing_request.uuid + exception = None + try: + self.landing_manager.get_tool_landing_request(self.trans, uuid) + except InsufficientPermissionsException as e: + exception = e + assert exception is not None + + def test_cannot_reclaim_tool_landing(self): + landing_request: ToolLandingRequest = self.landing_manager.create_tool_landing_request(self._tool_request) + assert landing_request.state == LandingRequestState.UNCLAIMED + uuid = landing_request.uuid + claim_payload = ClaimLandingPayload(client_secret=CLIENT_SECRET) + landing_request = self.landing_manager.claim_tool_landing_request(self.trans, uuid, claim_payload) + assert landing_request.state == LandingRequestState.CLAIMED + exception = None + try: + self.landing_manager.claim_tool_landing_request(self.trans, uuid, claim_payload) + except ItemAlreadyClaimedException as e: + exception = e + assert exception + + def test_get_tool_unknown_claim(self): + exception = None + try: + self.landing_manager.get_tool_landing_request(self.trans, uuid4()) + except ObjectNotFound as e: + exception = e + assert exception + + def test_stored_workflow_landing_requests_typical_flow(self): + landing_request: WorkflowLandingRequest = self.landing_manager.create_workflow_landing_request( + self._stored_workflow_request + ) + assert landing_request.state == LandingRequestState.UNCLAIMED + assert landing_request.uuid is not None + assert landing_request.workflow_target_type == "stored_workflow" + uuid = landing_request.uuid + claim_payload = ClaimLandingPayload(client_secret=CLIENT_SECRET) + landing_request = self.landing_manager.claim_workflow_landing_request(self.trans, uuid, claim_payload) + assert landing_request.state == LandingRequestState.CLAIMED + assert landing_request.uuid == uuid + assert landing_request.workflow_target_type == "stored_workflow" + landing_request = self.landing_manager.get_workflow_landing_request(self.trans, uuid) + assert landing_request.state == LandingRequestState.CLAIMED + assert landing_request.uuid == uuid + assert landing_request.workflow_target_type == "stored_workflow" + + def test_workflow_landing_requests_typical_flow(self): + landing_request: WorkflowLandingRequest = self.landing_manager.create_workflow_landing_request( + self._workflow_request + ) + assert landing_request.state == LandingRequestState.UNCLAIMED + assert landing_request.uuid is not None + assert landing_request.workflow_target_type == "workflow" + uuid = landing_request.uuid + claim_payload = ClaimLandingPayload(client_secret=CLIENT_SECRET) + landing_request = self.landing_manager.claim_workflow_landing_request(self.trans, uuid, claim_payload) + assert landing_request.state == LandingRequestState.CLAIMED + assert landing_request.uuid == uuid + assert landing_request.workflow_target_type == "workflow" + landing_request = self.landing_manager.get_workflow_landing_request(self.trans, uuid) + assert landing_request.state == LandingRequestState.CLAIMED + assert landing_request.uuid == uuid + assert landing_request.workflow_target_type == "workflow" + + @property + def _tool_request(self) -> CreateToolLandingRequestPayload: + return CreateToolLandingRequestPayload( + tool_id=TEST_TOOL_ID, + tool_version=TEST_TOOL_VERSION, + request_state=TEST_STATE.copy(), + client_secret=CLIENT_SECRET, + ) + + @property + def _stored_workflow_request(self) -> CreateWorkflowLandingRequestPayload: + sa_session = self.app.model.context + stored_workflow = StoredWorkflow() + stored_workflow.user = self.trans.user + sa_session.add(stored_workflow) + with transaction(sa_session): + sa_session.commit() + + return CreateWorkflowLandingRequestPayload( + workflow_id=self.app.security.encode_id(stored_workflow.id), + workflow_target_type="stored_workflow", + request_state=TEST_STATE.copy(), + client_secret=CLIENT_SECRET, + ) + + @property + def _workflow_request(self) -> CreateWorkflowLandingRequestPayload: + sa_session = self.app.model.context + stored_workflow = StoredWorkflow() + stored_workflow.user = self.trans.user + workflow = Workflow() + workflow.stored_workflow = stored_workflow + sa_session.add(stored_workflow) + sa_session.add(workflow) + with transaction(sa_session): + sa_session.commit() + + return CreateWorkflowLandingRequestPayload( + workflow_id=self.app.security.encode_id(workflow.id), + workflow_target_type="workflow", + request_state=TEST_STATE.copy(), + client_secret=CLIENT_SECRET, + ) diff --git a/test/unit/app/tools/test_data_fetch.py b/test/unit/app/tools/test_data_fetch.py index 58e13e5d1549..ee7cdd61536f 100644 --- a/test/unit/app/tools/test_data_fetch.py +++ b/test/unit/app/tools/test_data_fetch.py @@ -1,6 +1,7 @@ import json import os import tempfile +from base64 import b64encode from contextlib import contextmanager from shutil import rmtree from tempfile import mkdtemp @@ -8,6 +9,9 @@ from galaxy.tools.data_fetch import main from galaxy.util.unittest_utils import skip_if_github_down +B64_FOR_1_2_3 = b64encode(b"1 2 3").decode("utf-8") +URI_FOR_1_2_3 = f"base64://{B64_FOR_1_2_3}" + def test_simple_path_get(): with _execute_context() as execute_context: @@ -55,6 +59,134 @@ def test_simple_uri_get(): assert hda_result["ext"] == "bed" +def test_correct_md5(): + with _execute_context() as execute_context: + request = { + "targets": [ + { + "destination": { + "type": "hdas", + }, + "elements": [ + { + "src": "url", + "url": URI_FOR_1_2_3, + "hashes": [ + { + "hash_function": "MD5", + "hash_value": "5ba48b6e5a7c4d4930fda256f411e55b", + } + ], + } + ], + } + ], + "validate_hashes": True, + } + execute_context.execute_request(request) + output = _unnamed_output(execute_context) + hda_result = output["elements"][0] + assert hda_result["state"] == "ok" + assert hda_result["ext"] == "txt" + + +def test_incorrect_md5(): + with _execute_context() as execute_context: + request = { + "targets": [ + { + "destination": { + "type": "hdas", + }, + "elements": [ + { + "src": "url", + "url": URI_FOR_1_2_3, + "hashes": [ + { + "hash_function": "MD5", + "hash_value": "thisisbad", + } + ], + } + ], + } + ], + "validate_hashes": True, + } + execute_context.execute_request(request) + output = _unnamed_output(execute_context) + hda_result = output["elements"][0] + assert ( + hda_result["error_message"] + == "Failed to validate upload with [MD5] - expected [thisisbad] got [5ba48b6e5a7c4d4930fda256f411e55b]" + ) + + +def test_correct_sha1(): + with _execute_context() as execute_context: + request = { + "targets": [ + { + "destination": { + "type": "hdas", + }, + "elements": [ + { + "src": "url", + "url": URI_FOR_1_2_3, + "hashes": [ + { + "hash_function": "SHA-1", + "hash_value": "65e9d53484d28eef5447bc06fe2d754d1090975a", + } + ], + } + ], + } + ], + "validate_hashes": True, + } + execute_context.execute_request(request) + output = _unnamed_output(execute_context) + hda_result = output["elements"][0] + assert hda_result["state"] == "ok" + assert hda_result["ext"] == "txt" + + +def test_incorrect_sha1(): + with _execute_context() as execute_context: + request = { + "targets": [ + { + "destination": { + "type": "hdas", + }, + "elements": [ + { + "src": "url", + "url": URI_FOR_1_2_3, + "hashes": [ + { + "hash_function": "SHA-1", + "hash_value": "thisisbad", + } + ], + } + ], + } + ], + "validate_hashes": True, + } + execute_context.execute_request(request) + output = _unnamed_output(execute_context) + hda_result = output["elements"][0] + assert ( + hda_result["error_message"] + == "Failed to validate upload with [SHA-1] - expected [thisisbad] got [65e9d53484d28eef5447bc06fe2d754d1090975a]" + ) + + @skip_if_github_down def test_deferred_uri_get(): with _execute_context() as execute_context: diff --git a/test/unit/data/datatypes/test_check_required.py b/test/unit/data/datatypes/test_check_required.py index e5f8d1c880e3..8351f1c08bf5 100644 --- a/test/unit/data/datatypes/test_check_required.py +++ b/test/unit/data/datatypes/test_check_required.py @@ -22,21 +22,21 @@ class CheckRequiredInherited(CheckRequiredTrue): def test_check_required_metadata_false(): app = GalaxyDataTestApp() - app.datatypes_registry.datatypes_by_extension["false"] = CheckRequiredFalse + app.datatypes_registry.datatypes_by_extension["false"] = CheckRequiredFalse() hda = HistoryDatasetAssociation(sa_session=app.model.session, extension="false") assert not hda.metadata.spec["columns"].check_required_metadata def test_check_required_metadata_true(): app = GalaxyDataTestApp() - app.datatypes_registry.datatypes_by_extension["true"] = CheckRequiredTrue + app.datatypes_registry.datatypes_by_extension["true"] = CheckRequiredTrue() hda = HistoryDatasetAssociation(sa_session=app.model.session, extension="true") assert hda.metadata.spec["columns"].check_required_metadata def test_check_required_metadata_inherited(): app = GalaxyDataTestApp() - app.datatypes_registry.datatypes_by_extension["inherited"] = CheckRequiredInherited + app.datatypes_registry.datatypes_by_extension["inherited"] = CheckRequiredInherited() hda = HistoryDatasetAssociation(sa_session=app.model.session, extension="inherited") assert hda.metadata.spec["columns"].check_required_metadata assert not hda.metadata.spec["something"].check_required_metadata diff --git a/test/unit/data/test_dataset_materialization.py b/test/unit/data/test_dataset_materialization.py index 9015b107539f..e002b439726d 100644 --- a/test/unit/data/test_dataset_materialization.py +++ b/test/unit/data/test_dataset_materialization.py @@ -62,6 +62,69 @@ def test_deferred_hdas_basic_attached(): _assert_2_bed_metadata(materialized_hda) +def test_hash_validate(): + fixture_context = setup_fixture_context_with_history() + store_dict = deferred_hda_model_store_dict() + perform_import_from_store_dict(fixture_context, store_dict) + deferred_hda = fixture_context.history.datasets[0] + assert deferred_hda + _assert_2_bed_metadata(deferred_hda) + assert deferred_hda.dataset.state == "deferred" + materializer = materializer_factory(True, object_store=fixture_context.app.object_store) + materialized_hda = materializer.ensure_materialized(deferred_hda, validate_hashes=True) + materialized_dataset = materialized_hda.dataset + assert materialized_dataset.state == "ok" + + +def test_hash_invalid(): + fixture_context = setup_fixture_context_with_history() + store_dict = deferred_hda_model_store_dict() + store_dict["datasets"][0]["file_metadata"]["hashes"][0]["hash_value"] = "invalidhash" + perform_import_from_store_dict(fixture_context, store_dict) + deferred_hda = fixture_context.history.datasets[0] + assert deferred_hda + _assert_2_bed_metadata(deferred_hda) + assert deferred_hda.dataset.state == "deferred" + materializer = materializer_factory(True, object_store=fixture_context.app.object_store) + materialized_hda = materializer.ensure_materialized(deferred_hda, validate_hashes=True) + materialized_dataset = materialized_hda.dataset + assert materialized_dataset.state == "error" + + +def test_hash_validate_source_of_download(): + fixture_context = setup_fixture_context_with_history() + store_dict = deferred_hda_model_store_dict() + store_dict["datasets"][0]["file_metadata"]["sources"][0]["hashes"] = [ + {"model_class": "DatasetSourceHash", "hash_function": "MD5", "hash_value": "f568c29421792b1b1df4474dafae01f1"} + ] + perform_import_from_store_dict(fixture_context, store_dict) + deferred_hda = fixture_context.history.datasets[0] + assert deferred_hda + _assert_2_bed_metadata(deferred_hda) + assert deferred_hda.dataset.state == "deferred" + materializer = materializer_factory(True, object_store=fixture_context.app.object_store) + materialized_hda = materializer.ensure_materialized(deferred_hda, validate_hashes=True) + materialized_dataset = materialized_hda.dataset + assert materialized_dataset.state == "ok", materialized_hda.info + + +def test_hash_invalid_source_of_download(): + fixture_context = setup_fixture_context_with_history() + store_dict = deferred_hda_model_store_dict() + store_dict["datasets"][0]["file_metadata"]["sources"][0]["hashes"] = [ + {"model_class": "DatasetSourceHash", "hash_function": "MD5", "hash_value": "invalidhash"} + ] + perform_import_from_store_dict(fixture_context, store_dict) + deferred_hda = fixture_context.history.datasets[0] + assert deferred_hda + _assert_2_bed_metadata(deferred_hda) + assert deferred_hda.dataset.state == "deferred" + materializer = materializer_factory(True, object_store=fixture_context.app.object_store) + materialized_hda = materializer.ensure_materialized(deferred_hda, validate_hashes=True) + materialized_dataset = materialized_hda.dataset + assert materialized_dataset.state == "error", materialized_hda.info + + def test_deferred_hdas_basic_attached_store_by_uuid(): # skip a flush here so this is a different path... fixture_context = setup_fixture_context_with_history(store_by="uuid") diff --git a/test/unit/data/test_dereference.py b/test/unit/data/test_dereference.py new file mode 100644 index 000000000000..8448b731a081 --- /dev/null +++ b/test/unit/data/test_dereference.py @@ -0,0 +1,57 @@ +from base64 import b64encode + +from galaxy.model.dereference import dereference_to_model +from galaxy.tool_util.parameters import DataRequestUri +from .model.test_model_store import setup_fixture_context_with_history + +B64_FOR_1_2_3 = b64encode(b"1 2 3").decode("utf-8") +TEST_URI = "gxfiles://test/1.bed" +TEST_BASE64_URI = f"base64://{B64_FOR_1_2_3}" + + +def test_dereference(): + app, sa_session, user, history = setup_fixture_context_with_history() + uri_request = DataRequestUri(url=TEST_URI, ext="bed") + hda = dereference_to_model(sa_session, user, history, uri_request) + assert hda.name == "1.bed" + assert hda.dataset.sources[0].source_uri == TEST_URI + assert hda.ext == "bed" + + +def test_dereference_dbkey(): + app, sa_session, user, history = setup_fixture_context_with_history() + uri_request = DataRequestUri(url=TEST_URI, ext="bed", dbkey="hg19") + hda = dereference_to_model(sa_session, user, history, uri_request) + assert hda.name == "1.bed" + assert hda.dataset.sources[0].source_uri == TEST_URI + assert hda.dbkey == "hg19" + + +def test_dereference_md5(): + app, sa_session, user, history = setup_fixture_context_with_history() + md5 = "f2b33fb7b3d0eb95090a16060e6a24f9" + uri_request = DataRequestUri.model_validate( + { + "url": TEST_BASE64_URI, + "name": "foobar.txt", + "ext": "txt", + "hashes": [{"hash_function": "MD5", "hash_value": md5}], + } + ) + hda = dereference_to_model(sa_session, user, history, uri_request) + assert hda.name == "foobar.txt" + assert hda.dataset.sources[0].source_uri == TEST_BASE64_URI + assert hda.dataset.sources[0].hashes[0] + assert hda.dataset.sources[0].hashes[0].hash_function == "MD5" + assert hda.dataset.sources[0].hashes[0].hash_value == md5 + + +def test_dereference_to_posix(): + app, sa_session, user, history = setup_fixture_context_with_history() + uri_request = DataRequestUri.model_validate( + {"url": TEST_BASE64_URI, "name": "foobar.txt", "ext": "txt", "space_to_tab": True} + ) + hda = dereference_to_model(sa_session, user, history, uri_request) + assert hda.name == "foobar.txt" + assert hda.dataset.sources[0].source_uri == TEST_BASE64_URI + assert hda.dataset.sources[0].transform[0]["action"] == "space_to_tab" diff --git a/test/unit/tool_util/framework_tool_checks.yml b/test/unit/tool_util/framework_tool_checks.yml index c9a6cbf661df..883fd3d40ffc 100644 --- a/test/unit/tool_util/framework_tool_checks.yml +++ b/test/unit/tool_util/framework_tool_checks.yml @@ -6,3 +6,57 @@ empty_select: - {} request_invalid: - select_optional: anyoption + +select_from_dataset_in_conditional: + request_valid: + - single: {src: hda, id: abcde133543d} + request_invalid: + - single: 7 + request_internal_valid: + - single: {src: hda, id: 7} + request_internal_invalid: + - single: 7 + job_internal_valid: + - single: {src: hda, id: 7} + job_internal_valid: + - single: {src: hda, id: 7} + cond: + cond: single + select_single: chr10 + inner_cond: + inner_cond: single + select_single: chr10 + job_internal_invalid: + - single: {src: hda, id: 7} + cond: + cond: single + select_single: chr10 + inner_cond: + inner_cond: single + select_single: chr10 + badoption: true + - single: {src: hda, id: 7} + cond: + cond: single + select_single: chr10 + +column_param: + request_valid: + - input1: {src: hda, id: abcde133543d} + col: 1 + col_names: 1 + request_invalid: + - input1: {src: hda, id: abcde133543d} + col: moocow + col_names: moocow + request_internal_valid: + - input1: {src: hda, id: 7} + col: 1 + col_names: 1 + request_internal_invalid: + - input1: {src: hda, id: abcde133543d} + col: 1 + col_names: 1 + + + diff --git a/test/unit/tool_util/parameter_specification.yml b/test/unit/tool_util/parameter_specification.yml index 8497843a427c..761a1c3fb680 100644 --- a/test/unit/tool_util/parameter_specification.yml +++ b/test/unit/tool_util/parameter_specification.yml @@ -27,6 +27,11 @@ gx_int: - parameter: "None" - parameter: { 5 } - parameter: {__class__: 'ConnectedValue'} + job_internal_valid: + - parameter: 5 + job_internal_invalid: + - parameter: null + - {} test_case_xml_valid: - parameter: 5 - {} @@ -61,6 +66,11 @@ gx_boolean: - parameter: "mytrue" - parameter: null - parameter: {__class__: 'ConnectedValue'} + job_internal_valid: + - parameter: true + - parameter: false + job_internal_invalid: + - {} workflow_step_valid: - parameter: True - {} @@ -87,6 +97,11 @@ gx_int_optional: - parameter: "null" - parameter: [5] - parameter: {__class__: 'ConnectedValue'} + job_internal_valid: + - parameter: 5 + - parameter: null + job_internal_invalid: + - {} workflow_step_valid: - parameter: 5 - parameter: null @@ -113,22 +128,42 @@ gx_int_required: &gx_int_required - parameter: "None" - parameter: { 5 } - parameter: {__class__: 'ConnectedValue'} + job_internal_valid: + - parameter: 5 + job_internal_invalid: + - {} gx_int_required_via_empty_string: <<: *gx_int_required gx_text: - request_valid: + request_valid: &gx_text_request_valid - parameter: moocow - parameter: 'some spaces' - parameter: '' - {} - request_invalid: + request_invalid: &gx_text_request_invalid - parameter: 5 - parameter: null - parameter: {} - parameter: { "moo": "cow" } - parameter: {__class__: 'ConnectedValue'} + request_internal_valid: + *gx_text_request_valid + request_internal_invalid: + *gx_text_request_invalid + request_internal_dereferenced_valid: + *gx_text_request_valid + request_internal_dereferenced_invalid: + *gx_text_request_invalid + job_internal_valid: + - parameter: moocow + - parameter: 'some spaces' + - parameter: '' + job_internal_invalid: + - {} + - parameter: null + - parameter: { "moo": "cow" } workflow_step_valid: - parameter: moocow - parameter: 'some spaces' @@ -199,11 +234,20 @@ gx_select: - parameter: null - parameter: {} - parameter: 5 - request_internal_valid: + request_internal_valid: &gx_select_request_valid - parameter: "--ex1" - parameter: "ex2" - request_internal_invalid: + request_internal_invalid: &gx_select_request_invalid - parameter: {} + request_internal_dereferenced_valid: + *gx_select_request_valid + request_internal_dereferenced_invalid: + *gx_select_request_invalid + job_internal_valid: + - parameter: '--ex1' + - parameter: 'ex2' + job_internal_invalid: + - {} test_case_xml_valid: - parameter: 'ex2' - parameter: '--ex1' @@ -239,6 +283,15 @@ gx_select_optional: - parameter: ["ex2"] - parameter: {} - parameter: 5 + job_internal_valid: + - parameter: '--ex1' + - parameter: 'ex2' + - parameter: null + job_internal_invalid: + - parameter: 'invalid' + - {} + test_case_xml_valid: + - {} workflow_step_valid: - parameter: "--ex1" - parameter: "ex2" @@ -266,19 +319,26 @@ gx_select_multiple: request_valid: - parameter: ["--ex1"] - parameter: ["ex2"] + # ugh... but these aren't optional... + - parameter: null + - {} request_invalid: - parameter: ["Ex1"] - # DIVERGES_FROM_CURRENT_API - - parameter: null - parameter: {} - parameter: 5 - - {} workflow_step_valid: - parameter: ["--ex1"] - parameter: ["ex2"] - {} # could come in linked... # ... hmmm? this should maybe be invalid right? - parameter: null + job_internal_valid: + - parameter: ["--ex1"] + # itd be coool if this was forced to empty list - probably breaks backward compat though... + - parameter: null + job_internal_invalid: + - {} + - parameter: "--ex1" workflow_step_invalid: - parameter: ["Ex1"] - parameter: {} @@ -289,16 +349,15 @@ gx_select_multiple: - parameter: ["--ex1"] - parameter: ["ex2"] - parameter: [{__class__: 'ConnectedValue'}] + - {} + - parameter: null workflow_step_linked_invalid: - parameter: ["Ex1"] - parameter: {} - parameter: 5 - - {} # might be wrong? I guess we would expect the semantic of this to do like a map-over # but as far as I am aware that is not implemented https://github.com/galaxyproject/galaxy/issues/18541 - parameter: {__class__: 'ConnectedValue'} - # they are non-optinoal right? - - parameter: null gx_select_multiple_optional: request_valid: @@ -320,6 +379,10 @@ gx_genomebuild: request_invalid: - parameter: null - parameter: 9 + job_internal_valid: + - parameter: hg19 + job_internal_invalid: + - {} gx_genomebuild_optional: request_valid: @@ -331,6 +394,10 @@ gx_genomebuild_optional: request_invalid: - parameter: 8 - parameter: true + job_internal_valid: + - parameter: null + job_internal_invalid: + - {} gx_genomebuild_multiple: request_valid: @@ -348,6 +415,13 @@ gx_directory_uri: - parameter: "justsomestring" - parameter: true - parameter: null + job_internal_valid: + - parameter: "gxfiles://foobar/" + - parameter: "gxfiles://foobar" + job_internal_invalid: + - parameter: "justsomestring" + - parameter: true + - parameter: null gx_hidden: request_valid: @@ -361,6 +435,12 @@ gx_hidden: - parameter: 5 - parameter: {} - parameter: { "moo": "cow" } + job_internal_valid: + - parameter: moocow + - parameter: 'some spaces' + - parameter: '' + job_internal_invalid: + - {} workflow_step_valid: - parameter: moocow - {} @@ -501,6 +581,7 @@ gx_color: gx_data: request_valid: - parameter: {src: hda, id: abcdabcd} + - parameter: {src: url, url: "https://raw.githubusercontent.com/galaxyproject/planemo/7be1bf5b3971a43eaa73f483125bfb8cabf1c440/tests/data/hello.txt", "ext": "txt"} - parameter: {__class__: "Batch", values: [{src: hdca, id: abcdabcd}]} request_invalid: - parameter: {__class__: "Batch", values: [{src: hdca, id: 5}]} @@ -518,6 +599,7 @@ gx_data: - parameter: {__class__: "Batch", values: [{src: hdca, id: 5}]} - parameter: {src: hda, id: 5} - parameter: {src: hda, id: 0} + - parameter: {src: url, url: "https://raw.githubusercontent.com/galaxyproject/planemo/7be1bf5b3971a43eaa73f483125bfb8cabf1c440/tests/data/hello.txt", ext: "txt"} request_internal_invalid: - parameter: {__class__: "Batch", values: [{src: hdca, id: abcdabcd}]} - parameter: {src: hda, id: abcdabcd} @@ -526,6 +608,14 @@ gx_data: - parameter: true - parameter: 5 - parameter: "5" + request_internal_dereferenced_valid: + - parameter: {__class__: "Batch", values: [{src: hdca, id: 5}]} + - parameter: {src: hda, id: 5} + - parameter: {src: hda, id: 0} + request_internal_dereferenced_invalid: + # the difference between request internal and request internal dereferenced is that these have been converted + # to datasets. + - parameter: {src: url, url: "https://raw.githubusercontent.com/galaxyproject/planemo/7be1bf5b3971a43eaa73f483125bfb8cabf1c440/tests/data/hello.txt", ext: "txt"} job_internal_valid: - parameter: {src: hda, id: 7} job_internal_invalid: @@ -533,6 +623,9 @@ gx_data: # expanded out. - parameter: {__class__: "Batch", values: [{src: hdca, id: 5}]} - parameter: {src: hda, id: abcdabcd} + # url parameters should be dereferrenced into datasets by this point... + - parameter: {src: url, url: "https://raw.githubusercontent.com/galaxyproject/planemo/7be1bf5b3971a43eaa73f483125bfb8cabf1c440/tests/data/hello.txt", "ext": "txt"} + - {} test_case_xml_valid: - parameter: {class: File, path: foo.bed} - parameter: {class: File, location: "https://raw.githubusercontent.com/galaxyproject/planemo/7be1bf5b3971a43eaa73f483125bfb8cabf1c440/tests/data/hello.txt"} @@ -584,6 +677,11 @@ gx_data_optional: - parameter: true - parameter: 5 - parameter: "5" + job_internal_valid: + - parameter: null + - parameter: {src: hda, id: 5} + job_internal_invalid: + - {} workflow_step_valid: - {} workflow_step_invalid: @@ -627,6 +725,8 @@ gx_data_multiple: - parameter: [{src: hda, id: 5}] - parameter: [{src: hdca, id: 5}] - parameter: [{src: hdca, id: 5}, {src: hda, id: 5}] + - parameter: [{src: url, url: "https://raw.githubusercontent.com/galaxyproject/planemo/7be1bf5b3971a43eaa73f483125bfb8cabf1c440/tests/data/hello.txt", ext: "txt"}] + - parameter: {__class__: "Batch", values: [{src: hdca, id: 5}]} request_internal_invalid: - parameter: {src: hda, id: abcdabcd} - parameter: [{src: hdca, id: abcdabcd}, {src: hda, id: abcdabcd}] @@ -636,6 +736,19 @@ gx_data_multiple: - parameter: true - parameter: 5 - parameter: "5" + request_internal_dereferenced_valid: + - parameter: {__class__: "Batch", values: [{src: hdca, id: 5}]} + - parameter: [{src: hda, id: 5}] + - parameter: [{src: hda, id: 0}] + request_internal_dereferenced_invalid: + # the difference between request internal and request internal dereferenced is that these have been converted + # to datasets. + - parameter: [{src: url, url: "https://raw.githubusercontent.com/galaxyproject/planemo/7be1bf5b3971a43eaa73f483125bfb8cabf1c440/tests/data/hello.txt", ext: "txt"}] + job_internal_valid: + - parameter: {src: hda, id: 5} + - parameter: [{src: hda, id: 5}, {src: hda, id: 6}] + job_internal_invalid: + - {} gx_data_multiple_optional: request_valid: @@ -646,6 +759,8 @@ gx_data_multiple_optional: - parameter: [{src: hdca, id: abcdabcd}, {src: hda, id: abcdabcd}] - parameter: null - {} + - parameter: {src: url, url: "https://raw.githubusercontent.com/galaxyproject/planemo/7be1bf5b3971a43eaa73f483125bfb8cabf1c440/tests/data/hello.txt", ext: "txt"} + - parameter: [{src: url, url: "https://raw.githubusercontent.com/galaxyproject/planemo/7be1bf5b3971a43eaa73f483125bfb8cabf1c440/tests/data/hello.txt", ext: "txt"}] request_invalid: - parameter: {src: hda, id: 5} - parameter: {} @@ -660,12 +775,25 @@ gx_data_multiple_optional: - parameter: [{src: hdca, id: 5}, {src: hda, id: 5}] - parameter: null - {} + - parameter: [{src: url, url: "https://raw.githubusercontent.com/galaxyproject/planemo/7be1bf5b3971a43eaa73f483125bfb8cabf1c440/tests/data/hello.txt", ext: "txt"}] request_internal_invalid: - parameter: {src: hda, id: abcdabcd} - parameter: {} - parameter: true - parameter: 5 - parameter: "5" + request_internal_dereferenced_valid: + - parameter: {src: hda, id: 5} + request_internal_dereferenced_invalid: + - parameter: {src: hda, id: abcdabcd} + - parameter: [{src: url, url: "https://raw.githubusercontent.com/galaxyproject/planemo/7be1bf5b3971a43eaa73f483125bfb8cabf1c440/tests/data/hello.txt", ext: "txt"}] + - parameter: {src: url, url: "https://raw.githubusercontent.com/galaxyproject/planemo/7be1bf5b3971a43eaa73f483125bfb8cabf1c440/tests/data/hello.txt", ext: "txt"} + job_internal_valid: + - parameter: {src: hda, id: 5} + - parameter: [{src: hda, id: 5}, {src: hda, id: 6}] + - parameter: null + job_internal_invalid: + - {} gx_data_collection: request_valid: @@ -692,6 +820,10 @@ gx_data_collection: - parameter: true - parameter: 5 - parameter: "5" + request_internal_dereferenced_valid: + - parameter: {src: hdca, id: 5} + request_internal_dereferenced_invalid: + - parameter: {src: hdca, id: abcdabcd} workflow_step_valid: - {} workflow_step_invalid: @@ -745,6 +877,12 @@ gx_data_collection: elements: - {identifier: "forward", path: "1_f.bed", class: File} - {identifier: "reverse", path: "1_r.bed", class: FileX} + job_internal_valid: + - parameter: {src: hdca, id: 5} + job_internal_invalid: + - parameter: {src: hdca, id: abcdabcd} + - parameter: null + - {} gx_data_collection_optional: request_valid: @@ -771,6 +909,12 @@ gx_data_collection_optional: - parameter: 5 - parameter: "5" - parameter: {} + job_internal_valid: + - parameter: {src: hdca, id: 5} + - parameter: null + job_internal_invalid: + - parameter: {src: hdca, id: abcdabcd} + - {} gx_conditional_boolean: request_valid: @@ -813,6 +957,22 @@ gx_conditional_boolean: # in that case having an integer_parameter is not acceptable. - conditional_parameter: integer_parameter: 5 + job_internal_valid: + - conditional_parameter: + test_parameter: true + integer_parameter: 1 + - conditional_parameter: + test_parameter: false + boolean_parameter: false + job_internal_invalid: + # missing defaults are fine in request, but job_internal records parameters used, + # must include defaults. + - {} + - conditional_parameter: {} + - conditional_parameter: + test_parameter: true + - conditional_parameter: + boolean_parameter: true gx_conditional_boolean_checked: request_valid: @@ -829,6 +989,10 @@ gx_conditional_boolean_checked: - conditional_parameter: boolean_parameter: false + job_internal_invalid: + - conditional_parameter: {} + - {} + gx_conditional_conditional_boolean: request_valid: - outer_conditional_parameter: @@ -846,6 +1010,12 @@ gx_conditional_conditional_boolean: boolean_parameter: true # Test parameter has default and so does it "case" - so this should be fine - {} + - outer_conditional_parameter: {} + - outer_conditional_parameter: + outer_test_parameter: true + inner_conditional_parameter: {} + - outer_conditional_parameter: + outer_test_parameter: true request_invalid: - outer_conditional_parameter: outer_test_parameter: true @@ -860,6 +1030,15 @@ gx_conditional_conditional_boolean: inner_conditional_parameter: inner_test_parameter: true integer_parameter: true + job_internal_invalid: + - {} + - outer_conditional_parameter: {} + - outer_conditional_parameter: {} + - outer_conditional_parameter: + outer_test_parameter: true + inner_conditional_parameter: {} + - outer_conditional_parameter: + outer_test_parameter: true gx_conditional_select: request_valid: @@ -874,15 +1053,9 @@ gx_conditional_select: boolean_parameter: true # Test parameter has default and so does it "case" - so this should be fine - {} - # # The boolean_parameter is optional so just setting a test_parameter is fine - # - conditional_parameter: - # test_parameter: b - # - conditional_parameter: - # test_parameter: a - # # if test parameter is missing, it should be 'a' in this case - # - conditional_parameter: - # integer_parameter: 4 - # - conditional_parameter: {} + # # The select is optional so just setting a test_parameter is fine + - conditional_parameter: + integer_parameter: 1 request_invalid: - conditional_parameter: test_parameter: b @@ -899,6 +1072,19 @@ gx_conditional_select: # in that case having an integer_parameter is not acceptable. - conditional_parameter: boolean_parameter: true + job_internal_valid: + - conditional_parameter: + test_parameter: a + integer_parameter: 1 + - conditional_parameter: + test_parameter: b + boolean_parameter: true + job_internal_invalid: + - {} + - conditional_parameter: + integer_parameter: 1 + - conditional_parameter: null + - conditional_parameter: {} gx_repeat_boolean: request_valid: @@ -920,6 +1106,17 @@ gx_repeat_boolean: - { boolean_parameter: false } - { boolean_parameter: 4 } - parameter: 5 + job_internal_valid: + - parameter: + - { boolean_parameter: true } + - parameter: [] + - parameter: + - { boolean_parameter: true } + - { boolean_parameter: false } + job_internal_invalid: + - parameter: [{}] + - parameter: [{}, {}] + gx_repeat_boolean_min: request_valid: @@ -945,6 +1142,13 @@ gx_repeat_boolean_min: - { boolean_parameter: false } - { boolean_parameter: 4 } - parameter: 5 + job_internal_valid: + - parameter: + - { boolean_parameter: true } + - { boolean_parameter: false } + job_internal_invalid: + - parameter: [{}, {}] + - parameter: [] gx_repeat_data: request_valid: @@ -956,6 +1160,8 @@ gx_repeat_data: - { data_parameter: {src: hda, id: abcdabcd} } # an empty repeat is fine - {} + - parameter: + - { data_parameter: {src: url, url: "https://raw.githubusercontent.com/galaxyproject/planemo/7be1bf5b3971a43eaa73f483125bfb8cabf1c440/tests/data/hello.txt", ext: "txt"} } request_invalid: - parameter: [{}, {}] - parameter: [{}] @@ -963,9 +1169,19 @@ gx_repeat_data: request_internal_valid: - parameter: - { data_parameter: { src: hda, id: 5 } } + - parameter: + - { data_parameter: {src: url, url: "https://raw.githubusercontent.com/galaxyproject/planemo/7be1bf5b3971a43eaa73f483125bfb8cabf1c440/tests/data/hello.txt", ext: "txt"} } request_internal_invalid: - parameter: - { data_parameter: { src: hda, id: abcdabcd } } + job_internal_valid: + - parameter: + - { data_parameter: { src: hda, id: 5 } } + job_internal_invalid: + - parameter: null + - parameter: {} + - parameter: + - { data_parameter: {src: url, url: "https://raw.githubusercontent.com/galaxyproject/planemo/7be1bf5b3971a43eaa73f483125bfb8cabf1c440/tests/data/hello.txt", ext: "txt"} } gx_repeat_data_min: request_valid: @@ -990,6 +1206,17 @@ gx_repeat_data_min: - { data_parameter: { src: hda, id: abcdabcd } } - parameter: - { data_parameter: { src: hda, id: 5 } } + job_internal_valid: + - parameter: + - { data_parameter: { src: hda, id: 5 } } + - { data_parameter: { src: hda, id: 6 } } + job_internal_invalid: + - parameter: + - { data_parameter: { src: hda, id: 5 } } + - parameter: + - {} + - parameter: {} + gx_section_boolean: request_valid: @@ -998,6 +1225,12 @@ gx_section_boolean: - {} request_invalid: - parameter: { boolean_parameter: 4 } + job_internal_valid: + - parameter: { boolean_parameter: true } + job_internal_invalid: + - {} + - parameter: { boolean_parameter: 4 } + gx_section_data: request_valid: @@ -1005,39 +1238,68 @@ gx_section_data: request_invalid: - parameter: { data_parameter: 4 } - parameter: { data_parameter: { src: hda, id: 5 } } - # data parameter is non-optional, so this should be invalid (unlike boolean parameter above) - # - {} + - {} request_internal_valid: - parameter: { data_parameter: { src: hda, id: 5 } } request_internal_invalid: - parameter: { data_parameter: { src: hda, id: abcdabcd } } + - {} + job_internal_valid: + - parameter: { data_parameter: { src: hda, id: 5 } } + job_internal_invalid: + - {} gx_drill_down_exact: request_valid: - parameter: aa - parameter: bbb - parameter: ba - request_invalid: - # not multiple so cannot choose a non-leaf + # non-leaf nodes seem to be selectable in exact mode - parameter: a + request_invalid: - parameter: c - parameter: {} # no implicit default currently - see test_drill_down_first_by_default in API test test_tools.py. - {} - parameter: null + job_internal_valid: + - parameter: aa + job_internal_invalid: + - parameter: c + - {} gx_drill_down_exact_with_selection: request_valid: - parameter: aa - parameter: bbb - parameter: ba - # - {} - request_invalid: - # not multiple so cannot choose a non-leaf + # non-leaf nodes seem to be selectable in exact mode - parameter: a + request_invalid: - parameter: c + # see note above no implicit selection - parameter: {} - parameter: null + job_internal_valid: + - parameter: aa + job_internal_invalid: + - parameter: c + - parameter: null + - {} + +gx_drill_down_recurse: + request_valid: + - parameter: bba + request_invalid: + - parameter: a + - parameter: c + +gx_drill_down_recurse_multiple: + request_valid: + - parameter: [bba] + - parameter: [a] + request_invalid: + - parameter: c gx_data_column: request_valid: @@ -1049,6 +1311,11 @@ gx_data_column: - { ref_parameter: {src: hda, id: 123}, parameter: 0 } request_internal_invalid: - { ref_parameter: {src: hda, id: 123}, parameter: "0" } + job_internal_valid: + - { ref_parameter: {src: hda, id: 123}, parameter: 0 } + job_internal_invalid: + - { ref_parameter: {src: hda, id: 123} } + - { } test_case_xml_valid: - { ref_parameter: {class: "File", path: "1.bed"}, parameter: 3 } test_case_xml_invalid: @@ -1064,6 +1331,13 @@ gx_data_column_optional: request_invalid: - { ref_parameter: {src: hda, id: abcdabcd}, parameter: "0" } - { ref_parameter: {src: hda, id: abcdabcd}, parameter: [ 0 ] } + job_internal_valid: + - { ref_parameter: {src: hda, id: 1}, parameter: 0 } + - { ref_parameter: {src: hda, id: 1}, parameter: null } + job_internal_invalid: + - { ref_parameter: {src: hda, id: 1} } + - { } + - { ref_parameter: {src: hda, id: 1}, parameter: "0" } request_internal_valid: - { ref_parameter: {src: hda, id: 123}, parameter: 0 } request_internal_invalid: @@ -1077,6 +1351,11 @@ gx_data_column_multiple: - { ref_parameter: {src: hda, id: abcdabcd}, parameter: ["0"] } request_internal_valid: - { ref_parameter: {src: hda, id: 123}, parameter: [0] } + job_internal_valid: + - { ref_parameter: {src: hda, id: 123}, parameter: [0] } + job_internal_invalid: + - { ref_parameter: {src: hda, id: 123}, parameter: ["0"] } + - { ref_parameter: {src: hda, id: 123}, parameter: null } request_internal_invalid: - { ref_parameter: {src: hda, id: 123}, parameter: "0" } - { ref_parameter: {src: hda, id: 123}, parameter: 0 } @@ -1109,6 +1388,11 @@ gx_group_tag_optional: - { ref_parameter: {src: hdca, id: 123}, parameter: null } request_internal_invalid: - { ref_parameter: {src: hdca, id: 123}, parameter: 8 } + job_internal_valid: + - { ref_parameter: {src: hdca, id: 123}, parameter: null } + - { ref_parameter: { src: hdca, id: 123}, parameter: "matched" } + job_internal_invalid: + - { ref_parameter: {src: hdca, id: 123} } gx_group_tag_multiple: @@ -1127,6 +1411,12 @@ gx_group_tag_multiple: request_internal_invalid: - { ref_parameter: {src: hdca, id: 123}, parameter: 8 } - { ref_parameter: {src: hdca, id: 123}, parameter: null } + job_internal_valid: + - { ref_parameter: { src: hdca, id: 123}, parameter: ['matched'] } + job_internal_invalid: + - { ref_parameter: { src: hdca, id: 123} } + - { ref_parameter: { src: hdca, id: 123}, parameter: null } + cwl_int: diff --git a/test/unit/tool_util/test_parameter_convert.py b/test/unit/tool_util/test_parameter_convert.py new file mode 100644 index 000000000000..f86750026766 --- /dev/null +++ b/test/unit/tool_util/test_parameter_convert.py @@ -0,0 +1,221 @@ +from typing import ( + Any, + Dict, + Optional, +) + +from galaxy.tool_util.parameters import ( + DataRequestInternalHda, + DataRequestUri, + decode, + dereference, + encode, + fill_static_defaults, + input_models_for_tool_source, + RequestInternalDereferencedToolState, + RequestInternalToolState, + RequestToolState, +) +from galaxy.tool_util.parser.util import parse_profile_version +from .test_parameter_test_cases import tool_source_for + +EXAMPLE_ID_1_ENCODED = "123456789abcde" +EXAMPLE_ID_1 = 13 +EXAMPLE_ID_2_ENCODED = "123456789abcd2" +EXAMPLE_ID_2 = 14 + +ID_MAP: Dict[int, str] = { + EXAMPLE_ID_1: EXAMPLE_ID_1_ENCODED, + EXAMPLE_ID_2: EXAMPLE_ID_2_ENCODED, +} + + +def test_encode_data(): + tool_source = tool_source_for("parameters/gx_data") + bundle = input_models_for_tool_source(tool_source) + request_state = RequestToolState({"parameter": {"src": "hda", "id": EXAMPLE_ID_1_ENCODED}}) + request_state.validate(bundle) + decoded_state = decode(request_state, bundle, _fake_decode) + assert decoded_state.input_state["parameter"]["src"] == "hda" + assert decoded_state.input_state["parameter"]["id"] == EXAMPLE_ID_1 + + +def test_encode_collection(): + tool_source = tool_source_for("parameters/gx_data_collection") + bundle = input_models_for_tool_source(tool_source) + request_state = RequestToolState({"parameter": {"src": "hdca", "id": EXAMPLE_ID_1_ENCODED}}) + request_state.validate(bundle) + decoded_state = decode(request_state, bundle, _fake_decode) + assert decoded_state.input_state["parameter"]["src"] == "hdca" + assert decoded_state.input_state["parameter"]["id"] == EXAMPLE_ID_1 + + +def test_encode_repeat(): + tool_source = tool_source_for("parameters/gx_repeat_data") + bundle = input_models_for_tool_source(tool_source) + request_state = RequestToolState({"parameter": [{"data_parameter": {"src": "hda", "id": EXAMPLE_ID_1_ENCODED}}]}) + request_state.validate(bundle) + decoded_state = decode(request_state, bundle, _fake_decode) + assert decoded_state.input_state["parameter"][0]["data_parameter"]["src"] == "hda" + assert decoded_state.input_state["parameter"][0]["data_parameter"]["id"] == EXAMPLE_ID_1 + + +def test_encode_section(): + tool_source = tool_source_for("parameters/gx_section_data") + bundle = input_models_for_tool_source(tool_source) + request_state = RequestToolState({"parameter": {"data_parameter": {"src": "hda", "id": EXAMPLE_ID_1_ENCODED}}}) + request_state.validate(bundle) + decoded_state = decode(request_state, bundle, _fake_decode) + assert decoded_state.input_state["parameter"]["data_parameter"]["src"] == "hda" + assert decoded_state.input_state["parameter"]["data_parameter"]["id"] == EXAMPLE_ID_1 + + +def test_encode_conditional(): + tool_source = tool_source_for("identifier_in_conditional") + bundle = input_models_for_tool_source(tool_source) + request_state = RequestToolState( + {"outer_cond": {"multi_input": False, "input1": {"src": "hda", "id": EXAMPLE_ID_1_ENCODED}}} + ) + request_state.validate(bundle) + decoded_state = decode(request_state, bundle, _fake_decode) + assert decoded_state.input_state["outer_cond"]["input1"]["src"] == "hda" + assert decoded_state.input_state["outer_cond"]["input1"]["id"] == EXAMPLE_ID_1 + + +def test_multi_data(): + tool_source = tool_source_for("parameters/gx_data_multiple") + bundle = input_models_for_tool_source(tool_source) + request_state = RequestToolState( + {"parameter": [{"src": "hda", "id": EXAMPLE_ID_1_ENCODED}, {"src": "hda", "id": EXAMPLE_ID_2_ENCODED}]} + ) + request_state.validate(bundle) + decoded_state = decode(request_state, bundle, _fake_decode) + assert decoded_state.input_state["parameter"][0]["src"] == "hda" + assert decoded_state.input_state["parameter"][0]["id"] == EXAMPLE_ID_1 + assert decoded_state.input_state["parameter"][1]["src"] == "hda" + assert decoded_state.input_state["parameter"][1]["id"] == EXAMPLE_ID_2 + + encoded_state = encode(decoded_state, bundle, _fake_encode) + assert encoded_state.input_state["parameter"][0]["src"] == "hda" + assert encoded_state.input_state["parameter"][0]["id"] == EXAMPLE_ID_1_ENCODED + assert encoded_state.input_state["parameter"][1]["src"] == "hda" + assert encoded_state.input_state["parameter"][1]["id"] == EXAMPLE_ID_2_ENCODED + + +def test_dereference(): + tool_source = tool_source_for("parameters/gx_data") + bundle = input_models_for_tool_source(tool_source) + raw_request_state = {"parameter": {"src": "url", "url": "gxfiles://mystorage/1.bed", "ext": "bed"}} + request_state = RequestInternalToolState(raw_request_state) + request_state.validate(bundle) + + exception: Optional[Exception] = None + try: + # quickly verify this request needs to be dereferenced + bad_state = RequestInternalDereferencedToolState(raw_request_state) + bad_state.validate(bundle) + except Exception as e: + exception = e + assert exception is not None + + dereferenced_state = dereference(request_state, bundle, _fake_dereference) + assert isinstance(dereferenced_state, RequestInternalDereferencedToolState) + dereferenced_state.validate(bundle) + + +def test_fill_defaults(): + with_defaults = fill_state_for({}, "parameters/gx_int") + assert with_defaults["parameter"] == 1 + with_defaults = fill_state_for({}, "parameters/gx_float") + assert with_defaults["parameter"] == 1.0 + with_defaults = fill_state_for({}, "parameters/gx_boolean") + assert with_defaults["parameter"] is False + with_defaults = fill_state_for({}, "parameters/gx_boolean_optional") + # This is False unfortunately - see comments in gx_boolean_optional XML. + assert with_defaults["parameter"] is False + with_defaults = fill_state_for({}, "parameters/gx_boolean_checked") + assert with_defaults["parameter"] is True + with_defaults = fill_state_for({}, "parameters/gx_boolean_optional_checked") + assert with_defaults["parameter"] is True + + with_defaults = fill_state_for({}, "parameters/gx_conditional_boolean") + assert with_defaults["conditional_parameter"]["test_parameter"] is False + assert with_defaults["conditional_parameter"]["boolean_parameter"] is False + + with_defaults = fill_state_for({"conditional_parameter": {}}, "parameters/gx_conditional_boolean") + assert with_defaults["conditional_parameter"]["test_parameter"] is False + assert with_defaults["conditional_parameter"]["boolean_parameter"] is False + + with_defaults = fill_state_for({}, "parameters/gx_repeat_boolean") + assert len(with_defaults["parameter"]) == 0 + with_defaults = fill_state_for({"parameter": [{}]}, "parameters/gx_repeat_boolean") + assert len(with_defaults["parameter"]) == 1 + instance_state = with_defaults["parameter"][0] + assert instance_state["boolean_parameter"] is False + + with_defaults = fill_state_for({}, "parameters/gx_repeat_boolean_min") + assert len(with_defaults["parameter"]) == 2 + assert with_defaults["parameter"][0]["boolean_parameter"] is False + assert with_defaults["parameter"][1]["boolean_parameter"] is False + with_defaults = fill_state_for({"parameter": [{}, {}]}, "parameters/gx_repeat_boolean_min") + assert len(with_defaults["parameter"]) == 2 + assert with_defaults["parameter"][0]["boolean_parameter"] is False + assert with_defaults["parameter"][1]["boolean_parameter"] is False + with_defaults = fill_state_for({"parameter": [{}]}, "parameters/gx_repeat_boolean_min") + assert with_defaults["parameter"][0]["boolean_parameter"] is False + assert with_defaults["parameter"][1]["boolean_parameter"] is False + + with_defaults = fill_state_for({}, "parameters/gx_section_boolean") + assert with_defaults["parameter"]["boolean_parameter"] is False + + with_defaults = fill_state_for({}, "parameters/gx_drill_down_exact_with_selection") + assert with_defaults["parameter"] == "aba" + + with_defaults = fill_state_for({}, "parameters/gx_hidden") + assert with_defaults["parameter"] == "moo" + + with_defaults = fill_state_for({}, "parameters/gx_genomebuild_optional") + assert with_defaults["parameter"] is None + + with_defaults = fill_state_for({}, "parameters/gx_select") + assert with_defaults["parameter"] == "--ex1" + + with_defaults = fill_state_for({}, "parameters/gx_select_optional") + assert with_defaults["parameter"] is None + + # Not ideal but matching current behavior + with_defaults = fill_state_for({}, "parameters/gx_select_multiple") + assert with_defaults["parameter"] is None + + with_defaults = fill_state_for({}, "parameters/gx_select_multiple_optional") + assert with_defaults["parameter"] is None + + # Do not fill in dynamic defaults... these require a Galaxy runtime. + with_defaults = fill_state_for({}, "remove_value", partial=True) + assert "choose_value" not in with_defaults + + with_defaults = fill_state_for( + {"single": {"src": "hda", "id": 4}}, "select_from_dataset_in_conditional", partial=True + ) + assert with_defaults["cond"]["cond"] == "single" + assert with_defaults["cond"]["inner_cond"]["inner_cond"] == "single" + + +def _fake_dereference(input: DataRequestUri) -> DataRequestInternalHda: + return DataRequestInternalHda(id=EXAMPLE_ID_1) + + +def _fake_decode(input: str) -> int: + return next(key for key, value in ID_MAP.items() if value == input) + + +def _fake_encode(input: int) -> str: + return ID_MAP[input] + + +def fill_state_for(tool_state: Dict[str, Any], tool_path: str, partial: bool = False) -> Dict[str, Any]: + tool_source = tool_source_for(tool_path) + bundle = input_models_for_tool_source(tool_source) + profile = parse_profile_version(tool_source) + internal_state = fill_static_defaults(tool_state, bundle, profile, partial=partial) + return internal_state diff --git a/test/unit/tool_util/test_parameter_specification.py b/test/unit/tool_util/test_parameter_specification.py index 6a54d78cfabf..dd54ac7276be 100644 --- a/test/unit/tool_util/test_parameter_specification.py +++ b/test/unit/tool_util/test_parameter_specification.py @@ -1,3 +1,4 @@ +import sys from functools import partial from typing import ( Any, @@ -7,6 +8,7 @@ Optional, ) +import pytest import yaml from galaxy.exceptions import RequestParameterInvalidException @@ -18,6 +20,7 @@ ToolParameterBundleModel, validate_internal_job, validate_internal_request, + validate_internal_request_dereferenced, validate_request, validate_test_case, validate_workflow_step, @@ -33,6 +36,10 @@ RawStateDict = Dict[str, Any] +if sys.version_info < (3, 8): # noqa: UP036 + pytest.skip(reason="Pydantic tool parameter models require python3.8 or higher", allow_module_level=True) + + def specification_object(): try: yaml_str = resource_string(__package__, "parameter_specification.yml") @@ -91,6 +98,8 @@ def _test_file(file: str, specification=None, parameter_bundle: Optional[ToolPar "request_invalid": _assert_requests_invalid, "request_internal_valid": _assert_internal_requests_validate, "request_internal_invalid": _assert_internal_requests_invalid, + "request_internal_dereferenced_valid": _assert_internal_requests_dereferenced_validate, + "request_internal_dereferenced_invalid": _assert_internal_requests_dereferenced_invalid, "job_internal_valid": _assert_internal_jobs_validate, "job_internal_invalid": _assert_internal_jobs_invalid, "test_case_xml_valid": _assert_test_cases_validate, @@ -153,6 +162,26 @@ def _assert_internal_request_invalid(parameters: ToolParameterBundleModel, reque ), f"Parameters {parameters} didn't result in validation error on internal request {request} as expected." +def _assert_internal_request_dereferenced_validates( + parameters: ToolParameterBundleModel, request: RawStateDict +) -> None: + try: + validate_internal_request_dereferenced(parameters, request) + except RequestParameterInvalidException as e: + raise AssertionError(f"Parameters {parameters} failed to validate dereferenced internal request {request}. {e}") + + +def _assert_internal_request_dereferenced_invalid(parameters: ToolParameterBundleModel, request: RawStateDict) -> None: + exc = None + try: + validate_internal_request_dereferenced(parameters, request) + except RequestParameterInvalidException as e: + exc = e + assert ( + exc is not None + ), f"Parameters {parameters} didn't result in validation error on dereferenced internal request {request} as expected." + + def _assert_internal_job_validates(parameters: ToolParameterBundleModel, request: RawStateDict) -> None: try: validate_internal_job(parameters, request) @@ -235,6 +264,8 @@ def _assert_workflow_step_linked_invalid( _assert_requests_invalid = partial(_for_each, _assert_request_invalid) _assert_internal_requests_validate = partial(_for_each, _assert_internal_request_validates) _assert_internal_requests_invalid = partial(_for_each, _assert_internal_request_invalid) +_assert_internal_requests_dereferenced_validate = partial(_for_each, _assert_internal_request_dereferenced_validates) +_assert_internal_requests_dereferenced_invalid = partial(_for_each, _assert_internal_request_dereferenced_invalid) _assert_internal_jobs_validate = partial(_for_each, _assert_internal_job_validates) _assert_internal_jobs_invalid = partial(_for_each, _assert_internal_job_invalid) _assert_test_cases_validate = partial(_for_each, _assert_test_case_validates) diff --git a/test/unit/tool_util/test_parameter_test_cases.py b/test/unit/tool_util/test_parameter_test_cases.py index 909c0bfd2da5..3ff5c3a0fdbf 100644 --- a/test/unit/tool_util/test_parameter_test_cases.py +++ b/test/unit/tool_util/test_parameter_test_cases.py @@ -1,11 +1,22 @@ import os import re import sys -from typing import List +from typing import ( + Any, + List, + Optional, + Tuple, +) import pytest from galaxy.tool_util.models import parse_tool +from galaxy.tool_util.parameters import ( + DataCollectionRequest, + DataRequestHda, + encode_test, + input_models_for_tool_source, +) from galaxy.tool_util.parameters.case import ( test_case_state as case_state, TestCaseStateAndWarnings, @@ -14,6 +25,8 @@ ) from galaxy.tool_util.parser.factory import get_tool_source from galaxy.tool_util.parser.interface import ( + JsonTestCollectionDefDict, + JsonTestDatasetDefDict, ToolSource, ToolSourceTest, ) @@ -59,6 +72,8 @@ ] ) +MOCK_ID = "thisisafakeid" + if sys.version_info < (3, 8): # noqa: UP036 pytest.skip(reason="Pydantic tool parameter models require python3.8 or higher", allow_module_level=True) @@ -76,7 +91,7 @@ def test_parameter_test_cases_validate(): def test_legacy_features_fail_validation_with_24_2(tmp_path): for filename in TOOLS_THAT_USE_UNQUALIFIED_PARAMETER_ACCESS + TOOLS_THAT_USE_TRUE_FALSE_VALUE_BOOLEAN_SPECIFICATION: - _assert_tool_test_parsing_only_fails_with_newer_profile(tmp_path, filename) + _assert_tool_test_parsing_only_fails_with_newer_profile(tmp_path, filename, index=None) # column parameters need to be indexes _assert_tool_test_parsing_only_fails_with_newer_profile(tmp_path, "column_param.xml", index=2) @@ -85,7 +100,7 @@ def test_legacy_features_fail_validation_with_24_2(tmp_path): _assert_tool_test_parsing_only_fails_with_newer_profile(tmp_path, "multi_select.xml", index=1) -def _assert_tool_test_parsing_only_fails_with_newer_profile(tmp_path, filename: str, index: int = 0): +def _assert_tool_test_parsing_only_fails_with_newer_profile(tmp_path, filename: str, index: Optional[int] = 0): test_tool_directory = functional_test_tool_directory() original_path = os.path.join(test_tool_directory, filename) new_path = tmp_path / filename @@ -96,11 +111,19 @@ def _assert_tool_test_parsing_only_fails_with_newer_profile(tmp_path, filename: with open(new_path, "w") as wf: wf.write(new_profile_contents) test_cases = list(parse_tool_test_descriptions(get_tool_source(original_path))) - assert test_cases[index].to_dict()["error"] is False + if index is not None: + assert test_cases[index].to_dict()["error"] is False + else: + # just make sure there is at least one failure... + assert not any(c.to_dict()["error"] is True for c in test_cases) + test_cases = list(parse_tool_test_descriptions(get_tool_source(new_path))) - assert ( - test_cases[index].to_dict()["error"] is True - ), f"expected {filename} to have validation failure preventing loading of tools" + if index is not None: + assert ( + test_cases[index].to_dict()["error"] is True + ), f"expected {filename} to have validation failure preventing loading of tools" + else: + assert any(c.to_dict()["error"] is True for c in test_cases) def test_validate_framework_test_tools(): @@ -125,6 +148,7 @@ def test_test_case_state_conversion(): tool_source = tool_source_for("collection_nested_test") test_cases: List[ToolSourceTest] = tool_source.parse_tests_to_dict()["tests"] state = case_state_for(tool_source, test_cases[0]) + expectations: List[Tuple[List[Any], Optional[Any]]] expectations = [ (["f1", "collection_type"], "list:paired"), (["f1", "class"], "Collection"), @@ -187,6 +211,77 @@ def test_test_case_state_conversion(): ] dict_verify_each(state.tool_state.input_state, expectations) + index = 2 + tool_source = tool_source_for("filter_param_value_ref_attribute") + test_cases = tool_source.parse_tests_to_dict()["tests"] + state = case_state_for(tool_source, test_cases[index]) + expectations = [ + (["data_mult", 0, "path"], "1.bed"), + (["data_mult", 0, "dbkey"], "hg19"), + (["data_mult", 1, "path"], "2.bed"), + (["data_mult", 0, "dbkey"], "hg19"), + ] + dict_verify_each(state.tool_state.input_state, expectations) + + index = 1 + tool_source = tool_source_for("expression_pick_larger_file") + test_cases = tool_source.parse_tests_to_dict()["tests"] + state = case_state_for(tool_source, test_cases[index]) + expectations = [ + (["input1", "path"], "simple_line_alternative.txt"), + (["input2"], None), + ] + dict_verify_each(state.tool_state.input_state, expectations) + + index = 2 + state = case_state_for(tool_source, test_cases[index]) + expectations = [ + (["input1"], None), + (["input2", "path"], "simple_line.txt"), + ] + dict_verify_each(state.tool_state.input_state, expectations) + + index = 0 + tool_source = tool_source_for("composite_shapefile") + test_cases = tool_source.parse_tests_to_dict()["tests"] + state = case_state_for(tool_source, test_cases[index]) + expectations = [ + (["input", "filetype"], "shp"), + (["input", "composite_data", 0], "shapefile/shapefile.shp"), + ] + dict_verify_each(state.tool_state.input_state, expectations) + + +def test_convert_to_requests(): + tools = [ + "parameters/gx_drill_down_recurse_multiple", + "parameters/gx_conditional_select", + "expression_pick_larger_file", + "identifier_in_conditional", + "column_param_list", + "composite_shapefile", + ] + for tool_path in tools: + tool_source = tool_source_for(tool_path) + parameters = input_models_for_tool_source(tool_source) + parsed_tool = parse_tool(tool_source) + profile = tool_source.parse_profile() + test_cases: List[ToolSourceTest] = tool_source.parse_tests_to_dict()["tests"] + + def mock_adapt_datasets(input: JsonTestDatasetDefDict) -> DataRequestHda: + return DataRequestHda(src="hda", id=MOCK_ID) + + def mock_adapt_collections(input: JsonTestCollectionDefDict) -> DataCollectionRequest: + return DataCollectionRequest(src="hdca", id=MOCK_ID) + + for test_case in test_cases: + if test_case.get("expect_failure"): + continue + test_case_state_and_warnings = case_state(test_case, parsed_tool.inputs, profile) + test_case_state = test_case_state_and_warnings.tool_state + + encode_test(test_case_state, parameters, mock_adapt_datasets, mock_adapt_collections) + def _validate_path(tool_path: str): tool_source = get_tool_source(tool_path) diff --git a/test/unit/tool_util/util.py b/test/unit/tool_util/util.py index b03aff0fc15f..27f0ee677fbd 100644 --- a/test/unit/tool_util/util.py +++ b/test/unit/tool_util/util.py @@ -19,7 +19,7 @@ def dict_verify_each(target_dict: dict, expectations: List[Any]): dict_verify(target_dict, path, expectation) -def dict_verify(target_dict: dict, expectation_path: List[str], expectation: Any): +def dict_verify(target_dict: dict, expectation_path: List[Any], expectation: Any): rest = target_dict for path_part in expectation_path: rest = rest[path_part] diff --git a/tools/filters/sff_extract.py b/tools/filters/sff_extract.py index 9c67a644c8ff..d7bcf05bf196 100644 --- a/tools/filters/sff_extract.py +++ b/tools/filters/sff_extract.py @@ -42,9 +42,9 @@ fake_sff_name = "fake_sff_name" # readname as key: lines with matches from SSAHA, one best match -ssahapematches = {} # type: ignore +ssahapematches = {} # type: ignore[var-annotated] # linker readname as key: length of linker sequence -linkerlengths = {} # type: ignore +linkerlengths = {} # type: ignore[var-annotated] # set to true if something really fishy is going on with the sequences stern_warning = False