From 59d3bd054e97dde80af8ccb6f6598ce365c15ba9 Mon Sep 17 00:00:00 2001 From: camlyall Date: Mon, 11 Sep 2023 13:00:48 +0000 Subject: [PATCH] Add option to exclude hidden files from the transfer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jesús García Crespo --- Makefile | 1 + enduro.toml | 2 + internal/api/design/batch.go | 1 + internal/api/gen/batch/service.go | 1 + internal/api/gen/http/batch/client/cli.go | 9 +- internal/api/gen/http/batch/client/types.go | 8 + internal/api/gen/http/batch/server/types.go | 7 + internal/api/gen/http/cli/enduro/cli.go | 2 + internal/api/gen/http/openapi.json | 6 + internal/api/gen/http/openapi.yaml | 5 + internal/api/gen/http/openapi3.json | 7 + internal/api/gen/http/openapi3.yaml | 6 + internal/batch/service.go | 3 +- internal/batch/workflow.go | 40 +- internal/collection/workflow.go | 3 + internal/watcher/config.go | 24 +- internal/watcher/event.go | 22 +- internal/watcher/fake/mock_watcher_unit.go | 458 ++++++++++++++++++ internal/watcher/filesystem.go | 15 +- internal/watcher/minio.go | 13 +- internal/watcher/watcher.go | 20 +- internal/workflow/activities/bundle.go | 34 +- internal/workflow/activities/bundle_test.go | 80 +++ internal/workflow/processing.go | 45 +- main.go | 23 +- .../models/SubmitRequestBody.ts | 8 + ui/src/views/Batch.vue | 12 +- .../en/docs/user-manual/configuration.md | 18 + 28 files changed, 769 insertions(+), 104 deletions(-) create mode 100644 internal/watcher/fake/mock_watcher_unit.go diff --git a/Makefile b/Makefile index 613c6194..5091ce34 100644 --- a/Makefile +++ b/Makefile @@ -154,6 +154,7 @@ gen-mock: $(MOCKGEN) mockgen -typed -destination=./internal/collection/fake/mock_collection.go -package=fake github.com/artefactual-labs/enduro/internal/collection Service mockgen -typed -destination=./internal/pipeline/fake/mock_pipeline.go -package=fake github.com/artefactual-labs/enduro/internal/pipeline Service mockgen -typed -destination=./internal/watcher/fake/mock_watcher.go -package=fake github.com/artefactual-labs/enduro/internal/watcher Service + mockgen -typed -destination=./internal/watcher/fake/mock_watcher_unit.go -package=fake github.com/artefactual-labs/enduro/internal/watcher Watcher temporal: # @HELP Runs a development instance of Temporal. temporal: PORT := 55555 diff --git a/enduro.toml b/enduro.toml index c3037a68..27d5bf66 100644 --- a/enduro.toml +++ b/enduro.toml @@ -35,6 +35,7 @@ pipeline = "am" retentionPeriod = "10s" stripTopLevelDir = true rejectDuplicates = false +excludeHiddenFiles = false transferType = "standard" [[watcher.filesystem]] @@ -46,6 +47,7 @@ completedDir = "./hack/landfill" ignore = '(^\.gitkeep)|(^*\.mft)$' stripTopLevelDir = true rejectDuplicates = false +excludeHiddenFiles = false transferType = "standard" [[pipeline]] diff --git a/internal/api/design/batch.go b/internal/api/design/batch.go index 06c8f62d..304d6d94 100644 --- a/internal/api/design/batch.go +++ b/internal/api/design/batch.go @@ -18,6 +18,7 @@ var _ = Service("batch", func() { Attribute("completed_dir", String) Attribute("retention_period", String) Attribute("reject_duplicates", Boolean, func() { Default(false) }) + Attribute("exclude_hidden_files", Boolean, func() { Default(false) }) Attribute("transfer_type", String) Attribute("process_name_metadata", Boolean, func() { Default(false) }) Attribute("depth", Int, func() { diff --git a/internal/api/gen/batch/service.go b/internal/api/gen/batch/service.go index 0936926f..11f0b16d 100644 --- a/internal/api/gen/batch/service.go +++ b/internal/api/gen/batch/service.go @@ -62,6 +62,7 @@ type SubmitPayload struct { CompletedDir *string RetentionPeriod *string RejectDuplicates bool + ExcludeHiddenFiles bool TransferType *string ProcessNameMetadata bool Depth int diff --git a/internal/api/gen/http/batch/client/cli.go b/internal/api/gen/http/batch/client/cli.go index b671a632..b46d346a 100644 --- a/internal/api/gen/http/batch/client/cli.go +++ b/internal/api/gen/http/batch/client/cli.go @@ -24,7 +24,7 @@ func BuildSubmitPayload(batchSubmitBody string) (*batch.SubmitPayload, error) { { err = json.Unmarshal([]byte(batchSubmitBody), &body) if err != nil { - return nil, fmt.Errorf("invalid JSON for body, \nerror: %s, \nexample of valid JSON:\n%s", err, "'{\n \"completed_dir\": \"abc123\",\n \"depth\": 1,\n \"path\": \"abc123\",\n \"pipeline\": \"abc123\",\n \"process_name_metadata\": false,\n \"processing_config\": \"abc123\",\n \"reject_duplicates\": false,\n \"retention_period\": \"abc123\",\n \"transfer_type\": \"abc123\"\n }'") + return nil, fmt.Errorf("invalid JSON for body, \nerror: %s, \nexample of valid JSON:\n%s", err, "'{\n \"completed_dir\": \"abc123\",\n \"depth\": 1,\n \"exclude_hidden_files\": false,\n \"path\": \"abc123\",\n \"pipeline\": \"abc123\",\n \"process_name_metadata\": false,\n \"processing_config\": \"abc123\",\n \"reject_duplicates\": false,\n \"retention_period\": \"abc123\",\n \"transfer_type\": \"abc123\"\n }'") } if body.Depth < 0 { err = goa.MergeErrors(err, goa.InvalidRangeError("body.depth", body.Depth, 0, true)) @@ -40,6 +40,7 @@ func BuildSubmitPayload(batchSubmitBody string) (*batch.SubmitPayload, error) { CompletedDir: body.CompletedDir, RetentionPeriod: body.RetentionPeriod, RejectDuplicates: body.RejectDuplicates, + ExcludeHiddenFiles: body.ExcludeHiddenFiles, TransferType: body.TransferType, ProcessNameMetadata: body.ProcessNameMetadata, Depth: body.Depth, @@ -50,6 +51,12 @@ func BuildSubmitPayload(batchSubmitBody string) (*batch.SubmitPayload, error) { v.RejectDuplicates = false } } + { + var zero bool + if v.ExcludeHiddenFiles == zero { + v.ExcludeHiddenFiles = false + } + } { var zero bool if v.ProcessNameMetadata == zero { diff --git a/internal/api/gen/http/batch/client/types.go b/internal/api/gen/http/batch/client/types.go index d7becbaf..338a1a08 100644 --- a/internal/api/gen/http/batch/client/types.go +++ b/internal/api/gen/http/batch/client/types.go @@ -22,6 +22,7 @@ type SubmitRequestBody struct { CompletedDir *string `form:"completed_dir,omitempty" json:"completed_dir,omitempty" xml:"completed_dir,omitempty"` RetentionPeriod *string `form:"retention_period,omitempty" json:"retention_period,omitempty" xml:"retention_period,omitempty"` RejectDuplicates bool `form:"reject_duplicates" json:"reject_duplicates" xml:"reject_duplicates"` + ExcludeHiddenFiles bool `form:"exclude_hidden_files" json:"exclude_hidden_files" xml:"exclude_hidden_files"` TransferType *string `form:"transfer_type,omitempty" json:"transfer_type,omitempty" xml:"transfer_type,omitempty"` ProcessNameMetadata bool `form:"process_name_metadata" json:"process_name_metadata" xml:"process_name_metadata"` Depth int `form:"depth" json:"depth" xml:"depth"` @@ -96,6 +97,7 @@ func NewSubmitRequestBody(p *batch.SubmitPayload) *SubmitRequestBody { CompletedDir: p.CompletedDir, RetentionPeriod: p.RetentionPeriod, RejectDuplicates: p.RejectDuplicates, + ExcludeHiddenFiles: p.ExcludeHiddenFiles, TransferType: p.TransferType, ProcessNameMetadata: p.ProcessNameMetadata, Depth: p.Depth, @@ -106,6 +108,12 @@ func NewSubmitRequestBody(p *batch.SubmitPayload) *SubmitRequestBody { body.RejectDuplicates = false } } + { + var zero bool + if body.ExcludeHiddenFiles == zero { + body.ExcludeHiddenFiles = false + } + } { var zero bool if body.ProcessNameMetadata == zero { diff --git a/internal/api/gen/http/batch/server/types.go b/internal/api/gen/http/batch/server/types.go index 81842c1f..a132cf54 100644 --- a/internal/api/gen/http/batch/server/types.go +++ b/internal/api/gen/http/batch/server/types.go @@ -22,6 +22,7 @@ type SubmitRequestBody struct { CompletedDir *string `form:"completed_dir,omitempty" json:"completed_dir,omitempty" xml:"completed_dir,omitempty"` RetentionPeriod *string `form:"retention_period,omitempty" json:"retention_period,omitempty" xml:"retention_period,omitempty"` RejectDuplicates *bool `form:"reject_duplicates,omitempty" json:"reject_duplicates,omitempty" xml:"reject_duplicates,omitempty"` + ExcludeHiddenFiles *bool `form:"exclude_hidden_files,omitempty" json:"exclude_hidden_files,omitempty" xml:"exclude_hidden_files,omitempty"` TransferType *string `form:"transfer_type,omitempty" json:"transfer_type,omitempty" xml:"transfer_type,omitempty"` ProcessNameMetadata *bool `form:"process_name_metadata,omitempty" json:"process_name_metadata,omitempty" xml:"process_name_metadata,omitempty"` Depth *int `form:"depth,omitempty" json:"depth,omitempty" xml:"depth,omitempty"` @@ -162,6 +163,9 @@ func NewSubmitPayload(body *SubmitRequestBody) *batch.SubmitPayload { if body.RejectDuplicates != nil { v.RejectDuplicates = *body.RejectDuplicates } + if body.ExcludeHiddenFiles != nil { + v.ExcludeHiddenFiles = *body.ExcludeHiddenFiles + } if body.ProcessNameMetadata != nil { v.ProcessNameMetadata = *body.ProcessNameMetadata } @@ -171,6 +175,9 @@ func NewSubmitPayload(body *SubmitRequestBody) *batch.SubmitPayload { if body.RejectDuplicates == nil { v.RejectDuplicates = false } + if body.ExcludeHiddenFiles == nil { + v.ExcludeHiddenFiles = false + } if body.ProcessNameMetadata == nil { v.ProcessNameMetadata = false } diff --git a/internal/api/gen/http/cli/enduro/cli.go b/internal/api/gen/http/cli/enduro/cli.go index 0d82b391..18590511 100644 --- a/internal/api/gen/http/cli/enduro/cli.go +++ b/internal/api/gen/http/cli/enduro/cli.go @@ -37,6 +37,7 @@ func UsageExamples() string { os.Args[0] + ` batch submit --body '{ "completed_dir": "abc123", "depth": 1, + "exclude_hidden_files": false, "path": "abc123", "pipeline": "abc123", "process_name_metadata": false, @@ -412,6 +413,7 @@ Example: %[1]s batch submit --body '{ "completed_dir": "abc123", "depth": 1, + "exclude_hidden_files": false, "path": "abc123", "pipeline": "abc123", "process_name_metadata": false, diff --git a/internal/api/gen/http/openapi.json b/internal/api/gen/http/openapi.json index 90df0b66..fa175be9 100644 --- a/internal/api/gen/http/openapi.json +++ b/internal/api/gen/http/openapi.json @@ -168,6 +168,7 @@ "example": { "completed_dir": "abc123", "depth": 1, + "exclude_hidden_files": false, "path": "abc123", "pipeline": "abc123", "process_name_metadata": false, @@ -187,6 +188,11 @@ "minimum": 0, "type": "integer" }, + "exclude_hidden_files": { + "default": false, + "example": false, + "type": "boolean" + }, "path": { "example": "abc123", "type": "string" diff --git a/internal/api/gen/http/openapi.yaml b/internal/api/gen/http/openapi.yaml index 2dbda671..6b2cb3f8 100644 --- a/internal/api/gen/http/openapi.yaml +++ b/internal/api/gen/http/openapi.yaml @@ -650,6 +650,10 @@ definitions: default: 0 example: 1 minimum: 0 + exclude_hidden_files: + type: boolean + default: false + example: false path: type: string example: abc123 @@ -676,6 +680,7 @@ definitions: example: completed_dir: abc123 depth: 1 + exclude_hidden_files: false path: abc123 pipeline: abc123 process_name_metadata: false diff --git a/internal/api/gen/http/openapi3.json b/internal/api/gen/http/openapi3.json index 4d582181..6cb043be 100644 --- a/internal/api/gen/http/openapi3.json +++ b/internal/api/gen/http/openapi3.json @@ -565,6 +565,7 @@ "example": { "completed_dir": "abc123", "depth": 1, + "exclude_hidden_files": false, "path": "abc123", "pipeline": "abc123", "process_name_metadata": false, @@ -584,6 +585,11 @@ "minimum": 0, "type": "integer" }, + "exclude_hidden_files": { + "default": false, + "example": false, + "type": "boolean" + }, "path": { "example": "abc123", "type": "string" @@ -664,6 +670,7 @@ "example": { "completed_dir": "abc123", "depth": 1, + "exclude_hidden_files": false, "path": "abc123", "pipeline": "abc123", "process_name_metadata": false, diff --git a/internal/api/gen/http/openapi3.yaml b/internal/api/gen/http/openapi3.yaml index d18f2e2d..0179a9b0 100644 --- a/internal/api/gen/http/openapi3.yaml +++ b/internal/api/gen/http/openapi3.yaml @@ -39,6 +39,7 @@ paths: example: completed_dir: abc123 depth: 1 + exclude_hidden_files: false path: abc123 pipeline: abc123 process_name_metadata: false @@ -1154,6 +1155,10 @@ components: default: 0 example: 1 minimum: 0 + exclude_hidden_files: + type: boolean + default: false + example: false path: type: string example: abc123 @@ -1180,6 +1185,7 @@ components: example: completed_dir: abc123 depth: 1 + exclude_hidden_files: false path: abc123 pipeline: abc123 process_name_metadata: false diff --git a/internal/batch/service.go b/internal/batch/service.go index 6642f5c6..2ffce631 100644 --- a/internal/batch/service.go +++ b/internal/batch/service.go @@ -72,10 +72,11 @@ func (s *batchImpl) Submit(ctx context.Context, payload *goabatch.SubmitPayload) } input.RetentionPeriod = &dur } - input.RejectDuplicates = payload.RejectDuplicates if payload.TransferType != nil { input.TransferType = *payload.TransferType } + input.RejectDuplicates = payload.RejectDuplicates + input.ExcludeHiddenFiles = payload.ExcludeHiddenFiles input.MetadataConfig.ProcessNameMetadata = payload.ProcessNameMetadata input.Depth = int32(payload.Depth) opts := temporalsdk_client.StartWorkflowOptions{ diff --git a/internal/batch/workflow.go b/internal/batch/workflow.go index 0d9d2e03..ac45037f 100644 --- a/internal/batch/workflow.go +++ b/internal/batch/workflow.go @@ -26,15 +26,16 @@ type BatchProgress struct { } type BatchWorkflowInput struct { - Path string - PipelineName string - ProcessingConfig string - CompletedDir string - RetentionPeriod *time.Duration - RejectDuplicates bool - TransferType string - MetadataConfig metadata.Config - Depth int32 + Path string + PipelineName string + ProcessingConfig string + CompletedDir string + RetentionPeriod *time.Duration + RejectDuplicates bool + ExcludeHiddenFiles bool + TransferType string + MetadataConfig metadata.Config + Depth int32 } func BatchWorkflow(ctx temporalsdk_workflow.Context, params BatchWorkflowInput) error { @@ -89,16 +90,17 @@ func (a *BatchActivity) Execute(ctx context.Context, params BatchWorkflowInput) } req := collection.ProcessingWorkflowRequest{ - BatchDir: filepath.Dir(path), - Key: entry.Name(), - IsDir: entry.IsDir(), - PipelineNames: pipelines, - ProcessingConfig: params.ProcessingConfig, - CompletedDir: params.CompletedDir, - RetentionPeriod: params.RetentionPeriod, - RejectDuplicates: params.RejectDuplicates, - TransferType: params.TransferType, - MetadataConfig: params.MetadataConfig, + BatchDir: filepath.Dir(path), + Key: entry.Name(), + IsDir: entry.IsDir(), + PipelineNames: pipelines, + ProcessingConfig: params.ProcessingConfig, + CompletedDir: params.CompletedDir, + RetentionPeriod: params.RetentionPeriod, + RejectDuplicates: params.RejectDuplicates, + ExcludeHiddenFiles: params.ExcludeHiddenFiles, + TransferType: params.TransferType, + MetadataConfig: params.MetadataConfig, } _ = a.batchsvc.InitProcessingWorkflow(ctx, &req) diff --git a/internal/collection/workflow.go b/internal/collection/workflow.go index faf4643b..028db3dc 100644 --- a/internal/collection/workflow.go +++ b/internal/collection/workflow.go @@ -62,6 +62,9 @@ type ProcessingWorkflowRequest struct { // Whether we reject duplicates based on name (key). RejectDuplicates bool + // Whether we exclude hidden files from submission. + ExcludeHiddenFiles bool + // Transfer type. TransferType string diff --git a/internal/watcher/config.go b/internal/watcher/config.go index e3a3214d..9cbb3f99 100644 --- a/internal/watcher/config.go +++ b/internal/watcher/config.go @@ -33,12 +33,13 @@ type FilesystemConfig struct { Inotify bool Ignore string - Pipeline []string - RetentionPeriod *time.Duration - CompletedDir string - StripTopLevelDir bool - RejectDuplicates bool - TransferType string + Pipeline []string + RetentionPeriod *time.Duration + CompletedDir string + StripTopLevelDir bool + RejectDuplicates bool + ExcludeHiddenFiles bool + TransferType string } // See minio.go for more. @@ -55,9 +56,10 @@ type MinioConfig struct { Token string Bucket string - Pipeline []string - RetentionPeriod *time.Duration - StripTopLevelDir bool - RejectDuplicates bool - TransferType string + Pipeline []string + RetentionPeriod *time.Duration + StripTopLevelDir bool + RejectDuplicates bool + ExcludeHiddenFiles bool + TransferType string } diff --git a/internal/watcher/event.go b/internal/watcher/event.go index adca33fa..472a3dbe 100644 --- a/internal/watcher/event.go +++ b/internal/watcher/event.go @@ -32,6 +32,9 @@ type BlobEvent struct { // Whether duplicates are rejected or not. RejectDuplicates bool + // Whether hidden files are exluded or not. + ExcludeHiddenFiles bool + // Which transfer type to use in Archivemaitca. TransferType string @@ -47,15 +50,16 @@ type BlobEvent struct { func NewBlobEvent(w Watcher, key string, isDir bool) *BlobEvent { return &BlobEvent{ - WatcherName: w.String(), - PipelineName: w.Pipelines(), - RetentionPeriod: w.RetentionPeriod(), - CompletedDir: w.CompletedDir(), - StripTopLevelDir: w.StripTopLevelDir(), - RejectDuplicates: w.RejectDuplicates(), - TransferType: w.TransferType(), - Key: key, - IsDir: isDir, + WatcherName: w.String(), + PipelineName: w.Pipelines(), + RetentionPeriod: w.RetentionPeriod(), + CompletedDir: w.CompletedDir(), + StripTopLevelDir: w.StripTopLevelDir(), + RejectDuplicates: w.RejectDuplicates(), + ExcludeHiddenFiles: w.ExcludeHiddenFiles(), + TransferType: w.TransferType(), + Key: key, + IsDir: isDir, } } diff --git a/internal/watcher/fake/mock_watcher_unit.go b/internal/watcher/fake/mock_watcher_unit.go new file mode 100644 index 00000000..0ea9e057 --- /dev/null +++ b/internal/watcher/fake/mock_watcher_unit.go @@ -0,0 +1,458 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/artefactual-labs/enduro/internal/watcher (interfaces: Watcher) + +// Package fake is a generated GoMock package. +package fake + +import ( + context "context" + reflect "reflect" + time "time" + + watcher "github.com/artefactual-labs/enduro/internal/watcher" + gomock "go.uber.org/mock/gomock" + blob "gocloud.dev/blob" +) + +// MockWatcher is a mock of Watcher interface. +type MockWatcher struct { + ctrl *gomock.Controller + recorder *MockWatcherMockRecorder +} + +// MockWatcherMockRecorder is the mock recorder for MockWatcher. +type MockWatcherMockRecorder struct { + mock *MockWatcher +} + +// NewMockWatcher creates a new mock instance. +func NewMockWatcher(ctrl *gomock.Controller) *MockWatcher { + mock := &MockWatcher{ctrl: ctrl} + mock.recorder = &MockWatcherMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockWatcher) EXPECT() *MockWatcherMockRecorder { + return m.recorder +} + +// CompletedDir mocks base method. +func (m *MockWatcher) CompletedDir() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CompletedDir") + ret0, _ := ret[0].(string) + return ret0 +} + +// CompletedDir indicates an expected call of CompletedDir. +func (mr *MockWatcherMockRecorder) CompletedDir() *WatcherCompletedDirCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CompletedDir", reflect.TypeOf((*MockWatcher)(nil).CompletedDir)) + return &WatcherCompletedDirCall{Call: call} +} + +// WatcherCompletedDirCall wrap *gomock.Call +type WatcherCompletedDirCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *WatcherCompletedDirCall) Return(arg0 string) *WatcherCompletedDirCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *WatcherCompletedDirCall) Do(f func() string) *WatcherCompletedDirCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *WatcherCompletedDirCall) DoAndReturn(f func() string) *WatcherCompletedDirCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// ExcludeHiddenFiles mocks base method. +func (m *MockWatcher) ExcludeHiddenFiles() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ExcludeHiddenFiles") + ret0, _ := ret[0].(bool) + return ret0 +} + +// ExcludeHiddenFiles indicates an expected call of ExcludeHiddenFiles. +func (mr *MockWatcherMockRecorder) ExcludeHiddenFiles() *WatcherExcludeHiddenFilesCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExcludeHiddenFiles", reflect.TypeOf((*MockWatcher)(nil).ExcludeHiddenFiles)) + return &WatcherExcludeHiddenFilesCall{Call: call} +} + +// WatcherExcludeHiddenFilesCall wrap *gomock.Call +type WatcherExcludeHiddenFilesCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *WatcherExcludeHiddenFilesCall) Return(arg0 bool) *WatcherExcludeHiddenFilesCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *WatcherExcludeHiddenFilesCall) Do(f func() bool) *WatcherExcludeHiddenFilesCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *WatcherExcludeHiddenFilesCall) DoAndReturn(f func() bool) *WatcherExcludeHiddenFilesCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// OpenBucket mocks base method. +func (m *MockWatcher) OpenBucket(arg0 context.Context) (*blob.Bucket, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "OpenBucket", arg0) + ret0, _ := ret[0].(*blob.Bucket) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// OpenBucket indicates an expected call of OpenBucket. +func (mr *MockWatcherMockRecorder) OpenBucket(arg0 interface{}) *WatcherOpenBucketCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OpenBucket", reflect.TypeOf((*MockWatcher)(nil).OpenBucket), arg0) + return &WatcherOpenBucketCall{Call: call} +} + +// WatcherOpenBucketCall wrap *gomock.Call +type WatcherOpenBucketCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *WatcherOpenBucketCall) Return(arg0 *blob.Bucket, arg1 error) *WatcherOpenBucketCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *WatcherOpenBucketCall) Do(f func(context.Context) (*blob.Bucket, error)) *WatcherOpenBucketCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *WatcherOpenBucketCall) DoAndReturn(f func(context.Context) (*blob.Bucket, error)) *WatcherOpenBucketCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// Path mocks base method. +func (m *MockWatcher) Path() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Path") + ret0, _ := ret[0].(string) + return ret0 +} + +// Path indicates an expected call of Path. +func (mr *MockWatcherMockRecorder) Path() *WatcherPathCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Path", reflect.TypeOf((*MockWatcher)(nil).Path)) + return &WatcherPathCall{Call: call} +} + +// WatcherPathCall wrap *gomock.Call +type WatcherPathCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *WatcherPathCall) Return(arg0 string) *WatcherPathCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *WatcherPathCall) Do(f func() string) *WatcherPathCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *WatcherPathCall) DoAndReturn(f func() string) *WatcherPathCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// Pipelines mocks base method. +func (m *MockWatcher) Pipelines() []string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Pipelines") + ret0, _ := ret[0].([]string) + return ret0 +} + +// Pipelines indicates an expected call of Pipelines. +func (mr *MockWatcherMockRecorder) Pipelines() *WatcherPipelinesCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Pipelines", reflect.TypeOf((*MockWatcher)(nil).Pipelines)) + return &WatcherPipelinesCall{Call: call} +} + +// WatcherPipelinesCall wrap *gomock.Call +type WatcherPipelinesCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *WatcherPipelinesCall) Return(arg0 []string) *WatcherPipelinesCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *WatcherPipelinesCall) Do(f func() []string) *WatcherPipelinesCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *WatcherPipelinesCall) DoAndReturn(f func() []string) *WatcherPipelinesCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// RejectDuplicates mocks base method. +func (m *MockWatcher) RejectDuplicates() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RejectDuplicates") + ret0, _ := ret[0].(bool) + return ret0 +} + +// RejectDuplicates indicates an expected call of RejectDuplicates. +func (mr *MockWatcherMockRecorder) RejectDuplicates() *WatcherRejectDuplicatesCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RejectDuplicates", reflect.TypeOf((*MockWatcher)(nil).RejectDuplicates)) + return &WatcherRejectDuplicatesCall{Call: call} +} + +// WatcherRejectDuplicatesCall wrap *gomock.Call +type WatcherRejectDuplicatesCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *WatcherRejectDuplicatesCall) Return(arg0 bool) *WatcherRejectDuplicatesCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *WatcherRejectDuplicatesCall) Do(f func() bool) *WatcherRejectDuplicatesCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *WatcherRejectDuplicatesCall) DoAndReturn(f func() bool) *WatcherRejectDuplicatesCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// RetentionPeriod mocks base method. +func (m *MockWatcher) RetentionPeriod() *time.Duration { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RetentionPeriod") + ret0, _ := ret[0].(*time.Duration) + return ret0 +} + +// RetentionPeriod indicates an expected call of RetentionPeriod. +func (mr *MockWatcherMockRecorder) RetentionPeriod() *WatcherRetentionPeriodCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RetentionPeriod", reflect.TypeOf((*MockWatcher)(nil).RetentionPeriod)) + return &WatcherRetentionPeriodCall{Call: call} +} + +// WatcherRetentionPeriodCall wrap *gomock.Call +type WatcherRetentionPeriodCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *WatcherRetentionPeriodCall) Return(arg0 *time.Duration) *WatcherRetentionPeriodCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *WatcherRetentionPeriodCall) Do(f func() *time.Duration) *WatcherRetentionPeriodCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *WatcherRetentionPeriodCall) DoAndReturn(f func() *time.Duration) *WatcherRetentionPeriodCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// String mocks base method. +func (m *MockWatcher) String() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "String") + ret0, _ := ret[0].(string) + return ret0 +} + +// String indicates an expected call of String. +func (mr *MockWatcherMockRecorder) String() *WatcherStringCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "String", reflect.TypeOf((*MockWatcher)(nil).String)) + return &WatcherStringCall{Call: call} +} + +// WatcherStringCall wrap *gomock.Call +type WatcherStringCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *WatcherStringCall) Return(arg0 string) *WatcherStringCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *WatcherStringCall) Do(f func() string) *WatcherStringCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *WatcherStringCall) DoAndReturn(f func() string) *WatcherStringCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// StripTopLevelDir mocks base method. +func (m *MockWatcher) StripTopLevelDir() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StripTopLevelDir") + ret0, _ := ret[0].(bool) + return ret0 +} + +// StripTopLevelDir indicates an expected call of StripTopLevelDir. +func (mr *MockWatcherMockRecorder) StripTopLevelDir() *WatcherStripTopLevelDirCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StripTopLevelDir", reflect.TypeOf((*MockWatcher)(nil).StripTopLevelDir)) + return &WatcherStripTopLevelDirCall{Call: call} +} + +// WatcherStripTopLevelDirCall wrap *gomock.Call +type WatcherStripTopLevelDirCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *WatcherStripTopLevelDirCall) Return(arg0 bool) *WatcherStripTopLevelDirCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *WatcherStripTopLevelDirCall) Do(f func() bool) *WatcherStripTopLevelDirCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *WatcherStripTopLevelDirCall) DoAndReturn(f func() bool) *WatcherStripTopLevelDirCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// TransferType mocks base method. +func (m *MockWatcher) TransferType() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TransferType") + ret0, _ := ret[0].(string) + return ret0 +} + +// TransferType indicates an expected call of TransferType. +func (mr *MockWatcherMockRecorder) TransferType() *WatcherTransferTypeCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TransferType", reflect.TypeOf((*MockWatcher)(nil).TransferType)) + return &WatcherTransferTypeCall{Call: call} +} + +// WatcherTransferTypeCall wrap *gomock.Call +type WatcherTransferTypeCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *WatcherTransferTypeCall) Return(arg0 string) *WatcherTransferTypeCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *WatcherTransferTypeCall) Do(f func() string) *WatcherTransferTypeCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *WatcherTransferTypeCall) DoAndReturn(f func() string) *WatcherTransferTypeCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// Watch mocks base method. +func (m *MockWatcher) Watch(arg0 context.Context) (*watcher.BlobEvent, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Watch", arg0) + ret0, _ := ret[0].(*watcher.BlobEvent) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Watch indicates an expected call of Watch. +func (mr *MockWatcherMockRecorder) Watch(arg0 interface{}) *WatcherWatchCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Watch", reflect.TypeOf((*MockWatcher)(nil).Watch), arg0) + return &WatcherWatchCall{Call: call} +} + +// WatcherWatchCall wrap *gomock.Call +type WatcherWatchCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *WatcherWatchCall) Return(arg0 *watcher.BlobEvent, arg1 error) *WatcherWatchCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *WatcherWatchCall) Do(f func(context.Context) (*watcher.BlobEvent, error)) *WatcherWatchCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *WatcherWatchCall) DoAndReturn(f func(context.Context) (*watcher.BlobEvent, error)) *WatcherWatchCall { + c.Call = c.Call.DoAndReturn(f) + return c +} diff --git a/internal/watcher/filesystem.go b/internal/watcher/filesystem.go index c4943968..1d28d44d 100644 --- a/internal/watcher/filesystem.go +++ b/internal/watcher/filesystem.go @@ -71,13 +71,14 @@ func NewFilesystemWatcher(ctx context.Context, config *FilesystemConfig) (*files path: abspath, regex: regex, commonWatcherImpl: &commonWatcherImpl{ - name: config.Name, - pipeline: config.Pipeline, - retentionPeriod: config.RetentionPeriod, - completedDir: config.CompletedDir, - stripTopLevelDir: config.StripTopLevelDir, - rejectDuplicates: config.RejectDuplicates, - transferType: config.TransferType, + name: config.Name, + pipeline: config.Pipeline, + retentionPeriod: config.RetentionPeriod, + completedDir: config.CompletedDir, + stripTopLevelDir: config.StripTopLevelDir, + rejectDuplicates: config.RejectDuplicates, + excludeHiddenFiles: config.ExcludeHiddenFiles, + transferType: config.TransferType, }, } diff --git a/internal/watcher/minio.go b/internal/watcher/minio.go index ccf95d30..01d544a8 100644 --- a/internal/watcher/minio.go +++ b/internal/watcher/minio.go @@ -66,12 +66,13 @@ func NewMinioWatcher(ctx context.Context, config *MinioConfig) (*minioWatcher, e listName: config.RedisList, bucket: config.Bucket, commonWatcherImpl: &commonWatcherImpl{ - name: config.Name, - pipeline: config.Pipeline, - retentionPeriod: config.RetentionPeriod, - stripTopLevelDir: config.StripTopLevelDir, - rejectDuplicates: config.RejectDuplicates, - transferType: config.TransferType, + name: config.Name, + pipeline: config.Pipeline, + retentionPeriod: config.RetentionPeriod, + stripTopLevelDir: config.StripTopLevelDir, + rejectDuplicates: config.RejectDuplicates, + excludeHiddenFiles: config.ExcludeHiddenFiles, + transferType: config.TransferType, }, }, nil } diff --git a/internal/watcher/watcher.go b/internal/watcher/watcher.go index 06b20a83..75f32687 100644 --- a/internal/watcher/watcher.go +++ b/internal/watcher/watcher.go @@ -30,6 +30,7 @@ type Watcher interface { CompletedDir() string StripTopLevelDir() bool RejectDuplicates() bool + ExcludeHiddenFiles() bool TransferType() string // Full path of the watched bucket when available, empty string otherwise. @@ -39,13 +40,14 @@ type Watcher interface { } type commonWatcherImpl struct { - name string - pipeline []string - retentionPeriod *time.Duration - completedDir string - stripTopLevelDir bool - rejectDuplicates bool - transferType string + name string + pipeline []string + retentionPeriod *time.Duration + completedDir string + stripTopLevelDir bool + rejectDuplicates bool + excludeHiddenFiles bool + transferType string } func (w *commonWatcherImpl) String() string { @@ -72,6 +74,10 @@ func (w *commonWatcherImpl) RejectDuplicates() bool { return w.rejectDuplicates } +func (w *commonWatcherImpl) ExcludeHiddenFiles() bool { + return w.excludeHiddenFiles +} + func (w *commonWatcherImpl) TransferType() string { return w.transferType } diff --git a/internal/workflow/activities/bundle.go b/internal/workflow/activities/bundle.go index 16db4d93..f14a54c0 100644 --- a/internal/workflow/activities/bundle.go +++ b/internal/workflow/activities/bundle.go @@ -28,14 +28,15 @@ func NewBundleActivity(m *manager.Manager) *BundleActivity { } type BundleActivityParams struct { - WatcherName string - TransferDir string - Key string - TempFile string - StripTopLevelDir bool - IsDir bool - BatchDir string - Unbag bool + WatcherName string + TransferDir string + Key string + TempFile string + StripTopLevelDir bool + ExcludeHiddenFiles bool + IsDir bool + BatchDir string + Unbag bool } type BundleActivityResult struct { @@ -69,7 +70,7 @@ func (a *BundleActivity) Execute(ctx context.Context, params *BundleActivityPara } else { src := filepath.Join(params.BatchDir, params.Key) dst := params.TransferDir - res.FullPath, res.FullPathBeforeStrip, err = a.Copy(ctx, src, dst, params.StripTopLevelDir) + res.FullPath, res.FullPathBeforeStrip, err = a.Copy(ctx, src, dst, params.StripTopLevelDir, params.ExcludeHiddenFiles) } } else if params.IsDir { var w watcher.Watcher @@ -77,7 +78,7 @@ func (a *BundleActivity) Execute(ctx context.Context, params *BundleActivityPara if err == nil { src := filepath.Join(w.Path(), params.Key) dst := params.TransferDir - res.FullPath, res.FullPathBeforeStrip, err = a.Copy(ctx, src, dst, false) + res.FullPath, res.FullPathBeforeStrip, err = a.Copy(ctx, src, dst, false, params.ExcludeHiddenFiles) } } else { unar := a.Unarchiver(params.Key, params.TempFile) @@ -185,7 +186,7 @@ func (a *BundleActivity) Bundle(ctx context.Context, unar archiver.Unarchiver, t } // Copy a transfer in the given destination using an intermediate temp. directory. -func (a *BundleActivity) Copy(ctx context.Context, src, dst string, stripTopLevelDir bool) (string, string, error) { +func (a *BundleActivity) Copy(ctx context.Context, src, dst string, stripTopLevelDir bool, excludeHiddenFiles bool) (string, string, error) { const prefix = "enduro" tempDir, err := os.MkdirTemp(dst, prefix) if err != nil { @@ -193,7 +194,16 @@ func (a *BundleActivity) Copy(ctx context.Context, src, dst string, stripTopLeve } _ = os.Chmod(tempDir, os.FileMode(0o755)) - if err := copy.Copy(src, tempDir); err != nil { + if err := copy.Copy(src, tempDir, copy.Options{ + Skip: func(srcinfo os.FileInfo, src, dest string) (bool, error) { + // Exclude hidden files. + if excludeHiddenFiles && strings.HasPrefix(srcinfo.Name(), ".") { + return true, nil + } + + return false, nil + }, + }); err != nil { return "", "", fmt.Errorf("error copying transfer: %v", err) } diff --git a/internal/workflow/activities/bundle_test.go b/internal/workflow/activities/bundle_test.go index 830b0ddd..6a8cd322 100644 --- a/internal/workflow/activities/bundle_test.go +++ b/internal/workflow/activities/bundle_test.go @@ -1,14 +1,94 @@ package activities import ( + "os" + "path/filepath" "runtime" "syscall" "testing" + "github.com/go-logr/logr" + temporalsdk_testsuite "go.temporal.io/sdk/testsuite" + "go.uber.org/mock/gomock" "gotest.tools/v3/assert" "gotest.tools/v3/fs" + + collectionfake "github.com/artefactual-labs/enduro/internal/collection/fake" + "github.com/artefactual-labs/enduro/internal/pipeline" + "github.com/artefactual-labs/enduro/internal/watcher" + watcherfake "github.com/artefactual-labs/enduro/internal/watcher/fake" + "github.com/artefactual-labs/enduro/internal/workflow/manager" ) +func TestBundleActivity(t *testing.T) { + t.Parallel() + + t.Run("Excludes hidden files", func(t *testing.T) { + ctrl := gomock.NewController(t) + wsvc := watcherfake.NewMockService(ctrl) + m := manager.NewManager( + logr.Discard(), + collectionfake.NewMockService(ctrl), + wsvc, + &pipeline.Registry{}, + map[string]map[string]interface{}{ + "prod": {"disabled": "false"}, + "hari": {"disabled": "false"}, + }, + ) + activity := NewBundleActivity(m) + ts := &temporalsdk_testsuite.WorkflowTestSuite{} + env := ts.NewTestActivityEnvironment() + env.RegisterActivity(activity.Execute) + + transferDir := fs.NewDir( + t, "enduro", + fs.WithDir( + "transfer", + fs.WithFile("foobar.txt", "Hello world!\n"), + fs.WithFile(".hidden", ""), + ), + ) + + transferSourceDir := fs.NewDir(t, "enduro") + + wsvc.EXPECT().ByName("watcher").DoAndReturn(func(string) (watcher.Watcher, error) { + w := watcherfake.NewMockWatcher(ctrl) + w.EXPECT().Path().Return(transferDir.Path()) + return w, nil + }) + + fut, err := env.ExecuteActivity(activity.Execute, &BundleActivityParams{ + WatcherName: "watcher", + ExcludeHiddenFiles: true, + IsDir: true, + TransferDir: transferSourceDir.Path(), + Key: "transfer", + }) + assert.NilError(t, err) + + // Capture final destination directory within the transfer source + // directory, i.e. Copy method uses a random name. + items, err := os.ReadDir(transferSourceDir.Path()) + assert.NilError(t, err) + destDir := filepath.Join(transferSourceDir.Path(), items[0].Name()) + + res := BundleActivityResult{} + assert.NilError(t, fut.Get(&res)) + assert.DeepEqual(t, res, res) + assert.Assert(t, + fs.Equal( + destDir, + fs.Expected(t, + // .hidden is not expected because ExcludeHiddenFiles is enabled. + fs.WithFile("foobar.txt", "Hello world!\n"), + fs.MatchAnyFileMode, + ), + ), + ) + }) +} + func TestUnbag(t *testing.T) { if runtime.GOOS == "linux" || runtime.GOOS == "darwin" { syscall.Umask(2) diff --git a/internal/workflow/processing.go b/internal/workflow/processing.go index 2199203c..b9b39b59 100644 --- a/internal/workflow/processing.go +++ b/internal/workflow/processing.go @@ -88,6 +88,11 @@ type TransferInfo struct { // It is populated via the workflow request. StripTopLevelDir bool + // Whether hidden files are excluded from the transfer + // + // It is populated via the workflow request. + ExcludeHiddenFiles bool + // Key of the blob. // // It is populated via the workflow request. @@ -166,17 +171,18 @@ func (w *ProcessingWorkflow) Execute(ctx temporalsdk_workflow.Context, req *coll logger = temporalsdk_workflow.GetLogger(ctx) tinfo = &TransferInfo{ - CollectionID: req.CollectionID, - WatcherName: req.WatcherName, - RetentionPeriod: req.RetentionPeriod, - CompletedDir: req.CompletedDir, - StripTopLevelDir: req.StripTopLevelDir, - Key: req.Key, - IsDir: req.IsDir, - BatchDir: req.BatchDir, - ProcessingConfig: req.ProcessingConfig, - TransferType: req.TransferType, - MetadataConfig: req.MetadataConfig, + CollectionID: req.CollectionID, + WatcherName: req.WatcherName, + RetentionPeriod: req.RetentionPeriod, + CompletedDir: req.CompletedDir, + StripTopLevelDir: req.StripTopLevelDir, + ExcludeHiddenFiles: req.ExcludeHiddenFiles, + Key: req.Key, + IsDir: req.IsDir, + BatchDir: req.BatchDir, + ProcessingConfig: req.ProcessingConfig, + TransferType: req.TransferType, + MetadataConfig: req.MetadataConfig, } // Attributes inferred from the name of the transfer. Populated by parseNameLocalActivity. @@ -456,14 +462,15 @@ func (w *ProcessingWorkflow) SessionHandler(sessCtx temporalsdk_workflow.Context if tinfo.Bundle == (activities.BundleActivityResult{}) { activityOpts := withActivityOptsForLongLivedRequest(sessCtx) err := temporalsdk_workflow.ExecuteActivity(activityOpts, activities.BundleActivityName, &activities.BundleActivityParams{ - WatcherName: tinfo.WatcherName, - TransferDir: tinfo.PipelineConfig.TransferDir, - Key: tinfo.Key, - IsDir: tinfo.IsDir, - TempFile: tinfo.TempFile, - StripTopLevelDir: tinfo.StripTopLevelDir, - BatchDir: tinfo.BatchDir, - Unbag: tinfo.PipelineConfig.Unbag, + WatcherName: tinfo.WatcherName, + TransferDir: tinfo.PipelineConfig.TransferDir, + Key: tinfo.Key, + IsDir: tinfo.IsDir, + TempFile: tinfo.TempFile, + StripTopLevelDir: tinfo.StripTopLevelDir, + ExcludeHiddenFiles: tinfo.ExcludeHiddenFiles, + BatchDir: tinfo.BatchDir, + Unbag: tinfo.PipelineConfig.Unbag, }).Get(activityOpts, &tinfo.Bundle) if err != nil { return err diff --git a/main.go b/main.go index c1ee327f..92a9403b 100644 --- a/main.go +++ b/main.go @@ -218,17 +218,18 @@ func main() { ) logger.V(1).Info("Starting new workflow", "watcher", event.WatcherName, "bucket", event.Bucket, "key", event.Key, "dir", event.IsDir) req := collection.ProcessingWorkflowRequest{ - WatcherName: event.WatcherName, - PipelineNames: event.PipelineName, - RetentionPeriod: event.RetentionPeriod, - CompletedDir: event.CompletedDir, - StripTopLevelDir: event.StripTopLevelDir, - RejectDuplicates: event.RejectDuplicates, - TransferType: event.TransferType, - Key: event.Key, - IsDir: event.IsDir, - ValidationConfig: config.Validation, - MetadataConfig: config.Metadata, + WatcherName: event.WatcherName, + PipelineNames: event.PipelineName, + RetentionPeriod: event.RetentionPeriod, + CompletedDir: event.CompletedDir, + StripTopLevelDir: event.StripTopLevelDir, + RejectDuplicates: event.RejectDuplicates, + ExcludeHiddenFiles: event.ExcludeHiddenFiles, + TransferType: event.TransferType, + Key: event.Key, + IsDir: event.IsDir, + ValidationConfig: config.Validation, + MetadataConfig: config.Metadata, } if err := collection.InitProcessingWorkflow(ctx, tracer, temporalClient, config.Temporal.TaskQueue, &req); err != nil { logger.Error(err, "Error initializing processing workflow.") diff --git a/ui/src/openapi-generator/models/SubmitRequestBody.ts b/ui/src/openapi-generator/models/SubmitRequestBody.ts index 86a4ae2f..aa9a7251 100644 --- a/ui/src/openapi-generator/models/SubmitRequestBody.ts +++ b/ui/src/openapi-generator/models/SubmitRequestBody.ts @@ -31,6 +31,12 @@ export interface SubmitRequestBody { * @memberof SubmitRequestBody */ depth?: number; + /** + * + * @type {boolean} + * @memberof SubmitRequestBody + */ + excludeHiddenFiles?: boolean; /** * * @type {string} @@ -97,6 +103,7 @@ export function SubmitRequestBodyFromJSONTyped(json: any, ignoreDiscriminator: b 'completedDir': !exists(json, 'completed_dir') ? undefined : json['completed_dir'], 'depth': !exists(json, 'depth') ? undefined : json['depth'], + 'excludeHiddenFiles': !exists(json, 'exclude_hidden_files') ? undefined : json['exclude_hidden_files'], 'path': json['path'], 'pipeline': !exists(json, 'pipeline') ? undefined : json['pipeline'], 'processNameMetadata': !exists(json, 'process_name_metadata') ? undefined : json['process_name_metadata'], @@ -118,6 +125,7 @@ export function SubmitRequestBodyToJSON(value?: SubmitRequestBody | null): any { 'completed_dir': value.completedDir, 'depth': value.depth, + 'exclude_hidden_files': value.excludeHiddenFiles, 'path': value.path, 'pipeline': value.pipeline, 'process_name_metadata': value.processNameMetadata, diff --git a/ui/src/views/Batch.vue b/ui/src/views/Batch.vue index eee574f2..aa79449c 100644 --- a/ui/src/views/Batch.vue +++ b/ui/src/views/Batch.vue @@ -41,6 +41,12 @@ + + + Exclude hidden files. + + + Process transfer name metadata. @@ -103,6 +109,7 @@ import PipelineProcessingConfigurationDropdown from '@/components/PipelineProces type UserDefaults = { transferType: string | null; rejectDuplicates: boolean; + excludeHiddenFiles: boolean; processNameMetadata: boolean; completedDir: string | null; retentionPeriod: string | null; @@ -184,6 +191,7 @@ export default class Batch extends Vue { } this.form.transferType = defaults.transferType; this.form.rejectDuplicates = defaults.rejectDuplicates; + this.form.excludeHiddenFiles = defaults.excludeHiddenFiles; this.form.processNameMetadata = defaults.processNameMetadata; this.form.completedDir = defaults.completedDir; this.form.retentionPeriod = defaults.retentionPeriod; @@ -195,6 +203,7 @@ export default class Batch extends Vue { const defaults: UserDefaults = { transferType: this.form.transferType, rejectDuplicates: this.form.rejectDuplicates, + excludeHiddenFiles: this.form.excludeHiddenFiles, processNameMetadata: this.form.processNameMetadata, completedDir: this.form.completedDir, retentionPeriod: this.form.retentionPeriod, @@ -243,10 +252,11 @@ export default class Batch extends Vue { if (this.form.retentionPeriod && this.tabIndex === 1) { request.submitRequestBody.retentionPeriod = this.form.retentionPeriod; } - request.submitRequestBody.rejectDuplicates = this.form.rejectDuplicates; if (this.form.transferType) { request.submitRequestBody.transferType = this.form.transferType; } + request.submitRequestBody.rejectDuplicates = this.form.rejectDuplicates; + request.submitRequestBody.excludeHiddenFiles = this.form.excludeHiddenFiles; request.submitRequestBody.processNameMetadata = this.form.processNameMetadata; request.submitRequestBody.depth = Number(this.form.depth); return EnduroBatchClient.batchSubmit(request).then((response: api.BatchSubmitResponseBody) => { diff --git a/website/content/en/docs/user-manual/configuration.md b/website/content/en/docs/user-manual/configuration.md index a6b4e448..9271a034 100644 --- a/website/content/en/docs/user-manual/configuration.md +++ b/website/content/en/docs/user-manual/configuration.md @@ -164,6 +164,9 @@ stripTopLevelDir = true # Reject transfers with duplicate transfer names. rejectDuplicates = false +# Exclude hidden files from transfer. +excludeHiddenFiles = false + # Archivematica transfer type. transferType = "standard" ``` @@ -193,6 +196,12 @@ processing package. If it finds a duplicate the transfer will fail. E.g.: `false` +#### `excludeHiddenFiles` (Boolean) + +When enabled, the workflow will exclude hidden files from the transfer. + +E.g.: `false` + #### `transferType` (String) Archivematica submission transfer type. @@ -270,6 +279,9 @@ stripTopLevelDir = true # Reject transfers with duplicate transfer names. rejectDuplicates = false +# Exclude hidden files from transfer. +excludeHiddenFiles = false + # Archivematica transfer type. transferType = "standard" ``` @@ -301,6 +313,12 @@ processing package. If it finds a duplicate the transfer will fail. E.g.: `false` +#### `excludeHiddenFiles` (Boolean) + +When enabled, the workflow will exclude hidden files from the transfer. + +E.g.: `false` + #### `transferType` (String) Archivematica submission transfer type.