diff --git a/Makefile b/Makefile
index 613c6194..5091ce34 100644
--- a/Makefile
+++ b/Makefile
@@ -154,6 +154,7 @@ gen-mock: $(MOCKGEN)
mockgen -typed -destination=./internal/collection/fake/mock_collection.go -package=fake github.com/artefactual-labs/enduro/internal/collection Service
mockgen -typed -destination=./internal/pipeline/fake/mock_pipeline.go -package=fake github.com/artefactual-labs/enduro/internal/pipeline Service
mockgen -typed -destination=./internal/watcher/fake/mock_watcher.go -package=fake github.com/artefactual-labs/enduro/internal/watcher Service
+ mockgen -typed -destination=./internal/watcher/fake/mock_watcher_unit.go -package=fake github.com/artefactual-labs/enduro/internal/watcher Watcher
temporal: # @HELP Runs a development instance of Temporal.
temporal: PORT := 55555
diff --git a/enduro.toml b/enduro.toml
index c3037a68..27d5bf66 100644
--- a/enduro.toml
+++ b/enduro.toml
@@ -35,6 +35,7 @@ pipeline = "am"
retentionPeriod = "10s"
stripTopLevelDir = true
rejectDuplicates = false
+excludeHiddenFiles = false
transferType = "standard"
[[watcher.filesystem]]
@@ -46,6 +47,7 @@ completedDir = "./hack/landfill"
ignore = '(^\.gitkeep)|(^*\.mft)$'
stripTopLevelDir = true
rejectDuplicates = false
+excludeHiddenFiles = false
transferType = "standard"
[[pipeline]]
diff --git a/internal/api/design/batch.go b/internal/api/design/batch.go
index 06c8f62d..304d6d94 100644
--- a/internal/api/design/batch.go
+++ b/internal/api/design/batch.go
@@ -18,6 +18,7 @@ var _ = Service("batch", func() {
Attribute("completed_dir", String)
Attribute("retention_period", String)
Attribute("reject_duplicates", Boolean, func() { Default(false) })
+ Attribute("exclude_hidden_files", Boolean, func() { Default(false) })
Attribute("transfer_type", String)
Attribute("process_name_metadata", Boolean, func() { Default(false) })
Attribute("depth", Int, func() {
diff --git a/internal/api/gen/batch/service.go b/internal/api/gen/batch/service.go
index 0936926f..11f0b16d 100644
--- a/internal/api/gen/batch/service.go
+++ b/internal/api/gen/batch/service.go
@@ -62,6 +62,7 @@ type SubmitPayload struct {
CompletedDir *string
RetentionPeriod *string
RejectDuplicates bool
+ ExcludeHiddenFiles bool
TransferType *string
ProcessNameMetadata bool
Depth int
diff --git a/internal/api/gen/http/batch/client/cli.go b/internal/api/gen/http/batch/client/cli.go
index b671a632..b46d346a 100644
--- a/internal/api/gen/http/batch/client/cli.go
+++ b/internal/api/gen/http/batch/client/cli.go
@@ -24,7 +24,7 @@ func BuildSubmitPayload(batchSubmitBody string) (*batch.SubmitPayload, error) {
{
err = json.Unmarshal([]byte(batchSubmitBody), &body)
if err != nil {
- return nil, fmt.Errorf("invalid JSON for body, \nerror: %s, \nexample of valid JSON:\n%s", err, "'{\n \"completed_dir\": \"abc123\",\n \"depth\": 1,\n \"path\": \"abc123\",\n \"pipeline\": \"abc123\",\n \"process_name_metadata\": false,\n \"processing_config\": \"abc123\",\n \"reject_duplicates\": false,\n \"retention_period\": \"abc123\",\n \"transfer_type\": \"abc123\"\n }'")
+ return nil, fmt.Errorf("invalid JSON for body, \nerror: %s, \nexample of valid JSON:\n%s", err, "'{\n \"completed_dir\": \"abc123\",\n \"depth\": 1,\n \"exclude_hidden_files\": false,\n \"path\": \"abc123\",\n \"pipeline\": \"abc123\",\n \"process_name_metadata\": false,\n \"processing_config\": \"abc123\",\n \"reject_duplicates\": false,\n \"retention_period\": \"abc123\",\n \"transfer_type\": \"abc123\"\n }'")
}
if body.Depth < 0 {
err = goa.MergeErrors(err, goa.InvalidRangeError("body.depth", body.Depth, 0, true))
@@ -40,6 +40,7 @@ func BuildSubmitPayload(batchSubmitBody string) (*batch.SubmitPayload, error) {
CompletedDir: body.CompletedDir,
RetentionPeriod: body.RetentionPeriod,
RejectDuplicates: body.RejectDuplicates,
+ ExcludeHiddenFiles: body.ExcludeHiddenFiles,
TransferType: body.TransferType,
ProcessNameMetadata: body.ProcessNameMetadata,
Depth: body.Depth,
@@ -50,6 +51,12 @@ func BuildSubmitPayload(batchSubmitBody string) (*batch.SubmitPayload, error) {
v.RejectDuplicates = false
}
}
+ {
+ var zero bool
+ if v.ExcludeHiddenFiles == zero {
+ v.ExcludeHiddenFiles = false
+ }
+ }
{
var zero bool
if v.ProcessNameMetadata == zero {
diff --git a/internal/api/gen/http/batch/client/types.go b/internal/api/gen/http/batch/client/types.go
index d7becbaf..338a1a08 100644
--- a/internal/api/gen/http/batch/client/types.go
+++ b/internal/api/gen/http/batch/client/types.go
@@ -22,6 +22,7 @@ type SubmitRequestBody struct {
CompletedDir *string `form:"completed_dir,omitempty" json:"completed_dir,omitempty" xml:"completed_dir,omitempty"`
RetentionPeriod *string `form:"retention_period,omitempty" json:"retention_period,omitempty" xml:"retention_period,omitempty"`
RejectDuplicates bool `form:"reject_duplicates" json:"reject_duplicates" xml:"reject_duplicates"`
+ ExcludeHiddenFiles bool `form:"exclude_hidden_files" json:"exclude_hidden_files" xml:"exclude_hidden_files"`
TransferType *string `form:"transfer_type,omitempty" json:"transfer_type,omitempty" xml:"transfer_type,omitempty"`
ProcessNameMetadata bool `form:"process_name_metadata" json:"process_name_metadata" xml:"process_name_metadata"`
Depth int `form:"depth" json:"depth" xml:"depth"`
@@ -96,6 +97,7 @@ func NewSubmitRequestBody(p *batch.SubmitPayload) *SubmitRequestBody {
CompletedDir: p.CompletedDir,
RetentionPeriod: p.RetentionPeriod,
RejectDuplicates: p.RejectDuplicates,
+ ExcludeHiddenFiles: p.ExcludeHiddenFiles,
TransferType: p.TransferType,
ProcessNameMetadata: p.ProcessNameMetadata,
Depth: p.Depth,
@@ -106,6 +108,12 @@ func NewSubmitRequestBody(p *batch.SubmitPayload) *SubmitRequestBody {
body.RejectDuplicates = false
}
}
+ {
+ var zero bool
+ if body.ExcludeHiddenFiles == zero {
+ body.ExcludeHiddenFiles = false
+ }
+ }
{
var zero bool
if body.ProcessNameMetadata == zero {
diff --git a/internal/api/gen/http/batch/server/types.go b/internal/api/gen/http/batch/server/types.go
index 81842c1f..a132cf54 100644
--- a/internal/api/gen/http/batch/server/types.go
+++ b/internal/api/gen/http/batch/server/types.go
@@ -22,6 +22,7 @@ type SubmitRequestBody struct {
CompletedDir *string `form:"completed_dir,omitempty" json:"completed_dir,omitempty" xml:"completed_dir,omitempty"`
RetentionPeriod *string `form:"retention_period,omitempty" json:"retention_period,omitempty" xml:"retention_period,omitempty"`
RejectDuplicates *bool `form:"reject_duplicates,omitempty" json:"reject_duplicates,omitempty" xml:"reject_duplicates,omitempty"`
+ ExcludeHiddenFiles *bool `form:"exclude_hidden_files,omitempty" json:"exclude_hidden_files,omitempty" xml:"exclude_hidden_files,omitempty"`
TransferType *string `form:"transfer_type,omitempty" json:"transfer_type,omitempty" xml:"transfer_type,omitempty"`
ProcessNameMetadata *bool `form:"process_name_metadata,omitempty" json:"process_name_metadata,omitempty" xml:"process_name_metadata,omitempty"`
Depth *int `form:"depth,omitempty" json:"depth,omitempty" xml:"depth,omitempty"`
@@ -162,6 +163,9 @@ func NewSubmitPayload(body *SubmitRequestBody) *batch.SubmitPayload {
if body.RejectDuplicates != nil {
v.RejectDuplicates = *body.RejectDuplicates
}
+ if body.ExcludeHiddenFiles != nil {
+ v.ExcludeHiddenFiles = *body.ExcludeHiddenFiles
+ }
if body.ProcessNameMetadata != nil {
v.ProcessNameMetadata = *body.ProcessNameMetadata
}
@@ -171,6 +175,9 @@ func NewSubmitPayload(body *SubmitRequestBody) *batch.SubmitPayload {
if body.RejectDuplicates == nil {
v.RejectDuplicates = false
}
+ if body.ExcludeHiddenFiles == nil {
+ v.ExcludeHiddenFiles = false
+ }
if body.ProcessNameMetadata == nil {
v.ProcessNameMetadata = false
}
diff --git a/internal/api/gen/http/cli/enduro/cli.go b/internal/api/gen/http/cli/enduro/cli.go
index 0d82b391..18590511 100644
--- a/internal/api/gen/http/cli/enduro/cli.go
+++ b/internal/api/gen/http/cli/enduro/cli.go
@@ -37,6 +37,7 @@ func UsageExamples() string {
os.Args[0] + ` batch submit --body '{
"completed_dir": "abc123",
"depth": 1,
+ "exclude_hidden_files": false,
"path": "abc123",
"pipeline": "abc123",
"process_name_metadata": false,
@@ -412,6 +413,7 @@ Example:
%[1]s batch submit --body '{
"completed_dir": "abc123",
"depth": 1,
+ "exclude_hidden_files": false,
"path": "abc123",
"pipeline": "abc123",
"process_name_metadata": false,
diff --git a/internal/api/gen/http/openapi.json b/internal/api/gen/http/openapi.json
index 90df0b66..fa175be9 100644
--- a/internal/api/gen/http/openapi.json
+++ b/internal/api/gen/http/openapi.json
@@ -168,6 +168,7 @@
"example": {
"completed_dir": "abc123",
"depth": 1,
+ "exclude_hidden_files": false,
"path": "abc123",
"pipeline": "abc123",
"process_name_metadata": false,
@@ -187,6 +188,11 @@
"minimum": 0,
"type": "integer"
},
+ "exclude_hidden_files": {
+ "default": false,
+ "example": false,
+ "type": "boolean"
+ },
"path": {
"example": "abc123",
"type": "string"
diff --git a/internal/api/gen/http/openapi.yaml b/internal/api/gen/http/openapi.yaml
index 2dbda671..6b2cb3f8 100644
--- a/internal/api/gen/http/openapi.yaml
+++ b/internal/api/gen/http/openapi.yaml
@@ -650,6 +650,10 @@ definitions:
default: 0
example: 1
minimum: 0
+ exclude_hidden_files:
+ type: boolean
+ default: false
+ example: false
path:
type: string
example: abc123
@@ -676,6 +680,7 @@ definitions:
example:
completed_dir: abc123
depth: 1
+ exclude_hidden_files: false
path: abc123
pipeline: abc123
process_name_metadata: false
diff --git a/internal/api/gen/http/openapi3.json b/internal/api/gen/http/openapi3.json
index 4d582181..6cb043be 100644
--- a/internal/api/gen/http/openapi3.json
+++ b/internal/api/gen/http/openapi3.json
@@ -565,6 +565,7 @@
"example": {
"completed_dir": "abc123",
"depth": 1,
+ "exclude_hidden_files": false,
"path": "abc123",
"pipeline": "abc123",
"process_name_metadata": false,
@@ -584,6 +585,11 @@
"minimum": 0,
"type": "integer"
},
+ "exclude_hidden_files": {
+ "default": false,
+ "example": false,
+ "type": "boolean"
+ },
"path": {
"example": "abc123",
"type": "string"
@@ -664,6 +670,7 @@
"example": {
"completed_dir": "abc123",
"depth": 1,
+ "exclude_hidden_files": false,
"path": "abc123",
"pipeline": "abc123",
"process_name_metadata": false,
diff --git a/internal/api/gen/http/openapi3.yaml b/internal/api/gen/http/openapi3.yaml
index d18f2e2d..0179a9b0 100644
--- a/internal/api/gen/http/openapi3.yaml
+++ b/internal/api/gen/http/openapi3.yaml
@@ -39,6 +39,7 @@ paths:
example:
completed_dir: abc123
depth: 1
+ exclude_hidden_files: false
path: abc123
pipeline: abc123
process_name_metadata: false
@@ -1154,6 +1155,10 @@ components:
default: 0
example: 1
minimum: 0
+ exclude_hidden_files:
+ type: boolean
+ default: false
+ example: false
path:
type: string
example: abc123
@@ -1180,6 +1185,7 @@ components:
example:
completed_dir: abc123
depth: 1
+ exclude_hidden_files: false
path: abc123
pipeline: abc123
process_name_metadata: false
diff --git a/internal/batch/service.go b/internal/batch/service.go
index 6642f5c6..2ffce631 100644
--- a/internal/batch/service.go
+++ b/internal/batch/service.go
@@ -72,10 +72,11 @@ func (s *batchImpl) Submit(ctx context.Context, payload *goabatch.SubmitPayload)
}
input.RetentionPeriod = &dur
}
- input.RejectDuplicates = payload.RejectDuplicates
if payload.TransferType != nil {
input.TransferType = *payload.TransferType
}
+ input.RejectDuplicates = payload.RejectDuplicates
+ input.ExcludeHiddenFiles = payload.ExcludeHiddenFiles
input.MetadataConfig.ProcessNameMetadata = payload.ProcessNameMetadata
input.Depth = int32(payload.Depth)
opts := temporalsdk_client.StartWorkflowOptions{
diff --git a/internal/batch/workflow.go b/internal/batch/workflow.go
index 0d9d2e03..ac45037f 100644
--- a/internal/batch/workflow.go
+++ b/internal/batch/workflow.go
@@ -26,15 +26,16 @@ type BatchProgress struct {
}
type BatchWorkflowInput struct {
- Path string
- PipelineName string
- ProcessingConfig string
- CompletedDir string
- RetentionPeriod *time.Duration
- RejectDuplicates bool
- TransferType string
- MetadataConfig metadata.Config
- Depth int32
+ Path string
+ PipelineName string
+ ProcessingConfig string
+ CompletedDir string
+ RetentionPeriod *time.Duration
+ RejectDuplicates bool
+ ExcludeHiddenFiles bool
+ TransferType string
+ MetadataConfig metadata.Config
+ Depth int32
}
func BatchWorkflow(ctx temporalsdk_workflow.Context, params BatchWorkflowInput) error {
@@ -89,16 +90,17 @@ func (a *BatchActivity) Execute(ctx context.Context, params BatchWorkflowInput)
}
req := collection.ProcessingWorkflowRequest{
- BatchDir: filepath.Dir(path),
- Key: entry.Name(),
- IsDir: entry.IsDir(),
- PipelineNames: pipelines,
- ProcessingConfig: params.ProcessingConfig,
- CompletedDir: params.CompletedDir,
- RetentionPeriod: params.RetentionPeriod,
- RejectDuplicates: params.RejectDuplicates,
- TransferType: params.TransferType,
- MetadataConfig: params.MetadataConfig,
+ BatchDir: filepath.Dir(path),
+ Key: entry.Name(),
+ IsDir: entry.IsDir(),
+ PipelineNames: pipelines,
+ ProcessingConfig: params.ProcessingConfig,
+ CompletedDir: params.CompletedDir,
+ RetentionPeriod: params.RetentionPeriod,
+ RejectDuplicates: params.RejectDuplicates,
+ ExcludeHiddenFiles: params.ExcludeHiddenFiles,
+ TransferType: params.TransferType,
+ MetadataConfig: params.MetadataConfig,
}
_ = a.batchsvc.InitProcessingWorkflow(ctx, &req)
diff --git a/internal/collection/workflow.go b/internal/collection/workflow.go
index faf4643b..028db3dc 100644
--- a/internal/collection/workflow.go
+++ b/internal/collection/workflow.go
@@ -62,6 +62,9 @@ type ProcessingWorkflowRequest struct {
// Whether we reject duplicates based on name (key).
RejectDuplicates bool
+ // Whether we exclude hidden files from submission.
+ ExcludeHiddenFiles bool
+
// Transfer type.
TransferType string
diff --git a/internal/watcher/config.go b/internal/watcher/config.go
index e3a3214d..9cbb3f99 100644
--- a/internal/watcher/config.go
+++ b/internal/watcher/config.go
@@ -33,12 +33,13 @@ type FilesystemConfig struct {
Inotify bool
Ignore string
- Pipeline []string
- RetentionPeriod *time.Duration
- CompletedDir string
- StripTopLevelDir bool
- RejectDuplicates bool
- TransferType string
+ Pipeline []string
+ RetentionPeriod *time.Duration
+ CompletedDir string
+ StripTopLevelDir bool
+ RejectDuplicates bool
+ ExcludeHiddenFiles bool
+ TransferType string
}
// See minio.go for more.
@@ -55,9 +56,10 @@ type MinioConfig struct {
Token string
Bucket string
- Pipeline []string
- RetentionPeriod *time.Duration
- StripTopLevelDir bool
- RejectDuplicates bool
- TransferType string
+ Pipeline []string
+ RetentionPeriod *time.Duration
+ StripTopLevelDir bool
+ RejectDuplicates bool
+ ExcludeHiddenFiles bool
+ TransferType string
}
diff --git a/internal/watcher/event.go b/internal/watcher/event.go
index adca33fa..472a3dbe 100644
--- a/internal/watcher/event.go
+++ b/internal/watcher/event.go
@@ -32,6 +32,9 @@ type BlobEvent struct {
// Whether duplicates are rejected or not.
RejectDuplicates bool
+ // Whether hidden files are exluded or not.
+ ExcludeHiddenFiles bool
+
// Which transfer type to use in Archivemaitca.
TransferType string
@@ -47,15 +50,16 @@ type BlobEvent struct {
func NewBlobEvent(w Watcher, key string, isDir bool) *BlobEvent {
return &BlobEvent{
- WatcherName: w.String(),
- PipelineName: w.Pipelines(),
- RetentionPeriod: w.RetentionPeriod(),
- CompletedDir: w.CompletedDir(),
- StripTopLevelDir: w.StripTopLevelDir(),
- RejectDuplicates: w.RejectDuplicates(),
- TransferType: w.TransferType(),
- Key: key,
- IsDir: isDir,
+ WatcherName: w.String(),
+ PipelineName: w.Pipelines(),
+ RetentionPeriod: w.RetentionPeriod(),
+ CompletedDir: w.CompletedDir(),
+ StripTopLevelDir: w.StripTopLevelDir(),
+ RejectDuplicates: w.RejectDuplicates(),
+ ExcludeHiddenFiles: w.ExcludeHiddenFiles(),
+ TransferType: w.TransferType(),
+ Key: key,
+ IsDir: isDir,
}
}
diff --git a/internal/watcher/fake/mock_watcher_unit.go b/internal/watcher/fake/mock_watcher_unit.go
new file mode 100644
index 00000000..0ea9e057
--- /dev/null
+++ b/internal/watcher/fake/mock_watcher_unit.go
@@ -0,0 +1,458 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/artefactual-labs/enduro/internal/watcher (interfaces: Watcher)
+
+// Package fake is a generated GoMock package.
+package fake
+
+import (
+ context "context"
+ reflect "reflect"
+ time "time"
+
+ watcher "github.com/artefactual-labs/enduro/internal/watcher"
+ gomock "go.uber.org/mock/gomock"
+ blob "gocloud.dev/blob"
+)
+
+// MockWatcher is a mock of Watcher interface.
+type MockWatcher struct {
+ ctrl *gomock.Controller
+ recorder *MockWatcherMockRecorder
+}
+
+// MockWatcherMockRecorder is the mock recorder for MockWatcher.
+type MockWatcherMockRecorder struct {
+ mock *MockWatcher
+}
+
+// NewMockWatcher creates a new mock instance.
+func NewMockWatcher(ctrl *gomock.Controller) *MockWatcher {
+ mock := &MockWatcher{ctrl: ctrl}
+ mock.recorder = &MockWatcherMockRecorder{mock}
+ return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockWatcher) EXPECT() *MockWatcherMockRecorder {
+ return m.recorder
+}
+
+// CompletedDir mocks base method.
+func (m *MockWatcher) CompletedDir() string {
+ m.ctrl.T.Helper()
+ ret := m.ctrl.Call(m, "CompletedDir")
+ ret0, _ := ret[0].(string)
+ return ret0
+}
+
+// CompletedDir indicates an expected call of CompletedDir.
+func (mr *MockWatcherMockRecorder) CompletedDir() *WatcherCompletedDirCall {
+ mr.mock.ctrl.T.Helper()
+ call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CompletedDir", reflect.TypeOf((*MockWatcher)(nil).CompletedDir))
+ return &WatcherCompletedDirCall{Call: call}
+}
+
+// WatcherCompletedDirCall wrap *gomock.Call
+type WatcherCompletedDirCall struct {
+ *gomock.Call
+}
+
+// Return rewrite *gomock.Call.Return
+func (c *WatcherCompletedDirCall) Return(arg0 string) *WatcherCompletedDirCall {
+ c.Call = c.Call.Return(arg0)
+ return c
+}
+
+// Do rewrite *gomock.Call.Do
+func (c *WatcherCompletedDirCall) Do(f func() string) *WatcherCompletedDirCall {
+ c.Call = c.Call.Do(f)
+ return c
+}
+
+// DoAndReturn rewrite *gomock.Call.DoAndReturn
+func (c *WatcherCompletedDirCall) DoAndReturn(f func() string) *WatcherCompletedDirCall {
+ c.Call = c.Call.DoAndReturn(f)
+ return c
+}
+
+// ExcludeHiddenFiles mocks base method.
+func (m *MockWatcher) ExcludeHiddenFiles() bool {
+ m.ctrl.T.Helper()
+ ret := m.ctrl.Call(m, "ExcludeHiddenFiles")
+ ret0, _ := ret[0].(bool)
+ return ret0
+}
+
+// ExcludeHiddenFiles indicates an expected call of ExcludeHiddenFiles.
+func (mr *MockWatcherMockRecorder) ExcludeHiddenFiles() *WatcherExcludeHiddenFilesCall {
+ mr.mock.ctrl.T.Helper()
+ call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExcludeHiddenFiles", reflect.TypeOf((*MockWatcher)(nil).ExcludeHiddenFiles))
+ return &WatcherExcludeHiddenFilesCall{Call: call}
+}
+
+// WatcherExcludeHiddenFilesCall wrap *gomock.Call
+type WatcherExcludeHiddenFilesCall struct {
+ *gomock.Call
+}
+
+// Return rewrite *gomock.Call.Return
+func (c *WatcherExcludeHiddenFilesCall) Return(arg0 bool) *WatcherExcludeHiddenFilesCall {
+ c.Call = c.Call.Return(arg0)
+ return c
+}
+
+// Do rewrite *gomock.Call.Do
+func (c *WatcherExcludeHiddenFilesCall) Do(f func() bool) *WatcherExcludeHiddenFilesCall {
+ c.Call = c.Call.Do(f)
+ return c
+}
+
+// DoAndReturn rewrite *gomock.Call.DoAndReturn
+func (c *WatcherExcludeHiddenFilesCall) DoAndReturn(f func() bool) *WatcherExcludeHiddenFilesCall {
+ c.Call = c.Call.DoAndReturn(f)
+ return c
+}
+
+// OpenBucket mocks base method.
+func (m *MockWatcher) OpenBucket(arg0 context.Context) (*blob.Bucket, error) {
+ m.ctrl.T.Helper()
+ ret := m.ctrl.Call(m, "OpenBucket", arg0)
+ ret0, _ := ret[0].(*blob.Bucket)
+ ret1, _ := ret[1].(error)
+ return ret0, ret1
+}
+
+// OpenBucket indicates an expected call of OpenBucket.
+func (mr *MockWatcherMockRecorder) OpenBucket(arg0 interface{}) *WatcherOpenBucketCall {
+ mr.mock.ctrl.T.Helper()
+ call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OpenBucket", reflect.TypeOf((*MockWatcher)(nil).OpenBucket), arg0)
+ return &WatcherOpenBucketCall{Call: call}
+}
+
+// WatcherOpenBucketCall wrap *gomock.Call
+type WatcherOpenBucketCall struct {
+ *gomock.Call
+}
+
+// Return rewrite *gomock.Call.Return
+func (c *WatcherOpenBucketCall) Return(arg0 *blob.Bucket, arg1 error) *WatcherOpenBucketCall {
+ c.Call = c.Call.Return(arg0, arg1)
+ return c
+}
+
+// Do rewrite *gomock.Call.Do
+func (c *WatcherOpenBucketCall) Do(f func(context.Context) (*blob.Bucket, error)) *WatcherOpenBucketCall {
+ c.Call = c.Call.Do(f)
+ return c
+}
+
+// DoAndReturn rewrite *gomock.Call.DoAndReturn
+func (c *WatcherOpenBucketCall) DoAndReturn(f func(context.Context) (*blob.Bucket, error)) *WatcherOpenBucketCall {
+ c.Call = c.Call.DoAndReturn(f)
+ return c
+}
+
+// Path mocks base method.
+func (m *MockWatcher) Path() string {
+ m.ctrl.T.Helper()
+ ret := m.ctrl.Call(m, "Path")
+ ret0, _ := ret[0].(string)
+ return ret0
+}
+
+// Path indicates an expected call of Path.
+func (mr *MockWatcherMockRecorder) Path() *WatcherPathCall {
+ mr.mock.ctrl.T.Helper()
+ call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Path", reflect.TypeOf((*MockWatcher)(nil).Path))
+ return &WatcherPathCall{Call: call}
+}
+
+// WatcherPathCall wrap *gomock.Call
+type WatcherPathCall struct {
+ *gomock.Call
+}
+
+// Return rewrite *gomock.Call.Return
+func (c *WatcherPathCall) Return(arg0 string) *WatcherPathCall {
+ c.Call = c.Call.Return(arg0)
+ return c
+}
+
+// Do rewrite *gomock.Call.Do
+func (c *WatcherPathCall) Do(f func() string) *WatcherPathCall {
+ c.Call = c.Call.Do(f)
+ return c
+}
+
+// DoAndReturn rewrite *gomock.Call.DoAndReturn
+func (c *WatcherPathCall) DoAndReturn(f func() string) *WatcherPathCall {
+ c.Call = c.Call.DoAndReturn(f)
+ return c
+}
+
+// Pipelines mocks base method.
+func (m *MockWatcher) Pipelines() []string {
+ m.ctrl.T.Helper()
+ ret := m.ctrl.Call(m, "Pipelines")
+ ret0, _ := ret[0].([]string)
+ return ret0
+}
+
+// Pipelines indicates an expected call of Pipelines.
+func (mr *MockWatcherMockRecorder) Pipelines() *WatcherPipelinesCall {
+ mr.mock.ctrl.T.Helper()
+ call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Pipelines", reflect.TypeOf((*MockWatcher)(nil).Pipelines))
+ return &WatcherPipelinesCall{Call: call}
+}
+
+// WatcherPipelinesCall wrap *gomock.Call
+type WatcherPipelinesCall struct {
+ *gomock.Call
+}
+
+// Return rewrite *gomock.Call.Return
+func (c *WatcherPipelinesCall) Return(arg0 []string) *WatcherPipelinesCall {
+ c.Call = c.Call.Return(arg0)
+ return c
+}
+
+// Do rewrite *gomock.Call.Do
+func (c *WatcherPipelinesCall) Do(f func() []string) *WatcherPipelinesCall {
+ c.Call = c.Call.Do(f)
+ return c
+}
+
+// DoAndReturn rewrite *gomock.Call.DoAndReturn
+func (c *WatcherPipelinesCall) DoAndReturn(f func() []string) *WatcherPipelinesCall {
+ c.Call = c.Call.DoAndReturn(f)
+ return c
+}
+
+// RejectDuplicates mocks base method.
+func (m *MockWatcher) RejectDuplicates() bool {
+ m.ctrl.T.Helper()
+ ret := m.ctrl.Call(m, "RejectDuplicates")
+ ret0, _ := ret[0].(bool)
+ return ret0
+}
+
+// RejectDuplicates indicates an expected call of RejectDuplicates.
+func (mr *MockWatcherMockRecorder) RejectDuplicates() *WatcherRejectDuplicatesCall {
+ mr.mock.ctrl.T.Helper()
+ call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RejectDuplicates", reflect.TypeOf((*MockWatcher)(nil).RejectDuplicates))
+ return &WatcherRejectDuplicatesCall{Call: call}
+}
+
+// WatcherRejectDuplicatesCall wrap *gomock.Call
+type WatcherRejectDuplicatesCall struct {
+ *gomock.Call
+}
+
+// Return rewrite *gomock.Call.Return
+func (c *WatcherRejectDuplicatesCall) Return(arg0 bool) *WatcherRejectDuplicatesCall {
+ c.Call = c.Call.Return(arg0)
+ return c
+}
+
+// Do rewrite *gomock.Call.Do
+func (c *WatcherRejectDuplicatesCall) Do(f func() bool) *WatcherRejectDuplicatesCall {
+ c.Call = c.Call.Do(f)
+ return c
+}
+
+// DoAndReturn rewrite *gomock.Call.DoAndReturn
+func (c *WatcherRejectDuplicatesCall) DoAndReturn(f func() bool) *WatcherRejectDuplicatesCall {
+ c.Call = c.Call.DoAndReturn(f)
+ return c
+}
+
+// RetentionPeriod mocks base method.
+func (m *MockWatcher) RetentionPeriod() *time.Duration {
+ m.ctrl.T.Helper()
+ ret := m.ctrl.Call(m, "RetentionPeriod")
+ ret0, _ := ret[0].(*time.Duration)
+ return ret0
+}
+
+// RetentionPeriod indicates an expected call of RetentionPeriod.
+func (mr *MockWatcherMockRecorder) RetentionPeriod() *WatcherRetentionPeriodCall {
+ mr.mock.ctrl.T.Helper()
+ call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RetentionPeriod", reflect.TypeOf((*MockWatcher)(nil).RetentionPeriod))
+ return &WatcherRetentionPeriodCall{Call: call}
+}
+
+// WatcherRetentionPeriodCall wrap *gomock.Call
+type WatcherRetentionPeriodCall struct {
+ *gomock.Call
+}
+
+// Return rewrite *gomock.Call.Return
+func (c *WatcherRetentionPeriodCall) Return(arg0 *time.Duration) *WatcherRetentionPeriodCall {
+ c.Call = c.Call.Return(arg0)
+ return c
+}
+
+// Do rewrite *gomock.Call.Do
+func (c *WatcherRetentionPeriodCall) Do(f func() *time.Duration) *WatcherRetentionPeriodCall {
+ c.Call = c.Call.Do(f)
+ return c
+}
+
+// DoAndReturn rewrite *gomock.Call.DoAndReturn
+func (c *WatcherRetentionPeriodCall) DoAndReturn(f func() *time.Duration) *WatcherRetentionPeriodCall {
+ c.Call = c.Call.DoAndReturn(f)
+ return c
+}
+
+// String mocks base method.
+func (m *MockWatcher) String() string {
+ m.ctrl.T.Helper()
+ ret := m.ctrl.Call(m, "String")
+ ret0, _ := ret[0].(string)
+ return ret0
+}
+
+// String indicates an expected call of String.
+func (mr *MockWatcherMockRecorder) String() *WatcherStringCall {
+ mr.mock.ctrl.T.Helper()
+ call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "String", reflect.TypeOf((*MockWatcher)(nil).String))
+ return &WatcherStringCall{Call: call}
+}
+
+// WatcherStringCall wrap *gomock.Call
+type WatcherStringCall struct {
+ *gomock.Call
+}
+
+// Return rewrite *gomock.Call.Return
+func (c *WatcherStringCall) Return(arg0 string) *WatcherStringCall {
+ c.Call = c.Call.Return(arg0)
+ return c
+}
+
+// Do rewrite *gomock.Call.Do
+func (c *WatcherStringCall) Do(f func() string) *WatcherStringCall {
+ c.Call = c.Call.Do(f)
+ return c
+}
+
+// DoAndReturn rewrite *gomock.Call.DoAndReturn
+func (c *WatcherStringCall) DoAndReturn(f func() string) *WatcherStringCall {
+ c.Call = c.Call.DoAndReturn(f)
+ return c
+}
+
+// StripTopLevelDir mocks base method.
+func (m *MockWatcher) StripTopLevelDir() bool {
+ m.ctrl.T.Helper()
+ ret := m.ctrl.Call(m, "StripTopLevelDir")
+ ret0, _ := ret[0].(bool)
+ return ret0
+}
+
+// StripTopLevelDir indicates an expected call of StripTopLevelDir.
+func (mr *MockWatcherMockRecorder) StripTopLevelDir() *WatcherStripTopLevelDirCall {
+ mr.mock.ctrl.T.Helper()
+ call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StripTopLevelDir", reflect.TypeOf((*MockWatcher)(nil).StripTopLevelDir))
+ return &WatcherStripTopLevelDirCall{Call: call}
+}
+
+// WatcherStripTopLevelDirCall wrap *gomock.Call
+type WatcherStripTopLevelDirCall struct {
+ *gomock.Call
+}
+
+// Return rewrite *gomock.Call.Return
+func (c *WatcherStripTopLevelDirCall) Return(arg0 bool) *WatcherStripTopLevelDirCall {
+ c.Call = c.Call.Return(arg0)
+ return c
+}
+
+// Do rewrite *gomock.Call.Do
+func (c *WatcherStripTopLevelDirCall) Do(f func() bool) *WatcherStripTopLevelDirCall {
+ c.Call = c.Call.Do(f)
+ return c
+}
+
+// DoAndReturn rewrite *gomock.Call.DoAndReturn
+func (c *WatcherStripTopLevelDirCall) DoAndReturn(f func() bool) *WatcherStripTopLevelDirCall {
+ c.Call = c.Call.DoAndReturn(f)
+ return c
+}
+
+// TransferType mocks base method.
+func (m *MockWatcher) TransferType() string {
+ m.ctrl.T.Helper()
+ ret := m.ctrl.Call(m, "TransferType")
+ ret0, _ := ret[0].(string)
+ return ret0
+}
+
+// TransferType indicates an expected call of TransferType.
+func (mr *MockWatcherMockRecorder) TransferType() *WatcherTransferTypeCall {
+ mr.mock.ctrl.T.Helper()
+ call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TransferType", reflect.TypeOf((*MockWatcher)(nil).TransferType))
+ return &WatcherTransferTypeCall{Call: call}
+}
+
+// WatcherTransferTypeCall wrap *gomock.Call
+type WatcherTransferTypeCall struct {
+ *gomock.Call
+}
+
+// Return rewrite *gomock.Call.Return
+func (c *WatcherTransferTypeCall) Return(arg0 string) *WatcherTransferTypeCall {
+ c.Call = c.Call.Return(arg0)
+ return c
+}
+
+// Do rewrite *gomock.Call.Do
+func (c *WatcherTransferTypeCall) Do(f func() string) *WatcherTransferTypeCall {
+ c.Call = c.Call.Do(f)
+ return c
+}
+
+// DoAndReturn rewrite *gomock.Call.DoAndReturn
+func (c *WatcherTransferTypeCall) DoAndReturn(f func() string) *WatcherTransferTypeCall {
+ c.Call = c.Call.DoAndReturn(f)
+ return c
+}
+
+// Watch mocks base method.
+func (m *MockWatcher) Watch(arg0 context.Context) (*watcher.BlobEvent, error) {
+ m.ctrl.T.Helper()
+ ret := m.ctrl.Call(m, "Watch", arg0)
+ ret0, _ := ret[0].(*watcher.BlobEvent)
+ ret1, _ := ret[1].(error)
+ return ret0, ret1
+}
+
+// Watch indicates an expected call of Watch.
+func (mr *MockWatcherMockRecorder) Watch(arg0 interface{}) *WatcherWatchCall {
+ mr.mock.ctrl.T.Helper()
+ call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Watch", reflect.TypeOf((*MockWatcher)(nil).Watch), arg0)
+ return &WatcherWatchCall{Call: call}
+}
+
+// WatcherWatchCall wrap *gomock.Call
+type WatcherWatchCall struct {
+ *gomock.Call
+}
+
+// Return rewrite *gomock.Call.Return
+func (c *WatcherWatchCall) Return(arg0 *watcher.BlobEvent, arg1 error) *WatcherWatchCall {
+ c.Call = c.Call.Return(arg0, arg1)
+ return c
+}
+
+// Do rewrite *gomock.Call.Do
+func (c *WatcherWatchCall) Do(f func(context.Context) (*watcher.BlobEvent, error)) *WatcherWatchCall {
+ c.Call = c.Call.Do(f)
+ return c
+}
+
+// DoAndReturn rewrite *gomock.Call.DoAndReturn
+func (c *WatcherWatchCall) DoAndReturn(f func(context.Context) (*watcher.BlobEvent, error)) *WatcherWatchCall {
+ c.Call = c.Call.DoAndReturn(f)
+ return c
+}
diff --git a/internal/watcher/filesystem.go b/internal/watcher/filesystem.go
index c4943968..1d28d44d 100644
--- a/internal/watcher/filesystem.go
+++ b/internal/watcher/filesystem.go
@@ -71,13 +71,14 @@ func NewFilesystemWatcher(ctx context.Context, config *FilesystemConfig) (*files
path: abspath,
regex: regex,
commonWatcherImpl: &commonWatcherImpl{
- name: config.Name,
- pipeline: config.Pipeline,
- retentionPeriod: config.RetentionPeriod,
- completedDir: config.CompletedDir,
- stripTopLevelDir: config.StripTopLevelDir,
- rejectDuplicates: config.RejectDuplicates,
- transferType: config.TransferType,
+ name: config.Name,
+ pipeline: config.Pipeline,
+ retentionPeriod: config.RetentionPeriod,
+ completedDir: config.CompletedDir,
+ stripTopLevelDir: config.StripTopLevelDir,
+ rejectDuplicates: config.RejectDuplicates,
+ excludeHiddenFiles: config.ExcludeHiddenFiles,
+ transferType: config.TransferType,
},
}
diff --git a/internal/watcher/minio.go b/internal/watcher/minio.go
index ccf95d30..01d544a8 100644
--- a/internal/watcher/minio.go
+++ b/internal/watcher/minio.go
@@ -66,12 +66,13 @@ func NewMinioWatcher(ctx context.Context, config *MinioConfig) (*minioWatcher, e
listName: config.RedisList,
bucket: config.Bucket,
commonWatcherImpl: &commonWatcherImpl{
- name: config.Name,
- pipeline: config.Pipeline,
- retentionPeriod: config.RetentionPeriod,
- stripTopLevelDir: config.StripTopLevelDir,
- rejectDuplicates: config.RejectDuplicates,
- transferType: config.TransferType,
+ name: config.Name,
+ pipeline: config.Pipeline,
+ retentionPeriod: config.RetentionPeriod,
+ stripTopLevelDir: config.StripTopLevelDir,
+ rejectDuplicates: config.RejectDuplicates,
+ excludeHiddenFiles: config.ExcludeHiddenFiles,
+ transferType: config.TransferType,
},
}, nil
}
diff --git a/internal/watcher/watcher.go b/internal/watcher/watcher.go
index 06b20a83..75f32687 100644
--- a/internal/watcher/watcher.go
+++ b/internal/watcher/watcher.go
@@ -30,6 +30,7 @@ type Watcher interface {
CompletedDir() string
StripTopLevelDir() bool
RejectDuplicates() bool
+ ExcludeHiddenFiles() bool
TransferType() string
// Full path of the watched bucket when available, empty string otherwise.
@@ -39,13 +40,14 @@ type Watcher interface {
}
type commonWatcherImpl struct {
- name string
- pipeline []string
- retentionPeriod *time.Duration
- completedDir string
- stripTopLevelDir bool
- rejectDuplicates bool
- transferType string
+ name string
+ pipeline []string
+ retentionPeriod *time.Duration
+ completedDir string
+ stripTopLevelDir bool
+ rejectDuplicates bool
+ excludeHiddenFiles bool
+ transferType string
}
func (w *commonWatcherImpl) String() string {
@@ -72,6 +74,10 @@ func (w *commonWatcherImpl) RejectDuplicates() bool {
return w.rejectDuplicates
}
+func (w *commonWatcherImpl) ExcludeHiddenFiles() bool {
+ return w.excludeHiddenFiles
+}
+
func (w *commonWatcherImpl) TransferType() string {
return w.transferType
}
diff --git a/internal/workflow/activities/bundle.go b/internal/workflow/activities/bundle.go
index 16db4d93..f14a54c0 100644
--- a/internal/workflow/activities/bundle.go
+++ b/internal/workflow/activities/bundle.go
@@ -28,14 +28,15 @@ func NewBundleActivity(m *manager.Manager) *BundleActivity {
}
type BundleActivityParams struct {
- WatcherName string
- TransferDir string
- Key string
- TempFile string
- StripTopLevelDir bool
- IsDir bool
- BatchDir string
- Unbag bool
+ WatcherName string
+ TransferDir string
+ Key string
+ TempFile string
+ StripTopLevelDir bool
+ ExcludeHiddenFiles bool
+ IsDir bool
+ BatchDir string
+ Unbag bool
}
type BundleActivityResult struct {
@@ -69,7 +70,7 @@ func (a *BundleActivity) Execute(ctx context.Context, params *BundleActivityPara
} else {
src := filepath.Join(params.BatchDir, params.Key)
dst := params.TransferDir
- res.FullPath, res.FullPathBeforeStrip, err = a.Copy(ctx, src, dst, params.StripTopLevelDir)
+ res.FullPath, res.FullPathBeforeStrip, err = a.Copy(ctx, src, dst, params.StripTopLevelDir, params.ExcludeHiddenFiles)
}
} else if params.IsDir {
var w watcher.Watcher
@@ -77,7 +78,7 @@ func (a *BundleActivity) Execute(ctx context.Context, params *BundleActivityPara
if err == nil {
src := filepath.Join(w.Path(), params.Key)
dst := params.TransferDir
- res.FullPath, res.FullPathBeforeStrip, err = a.Copy(ctx, src, dst, false)
+ res.FullPath, res.FullPathBeforeStrip, err = a.Copy(ctx, src, dst, false, params.ExcludeHiddenFiles)
}
} else {
unar := a.Unarchiver(params.Key, params.TempFile)
@@ -185,7 +186,7 @@ func (a *BundleActivity) Bundle(ctx context.Context, unar archiver.Unarchiver, t
}
// Copy a transfer in the given destination using an intermediate temp. directory.
-func (a *BundleActivity) Copy(ctx context.Context, src, dst string, stripTopLevelDir bool) (string, string, error) {
+func (a *BundleActivity) Copy(ctx context.Context, src, dst string, stripTopLevelDir bool, excludeHiddenFiles bool) (string, string, error) {
const prefix = "enduro"
tempDir, err := os.MkdirTemp(dst, prefix)
if err != nil {
@@ -193,7 +194,16 @@ func (a *BundleActivity) Copy(ctx context.Context, src, dst string, stripTopLeve
}
_ = os.Chmod(tempDir, os.FileMode(0o755))
- if err := copy.Copy(src, tempDir); err != nil {
+ if err := copy.Copy(src, tempDir, copy.Options{
+ Skip: func(srcinfo os.FileInfo, src, dest string) (bool, error) {
+ // Exclude hidden files.
+ if excludeHiddenFiles && strings.HasPrefix(srcinfo.Name(), ".") {
+ return true, nil
+ }
+
+ return false, nil
+ },
+ }); err != nil {
return "", "", fmt.Errorf("error copying transfer: %v", err)
}
diff --git a/internal/workflow/activities/bundle_test.go b/internal/workflow/activities/bundle_test.go
index 830b0ddd..6a8cd322 100644
--- a/internal/workflow/activities/bundle_test.go
+++ b/internal/workflow/activities/bundle_test.go
@@ -1,14 +1,94 @@
package activities
import (
+ "os"
+ "path/filepath"
"runtime"
"syscall"
"testing"
+ "github.com/go-logr/logr"
+ temporalsdk_testsuite "go.temporal.io/sdk/testsuite"
+ "go.uber.org/mock/gomock"
"gotest.tools/v3/assert"
"gotest.tools/v3/fs"
+
+ collectionfake "github.com/artefactual-labs/enduro/internal/collection/fake"
+ "github.com/artefactual-labs/enduro/internal/pipeline"
+ "github.com/artefactual-labs/enduro/internal/watcher"
+ watcherfake "github.com/artefactual-labs/enduro/internal/watcher/fake"
+ "github.com/artefactual-labs/enduro/internal/workflow/manager"
)
+func TestBundleActivity(t *testing.T) {
+ t.Parallel()
+
+ t.Run("Excludes hidden files", func(t *testing.T) {
+ ctrl := gomock.NewController(t)
+ wsvc := watcherfake.NewMockService(ctrl)
+ m := manager.NewManager(
+ logr.Discard(),
+ collectionfake.NewMockService(ctrl),
+ wsvc,
+ &pipeline.Registry{},
+ map[string]map[string]interface{}{
+ "prod": {"disabled": "false"},
+ "hari": {"disabled": "false"},
+ },
+ )
+ activity := NewBundleActivity(m)
+ ts := &temporalsdk_testsuite.WorkflowTestSuite{}
+ env := ts.NewTestActivityEnvironment()
+ env.RegisterActivity(activity.Execute)
+
+ transferDir := fs.NewDir(
+ t, "enduro",
+ fs.WithDir(
+ "transfer",
+ fs.WithFile("foobar.txt", "Hello world!\n"),
+ fs.WithFile(".hidden", ""),
+ ),
+ )
+
+ transferSourceDir := fs.NewDir(t, "enduro")
+
+ wsvc.EXPECT().ByName("watcher").DoAndReturn(func(string) (watcher.Watcher, error) {
+ w := watcherfake.NewMockWatcher(ctrl)
+ w.EXPECT().Path().Return(transferDir.Path())
+ return w, nil
+ })
+
+ fut, err := env.ExecuteActivity(activity.Execute, &BundleActivityParams{
+ WatcherName: "watcher",
+ ExcludeHiddenFiles: true,
+ IsDir: true,
+ TransferDir: transferSourceDir.Path(),
+ Key: "transfer",
+ })
+ assert.NilError(t, err)
+
+ // Capture final destination directory within the transfer source
+ // directory, i.e. Copy method uses a random name.
+ items, err := os.ReadDir(transferSourceDir.Path())
+ assert.NilError(t, err)
+ destDir := filepath.Join(transferSourceDir.Path(), items[0].Name())
+
+ res := BundleActivityResult{}
+ assert.NilError(t, fut.Get(&res))
+ assert.DeepEqual(t, res, res)
+ assert.Assert(t,
+ fs.Equal(
+ destDir,
+ fs.Expected(t,
+ // .hidden is not expected because ExcludeHiddenFiles is enabled.
+ fs.WithFile("foobar.txt", "Hello world!\n"),
+ fs.MatchAnyFileMode,
+ ),
+ ),
+ )
+ })
+}
+
func TestUnbag(t *testing.T) {
if runtime.GOOS == "linux" || runtime.GOOS == "darwin" {
syscall.Umask(2)
diff --git a/internal/workflow/processing.go b/internal/workflow/processing.go
index 2199203c..b9b39b59 100644
--- a/internal/workflow/processing.go
+++ b/internal/workflow/processing.go
@@ -88,6 +88,11 @@ type TransferInfo struct {
// It is populated via the workflow request.
StripTopLevelDir bool
+ // Whether hidden files are excluded from the transfer
+ //
+ // It is populated via the workflow request.
+ ExcludeHiddenFiles bool
+
// Key of the blob.
//
// It is populated via the workflow request.
@@ -166,17 +171,18 @@ func (w *ProcessingWorkflow) Execute(ctx temporalsdk_workflow.Context, req *coll
logger = temporalsdk_workflow.GetLogger(ctx)
tinfo = &TransferInfo{
- CollectionID: req.CollectionID,
- WatcherName: req.WatcherName,
- RetentionPeriod: req.RetentionPeriod,
- CompletedDir: req.CompletedDir,
- StripTopLevelDir: req.StripTopLevelDir,
- Key: req.Key,
- IsDir: req.IsDir,
- BatchDir: req.BatchDir,
- ProcessingConfig: req.ProcessingConfig,
- TransferType: req.TransferType,
- MetadataConfig: req.MetadataConfig,
+ CollectionID: req.CollectionID,
+ WatcherName: req.WatcherName,
+ RetentionPeriod: req.RetentionPeriod,
+ CompletedDir: req.CompletedDir,
+ StripTopLevelDir: req.StripTopLevelDir,
+ ExcludeHiddenFiles: req.ExcludeHiddenFiles,
+ Key: req.Key,
+ IsDir: req.IsDir,
+ BatchDir: req.BatchDir,
+ ProcessingConfig: req.ProcessingConfig,
+ TransferType: req.TransferType,
+ MetadataConfig: req.MetadataConfig,
}
// Attributes inferred from the name of the transfer. Populated by parseNameLocalActivity.
@@ -456,14 +462,15 @@ func (w *ProcessingWorkflow) SessionHandler(sessCtx temporalsdk_workflow.Context
if tinfo.Bundle == (activities.BundleActivityResult{}) {
activityOpts := withActivityOptsForLongLivedRequest(sessCtx)
err := temporalsdk_workflow.ExecuteActivity(activityOpts, activities.BundleActivityName, &activities.BundleActivityParams{
- WatcherName: tinfo.WatcherName,
- TransferDir: tinfo.PipelineConfig.TransferDir,
- Key: tinfo.Key,
- IsDir: tinfo.IsDir,
- TempFile: tinfo.TempFile,
- StripTopLevelDir: tinfo.StripTopLevelDir,
- BatchDir: tinfo.BatchDir,
- Unbag: tinfo.PipelineConfig.Unbag,
+ WatcherName: tinfo.WatcherName,
+ TransferDir: tinfo.PipelineConfig.TransferDir,
+ Key: tinfo.Key,
+ IsDir: tinfo.IsDir,
+ TempFile: tinfo.TempFile,
+ StripTopLevelDir: tinfo.StripTopLevelDir,
+ ExcludeHiddenFiles: tinfo.ExcludeHiddenFiles,
+ BatchDir: tinfo.BatchDir,
+ Unbag: tinfo.PipelineConfig.Unbag,
}).Get(activityOpts, &tinfo.Bundle)
if err != nil {
return err
diff --git a/main.go b/main.go
index c1ee327f..92a9403b 100644
--- a/main.go
+++ b/main.go
@@ -218,17 +218,18 @@ func main() {
)
logger.V(1).Info("Starting new workflow", "watcher", event.WatcherName, "bucket", event.Bucket, "key", event.Key, "dir", event.IsDir)
req := collection.ProcessingWorkflowRequest{
- WatcherName: event.WatcherName,
- PipelineNames: event.PipelineName,
- RetentionPeriod: event.RetentionPeriod,
- CompletedDir: event.CompletedDir,
- StripTopLevelDir: event.StripTopLevelDir,
- RejectDuplicates: event.RejectDuplicates,
- TransferType: event.TransferType,
- Key: event.Key,
- IsDir: event.IsDir,
- ValidationConfig: config.Validation,
- MetadataConfig: config.Metadata,
+ WatcherName: event.WatcherName,
+ PipelineNames: event.PipelineName,
+ RetentionPeriod: event.RetentionPeriod,
+ CompletedDir: event.CompletedDir,
+ StripTopLevelDir: event.StripTopLevelDir,
+ RejectDuplicates: event.RejectDuplicates,
+ ExcludeHiddenFiles: event.ExcludeHiddenFiles,
+ TransferType: event.TransferType,
+ Key: event.Key,
+ IsDir: event.IsDir,
+ ValidationConfig: config.Validation,
+ MetadataConfig: config.Metadata,
}
if err := collection.InitProcessingWorkflow(ctx, tracer, temporalClient, config.Temporal.TaskQueue, &req); err != nil {
logger.Error(err, "Error initializing processing workflow.")
diff --git a/ui/src/openapi-generator/models/SubmitRequestBody.ts b/ui/src/openapi-generator/models/SubmitRequestBody.ts
index 86a4ae2f..aa9a7251 100644
--- a/ui/src/openapi-generator/models/SubmitRequestBody.ts
+++ b/ui/src/openapi-generator/models/SubmitRequestBody.ts
@@ -31,6 +31,12 @@ export interface SubmitRequestBody {
* @memberof SubmitRequestBody
*/
depth?: number;
+ /**
+ *
+ * @type {boolean}
+ * @memberof SubmitRequestBody
+ */
+ excludeHiddenFiles?: boolean;
/**
*
* @type {string}
@@ -97,6 +103,7 @@ export function SubmitRequestBodyFromJSONTyped(json: any, ignoreDiscriminator: b
'completedDir': !exists(json, 'completed_dir') ? undefined : json['completed_dir'],
'depth': !exists(json, 'depth') ? undefined : json['depth'],
+ 'excludeHiddenFiles': !exists(json, 'exclude_hidden_files') ? undefined : json['exclude_hidden_files'],
'path': json['path'],
'pipeline': !exists(json, 'pipeline') ? undefined : json['pipeline'],
'processNameMetadata': !exists(json, 'process_name_metadata') ? undefined : json['process_name_metadata'],
@@ -118,6 +125,7 @@ export function SubmitRequestBodyToJSON(value?: SubmitRequestBody | null): any {
'completed_dir': value.completedDir,
'depth': value.depth,
+ 'exclude_hidden_files': value.excludeHiddenFiles,
'path': value.path,
'pipeline': value.pipeline,
'process_name_metadata': value.processNameMetadata,
diff --git a/ui/src/views/Batch.vue b/ui/src/views/Batch.vue
index eee574f2..aa79449c 100644
--- a/ui/src/views/Batch.vue
+++ b/ui/src/views/Batch.vue
@@ -41,6 +41,12 @@
+
+
+ Exclude hidden files.
+
+
+
Process transfer name metadata.
@@ -103,6 +109,7 @@ import PipelineProcessingConfigurationDropdown from '@/components/PipelineProces
type UserDefaults = {
transferType: string | null;
rejectDuplicates: boolean;
+ excludeHiddenFiles: boolean;
processNameMetadata: boolean;
completedDir: string | null;
retentionPeriod: string | null;
@@ -184,6 +191,7 @@ export default class Batch extends Vue {
}
this.form.transferType = defaults.transferType;
this.form.rejectDuplicates = defaults.rejectDuplicates;
+ this.form.excludeHiddenFiles = defaults.excludeHiddenFiles;
this.form.processNameMetadata = defaults.processNameMetadata;
this.form.completedDir = defaults.completedDir;
this.form.retentionPeriod = defaults.retentionPeriod;
@@ -195,6 +203,7 @@ export default class Batch extends Vue {
const defaults: UserDefaults = {
transferType: this.form.transferType,
rejectDuplicates: this.form.rejectDuplicates,
+ excludeHiddenFiles: this.form.excludeHiddenFiles,
processNameMetadata: this.form.processNameMetadata,
completedDir: this.form.completedDir,
retentionPeriod: this.form.retentionPeriod,
@@ -243,10 +252,11 @@ export default class Batch extends Vue {
if (this.form.retentionPeriod && this.tabIndex === 1) {
request.submitRequestBody.retentionPeriod = this.form.retentionPeriod;
}
- request.submitRequestBody.rejectDuplicates = this.form.rejectDuplicates;
if (this.form.transferType) {
request.submitRequestBody.transferType = this.form.transferType;
}
+ request.submitRequestBody.rejectDuplicates = this.form.rejectDuplicates;
+ request.submitRequestBody.excludeHiddenFiles = this.form.excludeHiddenFiles;
request.submitRequestBody.processNameMetadata = this.form.processNameMetadata;
request.submitRequestBody.depth = Number(this.form.depth);
return EnduroBatchClient.batchSubmit(request).then((response: api.BatchSubmitResponseBody) => {
diff --git a/website/content/en/docs/user-manual/configuration.md b/website/content/en/docs/user-manual/configuration.md
index a6b4e448..9271a034 100644
--- a/website/content/en/docs/user-manual/configuration.md
+++ b/website/content/en/docs/user-manual/configuration.md
@@ -164,6 +164,9 @@ stripTopLevelDir = true
# Reject transfers with duplicate transfer names.
rejectDuplicates = false
+# Exclude hidden files from transfer.
+excludeHiddenFiles = false
+
# Archivematica transfer type.
transferType = "standard"
```
@@ -193,6 +196,12 @@ processing package. If it finds a duplicate the transfer will fail.
E.g.: `false`
+#### `excludeHiddenFiles` (Boolean)
+
+When enabled, the workflow will exclude hidden files from the transfer.
+
+E.g.: `false`
+
#### `transferType` (String)
Archivematica submission transfer type.
@@ -270,6 +279,9 @@ stripTopLevelDir = true
# Reject transfers with duplicate transfer names.
rejectDuplicates = false
+# Exclude hidden files from transfer.
+excludeHiddenFiles = false
+
# Archivematica transfer type.
transferType = "standard"
```
@@ -301,6 +313,12 @@ processing package. If it finds a duplicate the transfer will fail.
E.g.: `false`
+#### `excludeHiddenFiles` (Boolean)
+
+When enabled, the workflow will exclude hidden files from the transfer.
+
+E.g.: `false`
+
#### `transferType` (String)
Archivematica submission transfer type.