From d112675c71e013dfb4c19ed612261c34cbb25f4f Mon Sep 17 00:00:00 2001 From: Jason Bornstein <131717043+jasonbornsteinMOOV@users.noreply.github.com> Date: Thu, 21 Sep 2023 11:08:41 -0400 Subject: [PATCH 1/4] BNK-340 generate multiple batches in a recon file --- pkg/response/batch_mirror.go | 110 +++++++++++++----- pkg/response/file_transformer.go | 17 +-- pkg/response/file_transformer_test.go | 51 +++++++- .../20230809-144155-102000021-2Batches.ach | 59 ++++++++++ 4 files changed, 196 insertions(+), 41 deletions(-) create mode 100644 testdata/20230809-144155-102000021-2Batches.ach diff --git a/pkg/response/batch_mirror.go b/pkg/response/batch_mirror.go index a91e25eb..96493b49 100644 --- a/pkg/response/batch_mirror.go +++ b/pkg/response/batch_mirror.go @@ -2,9 +2,9 @@ package response import ( "bytes" - "errors" "fmt" "path/filepath" + "sort" "time" "github.com/moov-io/ach" @@ -13,51 +13,105 @@ import ( // batchMirror is an object that will save batches type batchMirror struct { + batches map[batchMirrorKey]map[string]*batchMirrorBatch // path+companyID -> (batch ID -> header+entries+control) + + writer FileWriter +} + +type batchMirrorKey struct { + path string + companyID string +} + +func (key *batchMirrorKey) getFilePathName() string { + timestamp := time.Now().Format("20060102-150405.00000") + filename := fmt.Sprintf("%s_%s.ach", key.companyID, timestamp) + return filepath.Join(key.path, filename) +} + +type batchMirrorBatch struct { header *ach.BatchHeader - entries map[string][]*ach.EntryDetail // filepath -> entries + entries []*ach.EntryDetail control *ach.BatchControl +} - writer FileWriter +func (batch *batchMirrorBatch) write(buf *bytes.Buffer) error { + buf.WriteString(batch.header.String() + "\n") + for _, entry := range batch.entries { + buf.WriteString(entry.String() + "\n") + } + control, err := calculateControl(batch.header, batch.entries) + if err != nil { + return fmt.Errorf("problem computing control: %v", err) + } + buf.WriteString(control + "\n") + + return nil } -func newBatchMirror(w FileWriter, b ach.Batcher) *batchMirror { +func newBatchMirror(w FileWriter) *batchMirror { return &batchMirror{ - header: b.GetHeader(), - entries: make(map[string][]*ach.EntryDetail), - control: b.GetControl(), + batches: make(map[batchMirrorKey]map[string]*batchMirrorBatch), writer: w, } } -func (bm *batchMirror) saveEntry(copy *service.Copy, ed *ach.EntryDetail) { - if copy == nil { +func (bm *batchMirror) saveEntry(b *ach.Batcher, copy *service.Copy, ed *ach.EntryDetail) { + if b == nil || copy == nil || ed == nil { return } - bm.entries[copy.Path] = append(bm.entries[copy.Path], ed) + + batcher := *b + // Get the batchMirrorKey + key := batchMirrorKey{ + path: copy.Path, + companyID: batcher.GetHeader().CompanyIdentification, + } + // Create a new batchMirrorBatch map if this key does not exist + if _, exists := bm.batches[key]; !exists { + bm.batches[key] = make(map[string]*batchMirrorBatch) + } + // Create an array of batchMirrorBatch if this batch ID does not exist + if _, exists := bm.batches[key][batcher.GetHeader().BatchNumberField()]; !exists { + bm.batches[key][batcher.GetHeader().BatchNumberField()] = &batchMirrorBatch{ + header: batcher.GetHeader(), + entries: make([]*ach.EntryDetail, 0), + control: batcher.GetControl(), + } + } + // Append this EntryDetail to the batchMirrorBatch's EntryDetails slice for the derived key + bm.batches[key][batcher.GetHeader().BatchNumberField()].entries = append(bm.batches[key][batcher.GetHeader().BatchNumberField()].entries, ed) } func (bm *batchMirror) saveFiles() error { - if bm.header == nil || len(bm.entries) == 0 { + if len(bm.batches) == 0 { return nil } - for path, entries := range bm.entries { - // Accumulate file contents + + // Write files by Path/CompanyID + for key, mirror := range bm.batches { var buf bytes.Buffer - buf.WriteString(bm.header.String() + "\n") - for i := range entries { - buf.WriteString(entries[i].String() + "\n") + + // sort the keys so that the batches appear in the correct order + keys := make([]string, 0, len(mirror)) + for number, _ := range mirror { + keys = append(keys, number) } - control, err := calculateControl(bm.header, entries) - if err != nil { - return fmt.Errorf("problem computing control: %v", err) + sort.Slice(keys, func(i, j int) bool { + return keys[i] < keys[j] + }) + + for _, val := range keys { + batch := mirror[val] + if err := batch.write(&buf); err != nil { + return err + } } - buf.WriteString(control) // Write the file out - if filename, err := bm.filename(); err != nil { - return fmt.Errorf("unable to get filename: %v", err) - } else { - bm.writer.Write(filepath.Join(path, filename), &buf, nil) + err := bm.writer.Write(key.getFilePathName(), &buf, nil) + if err != nil { + return fmt.Errorf("problem writing file: %v", err) } } return nil @@ -73,11 +127,3 @@ func calculateControl(bh *ach.BatchHeader, entries []*ach.EntryDetail) (string, } return batch.GetControl().String(), nil } - -func (bm *batchMirror) filename() (string, error) { - if bm.header == nil { - return "", errors.New("missing BatchHeader") - } - timestamp := time.Now().Format("20060102-150405.00000") - return fmt.Sprintf("%s_%s.ach", bm.header.CompanyIdentification, timestamp), nil -} diff --git a/pkg/response/file_transformer.go b/pkg/response/file_transformer.go index e25b87d8..c949d34f 100644 --- a/pkg/response/file_transformer.go +++ b/pkg/response/file_transformer.go @@ -41,9 +41,10 @@ func (ft *FileTransfomer) Transform(file *ach.File) error { // Track ach.File objects to write based on different delay durations, including a default of "0s" var outFiles = outFiles{} + // batchMirror is used for copying entires to the reconciliation file (if needed) + mirror := newBatchMirror(ft.Writer) + for i := range file.Batches { - // batchMirror is used for copying entires to the reconciliation file (if needed) - mirror := newBatchMirror(ft.Writer, file.Batches[i]) // Track ach.Batcher to write based on different delay durations and whether the batch is for NOC var outBatches = outBatches{} @@ -57,7 +58,7 @@ func (ft *FileTransfomer) Transform(file *ach.File) error { logger.Log("Processing matched action") // Save this Entry - mirror.saveEntry(copyAction.Copy, entries[j]) + mirror.saveEntry(&file.Batches[i], copyAction.Copy, entries[j]) } if processAction != nil { logger := ft.Matcher.Logger.With(processAction) @@ -89,11 +90,6 @@ func (ft *FileTransfomer) Transform(file *ach.File) error { } } - // Save off the entries as requested - if err := mirror.saveFiles(); err != nil { - return fmt.Errorf("problem saving entries: %v", err) - } - // Create our Batch's Control and other fields for delay, batchesByCategory := range outBatches { for _, batch := range batchesByCategory { @@ -111,6 +107,11 @@ func (ft *FileTransfomer) Transform(file *ach.File) error { } } + // Save off the entries as requested + if err := mirror.saveFiles(); err != nil { + return fmt.Errorf("problem saving entries: %v", err) + } + for delay, out := range outFiles { if out != nil && len(out.Batches) > 0 { if err := out.Create(); err != nil { diff --git a/pkg/response/file_transformer_test.go b/pkg/response/file_transformer_test.go index fd663656..ca191db2 100644 --- a/pkg/response/file_transformer_test.go +++ b/pkg/response/file_transformer_test.go @@ -3,6 +3,7 @@ package response import ( "os" "path/filepath" + "sort" "testing" "time" @@ -137,7 +138,7 @@ func TestFileTransformer_CopyOnly(t *testing.T) { } // debit & credit -func TestFileTransformer_CopyOnlyAndCopyOnly(t *testing.T) { +func TestFileTransformer_CopyOnlyAndCopyOnly_SingleBatch(t *testing.T) { fileTransformer, dir := testFileTransformer(t, respCopyDebit, respCopyCredit) achIn, err := ach.ReadFile(filepath.Join("..", "..", "testdata", "20230809-144155-102000021.ach")) @@ -167,6 +168,54 @@ func TestFileTransformer_CopyOnlyAndCopyOnly(t *testing.T) { require.Less(t, fInfo.ModTime(), time.Now()) } +// debit & credit +func TestFileTransformer_CopyOnlyAndCopyOnly_MultipleBatches(t *testing.T) { + fileTransformer, dir := testFileTransformer(t, respCopyDebit, respCopyCredit) + + achIn, err := ach.ReadFile(filepath.Join("..", "..", "testdata", "20230809-144155-102000021-2Batches.ach")) + require.NoError(t, err) + require.NotNil(t, achIn) + require.Len(t, achIn.Batches, 3) + + // transform the file + err = fileTransformer.Transform(achIn) + require.NoError(t, err) + + // verify no "returned" files created + retdir := filepath.Join(dir, "returned") + _, err = os.ReadDir(retdir) + require.Error(t, err) + + // verify the "reconciliation" file created + recondir := filepath.Join(dir, "reconciliation") + fds, err := os.ReadDir(recondir) + require.NoError(t, err) + require.Len(t, fds, 2) + // sort the files by name, so we can reliably compare them + sort.Slice(fds, func(i, j int) bool { + return fds[i].Name() < fds[j].Name() + }) + + read, _ := ach.ReadFile(filepath.Join(recondir, fds[0].Name())) // ignore the error b/c this file has no header or control record + require.Len(t, read.Batches, 2) + require.Equal(t, achIn.Batches[0], read.Batches[0]) + require.Equal(t, achIn.Batches[1], read.Batches[1]) + + // verify the timestamp on the file is in the past + fInfo, err := fds[0].Info() + require.NoError(t, err) + require.Less(t, fInfo.ModTime(), time.Now()) + + read, _ = ach.ReadFile(filepath.Join(recondir, fds[1].Name())) // ignore the error b/c this file has no header or control record + require.Len(t, read.Batches, 1) + require.Equal(t, achIn.Batches[2], read.Batches[0]) + + // verify the timestamp on the file is in the past + fInfo, err = fds[1].Info() + require.NoError(t, err) + require.Less(t, fInfo.ModTime(), time.Now()) +} + // credit func TestFileTransformer_ReturnOnly(t *testing.T) { resp := service.Response{ diff --git a/testdata/20230809-144155-102000021-2Batches.ach b/testdata/20230809-144155-102000021-2Batches.ach new file mode 100644 index 00000000..cdaf93e1 --- /dev/null +++ b/testdata/20230809-144155-102000021-2Batches.ach @@ -0,0 +1,59 @@ +101 04200001302313801041108052100A094101US BANK NA EXAMPLE COMPANY +5200EXAMPLE COMPANY 0231380104PPDBUY WIDGET110808110808 1042000010000001 +627021200025998412345 0000027000A271 JULIAN PRICE 0042000010000001 +627021200025998412345 0000062000A272 SYDNEY BUTLER 0042000010000002 +627021200025998412345 0000209000A273 KEVIN CASTILLO 0042000010000003 +627021200025998412345 0000139000A274 ELIZABETH PATTERSON 0042000010000004 +627021200025998412345 0000118000A275 MORGAN WALKER 0042000010000005 +627021200025998412345 0000206000A278 JULIA LYNCH 0042000010000006 +627021200025998412345 0000206000A279 JULIA LYNCH 0042000010000007 +627021200025998412345 0000122000A280 ALLISON COLE 0042000010000008 +627021200025998412345 0000122000A281 ALLISON COLE 0042000010000009 +820000000900190800180000012110000000000000000231380104 042000010000001 +5200EXAMPLE COMPANY 0231380104PPDVERIFY 110808110808 1042000010000002 +627021200025998412345 0000170000A282 BROOKLYN MURPHY 0042000010000010 +627021200025998412345 0000170000A283 BROOKLYN MURPHY 0042000010000011 +627021200025998412345 0000250000A284 DANIEL FOSTER 0042000010000012 +627021200025998412345 0000250000A285 DANIEL FOSTER 0042000010000013 +627021200025998412345 0000262000A286 LUKE CRAIG 0042000010000014 +627021200025998412345 0000262000A287 LUKE CRAIG 0042000010000015 +627021200025998412345 0000242000A288 LILLIAN JENSEN 0042000010000016 +627021200025998412345 0000242000A289 LILLIAN JENSEN 0042000010000017 +627021200025998412345 0000087000A290 CHRISTIAN COX 0042000010000018 +627021200025998412345 0000150000A291 BRYAN HORTON 0042000010000019 +627021200025998412345 0000237000A292 CARLOS COOK 0042000010000020 +627021200025998412345 0000237000A294 AUSTIN WATTS 0042000010000021 +627021200025998412345 0000219000A295 ALEXANDER ERRY 0042000010000022 +627021200025998412345 0000255000A296 RYAN FLETCHER 0042000010000023 +627021200025998412345 0000149000A297 PAYTON MCCOY 0042000010000024 +627021200025998412345 0000217000A298 DESTINY COLEMAN 0042000010000025 +820000001600339200320000033990000000000000000231380104 042000010000002 +5200EXAMPLE COMPANY 0231380105PPDVERIFY 110808110808 1042000010000003 +622021200025998412345 0000000008A251 NATHAN NELSON 0042000010000101 +622021200025998412345 0000000010A252 NATHAN NELSON 0042000010000102 +622021200025998412345 0000000002A253 CHARLES REYES 0042000010000103 +622021200025998412345 0000000012A254 CHARLES REYES 0042000010000104 +622021200025998412345 0000000002A255 SAMANTHA BYRD 0042000010000105 +622021200025998412345 0000000011A256 SAMANTHA BYRD 0042000010000106 +622021200025998412345 0000000013A259 JOSEPH RUSSELL 0042000010000107 +622021200025998412345 0000000004A260 JOSEPH RUSSELL 0042000010000108 +622021200025998412345 0000000009A261 CHASE BATES 0042000010000109 +622021200025998412345 0000000005A262 CHASE BATES 0042000010000110 +622021200025998412345 0000000019A263 HANNAH LEWIS 0042000010000111 +622021200025998412345 0000000015A264 HANNAH LEWIS 0042000010000112 +622021200025998412345 0000000006A265 MICHAEL GARZA 0042000010000113 +622021200025998412345 0000000015A266 MICHAEL GARZA 0042000010000114 +622021200025998412345 0000000014A267 WILLIAM HUDSON 0042000010000115 +622021200025998412345 0000000009A268 WILLIAM HUDSON 0042000010000116 +622021200025998412345 0000000013A269 JOSHUA OBRIEN 0042000010000117 +622021200025998412345 0000000009A270 JOSHUA OBRIEN 0042000010000118 +820000001800381600360000000000000000000001760231380105 042000010000003 +9000005000010000000830136685201000005101000000000000200000000000000000000000000000000000000000 +9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999 +9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999 +9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999 +9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999 +9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999 +9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999 +9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999 +9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999 From aaeb751964088569b65c147d4b09e8cbea0c0ec7 Mon Sep 17 00:00:00 2001 From: Jason Bornstein <131717043+jasonbornsteinMOOV@users.noreply.github.com> Date: Thu, 21 Sep 2023 12:49:03 -0400 Subject: [PATCH 2/4] fix formatting --- pkg/response/batch_mirror.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/response/batch_mirror.go b/pkg/response/batch_mirror.go index 96493b49..aea93636 100644 --- a/pkg/response/batch_mirror.go +++ b/pkg/response/batch_mirror.go @@ -3,12 +3,11 @@ package response import ( "bytes" "fmt" + "github.com/moov-io/ach" + "github.com/moov-io/ach-test-harness/pkg/service" "path/filepath" "sort" "time" - - "github.com/moov-io/ach" - "github.com/moov-io/ach-test-harness/pkg/service" ) // batchMirror is an object that will save batches From 7a4bdf6780d9506eac843c42c8d1c7e13b421a10 Mon Sep 17 00:00:00 2001 From: Jason Bornstein <131717043+jasonbornsteinMOOV@users.noreply.github.com> Date: Thu, 21 Sep 2023 12:51:57 -0400 Subject: [PATCH 3/4] fix formatting --- pkg/response/batch_mirror.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/response/batch_mirror.go b/pkg/response/batch_mirror.go index aea93636..96493b49 100644 --- a/pkg/response/batch_mirror.go +++ b/pkg/response/batch_mirror.go @@ -3,11 +3,12 @@ package response import ( "bytes" "fmt" - "github.com/moov-io/ach" - "github.com/moov-io/ach-test-harness/pkg/service" "path/filepath" "sort" "time" + + "github.com/moov-io/ach" + "github.com/moov-io/ach-test-harness/pkg/service" ) // batchMirror is an object that will save batches From f22cc2c17c5c7799a8013a3ba5c08f9dd411a701 Mon Sep 17 00:00:00 2001 From: Jason Bornstein <131717043+jasonbornsteinMOOV@users.noreply.github.com> Date: Thu, 21 Sep 2023 13:06:32 -0400 Subject: [PATCH 4/4] fix formatting --- pkg/response/batch_mirror.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/response/batch_mirror.go b/pkg/response/batch_mirror.go index 96493b49..5a01f736 100644 --- a/pkg/response/batch_mirror.go +++ b/pkg/response/batch_mirror.go @@ -94,7 +94,7 @@ func (bm *batchMirror) saveFiles() error { // sort the keys so that the batches appear in the correct order keys := make([]string, 0, len(mirror)) - for number, _ := range mirror { + for number := range mirror { keys = append(keys, number) } sort.Slice(keys, func(i, j int) bool {