Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

BNK-370 generate multiple batches in a recon file #177

Merged
merged 4 commits into from
Sep 22, 2023
Merged

BNK-370 generate multiple batches in a recon file #177

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d112675
BNK-340 generate multiple batches in a recon file
jasonbornsteinMOOV Sep 21, 2023
aaeb751
fix formatting
jasonbornsteinMOOV Sep 21, 2023
7a4bdf6
fix formatting
jasonbornsteinMOOV Sep 21, 2023
f22cc2c
fix formatting
jasonbornsteinMOOV Sep 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 78 additions & 32 deletions pkg/response/batch_mirror.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@ package response

import (
"bytes"
"errors"
"fmt"
"path/filepath"
"sort"
"time"

"github.com/moov-io/ach"
Expand All @@ -13,51 +13,105 @@ import (

// batchMirror is an object that will save batches
type batchMirror struct {
batches map[batchMirrorKey]map[string]*batchMirrorBatch // path+companyID -> (batch ID -> header+entries+control)

writer FileWriter
}

type batchMirrorKey struct {
path string
companyID string
}

func (key *batchMirrorKey) getFilePathName() string {
timestamp := time.Now().Format("20060102-150405.00000")
filename := fmt.Sprintf("%s_%s.ach", key.companyID, timestamp)
return filepath.Join(key.path, filename)
}

type batchMirrorBatch struct {
header *ach.BatchHeader
entries map[string][]*ach.EntryDetail // filepath -> entries
entries []*ach.EntryDetail
control *ach.BatchControl
}

writer FileWriter
func (batch *batchMirrorBatch) write(buf *bytes.Buffer) error {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How big are these batches likely to get? Is storing them in memory ever going to be a problem?

Copy link
Contributor Author

@jasonbornsteinMOOV jasonbornsteinMOOV Sep 22, 2023 •
edited by adamdecaf
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

in a way, that's going to get tested when we use this for simulating volume.

buf.WriteString(batch.header.String() + "\n")
for _, entry := range batch.entries {
buf.WriteString(entry.String() + "\n")
}
control, err := calculateControl(batch.header, batch.entries)
if err != nil {
return fmt.Errorf("problem computing control: %v", err)
}
buf.WriteString(control + "\n")

return nil
}

func newBatchMirror(w FileWriter, b ach.Batcher) *batchMirror {
func newBatchMirror(w FileWriter) *batchMirror {
return &batchMirror{
header: b.GetHeader(),
entries: make(map[string][]*ach.EntryDetail),
control: b.GetControl(),
batches: make(map[batchMirrorKey]map[string]*batchMirrorBatch),
writer: w,
}
}

func (bm *batchMirror) saveEntry(copy *service.Copy, ed *ach.EntryDetail) {
if copy == nil {
func (bm *batchMirror) saveEntry(b *ach.Batcher, copy *service.Copy, ed *ach.EntryDetail) {
if b == nil || copy == nil || ed == nil {
return
}
bm.entries[copy.Path] = append(bm.entries[copy.Path], ed)

batcher := *b
// Get the batchMirrorKey
key := batchMirrorKey{
path: copy.Path,
companyID: batcher.GetHeader().CompanyIdentification,
}
// Create a new batchMirrorBatch map if this key does not exist
if _, exists := bm.batches[key]; !exists {
bm.batches[key] = make(map[string]*batchMirrorBatch)
}
// Create an array of batchMirrorBatch if this batch ID does not exist
if _, exists := bm.batches[key][batcher.GetHeader().BatchNumberField()]; !exists {
bm.batches[key][batcher.GetHeader().BatchNumberField()] = &batchMirrorBatch{
header: batcher.GetHeader(),
entries: make([]*ach.EntryDetail, 0),
control: batcher.GetControl(),
}
}
// Append this EntryDetail to the batchMirrorBatch's EntryDetails slice for the derived key
bm.batches[key][batcher.GetHeader().BatchNumberField()].entries = append(bm.batches[key][batcher.GetHeader().BatchNumberField()].entries, ed)
}

func (bm *batchMirror) saveFiles() error {
if bm.header == nil || len(bm.entries) == 0 {
if len(bm.batches) == 0 {
return nil
}
for path, entries := range bm.entries {
// Accumulate file contents

// Write files by Path/CompanyID
for key, mirror := range bm.batches {
var buf bytes.Buffer
buf.WriteString(bm.header.String() + "\n")
for i := range entries {
buf.WriteString(entries[i].String() + "\n")

// sort the keys so that the batches appear in the correct order
keys := make([]string, 0, len(mirror))
for number := range mirror {
keys = append(keys, number)
}
control, err := calculateControl(bm.header, entries)
if err != nil {
return fmt.Errorf("problem computing control: %v", err)
sort.Slice(keys, func(i, j int) bool {
return keys[i] < keys[j]
})

for _, val := range keys {
batch := mirror[val]
if err := batch.write(&buf); err != nil {
return err
}
}
buf.WriteString(control)

// Write the file out
if filename, err := bm.filename(); err != nil {
return fmt.Errorf("unable to get filename: %v", err)
} else {
bm.writer.Write(filepath.Join(path, filename), &buf, nil)
err := bm.writer.Write(key.getFilePathName(), &buf, nil)
if err != nil {
return fmt.Errorf("problem writing file: %v", err)
}
}
return nil
Expand All @@ -73,11 +127,3 @@ func calculateControl(bh *ach.BatchHeader, entries []*ach.EntryDetail) (string,
}
return batch.GetControl().String(), nil
}

func (bm *batchMirror) filename() (string, error) {
if bm.header == nil {
return "", errors.New("missing BatchHeader")
}
timestamp := time.Now().Format("20060102-150405.00000")
return fmt.Sprintf("%s_%s.ach", bm.header.CompanyIdentification, timestamp), nil
}
17 changes: 9 additions & 8 deletions pkg/response/file_transformer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,10 @@ func (ft *FileTransfomer) Transform(file *ach.File) error {
// Track ach.File objects to write based on different delay durations, including a default of "0s"
var outFiles = outFiles{}

// batchMirror is used for copying entires to the reconciliation file (if needed)
mirror := newBatchMirror(ft.Writer)

for i := range file.Batches {
// batchMirror is used for copying entires to the reconciliation file (if needed)
mirror := newBatchMirror(ft.Writer, file.Batches[i])

// Track ach.Batcher to write based on different delay durations and whether the batch is for NOC
var outBatches = outBatches{}
Expand All @@ -57,7 +58,7 @@ func (ft *FileTransfomer) Transform(file *ach.File) error {
logger.Log("Processing matched action")

// Save this Entry
mirror.saveEntry(copyAction.Copy, entries[j])
mirror.saveEntry(&file.Batches[i], copyAction.Copy, entries[j])
}
if processAction != nil {
logger := ft.Matcher.Logger.With(processAction)
Expand Down Expand Up @@ -89,11 +90,6 @@ func (ft *FileTransfomer) Transform(file *ach.File) error {
}
}

// Save off the entries as requested
if err := mirror.saveFiles(); err != nil {
return fmt.Errorf("problem saving entries: %v", err)
}

// Create our Batch's Control and other fields
for delay, batchesByCategory := range outBatches {
for _, batch := range batchesByCategory {
Expand All @@ -111,6 +107,11 @@ func (ft *FileTransfomer) Transform(file *ach.File) error {
}
}

// Save off the entries as requested
if err := mirror.saveFiles(); err != nil {
return fmt.Errorf("problem saving entries: %v", err)
}

for delay, out := range outFiles {
if out != nil && len(out.Batches) > 0 {
if err := out.Create(); err != nil {
Expand Down
51 changes: 50 additions & 1 deletion pkg/response/file_transformer_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package response
import (
"os"
"path/filepath"
"sort"
"testing"
"time"

Expand Down Expand Up @@ -137,7 +138,7 @@ func TestFileTransformer_CopyOnly(t *testing.T) {
}

// debit & credit
func TestFileTransformer_CopyOnlyAndCopyOnly(t *testing.T) {
func TestFileTransformer_CopyOnlyAndCopyOnly_SingleBatch(t *testing.T) {
fileTransformer, dir := testFileTransformer(t, respCopyDebit, respCopyCredit)

achIn, err := ach.ReadFile(filepath.Join("..", "..", "testdata", "20230809-144155-102000021.ach"))
Expand Down Expand Up @@ -167,6 +168,54 @@ func TestFileTransformer_CopyOnlyAndCopyOnly(t *testing.T) {
require.Less(t, fInfo.ModTime(), time.Now())
}

// debit & credit
func TestFileTransformer_CopyOnlyAndCopyOnly_MultipleBatches(t *testing.T) {
fileTransformer, dir := testFileTransformer(t, respCopyDebit, respCopyCredit)

achIn, err := ach.ReadFile(filepath.Join("..", "..", "testdata", "20230809-144155-102000021-2Batches.ach"))
require.NoError(t, err)
require.NotNil(t, achIn)
require.Len(t, achIn.Batches, 3)

// transform the file
err = fileTransformer.Transform(achIn)
require.NoError(t, err)

// verify no "returned" files created
retdir := filepath.Join(dir, "returned")
_, err = os.ReadDir(retdir)
require.Error(t, err)

// verify the "reconciliation" file created
recondir := filepath.Join(dir, "reconciliation")
fds, err := os.ReadDir(recondir)
require.NoError(t, err)
require.Len(t, fds, 2)
// sort the files by name, so we can reliably compare them
sort.Slice(fds, func(i, j int) bool {
return fds[i].Name() < fds[j].Name()
})

read, _ := ach.ReadFile(filepath.Join(recondir, fds[0].Name())) // ignore the error b/c this file has no header or control record
require.Len(t, read.Batches, 2)
require.Equal(t, achIn.Batches[0], read.Batches[0])
require.Equal(t, achIn.Batches[1], read.Batches[1])

// verify the timestamp on the file is in the past
fInfo, err := fds[0].Info()
require.NoError(t, err)
require.Less(t, fInfo.ModTime(), time.Now())

read, _ = ach.ReadFile(filepath.Join(recondir, fds[1].Name())) // ignore the error b/c this file has no header or control record
require.Len(t, read.Batches, 1)
require.Equal(t, achIn.Batches[2], read.Batches[0])

// verify the timestamp on the file is in the past
fInfo, err = fds[1].Info()
require.NoError(t, err)
require.Less(t, fInfo.ModTime(), time.Now())
}

// credit
func TestFileTransformer_ReturnOnly(t *testing.T) {
resp := service.Response{
Expand Down
59 changes: 59 additions & 0 deletions testdata/20230809-144155-102000021-2Batches.ach
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
101 04200001302313801041108052100A094101US BANK NA EXAMPLE COMPANY
5200EXAMPLE COMPANY 0231380104PPDBUY WIDGET110808110808 1042000010000001
627021200025998412345 0000027000A271 JULIAN PRICE 0042000010000001
627021200025998412345 0000062000A272 SYDNEY BUTLER 0042000010000002
627021200025998412345 0000209000A273 KEVIN CASTILLO 0042000010000003
627021200025998412345 0000139000A274 ELIZABETH PATTERSON 0042000010000004
627021200025998412345 0000118000A275 MORGAN WALKER 0042000010000005
627021200025998412345 0000206000A278 JULIA LYNCH 0042000010000006
627021200025998412345 0000206000A279 JULIA LYNCH 0042000010000007
627021200025998412345 0000122000A280 ALLISON COLE 0042000010000008
627021200025998412345 0000122000A281 ALLISON COLE 0042000010000009
820000000900190800180000012110000000000000000231380104 042000010000001
5200EXAMPLE COMPANY 0231380104PPDVERIFY 110808110808 1042000010000002
627021200025998412345 0000170000A282 BROOKLYN MURPHY 0042000010000010
627021200025998412345 0000170000A283 BROOKLYN MURPHY 0042000010000011
627021200025998412345 0000250000A284 DANIEL FOSTER 0042000010000012
627021200025998412345 0000250000A285 DANIEL FOSTER 0042000010000013
627021200025998412345 0000262000A286 LUKE CRAIG 0042000010000014
627021200025998412345 0000262000A287 LUKE CRAIG 0042000010000015
627021200025998412345 0000242000A288 LILLIAN JENSEN 0042000010000016
627021200025998412345 0000242000A289 LILLIAN JENSEN 0042000010000017
627021200025998412345 0000087000A290 CHRISTIAN COX 0042000010000018
627021200025998412345 0000150000A291 BRYAN HORTON 0042000010000019
627021200025998412345 0000237000A292 CARLOS COOK 0042000010000020
627021200025998412345 0000237000A294 AUSTIN WATTS 0042000010000021
627021200025998412345 0000219000A295 ALEXANDER ERRY 0042000010000022
627021200025998412345 0000255000A296 RYAN FLETCHER 0042000010000023
627021200025998412345 0000149000A297 PAYTON MCCOY 0042000010000024
627021200025998412345 0000217000A298 DESTINY COLEMAN 0042000010000025
820000001600339200320000033990000000000000000231380104 042000010000002
5200EXAMPLE COMPANY 0231380105PPDVERIFY 110808110808 1042000010000003
622021200025998412345 0000000008A251 NATHAN NELSON 0042000010000101
622021200025998412345 0000000010A252 NATHAN NELSON 0042000010000102
622021200025998412345 0000000002A253 CHARLES REYES 0042000010000103
622021200025998412345 0000000012A254 CHARLES REYES 0042000010000104
622021200025998412345 0000000002A255 SAMANTHA BYRD 0042000010000105
622021200025998412345 0000000011A256 SAMANTHA BYRD 0042000010000106
622021200025998412345 0000000013A259 JOSEPH RUSSELL 0042000010000107
622021200025998412345 0000000004A260 JOSEPH RUSSELL 0042000010000108
622021200025998412345 0000000009A261 CHASE BATES 0042000010000109
622021200025998412345 0000000005A262 CHASE BATES 0042000010000110
622021200025998412345 0000000019A263 HANNAH LEWIS 0042000010000111
622021200025998412345 0000000015A264 HANNAH LEWIS 0042000010000112
622021200025998412345 0000000006A265 MICHAEL GARZA 0042000010000113
622021200025998412345 0000000015A266 MICHAEL GARZA 0042000010000114
622021200025998412345 0000000014A267 WILLIAM HUDSON 0042000010000115
622021200025998412345 0000000009A268 WILLIAM HUDSON 0042000010000116
622021200025998412345 0000000013A269 JOSHUA OBRIEN 0042000010000117
622021200025998412345 0000000009A270 JOSHUA OBRIEN 0042000010000118
820000001800381600360000000000000000000001760231380105 042000010000003
9000005000010000000830136685201000005101000000000000200000000000000000000000000000000000000000
9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999
9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999
9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999
9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999
9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999
9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999
9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999
9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999