From b22cd4659bcf9285e1bdd5c951162183964fd179 Mon Sep 17 00:00:00 2001 From: Vladyslav Pavlenko Date: Wed, 3 Jul 2024 21:33:57 +0300 Subject: [PATCH 1/9] refactor: restructure dbrepo (-> storage) package --- go.mod | 3 ++ go.sum | 41 +++++++++++++++++++ internal/app/run.go | 8 ++-- internal/app/setup.go | 31 +++++++------- internal/handlers/handlers.go | 9 +++- internal/handlers/notifier.go | 2 +- internal/handlers/notifier_test.go | 7 +++- internal/handlers/repository.go | 17 ++++++-- internal/handlers/repository_test.go | 25 +++++------ .../gormrepo/driver.go} | 28 ++++++------- .../gormrepo/subscription.go} | 27 +++--------- 11 files changed, 123 insertions(+), 75 deletions(-) rename internal/{dbrepo/gorm_driver.go => storage/gormrepo/driver.go} (58%) rename internal/{dbrepo/gorm_subscriber.go => storage/gormrepo/subscription.go} (57%) diff --git a/go.mod b/go.mod index be5ba22..d3dda9d 100644 --- a/go.mod +++ b/go.mod @@ -25,11 +25,14 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/klauspost/compress v1.15.9 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 // indirect + github.com/segmentio/kafka-go v0.4.47 // indirect github.com/stretchr/objx v0.5.2 // indirect golang.org/x/crypto v0.24.0 // indirect golang.org/x/mod v0.18.0 // indirect diff --git a/go.sum b/go.sum index 72e2614..b4c737e 100644 --- a/go.sum +++ b/go.sum @@ -31,12 +31,16 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -47,35 +51,55 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 h1:Lt9DzQALzHoDwMBGJ6v8ObDPR0dzr2a6sXTB1Fq7IHs= github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA= +github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0= +github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= github.com/streadway/quantile v0.0.0-20220407130108-4246515d968d h1:X4+kt6zM/OVO6gbJdAfJR60MGPsqCzbtXNnjoGqdfAs= github.com/streadway/quantile v0.0.0-20220407130108-4246515d968d/go.mod h1:lbP8tGiBjZ5YWIc2fzuRpTaz0b/53vT6PEs3QuAWzuU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tsenart/vegeta/v12 v12.11.1 h1:Rbwe7Zxr7sJ+BDTReemeQalYPvKiSV+O7nwmUs20B3E= github.com/tsenart/vegeta/v12 v12.11.1/go.mod h1:swiFmrgpqj2llHURgHYFRFN0tfrIrlnspg01HjwOnSQ= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -83,15 +107,32 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/app/run.go b/internal/app/run.go index 58335d8..0d900fd 100644 --- a/internal/app/run.go +++ b/internal/app/run.go @@ -31,11 +31,11 @@ type scheduler interface { Stop() } -// db defines an interface for the database. -type db interface { - Connect(dsn string) error +// dbConnection defines an interface for the database connection. +type dbConnection interface { + Setup(dsn string) error Close() error - Migrate() error + Migrate(models ...any) error } // Run initializes the application, sets up the database, schedules email tasks, and starts the diff --git a/internal/app/setup.go b/internal/app/setup.go index d35099f..1eb8dd4 100644 --- a/internal/app/setup.go +++ b/internal/app/setup.go @@ -6,7 +6,8 @@ import ( "log" "net/http" - "github.com/vladyslavpavlenko/genesis-api-project/internal/dbrepo" + "github.com/vladyslavpavlenko/genesis-api-project/internal/models" + "github.com/vladyslavpavlenko/genesis-api-project/internal/storage/gormrepo" "github.com/vladyslavpavlenko/genesis-api-project/internal/app/config" @@ -32,7 +33,7 @@ type ( } ) -func setup(app *config.AppConfig) (db, error) { +func setup(app *config.AppConfig) (dbConnection, error) { envs, err := readEnv() if err != nil { return nil, fmt.Errorf("error reading the .env file: %w", err) @@ -60,9 +61,9 @@ func setup(app *config.AppConfig) (db, error) { return nil, errors.New("error setting up email configuration") } - services := setupServices(&envs, dbConn, &http.Client{}) + services := setupServices(&envs) - repo := handlers.NewRepo(app, services) + repo := handlers.NewRepo(app, services, dbConn) handlers.NewHandlers(repo) return dbConn, nil @@ -79,22 +80,22 @@ func readEnv() (envVariables, error) { } // connectDB sets up a GORM database connection and returns an interface. -func connectDB(dsn string) (*dbrepo.GormDB, error) { - var db dbrepo.GormDB +func connectDB(dsn string) (*gormrepo.Connection, error) { + var conn gormrepo.Connection - err := db.Connect(dsn) + err := conn.Setup(dsn) if err != nil { return nil, err } - return &db, nil + return &conn, nil } // migrateDB runs database migrations. -func migrateDB(db *dbrepo.GormDB) error { +func migrateDB(conn *gormrepo.Connection) error { log.Println("Running migrations...") - err := db.Migrate() + err := conn.Migrate(&models.Subscription{}) if err != nil { return fmt.Errorf("error running migrations: %w", err) } @@ -105,17 +106,15 @@ func migrateDB(db *dbrepo.GormDB) error { } // setupServices sets up handlers.Services. -func setupServices(envs *envVariables, dbConn *dbrepo.GormDB, client *http.Client) *handlers.Services { - fetcher := setupFetchersChain(client) - subscriber := dbrepo.NewSubscriptionRepository(dbConn) +func setupServices(envs *envVariables) *handlers.Services { + fetcher := setupFetchersChain(&http.Client{}) sender := &email.GomailSender{ Dialer: gomail.NewDialer("smtp.gmail.com", 587, envs.EmailAddr, envs.EmailPass), } return &handlers.Services{ - Subscriber: subscriber, - Fetcher: fetcher, - Sender: sender, + Fetcher: fetcher, + Sender: sender, } } diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 96e01ae..70d76d3 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -5,6 +5,8 @@ import ( "fmt" "net/http" + "github.com/vladyslavpavlenko/genesis-api-project/internal/email" + "github.com/vladyslavpavlenko/genesis-api-project/pkg/jsonutils" ) @@ -60,8 +62,13 @@ func (m *Repository) Subscribe(w http.ResponseWriter, r *http.Request) { return } + if !email.Email(body.Email).Validate() { + _ = jsonutils.ErrorJSON(w, errors.New("invalid email")) + return + } + // Perform the subscription operation - err = m.Services.Subscriber.AddSubscription(body.Email) + err = m.DB.AddSubscription(body.Email) if err != nil { _ = jsonutils.ErrorJSON(w, err, http.StatusInternalServerError) return diff --git a/internal/handlers/notifier.go b/internal/handlers/notifier.go index 301c49f..07b6d4b 100644 --- a/internal/handlers/notifier.go +++ b/internal/handlers/notifier.go @@ -36,7 +36,7 @@ type ( func (m *Repository) NotifySubscribers() error { var offset int for { - subscriptions, err := m.Services.Subscriber.GetSubscriptions(batchSize, offset) + subscriptions, err := m.DB.GetSubscriptions(batchSize, offset) if err != nil { return err } diff --git a/internal/handlers/notifier_test.go b/internal/handlers/notifier_test.go index 9b63a64..e3b2240 100644 --- a/internal/handlers/notifier_test.go +++ b/internal/handlers/notifier_test.go @@ -3,6 +3,8 @@ package handlers_test import ( "testing" + "github.com/vladyslavpavlenko/genesis-api-project/internal/storage/gormrepo" + "github.com/vladyslavpavlenko/genesis-api-project/internal/app/config" "github.com/stretchr/testify/assert" @@ -16,8 +18,9 @@ func TestNotifySubscribers_Success(t *testing.T) { mockFetcher := new(MockFetcher) mockSender := new(MockSender) appConfig := &config.AppConfig{} - services := setupServicesWithMocks(mockSubscriber, mockFetcher, mockSender) - repo := handlers.NewRepo(appConfig, services) + dbConn := &gormrepo.Connection{} + services := setupServicesWithMocks(mockFetcher, mockSender) + repo := handlers.NewRepo(appConfig, services, dbConn) subscribers := []models.Subscription{{Email: "user@example.com"}} mockSubscriber.On("GetSubscriptions").Return(subscribers, nil) diff --git a/internal/handlers/repository.go b/internal/handlers/repository.go index c2d3dd8..0156026 100644 --- a/internal/handlers/repository.go +++ b/internal/handlers/repository.go @@ -2,30 +2,39 @@ package handlers import ( "github.com/vladyslavpavlenko/genesis-api-project/internal/app/config" + "github.com/vladyslavpavlenko/genesis-api-project/internal/models" ) type ( // Services is the repository type. Services struct { - Subscriber Subscriber - Fetcher Fetcher - Sender Sender + // Subscriber Subscriber + Fetcher Fetcher + Sender Sender } // Repository is the repository type Repository struct { App *config.AppConfig + DB dbConnection Services *Services } + + // dbConnection defines an interface for the database connection. + dbConnection interface { + AddSubscription(emailAddr string) error + GetSubscriptions(limit, offset int) ([]models.Subscription, error) + } ) // Repo the repository used by the handlers var Repo *Repository // NewRepo creates a new Repository -func NewRepo(a *config.AppConfig, services *Services) *Repository { +func NewRepo(a *config.AppConfig, services *Services, conn dbConnection) *Repository { return &Repository{ App: a, + DB: conn, Services: services, } } diff --git a/internal/handlers/repository_test.go b/internal/handlers/repository_test.go index a5e7854..2a7c47d 100644 --- a/internal/handlers/repository_test.go +++ b/internal/handlers/repository_test.go @@ -4,6 +4,8 @@ import ( "context" "testing" + "github.com/vladyslavpavlenko/genesis-api-project/internal/storage/gormrepo" + "github.com/stretchr/testify/mock" "github.com/vladyslavpavlenko/genesis-api-project/internal/email" "github.com/vladyslavpavlenko/genesis-api-project/internal/models" @@ -46,11 +48,10 @@ func (m *MockFetcher) Fetch(ctx context.Context, base, target string) (string, e return args.String(0), args.Error(1) } -func setupServicesWithMocks(subscriber *MockSubscriber, fetcher *MockFetcher, sender *MockSender) *handlers.Services { +func setupServicesWithMocks(fetcher *MockFetcher, sender *MockSender) *handlers.Services { return &handlers.Services{ - Subscriber: subscriber, - Fetcher: fetcher, - Sender: sender, + Fetcher: fetcher, + Sender: sender, } } @@ -58,12 +59,12 @@ func setupServicesWithMocks(subscriber *MockSubscriber, fetcher *MockFetcher, se func TestNewRepo(t *testing.T) { appConfig := &config.AppConfig{} services := &handlers.Services{ - Subscriber: &MockSubscriber{}, - Fetcher: &MockFetcher{}, - Sender: &MockSender{}, + Fetcher: &MockFetcher{}, + Sender: &MockSender{}, } + dbConn := &gormrepo.Connection{} - repo := handlers.NewRepo(appConfig, services) + repo := handlers.NewRepo(appConfig, services, dbConn) assert.NotNil(t, repo) assert.Equal(t, appConfig, repo.App) @@ -74,12 +75,12 @@ func TestNewRepo(t *testing.T) { func TestNewHandlers(t *testing.T) { appConfig := &config.AppConfig{} services := &handlers.Services{ - Subscriber: &MockSubscriber{}, - Fetcher: &MockFetcher{}, - Sender: &MockSender{}, + Fetcher: &MockFetcher{}, + Sender: &MockSender{}, } + dbConn := &gormrepo.Connection{} - repo := handlers.NewRepo(appConfig, services) + repo := handlers.NewRepo(appConfig, services, dbConn) handlers.NewHandlers(repo) assert.Equal(t, repo, handlers.Repo) diff --git a/internal/dbrepo/gorm_driver.go b/internal/storage/gormrepo/driver.go similarity index 58% rename from internal/dbrepo/gorm_driver.go rename to internal/storage/gormrepo/driver.go index 49c45ba..96a701b 100644 --- a/internal/dbrepo/gorm_driver.go +++ b/internal/storage/gormrepo/driver.go @@ -1,22 +1,20 @@ -package dbrepo +package gormrepo import ( "fmt" "log" "time" - "github.com/vladyslavpavlenko/genesis-api-project/internal/models" - "gorm.io/driver/postgres" "gorm.io/gorm" ) -type GormDB struct { - DB *gorm.DB +type Connection struct { + db *gorm.DB } -// Connect implements the DB interface for GormDB. -func (g *GormDB) Connect(dsn string) error { +// Setup sets up a new Connection. +func (c *Connection) Setup(dsn string) error { var counts int64 for { db, err := openDB(dsn) @@ -25,7 +23,7 @@ func (g *GormDB) Connect(dsn string) error { counts++ } else { log.Println("Connected to Postgres!") - g.DB = db + c.db = db return nil } @@ -39,7 +37,7 @@ func (g *GormDB) Connect(dsn string) error { } } -// openDB initializes a new database connection. +// openDB initializes a new gorm.DB database connection. func openDB(dsn string) (*gorm.DB, error) { db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{}) if err != nil { @@ -48,18 +46,20 @@ func openDB(dsn string) (*gorm.DB, error) { return db, nil } -func (g *GormDB) Close() error { - sqlDB, err := g.DB.DB() +// Close closes a database connection. +func (c *Connection) Close() error { + sqlDB, err := c.db.DB() if err != nil { return err } return sqlDB.Close() } -func (g *GormDB) Migrate() error { - err := g.DB.AutoMigrate(&models.Subscription{}) +// Migrate performs a database migration for given models. +func (c *Connection) Migrate(models ...any) error { + err := c.db.AutoMigrate(models...) if err != nil { - return fmt.Errorf("error during migration: %w", err) + return fmt.Errorf("error migrating models: %w", err) } return nil diff --git a/internal/dbrepo/gorm_subscriber.go b/internal/storage/gormrepo/subscription.go similarity index 57% rename from internal/dbrepo/gorm_subscriber.go rename to internal/storage/gormrepo/subscription.go index 4cfec84..07e400b 100644 --- a/internal/dbrepo/gorm_subscriber.go +++ b/internal/storage/gormrepo/subscription.go @@ -1,11 +1,9 @@ -package dbrepo +package gormrepo import ( "errors" "time" - "github.com/vladyslavpavlenko/genesis-api-project/internal/email" - "github.com/vladyslavpavlenko/genesis-api-project/internal/models" "github.com/jackc/pgx/v5/pgconn" @@ -13,26 +11,13 @@ import ( var ErrDuplicateSubscription = errors.New("subscription already exists") -// SubscriberService is a models.Subscription repository. -type SubscriberService struct { - *GormDB -} - -// NewSubscriptionRepository creates a new GormSubscriptionRepository. -func NewSubscriptionRepository(db *GormDB) *SubscriberService { - return &SubscriberService{db} -} - // AddSubscription creates a new Subscription record. -func (s *SubscriberService) AddSubscription(emailAddr string) error { - if !email.Email(emailAddr).Validate() { - return errors.New("invalid email") - } +func (c *Connection) AddSubscription(email string) error { subscription := models.Subscription{ - Email: emailAddr, + Email: email, CreatedAt: time.Now(), } - result := s.DB.Create(&subscription) + result := c.db.Create(&subscription) if result.Error != nil { var pgErr *pgconn.PgError if errors.As(result.Error, &pgErr) && pgErr.Code == "23505" { @@ -46,9 +31,9 @@ func (s *SubscriberService) AddSubscription(emailAddr string) error { // GetSubscriptions returns a paginated list of subscriptions. Limit specify the number of records to be retrieved // Limit conditions can be canceled by using `Limit(-1)`. Offset specify the number of records to skip before starting // to return the records. Offset conditions can be canceled by using `Offset(-1)`. -func (s *SubscriberService) GetSubscriptions(limit, offset int) ([]models.Subscription, error) { +func (c *Connection) GetSubscriptions(limit, offset int) ([]models.Subscription, error) { var subscriptions []models.Subscription - result := s.DB.Limit(limit).Offset(offset).Find(&subscriptions) + result := c.db.Limit(limit).Offset(offset).Find(&subscriptions) if result.Error != nil { return nil, result.Error } From 87562cbebbc0fbe5aa7c36a13d8172c80844b5fd Mon Sep 17 00:00:00 2001 From: Vladyslav Pavlenko Date: Thu, 4 Jul 2024 16:05:52 +0300 Subject: [PATCH 2/9] feat: implement transactional outbox pattern with kafka for email dispatch --- docker-compose.yml | 30 ++++++ go.mod | 3 +- go.sum | 5 + internal/app/config/config.go | 6 +- internal/app/run.go | 117 ++++++++++++++-------- internal/app/setup.go | 37 +++---- internal/email/gomail_sender.go | 9 +- internal/email/gomail_sender_test.go | 6 +- internal/email/sender.go | 9 ++ internal/handlers/handlers.go | 7 +- internal/handlers/notifier.go | 61 +++-------- internal/handlers/notifier_test.go | 13 +-- internal/handlers/repository.go | 5 +- internal/handlers/repository_test.go | 11 +- internal/outbox/event.go | 40 ++++++++ internal/outbox/gormoutbox/outbox.go | 57 +++++++++++ internal/outbox/kafka_consumer.go | 58 +++++++++++ internal/outbox/kafka_producer.go | 88 ++++++++++++++++ internal/storage/gormrepo/driver.go | 8 +- internal/storage/gormrepo/subscription.go | 4 +- 20 files changed, 432 insertions(+), 142 deletions(-) create mode 100644 internal/email/sender.go create mode 100644 internal/outbox/event.go create mode 100644 internal/outbox/gormoutbox/outbox.go create mode 100644 internal/outbox/kafka_consumer.go create mode 100644 internal/outbox/kafka_producer.go diff --git a/docker-compose.yml b/docker-compose.yml index f1be56a..87e666d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,34 @@ services: timeout: 2s retries: 5 + zookeeper: + image: 'wurstmeister/zookeeper:latest' + ports: + - 2181:2181 + restart: always + + kafka: + image: 'wurstmeister/kafka:latest' + ports: + - 9092:9092 + - 29092:29092 + restart: always + environment: + KAFKA_LISTENERS: EXTERNAL_SAME_HOST://:29092,INTERNAL://:9092 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:9092,EXTERNAL_SAME_HOST://localhost:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL_SAME_HOST:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true' + healthcheck: + test: [ "CMD", "kafka-topics.sh", "--list", "--zookeeper", "zookeeper:2181" ] + interval: 3s + timeout: 2s + retries: 10 + volumes: + - /etc/timezone:/etc/timezone:ro + - /etc/localtime:/etc/localtime:ro + api-app: build: context: . @@ -28,6 +56,8 @@ services: depends_on: postgres: condition: service_healthy + kafka: + condition: service_healthy ports: - "8080:8080" restart: always diff --git a/go.mod b/go.mod index d3dda9d..b4ba7eb 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,9 @@ require ( github.com/golang/mock v1.6.0 github.com/jackc/pgx/v5 v5.4.3 github.com/kelseyhightower/envconfig v1.4.0 + github.com/pkg/errors v0.9.1 github.com/robfig/cron/v3 v3.0.1 + github.com/segmentio/kafka-go v0.4.47 github.com/stretchr/testify v1.9.0 github.com/tsenart/vegeta/v12 v12.11.1 golang.org/x/tools v0.22.0 @@ -32,7 +34,6 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 // indirect - github.com/segmentio/kafka-go v0.4.47 // indirect github.com/stretchr/objx v0.5.2 // indirect golang.org/x/crypto v0.24.0 // indirect golang.org/x/mod v0.18.0 // indirect diff --git a/go.sum b/go.sum index b4c737e..569e833 100644 --- a/go.sum +++ b/go.sum @@ -42,6 +42,8 @@ github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= @@ -67,8 +69,11 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tsenart/vegeta/v12 v12.11.1 h1:Rbwe7Zxr7sJ+BDTReemeQalYPvKiSV+O7nwmUs20B3E= github.com/tsenart/vegeta/v12 v12.11.1/go.mod h1:swiFmrgpqj2llHURgHYFRFN0tfrIrlnspg01HjwOnSQ= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= diff --git a/internal/app/config/config.go b/internal/app/config/config.go index dc104f4..d9f33ff 100644 --- a/internal/app/config/config.go +++ b/internal/app/config/config.go @@ -1,15 +1,15 @@ package config import ( - "github.com/vladyslavpavlenko/genesis-api-project/internal/email" + "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox" ) // AppConfig holds the application config. type AppConfig struct { - EmailConfig email.Config + Outbox outbox.Outbox } // NewAppConfig creates a new AppConfig. func NewAppConfig() *AppConfig { - return &AppConfig{EmailConfig: email.Config{}} + return &AppConfig{} } diff --git a/internal/app/run.go b/internal/app/run.go index 0d900fd..5483572 100644 --- a/internal/app/run.go +++ b/internal/app/run.go @@ -11,6 +11,11 @@ import ( "syscall" "time" + "github.com/segmentio/kafka-go" + + "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox" + "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox/gormoutbox" + "github.com/robfig/cron/v3" "github.com/vladyslavpavlenko/genesis-api-project/internal/app/config" @@ -24,78 +29,92 @@ const ( schedule = "0 10 * * *" ) -// scheduler defines an interface for scheduling tasks. -type scheduler interface { - ScheduleTask(schedule string, task func()) (cron.EntryID, error) - Start() - Stop() -} - -// dbConnection defines an interface for the database connection. -type dbConnection interface { - Setup(dsn string) error - Close() error - Migrate(models ...any) error -} +type ( + scheduler interface { + ScheduleTask(schedule string, task func()) (cron.EntryID, error) + Start() + Stop() + } +) -// Run initializes the application, sets up the database, schedules email tasks, and starts the -// HTTP server with graceful shutdown. +// Run is the application running process. func Run(appConfig *config.AppConfig) error { dbConn, err := setup(appConfig) if err != nil { return err } - defer func() { - if closeErr := dbConn.Close(); closeErr != nil { - log.Printf("Error closing the database connection: %v\n", closeErr) - } - }() + defer dbConn.Close() s := schedulerpkg.NewCronScheduler() - err = scheduleEmails(s) - if err != nil { + if err = scheduleEmails(s); err != nil { return fmt.Errorf("failed to schedule emails: %w", err) } s.Start() defer s.Stop() - log.Printf("Running on port %d", webPort) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Start the event producer for Kafka + outboxService, err := gormoutbox.NewOutbox(dbConn.DB) + if err != nil { + return fmt.Errorf("failed to create outbox: %w", err) + } + + appConfig.Outbox = outboxService + + kafkaURL := os.Getenv("KAFKA_URL") + kafkaTopic := "events-topic" + kafkaGroupID := "email-sender" + kafkaWriter := outbox.NewKafkaWriter(kafkaURL, kafkaTopic) + defer kafkaWriter.Close() + go eventProducer(ctx, outboxService, kafkaWriter) + + // Start the event consumer for Kafka + kafkaReader := outbox.NewKafkaReader(kafkaURL, kafkaTopic, kafkaGroupID) + defer kafkaReader.Close() + go eventConsumer(ctx, kafkaReader) + + log.Printf("Running on port %d", webPort) srv := &http.Server{ Addr: fmt.Sprintf(":%d", webPort), Handler: routes.Routes(), ReadHeaderTimeout: 5 * time.Second, } - // Graceful shutdown + // Handle graceful shutdown + handleShutdown(srv, cancel) + + return nil +} + +// handleShutdown handles a graceful shutdown of the application. +func handleShutdown(srv *http.Server, cancelFunc context.CancelFunc) { stop := make(chan os.Signal, 1) signal.Notify(stop, os.Interrupt, syscall.SIGTERM) go func() { - if err = srv.ListenAndServe(); err != nil && !errors.Is(http.ErrServerClosed, err) { - log.Fatalf("HTTP server ListenAndServe: %v", err) + <-stop + cancelFunc() // Cancel context to shut down dispatcher + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + log.Println("Shutting down server...") + if err := srv.Shutdown(ctx); err != nil { + log.Printf("HTTP server shutdown failed: %v", err) } + log.Println("Server has been stopped") }() - // Block until a signal is received - <-stop - - // Set a deadline - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - log.Println("Shutting down...") - if err = srv.Shutdown(ctx); err != nil { - return fmt.Errorf("server shutdown failed: %v", err) + if err := srv.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) { + log.Fatalf("HTTP server ListenAndServe: %v", err) } - - log.Println("Server has been stopped") - return nil } -// scheduleEmails uses the provided Scheduler to set up the mailing function. +// scheduleEmails sets up a mailing process. func scheduleEmails(s scheduler) error { _, err := s.ScheduleTask(schedule, func() { - err := handlers.Repo.NotifySubscribers() + err := handlers.Repo.ProduceMailingEvents() if err != nil { log.Printf("Error notifying subscribers: %v", err) } @@ -106,3 +125,21 @@ func scheduleEmails(s scheduler) error { return nil } + +// eventProducer runs an event dispatcher. +func eventProducer(ctx context.Context, o outbox.Outbox, w *kafka.Writer) { + outbox.Worker(ctx, o, w) + + // Wait for context cancellation to handle graceful shutdown + <-ctx.Done() + log.Println("Shutting down event producer...") +} + +// eventProducer runs an event dispatcher. +func eventConsumer(ctx context.Context, r *kafka.Reader) { + go outbox.ConsumeMessages(ctx, r) + + // Wait for context cancellation to handle graceful shutdown + <-ctx.Done() + log.Println("Shutting down event consumer...") +} diff --git a/internal/app/setup.go b/internal/app/setup.go index 1eb8dd4..0a67404 100644 --- a/internal/app/setup.go +++ b/internal/app/setup.go @@ -1,15 +1,15 @@ package app import ( - "errors" "fmt" "log" "net/http" - "github.com/vladyslavpavlenko/genesis-api-project/internal/models" - "github.com/vladyslavpavlenko/genesis-api-project/internal/storage/gormrepo" + "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox" "github.com/vladyslavpavlenko/genesis-api-project/internal/app/config" + "github.com/vladyslavpavlenko/genesis-api-project/internal/models" + "github.com/vladyslavpavlenko/genesis-api-project/internal/storage/gormrepo" "github.com/vladyslavpavlenko/genesis-api-project/internal/rateapi" "github.com/vladyslavpavlenko/genesis-api-project/internal/rateapi/chain" @@ -33,7 +33,7 @@ type ( } ) -func setup(app *config.AppConfig) (dbConnection, error) { +func setup(app *config.AppConfig) (*gormrepo.Connection, error) { envs, err := readEnv() if err != nil { return nil, fmt.Errorf("error reading the .env file: %w", err) @@ -56,14 +56,16 @@ func setup(app *config.AppConfig) (dbConnection, error) { return nil, fmt.Errorf("error runnning database migrations: %w", err) } - app.EmailConfig, err = email.NewEmailConfig(envs.EmailAddr, envs.EmailPass) + fetcher := setupFetchersChain(&http.Client{}) + + sender, err := setupSender(&envs) if err != nil { - return nil, errors.New("error setting up email configuration") + return nil, fmt.Errorf("error setting up sender: %w", err) } - services := setupServices(&envs) + email.NewSenderService(sender) - repo := handlers.NewRepo(app, services, dbConn) + repo := handlers.NewRepo(app, &handlers.Services{Fetcher: fetcher}, dbConn) handlers.NewHandlers(repo) return dbConn, nil @@ -95,7 +97,7 @@ func connectDB(dsn string) (*gormrepo.Connection, error) { func migrateDB(conn *gormrepo.Connection) error { log.Println("Running migrations...") - err := conn.Migrate(&models.Subscription{}) + err := conn.Migrate(&models.Subscription{}, &outbox.Event{}) if err != nil { return fmt.Errorf("error running migrations: %w", err) } @@ -105,17 +107,16 @@ func migrateDB(conn *gormrepo.Connection) error { return nil } -// setupServices sets up handlers.Services. -func setupServices(envs *envVariables) *handlers.Services { - fetcher := setupFetchersChain(&http.Client{}) - sender := &email.GomailSender{ - Dialer: gomail.NewDialer("smtp.gmail.com", 587, envs.EmailAddr, envs.EmailPass), +func setupSender(envs *envVariables) (sender *email.GomailSender, err error) { + emailConfig, err := email.NewEmailConfig(envs.EmailAddr, envs.EmailPass) + if err != nil { + return nil, fmt.Errorf("error creating email config: %w", err) } - return &handlers.Services{ - Fetcher: fetcher, - Sender: sender, - } + return &email.GomailSender{ + Dialer: gomail.NewDialer("smtp.gmail.com", 587, envs.EmailAddr, envs.EmailPass), + Config: emailConfig, + }, nil } // setupServices sets up a chain of responsibility for fetchers. diff --git a/internal/email/gomail_sender.go b/internal/email/gomail_sender.go index 1f4a2b6..e781f28 100644 --- a/internal/email/gomail_sender.go +++ b/internal/email/gomail_sender.go @@ -4,9 +4,14 @@ import ( "gopkg.in/gomail.v2" ) +type Sender interface { + Send(params Params) error +} + // GomailSender implements the Sender interface for Gomail. type GomailSender struct { Dialer Dialer + Config Config } type GomailDialer struct { @@ -17,9 +22,9 @@ func (d *GomailDialer) DialAndSend(m ...*gomail.Message) error { return d.Dialer.DialAndSend(m...) } -func (gs *GomailSender) Send(cfg Config, params Params) error { +func (gs *GomailSender) Send(params Params) error { m := gomail.NewMessage() - m.SetHeader("From", cfg.Email) + m.SetHeader("From", gs.Config.Email) m.SetHeader("To", params.To) m.SetHeader("Subject", params.Subject) m.SetBody("text/plain", params.Body) diff --git a/internal/email/gomail_sender_test.go b/internal/email/gomail_sender_test.go index 4afe8d6..f5fd4c3 100644 --- a/internal/email/gomail_sender_test.go +++ b/internal/email/gomail_sender_test.go @@ -17,12 +17,11 @@ func TestSend(t *testing.T) { mockDialer := mocks.NewMockDialer(ctrl) gomailSender := email.GomailSender{Dialer: mockDialer} - config := email.Config{Email: "test@example.com", Password: "password"} params := email.Params{To: "recipient@example.com", Subject: "Test", Body: "Hello"} mockDialer.EXPECT().DialAndSend(gomock.Any()).Return(nil) - err := gomailSender.Send(config, params) + err := gomailSender.Send(params) if err != nil { t.Errorf("Send failed: %v", err) } @@ -35,13 +34,12 @@ func TestGomailSenderSendFailure(t *testing.T) { mockDialer := mocks.NewMockDialer(ctrl) sender := email.GomailSender{Dialer: mockDialer} - cfg := email.Config{Email: "test@example.com", Password: "password"} params := email.Params{To: "recipient@example.com", Subject: "Failure Test", Body: "This email should encounter a send error."} testError := errors.New("smtp error") mockDialer.EXPECT().DialAndSend(gomock.Any()).Return(testError) - err := sender.Send(cfg, params) + err := sender.Send(params) assert.Equal(t, testError, err, "Expected a specific error, but got a different one") } diff --git a/internal/email/sender.go b/internal/email/sender.go new file mode 100644 index 0000000..508fe10 --- /dev/null +++ b/internal/email/sender.go @@ -0,0 +1,9 @@ +package email + +// SenderService is the package-level instance of the Sender. +var SenderService Sender + +// NewSenderService sets the Sender for the package. +func NewSenderService(s Sender) { + SenderService = s +} diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 70d76d3..57aca98 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -86,8 +86,8 @@ func (m *Repository) Subscribe(w http.ResponseWriter, r *http.Request) { // SendEmails handles the `/sendEmails` request. func (m *Repository) SendEmails(w http.ResponseWriter, _ *http.Request) { - // Perform the mailing operation - err := m.NotifySubscribers() + // Produce mailing events + err := m.ProduceMailingEvents() if err != nil { _ = jsonutils.ErrorJSON(w, err, http.StatusInternalServerError) return @@ -95,8 +95,7 @@ func (m *Repository) SendEmails(w http.ResponseWriter, _ *http.Request) { // AddSubscription a response payload := jsonutils.Response{ - Error: false, - Message: "sent", + Error: false, } // Send the response back diff --git a/internal/handlers/notifier.go b/internal/handlers/notifier.go index 07b6d4b..dcf60c1 100644 --- a/internal/handlers/notifier.go +++ b/internal/handlers/notifier.go @@ -7,33 +7,25 @@ import ( "strconv" "sync" - "github.com/vladyslavpavlenko/genesis-api-project/internal/models" + "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox" - "github.com/vladyslavpavlenko/genesis-api-project/internal/email" + "github.com/vladyslavpavlenko/genesis-api-project/internal/models" ) const batchSize = 100 -type ( - // Sender defines an interface for sending emails. - Sender interface { - Send(emailConfig email.Config, params email.Params) error - } - - // Fetcher interface defines an interface for fetching rates. - Fetcher interface { - Fetch(ctx context.Context, base, target string) (string, error) +// ProduceMailingEvents handles producing events for currency rate update emails. +func (m *Repository) ProduceMailingEvents() error { + rate, err := m.Services.Fetcher.Fetch(context.Background(), "USD", "UAH") + if err != nil { + return fmt.Errorf("failed to retrieve rate: %w", err) } - // Subscriber interface defines methods to access models.Subscription data. - Subscriber interface { - AddSubscription(string) error - GetSubscriptions(limit, offset int) ([]models.Subscription, error) + floatRate, err := strconv.ParseFloat(rate, 32) + if err != nil { + return fmt.Errorf("failed to parse price: %w", err) } -) -// NotifySubscribers handles sending currency update emails to all the subscribers in batches. -func (m *Repository) NotifySubscribers() error { var offset int for { subscriptions, err := m.DB.GetSubscriptions(batchSize, offset) @@ -49,8 +41,11 @@ func (m *Repository) NotifySubscribers() error { wg.Add(1) go func(sub models.Subscription) { defer wg.Done() - if err = m.sendEmail(sub); err != nil { - log.Println(err) + + data := outbox.Data{Email: sub.Email, Rate: floatRate} + + if err = m.App.Outbox.AddEvent(data); err != nil { + log.Printf("error adding event: %v", err) } }(subscription) } @@ -61,29 +56,3 @@ func (m *Repository) NotifySubscribers() error { return nil } - -// sendEmail is a controller function to prepare and send emails -func (m *Repository) sendEmail(subscription models.Subscription) error { - price, err := m.Services.Fetcher.Fetch(context.Background(), "USD", "UAH") - if err != nil { - return fmt.Errorf("failed to retrieve rate: %w", err) - } - - floatPrice, err := strconv.ParseFloat(price, 32) - if err != nil { - return fmt.Errorf("failed to parse price: %w", err) - } - - params := email.Params{ - To: subscription.Email, - Subject: "USD to UAH Exchange Rate", - Body: fmt.Sprintf("The current exchange rate for USD to UAH is %.2f.", floatPrice), - } - - err = m.Services.Sender.Send(m.App.EmailConfig, params) - if err != nil { - return fmt.Errorf("failed to send email: %w", err) - } - - return nil -} diff --git a/internal/handlers/notifier_test.go b/internal/handlers/notifier_test.go index e3b2240..cc28698 100644 --- a/internal/handlers/notifier_test.go +++ b/internal/handlers/notifier_test.go @@ -14,24 +14,21 @@ import ( ) func TestNotifySubscribers_Success(t *testing.T) { - mockSubscriber := new(MockSubscriber) + mockDB := new(MockDB) mockFetcher := new(MockFetcher) - mockSender := new(MockSender) appConfig := &config.AppConfig{} dbConn := &gormrepo.Connection{} - services := setupServicesWithMocks(mockFetcher, mockSender) + services := setupServicesWithMocks(mockFetcher) repo := handlers.NewRepo(appConfig, services, dbConn) subscribers := []models.Subscription{{Email: "user@example.com"}} - mockSubscriber.On("GetSubscriptions").Return(subscribers, nil) + mockDB.On("GetSubscriptions").Return(subscribers, nil) mockFetcher.On("Fetch", mock.Anything, "USD", "UAH").Return("24.5", nil) - mockSender.On("Send", mock.AnythingOfType("email.Config"), mock.AnythingOfType("email.Params")).Return(nil) - err := repo.NotifySubscribers() + err := repo.ProduceMailingEvents() assert.NoError(t, err) - mockSubscriber.AssertExpectations(t) + mockDB.AssertExpectations(t) mockFetcher.AssertExpectations(t) - mockSender.AssertExpectations(t) } diff --git a/internal/handlers/repository.go b/internal/handlers/repository.go index 0156026..40e1aa2 100644 --- a/internal/handlers/repository.go +++ b/internal/handlers/repository.go @@ -3,14 +3,13 @@ package handlers import ( "github.com/vladyslavpavlenko/genesis-api-project/internal/app/config" "github.com/vladyslavpavlenko/genesis-api-project/internal/models" + "github.com/vladyslavpavlenko/genesis-api-project/internal/rateapi" ) type ( // Services is the repository type. Services struct { - // Subscriber Subscriber - Fetcher Fetcher - Sender Sender + Fetcher rateapi.Fetcher } // Repository is the repository type diff --git a/internal/handlers/repository_test.go b/internal/handlers/repository_test.go index 2a7c47d..ccb3bfe 100644 --- a/internal/handlers/repository_test.go +++ b/internal/handlers/repository_test.go @@ -25,16 +25,16 @@ func (m *MockSender) Send(cfg email.Config, params email.Params) error { return args.Error(0) } -type MockSubscriber struct { +type MockDB struct { mock.Mock } -func (m *MockSubscriber) GetSubscriptions(_, _ int) ([]models.Subscription, error) { +func (m *MockDB) GetSubscriptions(_, _ int) ([]models.Subscription, error) { args := m.Called() return args.Get(0).([]models.Subscription), args.Error(1) } -func (m *MockSubscriber) AddSubscription(emailAddr string) error { +func (m *MockDB) AddSubscription(emailAddr string) error { args := m.Called(emailAddr) return args.Error(0) } @@ -48,10 +48,9 @@ func (m *MockFetcher) Fetch(ctx context.Context, base, target string) (string, e return args.String(0), args.Error(1) } -func setupServicesWithMocks(fetcher *MockFetcher, sender *MockSender) *handlers.Services { +func setupServicesWithMocks(fetcher *MockFetcher) *handlers.Services { return &handlers.Services{ Fetcher: fetcher, - Sender: sender, } } @@ -60,7 +59,6 @@ func TestNewRepo(t *testing.T) { appConfig := &config.AppConfig{} services := &handlers.Services{ Fetcher: &MockFetcher{}, - Sender: &MockSender{}, } dbConn := &gormrepo.Connection{} @@ -76,7 +74,6 @@ func TestNewHandlers(t *testing.T) { appConfig := &config.AppConfig{} services := &handlers.Services{ Fetcher: &MockFetcher{}, - Sender: &MockSender{}, } dbConn := &gormrepo.Connection{} diff --git a/internal/outbox/event.go b/internal/outbox/event.go new file mode 100644 index 0000000..75e58f8 --- /dev/null +++ b/internal/outbox/event.go @@ -0,0 +1,40 @@ +package outbox + +import ( + "encoding/json" + "time" +) + +// Event is a query message model stored in the database. +type Event struct { + ID uint `gorm:"primaryKey" json:"id"` + Data string `json:"data"` + Published bool `json:"published"` + CreatedAt time.Time `json:"created_at"` +} + +// Data is an event data model. +type Data struct { + Email string `json:"email"` + Rate float64 `json:"rate"` +} + +// SerializeData takes a Data struct and serializes it to a JSON string for storage. +func (e *Event) SerializeData(data Data) error { + bytes, err := json.Marshal(data) + if err != nil { + return err + } + e.Data = string(bytes) + return nil +} + +// DeserializeData deserializes JSON string to Data struct +func DeserializeData(jsonData []byte) (Data, error) { + var data Data + err := json.Unmarshal(jsonData, &data) + if err != nil { + return Data{}, err + } + return data, nil +} diff --git a/internal/outbox/gormoutbox/outbox.go b/internal/outbox/gormoutbox/outbox.go new file mode 100644 index 0000000..502ec77 --- /dev/null +++ b/internal/outbox/gormoutbox/outbox.go @@ -0,0 +1,57 @@ +package gormoutbox + +import ( + "fmt" + "time" + + "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox" + + "github.com/pkg/errors" + "gorm.io/gorm" +) + +// Outbox is the repository for this package. +type Outbox struct { + db *gorm.DB +} + +// NewOutbox creates a new `events` table to serve as an outbox. +func NewOutbox(db *gorm.DB) (*Outbox, error) { + err := db.AutoMigrate(&outbox.Event{}) + if err != nil { + return nil, errors.Wrap(err, "failed to migrate events") + } + return &Outbox{db: db}, nil +} + +// AddEvent creates a new Event record. +func (o *Outbox) AddEvent(data outbox.Data) error { + event := &outbox.Event{ + Published: false, + CreatedAt: time.Now(), + } + + err := event.SerializeData(data) + if err != nil { + return fmt.Errorf("failed to serialize data: %w", err) + } + + return o.db.Create(event).Error +} + +// GetUnpublishedEvents retrieves all events that haven't been published yet. +func (o *Outbox) GetUnpublishedEvents() ([]outbox.Event, error) { + var events []outbox.Event + err := o.db.Where("published = ?", false).Find(&events).Error + return events, err +} + +// MarkEventAsPublished marks an event as published. +func (o *Outbox) MarkEventAsPublished(eventID uint) error { + return o.db.Model(&outbox.Event{}).Where("id = ?", eventID).Update("published", true).Error +} + +// Cleanup deletes all the Event records that have already been published or are outdated. +func (o *Outbox) Cleanup() { + o.db.Where("published = ? AND created_at <= ?", true, time.Now().AddDate(0, 0, -1)).Delete(&outbox.Event{}) +} diff --git a/internal/outbox/kafka_consumer.go b/internal/outbox/kafka_consumer.go new file mode 100644 index 0000000..0be89ad --- /dev/null +++ b/internal/outbox/kafka_consumer.go @@ -0,0 +1,58 @@ +package outbox + +import ( + "context" + "fmt" + "log" + + "github.com/vladyslavpavlenko/genesis-api-project/internal/email" + + "github.com/segmentio/kafka-go" +) + +// NewKafkaReader initializes a new kafka.Reader with a specific topic and group. +func NewKafkaReader(kafkaURL, topic, groupID string) *kafka.Reader { + return kafka.NewReader(kafka.ReaderConfig{ + Brokers: []string{kafkaURL}, + Topic: topic, + GroupID: groupID, + Partition: 0, + MinBytes: 10e3, + MaxBytes: 10e6, + }) +} + +// ConsumeMessages reads messages from Kafka, deserializes them into Event structs, and processes them. +func ConsumeMessages(ctx context.Context, reader *kafka.Reader) { + for { + m, err := reader.ReadMessage(ctx) + if err != nil { + log.Printf("Failed to read message: %v", err) + continue + } + + log.Printf("Message at offset %d: %s = %s\n", m.Offset, string(m.Key), string(m.Value)) + + data, err := DeserializeData(m.Value) + if err != nil { + log.Printf("Failed to deserialize data from message: %v", err) + } + + sendMessage(data) + } +} + +func sendMessage(data Data) { + params := email.Params{ + To: data.Email, + Subject: "USD to UAH Exchange Rate", + Body: fmt.Sprintf("The current exchange rate for USD to UAH is %.2f.", data.Rate), + } + + log.Printf("Sending email to %s", data.Email) + + err := email.SenderService.Send(params) + if err != nil { + log.Printf("Failed to send email: %v", err) + } +} diff --git a/internal/outbox/kafka_producer.go b/internal/outbox/kafka_producer.go new file mode 100644 index 0000000..7adf385 --- /dev/null +++ b/internal/outbox/kafka_producer.go @@ -0,0 +1,88 @@ +package outbox + +import ( + "context" + "fmt" + "log" + "time" + + "github.com/segmentio/kafka-go" +) + +type Outbox interface { + AddEvent(data Data) error + GetUnpublishedEvents() ([]Event, error) + MarkEventAsPublished(eventID uint) error + Cleanup() +} + +// NewKafkaWriter initializes a new kafka.Writer and returns a reference to it. +func NewKafkaWriter(kafkaURL, topic string) *kafka.Writer { + return &kafka.Writer{ + Addr: kafka.TCP(kafkaURL), + Topic: topic, + Balancer: &kafka.LeastBytes{}, + AllowAutoTopicCreation: true, + } +} + +// Worker fetches for unpublished events, publishes them, and marks them as published. +func Worker(ctx context.Context, outbox Outbox, writer *kafka.Writer) { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + log.Println("Shutting down worker...") + return + case <-ticker.C: + processEvents(outbox, writer) + } + } +} + +// processEvents handles the retrieval and publishing of events. +func processEvents(outbox Outbox, writer *kafka.Writer) { + events, err := outbox.GetUnpublishedEvents() + if err != nil { + log.Println("Error fetching events:", err) + return + } + + for _, event := range events { + if err := publishMessage(writer, []byte(event.Data)); err == nil { + if err = outbox.MarkEventAsPublished(event.ID); err != nil { + log.Printf("Failed to mark event %d as published: %v", event.ID, err) + continue + } + } else { + log.Printf("Failed to publish message for event %d: %v", event.ID, err) + } + } + + outbox.Cleanup() +} + +func publishMessage(writer *kafka.Writer, message []byte) error { + // Establishing connection to the leader of the partition + conn, err := kafka.DialLeader(context.Background(), "tcp", writer.Addr.String(), writer.Topic, 0) + if err != nil { + log.Printf("Failed to dial leader: %v", err) + return err + } + defer conn.Close() + + // Writing message to the leader + _, err = conn.WriteMessages( + kafka.Message{ + Key: []byte(fmt.Sprintf("Key-%d", time.Now().Unix())), + Value: message, + }, + ) + if err != nil { + log.Printf("Failed to write to leader: %v", err) + return err + } + return nil +} diff --git a/internal/storage/gormrepo/driver.go b/internal/storage/gormrepo/driver.go index 96a701b..c2802db 100644 --- a/internal/storage/gormrepo/driver.go +++ b/internal/storage/gormrepo/driver.go @@ -10,7 +10,7 @@ import ( ) type Connection struct { - db *gorm.DB + DB *gorm.DB } // Setup sets up a new Connection. @@ -23,7 +23,7 @@ func (c *Connection) Setup(dsn string) error { counts++ } else { log.Println("Connected to Postgres!") - c.db = db + c.DB = db return nil } @@ -48,7 +48,7 @@ func openDB(dsn string) (*gorm.DB, error) { // Close closes a database connection. func (c *Connection) Close() error { - sqlDB, err := c.db.DB() + sqlDB, err := c.DB.DB() if err != nil { return err } @@ -57,7 +57,7 @@ func (c *Connection) Close() error { // Migrate performs a database migration for given models. func (c *Connection) Migrate(models ...any) error { - err := c.db.AutoMigrate(models...) + err := c.DB.AutoMigrate(models...) if err != nil { return fmt.Errorf("error migrating models: %w", err) } diff --git a/internal/storage/gormrepo/subscription.go b/internal/storage/gormrepo/subscription.go index 07e400b..cbcf80a 100644 --- a/internal/storage/gormrepo/subscription.go +++ b/internal/storage/gormrepo/subscription.go @@ -17,7 +17,7 @@ func (c *Connection) AddSubscription(email string) error { Email: email, CreatedAt: time.Now(), } - result := c.db.Create(&subscription) + result := c.DB.Create(&subscription) if result.Error != nil { var pgErr *pgconn.PgError if errors.As(result.Error, &pgErr) && pgErr.Code == "23505" { @@ -33,7 +33,7 @@ func (c *Connection) AddSubscription(email string) error { // to return the records. Offset conditions can be canceled by using `Offset(-1)`. func (c *Connection) GetSubscriptions(limit, offset int) ([]models.Subscription, error) { var subscriptions []models.Subscription - result := c.db.Limit(limit).Offset(offset).Find(&subscriptions) + result := c.DB.Limit(limit).Offset(offset).Find(&subscriptions) if result.Error != nil { return nil, result.Error } From 91097f994a978c0790e3dc909e51f6fcb548d339 Mon Sep 17 00:00:00 2001 From: Vladyslav Pavlenko Date: Fri, 5 Jul 2024 17:45:15 +0300 Subject: [PATCH 3/9] feat: add /unsubscribe route & handler --- .../consumer}/kafka_consumer.go | 25 ++++---- internal/email/sender.go | 9 --- internal/email/sender_test.go | 35 ------------ internal/handlers/handlers.go | 57 ++++++++++++------- internal/handlers/repository.go | 1 + internal/handlers/repository_test.go | 5 ++ internal/handlers/routes/routes.go | 1 + internal/storage/gormrepo/subscription.go | 23 +++++++- 8 files changed, 80 insertions(+), 76 deletions(-) rename internal/{outbox => email/consumer}/kafka_consumer.go (66%) delete mode 100644 internal/email/sender.go delete mode 100644 internal/email/sender_test.go diff --git a/internal/outbox/kafka_consumer.go b/internal/email/consumer/kafka_consumer.go similarity index 66% rename from internal/outbox/kafka_consumer.go rename to internal/email/consumer/kafka_consumer.go index 0be89ad..df58bc2 100644 --- a/internal/outbox/kafka_consumer.go +++ b/internal/email/consumer/kafka_consumer.go @@ -11,34 +11,39 @@ import ( ) // NewKafkaReader initializes a new kafka.Reader with a specific topic and group. -func NewKafkaReader(kafkaURL, topic, groupID string) *kafka.Reader { +func NewKafkaReader(kafkaURL, topic string, partition int) *kafka.Reader { return kafka.NewReader(kafka.ReaderConfig{ - Brokers: []string{kafkaURL}, - Topic: topic, - GroupID: groupID, - Partition: 0, - MinBytes: 10e3, - MaxBytes: 10e6, + Brokers: []string{kafkaURL}, + Topic: topic, + Partition: partition, + CommitInterval: 0, // disable auto-commit }) } // ConsumeMessages reads messages from Kafka, deserializes them into Event structs, and processes them. func ConsumeMessages(ctx context.Context, reader *kafka.Reader) { for { - m, err := reader.ReadMessage(ctx) + // Read a message from Kafka + m, err := reader.FetchMessage(ctx) if err != nil { log.Printf("Failed to read message: %v", err) continue } - log.Printf("Message at offset %d: %s = %s\n", m.Offset, string(m.Key), string(m.Value)) - + // Deserialize the data from the message data, err := DeserializeData(m.Value) if err != nil { log.Printf("Failed to deserialize data from message: %v", err) + continue } + // Process the message sendMessage(data) + + // Commit the offset after processing the message + if err = reader.CommitMessages(ctx, m); err != nil { + log.Printf("Failed to commit message offset: %v", err) + } } } diff --git a/internal/email/sender.go b/internal/email/sender.go deleted file mode 100644 index 508fe10..0000000 --- a/internal/email/sender.go +++ /dev/null @@ -1,9 +0,0 @@ -package email - -// SenderService is the package-level instance of the Sender. -var SenderService Sender - -// NewSenderService sets the Sender for the package. -func NewSenderService(s Sender) { - SenderService = s -} diff --git a/internal/email/sender_test.go b/internal/email/sender_test.go deleted file mode 100644 index 0583bcf..0000000 --- a/internal/email/sender_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package email_test - -// -// import ( -// "errors" -// "sync" -// "testing" -// -// "github.com/vladyslavpavlenko/genesis-api-project/internal/email" -//) -// -// func TestSendEmail_InvalidEmail(_ *testing.T) { -// var wg sync.WaitGroup -// wg.Add(1) -// -// cfg := email.Config{ -// Email: "invalidemail", -// Password: "password", -// } -// -// params := email.Params{ -// To: "recipient@example.com", -// Subject: "Test Subject", -// Body: "Test Body", -// } -// -// mockSender := MockEmailSender{ -// SendFunc: func(_ email.Config, _ email.Params) error { -// return errors.New("invalid email address") -// }, -// } -// -// go email.SendEmail(mockSender, cfg, params) -// wg.Wait() -// } diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 57aca98..723b11c 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -5,7 +5,7 @@ import ( "fmt" "net/http" - "github.com/vladyslavpavlenko/genesis-api-project/internal/email" + emailpkg "github.com/vladyslavpavlenko/genesis-api-project/internal/email" "github.com/vladyslavpavlenko/genesis-api-project/pkg/jsonutils" ) @@ -40,47 +40,66 @@ func (m *Repository) GetRate(w http.ResponseWriter, r *http.Request) { _ = jsonutils.WriteJSON(w, http.StatusOK, payload) } -// subscriptionBody is the email subscription request body structure. -type subscriptionBody struct { - Email string `json:"email"` +// parseEmailFromRequest parses the email from the multipart form and validates it. +func parseEmailFromRequest(r *http.Request) (string, error) { + err := r.ParseMultipartForm(10 << 20) + if err != nil { + return "", errors.New("failed to parse form") + } + + emailAddr := r.FormValue("email") + if emailAddr == "" { + return "", errors.New("email is required") + } + + if !emailpkg.Email(emailAddr).Validate() { + return "", errors.New("invalid email") + } + + return emailAddr, nil } // Subscribe handles the `/subscribe` request. func (m *Repository) Subscribe(w http.ResponseWriter, r *http.Request) { - // Parse the form - var body subscriptionBody - - err := r.ParseMultipartForm(10 << 20) + email, err := parseEmailFromRequest(r) if err != nil { - _ = jsonutils.ErrorJSON(w, errors.New("failed to parse form")) + _ = jsonutils.ErrorJSON(w, err) return } - body.Email = r.FormValue("email") - if body.Email == "" { - _ = jsonutils.ErrorJSON(w, errors.New("email is required")) + err = m.DB.AddSubscription(email) + if err != nil { + _ = jsonutils.ErrorJSON(w, err, http.StatusInternalServerError) return } - if !email.Email(body.Email).Validate() { - _ = jsonutils.ErrorJSON(w, errors.New("invalid email")) + payload := jsonutils.Response{ + Error: false, + Message: "subscribed", + } + + _ = jsonutils.WriteJSON(w, http.StatusOK, payload) +} + +// Unsubscribe handles the `/unsubscribe` request. +func (m *Repository) Unsubscribe(w http.ResponseWriter, r *http.Request) { + email, err := parseEmailFromRequest(r) + if err != nil { + _ = jsonutils.ErrorJSON(w, err) return } - // Perform the subscription operation - err = m.DB.AddSubscription(body.Email) + err = m.DB.DeleteSubscription(email) if err != nil { _ = jsonutils.ErrorJSON(w, err, http.StatusInternalServerError) return } - // AddSubscription a response payload := jsonutils.Response{ Error: false, - Message: "subscribed", + Message: "unsubscribed", } - // Send the response back _ = jsonutils.WriteJSON(w, http.StatusOK, payload) } diff --git a/internal/handlers/repository.go b/internal/handlers/repository.go index 40e1aa2..ff9ed96 100644 --- a/internal/handlers/repository.go +++ b/internal/handlers/repository.go @@ -22,6 +22,7 @@ type ( // dbConnection defines an interface for the database connection. dbConnection interface { AddSubscription(emailAddr string) error + DeleteSubscription(emailAddr string) error GetSubscriptions(limit, offset int) ([]models.Subscription, error) } ) diff --git a/internal/handlers/repository_test.go b/internal/handlers/repository_test.go index ccb3bfe..2781d96 100644 --- a/internal/handlers/repository_test.go +++ b/internal/handlers/repository_test.go @@ -39,6 +39,11 @@ func (m *MockDB) AddSubscription(emailAddr string) error { return args.Error(0) } +func (m *MockDB) DeleteSubscription(emailAddr string) error { + args := m.Called(emailAddr) + return args.Error(0) +} + type MockFetcher struct { mock.Mock } diff --git a/internal/handlers/routes/routes.go b/internal/handlers/routes/routes.go index d6b5c28..0fcd615 100644 --- a/internal/handlers/routes/routes.go +++ b/internal/handlers/routes/routes.go @@ -18,6 +18,7 @@ func Routes() http.Handler { mux.Route("/v1", func(mux chi.Router) { mux.Get("/rate", handlers.Repo.GetRate) mux.Post("/subscribe", handlers.Repo.Subscribe) + mux.Post("/unsubscribe", handlers.Repo.Unsubscribe) mux.Post("/sendEmails", handlers.Repo.SendEmails) }) }) diff --git a/internal/storage/gormrepo/subscription.go b/internal/storage/gormrepo/subscription.go index cbcf80a..79cf96e 100644 --- a/internal/storage/gormrepo/subscription.go +++ b/internal/storage/gormrepo/subscription.go @@ -9,9 +9,12 @@ import ( "github.com/jackc/pgx/v5/pgconn" ) -var ErrDuplicateSubscription = errors.New("subscription already exists") +var ( + ErrorDuplicateSubscription = errors.New("subscription already exists") + ErrorNonExistentSubscription = errors.New("subscription does not exist") +) -// AddSubscription creates a new Subscription record. +// AddSubscription creates a new models.Subscription record. func (c *Connection) AddSubscription(email string) error { subscription := models.Subscription{ Email: email, @@ -21,13 +24,27 @@ func (c *Connection) AddSubscription(email string) error { if result.Error != nil { var pgErr *pgconn.PgError if errors.As(result.Error, &pgErr) && pgErr.Code == "23505" { - return ErrDuplicateSubscription + return ErrorDuplicateSubscription } return result.Error } return nil } +// DeleteSubscription deletes a models.Subscription record. +func (c *Connection) DeleteSubscription(email string) error { + result := c.DB.Where("email = ?", email).Delete(&models.Subscription{}) + if result.Error != nil { + return result.Error + } + + if result.RowsAffected == 0 { + return ErrorNonExistentSubscription + } + + return nil +} + // GetSubscriptions returns a paginated list of subscriptions. Limit specify the number of records to be retrieved // Limit conditions can be canceled by using `Limit(-1)`. Offset specify the number of records to skip before starting // to return the records. Offset conditions can be canceled by using `Offset(-1)`. From 0212fe456eb4a1aca0adc1cbd65420a95a334777 Mon Sep 17 00:00:00 2001 From: Vladyslav Pavlenko Date: Fri, 5 Jul 2024 17:46:55 +0300 Subject: [PATCH 4/9] refactor: reorganize email, outbox, & app packages --- internal/app/run.go | 25 +++++++++------- internal/app/setup.go | 17 +++++++---- internal/email/consumer/kafka_consumer.go | 35 ++++++++++++++++------- internal/email/gomail_sender.go | 4 --- internal/outbox/event.go | 1 - internal/outbox/gormoutbox/outbox.go | 1 - 6 files changed, 51 insertions(+), 32 deletions(-) diff --git a/internal/app/run.go b/internal/app/run.go index 5483572..e719c6d 100644 --- a/internal/app/run.go +++ b/internal/app/run.go @@ -11,6 +11,8 @@ import ( "syscall" "time" + consumerpkg "github.com/vladyslavpavlenko/genesis-api-project/internal/email/consumer" + "github.com/segmentio/kafka-go" "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox" @@ -35,15 +37,19 @@ type ( Start() Stop() } + + consumer interface { + ConsumeMessages(ctx context.Context) + } ) // Run is the application running process. func Run(appConfig *config.AppConfig) error { - dbConn, err := setup(appConfig) + appServices, err := setup(appConfig) if err != nil { return err } - defer dbConn.Close() + defer appServices.DBConn.Close() s := schedulerpkg.NewCronScheduler() if err = scheduleEmails(s); err != nil { @@ -56,7 +62,7 @@ func Run(appConfig *config.AppConfig) error { defer cancel() // Start the event producer for Kafka - outboxService, err := gormoutbox.NewOutbox(dbConn.DB) + outboxService, err := gormoutbox.NewOutbox(appServices.DBConn.DB) if err != nil { return fmt.Errorf("failed to create outbox: %w", err) } @@ -64,17 +70,16 @@ func Run(appConfig *config.AppConfig) error { appConfig.Outbox = outboxService kafkaURL := os.Getenv("KAFKA_URL") - kafkaTopic := "events-topic" - kafkaGroupID := "email-sender" + kafkaTopic := "email-topic" kafkaWriter := outbox.NewKafkaWriter(kafkaURL, kafkaTopic) defer kafkaWriter.Close() go eventProducer(ctx, outboxService, kafkaWriter) // Start the event consumer for Kafka - kafkaReader := outbox.NewKafkaReader(kafkaURL, kafkaTopic, kafkaGroupID) - defer kafkaReader.Close() - go eventConsumer(ctx, kafkaReader) + kafkaConsumer := consumerpkg.NewKafkaConsumer(kafkaURL, kafkaTopic, 1) + defer kafkaConsumer.Reader.Close() + go eventConsumer(ctx, kafkaConsumer) log.Printf("Running on port %d", webPort) srv := &http.Server{ @@ -136,8 +141,8 @@ func eventProducer(ctx context.Context, o outbox.Outbox, w *kafka.Writer) { } // eventProducer runs an event dispatcher. -func eventConsumer(ctx context.Context, r *kafka.Reader) { - go outbox.ConsumeMessages(ctx, r) +func eventConsumer(ctx context.Context, c consumer) { + go c.ConsumeMessages(ctx) // Wait for context cancellation to handle graceful shutdown <-ctx.Done() diff --git a/internal/app/setup.go b/internal/app/setup.go index 0a67404..3e057c0 100644 --- a/internal/app/setup.go +++ b/internal/app/setup.go @@ -31,9 +31,14 @@ type ( EmailAddr string `envconfig:"EMAIL_ADDR"` EmailPass string `envconfig:"EMAIL_PASS"` } + + services struct { + DBConn *gormrepo.Connection + Sender *email.GomailSender + } ) -func setup(app *config.AppConfig) (*gormrepo.Connection, error) { +func setup(app *config.AppConfig) (*services, error) { envs, err := readEnv() if err != nil { return nil, fmt.Errorf("error reading the .env file: %w", err) @@ -63,12 +68,13 @@ func setup(app *config.AppConfig) (*gormrepo.Connection, error) { return nil, fmt.Errorf("error setting up sender: %w", err) } - email.NewSenderService(sender) - repo := handlers.NewRepo(app, &handlers.Services{Fetcher: fetcher}, dbConn) handlers.NewHandlers(repo) - return dbConn, nil + return &services{ + DBConn: dbConn, + Sender: sender, + }, nil } // readEnv reads and returns the environmental variables as an envVariables object. @@ -107,6 +113,7 @@ func migrateDB(conn *gormrepo.Connection) error { return nil } +// setupSender sets up a Sender service. func setupSender(envs *envVariables) (sender *email.GomailSender, err error) { emailConfig, err := email.NewEmailConfig(envs.EmailAddr, envs.EmailPass) if err != nil { @@ -119,7 +126,7 @@ func setupSender(envs *envVariables) (sender *email.GomailSender, err error) { }, nil } -// setupServices sets up a chain of responsibility for fetchers. +// setupFetchersChain sets up a chain of responsibility for fetchers. func setupFetchersChain(client *http.Client) *chain.Node { coinbaseFetcher := rateapi.NewFetcherWithLogger("coinbase", rateapi.NewCoinbaseFetcher(client)) diff --git a/internal/email/consumer/kafka_consumer.go b/internal/email/consumer/kafka_consumer.go index df58bc2..83be4cb 100644 --- a/internal/email/consumer/kafka_consumer.go +++ b/internal/email/consumer/kafka_consumer.go @@ -1,53 +1,66 @@ -package outbox +package consumer import ( "context" "fmt" "log" + "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox" + "github.com/vladyslavpavlenko/genesis-api-project/internal/email" "github.com/segmentio/kafka-go" ) -// NewKafkaReader initializes a new kafka.Reader with a specific topic and group. -func NewKafkaReader(kafkaURL, topic string, partition int) *kafka.Reader { - return kafka.NewReader(kafka.ReaderConfig{ +type Sender interface { + Send(params email.Params) error +} + +type KafkaConsumer struct { + Reader *kafka.Reader + Sender Sender +} + +// NewKafkaConsumer initializes a new KafkaConsumer. +func NewKafkaConsumer(kafkaURL, topic string, partition int) *KafkaConsumer { + reader := kafka.NewReader(kafka.ReaderConfig{ Brokers: []string{kafkaURL}, Topic: topic, Partition: partition, CommitInterval: 0, // disable auto-commit }) + + return &KafkaConsumer{Reader: reader} } // ConsumeMessages reads messages from Kafka, deserializes them into Event structs, and processes them. -func ConsumeMessages(ctx context.Context, reader *kafka.Reader) { +func (c *KafkaConsumer) ConsumeMessages(ctx context.Context) { for { // Read a message from Kafka - m, err := reader.FetchMessage(ctx) + m, err := c.Reader.FetchMessage(ctx) if err != nil { log.Printf("Failed to read message: %v", err) continue } // Deserialize the data from the message - data, err := DeserializeData(m.Value) + data, err := outbox.DeserializeData(m.Value) if err != nil { log.Printf("Failed to deserialize data from message: %v", err) continue } // Process the message - sendMessage(data) + sendMessage(data, c.Sender) // Commit the offset after processing the message - if err = reader.CommitMessages(ctx, m); err != nil { + if err = c.Reader.CommitMessages(ctx, m); err != nil { log.Printf("Failed to commit message offset: %v", err) } } } -func sendMessage(data Data) { +func sendMessage(data outbox.Data, sender Sender) { params := email.Params{ To: data.Email, Subject: "USD to UAH Exchange Rate", @@ -56,7 +69,7 @@ func sendMessage(data Data) { log.Printf("Sending email to %s", data.Email) - err := email.SenderService.Send(params) + err := sender.Send(params) if err != nil { log.Printf("Failed to send email: %v", err) } diff --git a/internal/email/gomail_sender.go b/internal/email/gomail_sender.go index e781f28..3b2e191 100644 --- a/internal/email/gomail_sender.go +++ b/internal/email/gomail_sender.go @@ -4,10 +4,6 @@ import ( "gopkg.in/gomail.v2" ) -type Sender interface { - Send(params Params) error -} - // GomailSender implements the Sender interface for Gomail. type GomailSender struct { Dialer Dialer diff --git a/internal/outbox/event.go b/internal/outbox/event.go index 75e58f8..01aa689 100644 --- a/internal/outbox/event.go +++ b/internal/outbox/event.go @@ -9,7 +9,6 @@ import ( type Event struct { ID uint `gorm:"primaryKey" json:"id"` Data string `json:"data"` - Published bool `json:"published"` CreatedAt time.Time `json:"created_at"` } diff --git a/internal/outbox/gormoutbox/outbox.go b/internal/outbox/gormoutbox/outbox.go index 502ec77..89010e0 100644 --- a/internal/outbox/gormoutbox/outbox.go +++ b/internal/outbox/gormoutbox/outbox.go @@ -27,7 +27,6 @@ func NewOutbox(db *gorm.DB) (*Outbox, error) { // AddEvent creates a new Event record. func (o *Outbox) AddEvent(data outbox.Data) error { event := &outbox.Event{ - Published: false, CreatedAt: time.Now(), } From 6a768f917f737b1bb0a5fafbbb57bc3c5181cf47 Mon Sep 17 00:00:00 2001 From: Vladyslav Pavlenko Date: Sat, 6 Jul 2024 18:16:22 +0300 Subject: [PATCH 5/9] refactor: add repository for producer, add topic creation, minor tweaks --- internal/app/run.go | 52 ++++++++++++-------- internal/email/consumer/kafka_consumer.go | 4 +- internal/outbox/gormoutbox/outbox.go | 1 - internal/outbox/kafka_producer.go | 59 ++++++++++++++++++----- internal/outbox/offset.go | 7 +++ internal/scheduler/cron_scheduler.go | 4 +- internal/scheduler/cron_scheduler_test.go | 6 +-- 7 files changed, 93 insertions(+), 40 deletions(-) create mode 100644 internal/outbox/offset.go diff --git a/internal/app/run.go b/internal/app/run.go index e719c6d..c1efb16 100644 --- a/internal/app/run.go +++ b/internal/app/run.go @@ -12,9 +12,6 @@ import ( "time" consumerpkg "github.com/vladyslavpavlenko/genesis-api-project/internal/email/consumer" - - "github.com/segmentio/kafka-go" - "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox" "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox/gormoutbox" @@ -31,17 +28,24 @@ const ( schedule = "0 10 * * *" ) -type ( - scheduler interface { - ScheduleTask(schedule string, task func()) (cron.EntryID, error) - Start() - Stop() - } +// scheduler is an interface for task scheduling. +type scheduler interface { + Schedule(schedule string, task func()) (cron.EntryID, error) + Start() + Stop() +} - consumer interface { - ConsumeMessages(ctx context.Context) - } -) +// consumer is an interface for event consumption. +type consumer interface { + Consume(ctx context.Context) +} + +// producer is an interface for event producing. +type producer interface { + NewTopic(topic string, partitions int, replicationFactor int) error + SetTopic(topic string) + Produce(ctx context.Context) +} // Run is the application running process. func Run(appConfig *config.AppConfig) error { @@ -70,11 +74,17 @@ func Run(appConfig *config.AppConfig) error { appConfig.Outbox = outboxService kafkaURL := os.Getenv("KAFKA_URL") - kafkaTopic := "email-topic" + kafkaTopic := "emails-topic" - kafkaWriter := outbox.NewKafkaWriter(kafkaURL, kafkaTopic) - defer kafkaWriter.Close() - go eventProducer(ctx, outboxService, kafkaWriter) + kafkaProducer := outbox.NewKafkaProducer(kafkaURL, outboxService) + defer kafkaProducer.Writer.Close() + + err = kafkaProducer.NewTopic(kafkaTopic, 1, 1) + if err != nil { + return fmt.Errorf("failed to create topic: %w", err) + } + kafkaProducer.SetTopic(kafkaTopic) + go eventProducer(ctx, kafkaProducer) // Start the event consumer for Kafka kafkaConsumer := consumerpkg.NewKafkaConsumer(kafkaURL, kafkaTopic, 1) @@ -118,7 +128,7 @@ func handleShutdown(srv *http.Server, cancelFunc context.CancelFunc) { // scheduleEmails sets up a mailing process. func scheduleEmails(s scheduler) error { - _, err := s.ScheduleTask(schedule, func() { + _, err := s.Schedule(schedule, func() { err := handlers.Repo.ProduceMailingEvents() if err != nil { log.Printf("Error notifying subscribers: %v", err) @@ -132,8 +142,8 @@ func scheduleEmails(s scheduler) error { } // eventProducer runs an event dispatcher. -func eventProducer(ctx context.Context, o outbox.Outbox, w *kafka.Writer) { - outbox.Worker(ctx, o, w) +func eventProducer(ctx context.Context, p producer) { + p.Produce(ctx) // Wait for context cancellation to handle graceful shutdown <-ctx.Done() @@ -142,7 +152,7 @@ func eventProducer(ctx context.Context, o outbox.Outbox, w *kafka.Writer) { // eventProducer runs an event dispatcher. func eventConsumer(ctx context.Context, c consumer) { - go c.ConsumeMessages(ctx) + go c.Consume(ctx) // Wait for context cancellation to handle graceful shutdown <-ctx.Done() diff --git a/internal/email/consumer/kafka_consumer.go b/internal/email/consumer/kafka_consumer.go index 83be4cb..8e39824 100644 --- a/internal/email/consumer/kafka_consumer.go +++ b/internal/email/consumer/kafka_consumer.go @@ -33,8 +33,8 @@ func NewKafkaConsumer(kafkaURL, topic string, partition int) *KafkaConsumer { return &KafkaConsumer{Reader: reader} } -// ConsumeMessages reads messages from Kafka, deserializes them into Event structs, and processes them. -func (c *KafkaConsumer) ConsumeMessages(ctx context.Context) { +// Consume reads messages from Kafka, deserializes them into Event structs, and processes them. +func (c *KafkaConsumer) Consume(ctx context.Context) { for { // Read a message from Kafka m, err := c.Reader.FetchMessage(ctx) diff --git a/internal/outbox/gormoutbox/outbox.go b/internal/outbox/gormoutbox/outbox.go index 89010e0..362705c 100644 --- a/internal/outbox/gormoutbox/outbox.go +++ b/internal/outbox/gormoutbox/outbox.go @@ -10,7 +10,6 @@ import ( "gorm.io/gorm" ) -// Outbox is the repository for this package. type Outbox struct { db *gorm.DB } diff --git a/internal/outbox/kafka_producer.go b/internal/outbox/kafka_producer.go index 7adf385..f4ad05a 100644 --- a/internal/outbox/kafka_producer.go +++ b/internal/outbox/kafka_producer.go @@ -16,18 +16,50 @@ type Outbox interface { Cleanup() } -// NewKafkaWriter initializes a new kafka.Writer and returns a reference to it. -func NewKafkaWriter(kafkaURL, topic string) *kafka.Writer { - return &kafka.Writer{ +type KafkaProducer struct { + Writer *kafka.Writer + Outbox Outbox +} + +// NewKafkaProducer initializes a new KafkaProducer. +func NewKafkaProducer(kafkaURL string, outbox Outbox) *KafkaProducer { + writer := &kafka.Writer{ Addr: kafka.TCP(kafkaURL), - Topic: topic, Balancer: &kafka.LeastBytes{}, AllowAutoTopicCreation: true, } + + return &KafkaProducer{Writer: writer, Outbox: outbox} +} + +// NewTopic creates a new kafka.TopicConfig if it does not exist. +func (p *KafkaProducer) NewTopic(topic string, partitions, replicationFactor int) error { + conn, err := kafka.Dial("tcp", p.Writer.Addr.String()) + if err != nil { + return err + } + defer conn.Close() + + topicConfig := kafka.TopicConfig{ + Topic: topic, + NumPartitions: partitions, + ReplicationFactor: replicationFactor, + } + + err = conn.CreateTopics(topicConfig) + if err != nil { + return err + } + return nil +} + +// SetTopic changes the topic of the current kafka.Writer +func (p *KafkaProducer) SetTopic(topic string) { + p.Writer.Topic = topic } -// Worker fetches for unpublished events, publishes them, and marks them as published. -func Worker(ctx context.Context, outbox Outbox, writer *kafka.Writer) { +// Produce fetches for unpublished events, publishes them, and marks them as published. +func (p *KafkaProducer) Produce(ctx context.Context) { ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() @@ -37,13 +69,13 @@ func Worker(ctx context.Context, outbox Outbox, writer *kafka.Writer) { log.Println("Shutting down worker...") return case <-ticker.C: - processEvents(outbox, writer) + p.processEvents(p.Outbox) } } } // processEvents handles the retrieval and publishing of events. -func processEvents(outbox Outbox, writer *kafka.Writer) { +func (p *KafkaProducer) processEvents(outbox Outbox) { events, err := outbox.GetUnpublishedEvents() if err != nil { log.Println("Error fetching events:", err) @@ -51,7 +83,7 @@ func processEvents(outbox Outbox, writer *kafka.Writer) { } for _, event := range events { - if err := publishMessage(writer, []byte(event.Data)); err == nil { + if err := p.publishMessage([]byte(event.Data)); err == nil { if err = outbox.MarkEventAsPublished(event.ID); err != nil { log.Printf("Failed to mark event %d as published: %v", event.ID, err) continue @@ -64,9 +96,14 @@ func processEvents(outbox Outbox, writer *kafka.Writer) { outbox.Cleanup() } -func publishMessage(writer *kafka.Writer, message []byte) error { +func (p *KafkaProducer) publishMessage(message []byte) error { // Establishing connection to the leader of the partition - conn, err := kafka.DialLeader(context.Background(), "tcp", writer.Addr.String(), writer.Topic, 0) + conn, err := kafka.DialLeader( + context.Background(), + "tcp", + p.Writer.Addr.String(), + p.Writer.Topic, + 0) if err != nil { log.Printf("Failed to dial leader: %v", err) return err diff --git a/internal/outbox/offset.go b/internal/outbox/offset.go new file mode 100644 index 0000000..0edf1e1 --- /dev/null +++ b/internal/outbox/offset.go @@ -0,0 +1,7 @@ +package outbox + +// Offset represents the last published Event for a topic. +type Offset struct { + Topic string `gorm:"primaryKey"` + Offset int64 +} diff --git a/internal/scheduler/cron_scheduler.go b/internal/scheduler/cron_scheduler.go index 775d06e..3bb8e7c 100644 --- a/internal/scheduler/cron_scheduler.go +++ b/internal/scheduler/cron_scheduler.go @@ -14,8 +14,8 @@ func NewCronScheduler() *CronScheduler { } } -// ScheduleTask schedules a given task to run at the specified cron schedule. -func (s *CronScheduler) ScheduleTask(schedule string, task func()) (cron.EntryID, error) { +// Schedule schedules a given task to run at the specified cron schedule. +func (s *CronScheduler) Schedule(schedule string, task func()) (cron.EntryID, error) { id, err := s.Cron.AddFunc(schedule, task) if err != nil { return 0, err diff --git a/internal/scheduler/cron_scheduler_test.go b/internal/scheduler/cron_scheduler_test.go index 154943c..6e794ca 100644 --- a/internal/scheduler/cron_scheduler_test.go +++ b/internal/scheduler/cron_scheduler_test.go @@ -16,12 +16,12 @@ func TestNewCronScheduler(t *testing.T) { func TestScheduleTask(t *testing.T) { s := scheduler.NewCronScheduler() - _, err := s.ScheduleTask("* * * * *", func() { t.Log("Task executed") }) + _, err := s.Schedule("* * * * *", func() { t.Log("Task executed") }) if err != nil { t.Errorf("Failed to schedule task with valid cron schedule: %v", err) } - _, err = s.ScheduleTask("invalid schedule", func() {}) + _, err = s.Schedule("invalid schedule", func() {}) if err == nil { t.Error("Expected error when scheduling task with invalid cron schedule, got none") } @@ -32,7 +32,7 @@ func TestStart(t *testing.T) { done := make(chan bool) wasRun := false - _, err := s.ScheduleTask("@every 1s", func() { + _, err := s.Schedule("@every 1s", func() { wasRun = true done <- true }) From d039e415590aa6acf428dfe59e9d2057381a4a84 Mon Sep 17 00:00:00 2001 From: Vladyslav Pavlenko Date: Sat, 6 Jul 2024 23:35:06 +0300 Subject: [PATCH 6/9] refactor, feat: update consumer & producer, move scheduler to pkg --- Makefile | 4 + internal/app/config/config.go | 4 +- internal/app/run.go | 36 ++-- internal/app/setup.go | 30 ++-- internal/email/consumer/consumed_event.go | 11 ++ internal/email/consumer/kafka_consumer.go | 54 ++++-- internal/handlers/notifier.go | 25 ++- internal/outbox/event.go | 6 +- internal/outbox/gormoutbox/outbox.go | 39 ++-- internal/outbox/kafka_producer.go | 125 ------------- internal/outbox/offset.go | 7 - internal/outbox/producer/kafka_producer.go | 169 ++++++++++++++++++ internal/outbox/producer/offset.go | 8 + internal/storage/gormrepo/consumed_event.go | 19 ++ internal/storage/gormrepo/driver.go | 31 +++- internal/storage/gormrepo/event.go | 33 ++++ internal/storage/gormrepo/offset.go | 29 +++ internal/storage/gormrepo/subscription.go | 18 +- {internal => pkg}/scheduler/cron_scheduler.go | 0 .../scheduler/cron_scheduler_test.go | 2 +- 20 files changed, 432 insertions(+), 218 deletions(-) create mode 100644 internal/email/consumer/consumed_event.go delete mode 100644 internal/outbox/kafka_producer.go delete mode 100644 internal/outbox/offset.go create mode 100644 internal/outbox/producer/kafka_producer.go create mode 100644 internal/outbox/producer/offset.go create mode 100644 internal/storage/gormrepo/consumed_event.go create mode 100644 internal/storage/gormrepo/event.go create mode 100644 internal/storage/gormrepo/offset.go rename {internal => pkg}/scheduler/cron_scheduler.go (100%) rename {internal => pkg}/scheduler/cron_scheduler_test.go (93%) diff --git a/Makefile b/Makefile index 811f052..9ca08a2 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,10 @@ up_build: build_app docker-compose up --build -d @echo "Docker images built and started!" +## restart: stops and then rebuilds and restarts docker-compose. +restart: down up_build + @echo "Restarting docker images..." + ## down: stop docker compose. down: @echo "Stopping docker compose..." diff --git a/internal/app/config/config.go b/internal/app/config/config.go index d9f33ff..3f5b023 100644 --- a/internal/app/config/config.go +++ b/internal/app/config/config.go @@ -1,12 +1,12 @@ package config import ( - "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox" + "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox/producer" ) // AppConfig holds the application config. type AppConfig struct { - Outbox outbox.Outbox + Outbox producer.Outbox } // NewAppConfig creates a new AppConfig. diff --git a/internal/app/run.go b/internal/app/run.go index c1efb16..9d59b83 100644 --- a/internal/app/run.go +++ b/internal/app/run.go @@ -11,8 +11,11 @@ import ( "syscall" "time" + schedulerpkg "github.com/vladyslavpavlenko/genesis-api-project/pkg/scheduler" + + producerpkg "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox/producer" + consumerpkg "github.com/vladyslavpavlenko/genesis-api-project/internal/email/consumer" - "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox" "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox/gormoutbox" "github.com/robfig/cron/v3" @@ -20,7 +23,6 @@ import ( "github.com/vladyslavpavlenko/genesis-api-project/internal/handlers" "github.com/vladyslavpavlenko/genesis-api-project/internal/handlers/routes" - schedulerpkg "github.com/vladyslavpavlenko/genesis-api-project/internal/scheduler" ) const ( @@ -44,7 +46,7 @@ type consumer interface { type producer interface { NewTopic(topic string, partitions int, replicationFactor int) error SetTopic(topic string) - Produce(ctx context.Context) + Produce(ctx context.Context, frequency time.Duration, topic string, partition int) } // Run is the application running process. @@ -66,7 +68,7 @@ func Run(appConfig *config.AppConfig) error { defer cancel() // Start the event producer for Kafka - outboxService, err := gormoutbox.NewOutbox(appServices.DBConn.DB) + outboxService, err := gormoutbox.NewOutbox(appServices.DBConn) if err != nil { return fmt.Errorf("failed to create outbox: %w", err) } @@ -76,7 +78,10 @@ func Run(appConfig *config.AppConfig) error { kafkaURL := os.Getenv("KAFKA_URL") kafkaTopic := "emails-topic" - kafkaProducer := outbox.NewKafkaProducer(kafkaURL, outboxService) + kafkaProducer, err := producerpkg.NewKafkaProducer(kafkaURL, outboxService, appServices.DBConn) + if err != nil { + return fmt.Errorf("failed to create kafka producer: %w", err) + } defer kafkaProducer.Writer.Close() err = kafkaProducer.NewTopic(kafkaTopic, 1, 1) @@ -84,10 +89,21 @@ func Run(appConfig *config.AppConfig) error { return fmt.Errorf("failed to create topic: %w", err) } kafkaProducer.SetTopic(kafkaTopic) - go eventProducer(ctx, kafkaProducer) + go eventProducer(ctx, kafkaProducer, kafkaTopic, 1) // Start the event consumer for Kafka - kafkaConsumer := consumerpkg.NewKafkaConsumer(kafkaURL, kafkaTopic, 1) + kafkaGroupID := "emails-group" + + kafkaConsumer, err := consumerpkg.NewKafkaConsumer( + kafkaURL, + kafkaTopic, + 0, + kafkaGroupID, + appServices.Sender, + appServices.DBConn) + if err != nil { + return fmt.Errorf("failed to create kafka consumer: %w", err) + } defer kafkaConsumer.Reader.Close() go eventConsumer(ctx, kafkaConsumer) @@ -142,15 +158,15 @@ func scheduleEmails(s scheduler) error { } // eventProducer runs an event dispatcher. -func eventProducer(ctx context.Context, p producer) { - p.Produce(ctx) +func eventProducer(ctx context.Context, producer producer, topic string, partition int) { + producer.Produce(ctx, 10*time.Second, topic, partition) // Wait for context cancellation to handle graceful shutdown <-ctx.Done() log.Println("Shutting down event producer...") } -// eventProducer runs an event dispatcher. +// eventConsumer runs an event dispatcher. func eventConsumer(ctx context.Context, c consumer) { go c.Consume(ctx) diff --git a/internal/app/setup.go b/internal/app/setup.go index 3e057c0..62f257b 100644 --- a/internal/app/setup.go +++ b/internal/app/setup.go @@ -20,23 +20,21 @@ import ( "github.com/vladyslavpavlenko/genesis-api-project/internal/handlers" ) -type ( - // envVariables holds environment variables used in the application. - envVariables struct { - DBURL string `envconfig:"DB_URL"` - DBPort string `envconfig:"DB_PORT"` - DBUser string `envconfig:"DB_USER"` - DBPass string `envconfig:"DB_PASS"` - DBName string `envconfig:"DB_NAME"` - EmailAddr string `envconfig:"EMAIL_ADDR"` - EmailPass string `envconfig:"EMAIL_PASS"` - } +// envVariables holds environment variables used in the application. +type envVariables struct { + DBURL string `envconfig:"DB_URL"` + DBPort string `envconfig:"DB_PORT"` + DBUser string `envconfig:"DB_USER"` + DBPass string `envconfig:"DB_PASS"` + DBName string `envconfig:"DB_NAME"` + EmailAddr string `envconfig:"EMAIL_ADDR"` + EmailPass string `envconfig:"EMAIL_PASS"` +} - services struct { - DBConn *gormrepo.Connection - Sender *email.GomailSender - } -) +type services struct { + DBConn *gormrepo.Connection + Sender *email.GomailSender +} func setup(app *config.AppConfig) (*services, error) { envs, err := readEnv() diff --git a/internal/email/consumer/consumed_event.go b/internal/email/consumer/consumed_event.go new file mode 100644 index 0000000..7455bce --- /dev/null +++ b/internal/email/consumer/consumed_event.go @@ -0,0 +1,11 @@ +package consumer + +import "time" + +// ConsumedEvent represents an event consumed by the consumer, including the partition information. +type ConsumedEvent struct { + Topic string `gorm:"primaryKey"` + Partition int `gorm:"primaryKey"` + Offset int64 `gorm:"primaryKey"` + ConsumedAt time.Time +} diff --git a/internal/email/consumer/kafka_consumer.go b/internal/email/consumer/kafka_consumer.go index 8e39824..03b6a18 100644 --- a/internal/email/consumer/kafka_consumer.go +++ b/internal/email/consumer/kafka_consumer.go @@ -4,6 +4,9 @@ import ( "context" "fmt" "log" + "time" + + "github.com/pkg/errors" "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox" @@ -16,51 +19,78 @@ type Sender interface { Send(params email.Params) error } +type dbConnection interface { + Migrate(models ...any) error + AddConsumedEvent(event ConsumedEvent) error +} + type KafkaConsumer struct { + db dbConnection Reader *kafka.Reader Sender Sender } // NewKafkaConsumer initializes a new KafkaConsumer. -func NewKafkaConsumer(kafkaURL, topic string, partition int) *KafkaConsumer { +func NewKafkaConsumer(kafkaURL, topic string, partition int, groupID string, sender Sender, db dbConnection) (*KafkaConsumer, error) { reader := kafka.NewReader(kafka.ReaderConfig{ Brokers: []string{kafkaURL}, Topic: topic, Partition: partition, + GroupID: groupID, CommitInterval: 0, // disable auto-commit }) - return &KafkaConsumer{Reader: reader} + err := db.Migrate(&ConsumedEvent{}) + if err != nil { + return nil, errors.Wrap(err, "failed to migrate offset") + } + + return &KafkaConsumer{Reader: reader, Sender: sender, db: db}, nil } -// Consume reads messages from Kafka, deserializes them into Event structs, and processes them. func (c *KafkaConsumer) Consume(ctx context.Context) { for { - // Read a message from Kafka + // Attempt to fetch a message from Kafka m, err := c.Reader.FetchMessage(ctx) if err != nil { log.Printf("Failed to read message: %v", err) continue } - // Deserialize the data from the message + // Attempt to deserialize the fetched message data, err := outbox.DeserializeData(m.Value) if err != nil { - log.Printf("Failed to deserialize data from message: %v", err) + log.Printf("Failed to deserialize data from message at offset %d: %v", m.Offset, err) continue } - // Process the message - sendMessage(data, c.Sender) + // Send a message + c.sendMessage(data) + + // Create a record of the consumed event + consumedEvent := ConsumedEvent{ + Topic: m.Topic, + Partition: m.Partition, + Offset: m.Offset, + ConsumedAt: time.Now(), + } + + // Attempt to add the consumed event to the database + if err = c.db.AddConsumedEvent(consumedEvent); err != nil { + log.Printf("Failed to record consumed event at offset %d: %v", m.Offset, err) + continue + } - // Commit the offset after processing the message + // Commit the offset back to Kafka to mark the message as processed if err = c.Reader.CommitMessages(ctx, m); err != nil { - log.Printf("Failed to commit message offset: %v", err) + log.Printf("Failed to commit message offset %d: %v", m.Offset, err) + } else { + log.Printf("Offset committed successfully for message at offset: %d", m.Offset) } } } -func sendMessage(data outbox.Data, sender Sender) { +func (c *KafkaConsumer) sendMessage(data outbox.Data) { params := email.Params{ To: data.Email, Subject: "USD to UAH Exchange Rate", @@ -69,7 +99,7 @@ func sendMessage(data outbox.Data, sender Sender) { log.Printf("Sending email to %s", data.Email) - err := sender.Send(params) + err := c.Sender.Send(params) if err != nil { log.Printf("Failed to send email: %v", err) } diff --git a/internal/handlers/notifier.go b/internal/handlers/notifier.go index dcf60c1..79cc694 100644 --- a/internal/handlers/notifier.go +++ b/internal/handlers/notifier.go @@ -3,7 +3,6 @@ package handlers import ( "context" "fmt" - "log" "strconv" "sync" @@ -21,12 +20,13 @@ func (m *Repository) ProduceMailingEvents() error { return fmt.Errorf("failed to retrieve rate: %w", err) } - floatRate, err := strconv.ParseFloat(rate, 32) + floatRate, err := strconv.ParseFloat(rate, 64) if err != nil { - return fmt.Errorf("failed to parse price: %w", err) + return fmt.Errorf("failed to parse rate: %w", err) } var offset int + errChan := make(chan error, 1) for { subscriptions, err := m.DB.GetSubscriptions(batchSize, offset) if err != nil { @@ -37,22 +37,29 @@ func (m *Repository) ProduceMailingEvents() error { } var wg sync.WaitGroup - for _, subscription := range subscriptions { + for _, sub := range subscriptions { wg.Add(1) go func(sub models.Subscription) { defer wg.Done() data := outbox.Data{Email: sub.Email, Rate: floatRate} - - if err = m.App.Outbox.AddEvent(data); err != nil { - log.Printf("error adding event: %v", err) + if localErr := m.App.Outbox.AddEvent(data); localErr != nil { + select { + case errChan <- localErr: + default: + } } - }(subscription) + }(sub) } wg.Wait() + select { + case err := <-errChan: + return err + default: + } + offset += batchSize } - return nil } diff --git a/internal/outbox/event.go b/internal/outbox/event.go index 01aa689..a2f1a86 100644 --- a/internal/outbox/event.go +++ b/internal/outbox/event.go @@ -7,9 +7,9 @@ import ( // Event is a query message model stored in the database. type Event struct { - ID uint `gorm:"primaryKey" json:"id"` - Data string `json:"data"` - CreatedAt time.Time `json:"created_at"` + ID uint `gorm:"primaryKey"` + Data string + CreatedAt time.Time } // Data is an event data model. diff --git a/internal/outbox/gormoutbox/outbox.go b/internal/outbox/gormoutbox/outbox.go index 362705c..902758a 100644 --- a/internal/outbox/gormoutbox/outbox.go +++ b/internal/outbox/gormoutbox/outbox.go @@ -1,22 +1,28 @@ package gormoutbox import ( - "fmt" "time" "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox" "github.com/pkg/errors" - "gorm.io/gorm" ) +// dbConnection defines an interface for the database connection. +type dbConnection interface { + Migrate(models ...any) error + AddEvent(event *outbox.Event) error +} + +// Outbox defines an interface for the transactional outbox. type Outbox struct { - db *gorm.DB + db dbConnection } -// NewOutbox creates a new `events` table to serve as an outbox. -func NewOutbox(db *gorm.DB) (*Outbox, error) { - err := db.AutoMigrate(&outbox.Event{}) +// NewOutbox creates `events` table to implement a transactional outbox. +// `events` table stores all the events ever occurred. +func NewOutbox(db dbConnection) (*Outbox, error) { + err := db.Migrate(&outbox.Event{}) if err != nil { return nil, errors.Wrap(err, "failed to migrate events") } @@ -31,25 +37,8 @@ func (o *Outbox) AddEvent(data outbox.Data) error { err := event.SerializeData(data) if err != nil { - return fmt.Errorf("failed to serialize data: %w", err) + return errors.Wrap(err, "failed to serialize data") } - return o.db.Create(event).Error -} - -// GetUnpublishedEvents retrieves all events that haven't been published yet. -func (o *Outbox) GetUnpublishedEvents() ([]outbox.Event, error) { - var events []outbox.Event - err := o.db.Where("published = ?", false).Find(&events).Error - return events, err -} - -// MarkEventAsPublished marks an event as published. -func (o *Outbox) MarkEventAsPublished(eventID uint) error { - return o.db.Model(&outbox.Event{}).Where("id = ?", eventID).Update("published", true).Error -} - -// Cleanup deletes all the Event records that have already been published or are outdated. -func (o *Outbox) Cleanup() { - o.db.Where("published = ? AND created_at <= ?", true, time.Now().AddDate(0, 0, -1)).Delete(&outbox.Event{}) + return o.db.AddEvent(event) } diff --git a/internal/outbox/kafka_producer.go b/internal/outbox/kafka_producer.go deleted file mode 100644 index f4ad05a..0000000 --- a/internal/outbox/kafka_producer.go +++ /dev/null @@ -1,125 +0,0 @@ -package outbox - -import ( - "context" - "fmt" - "log" - "time" - - "github.com/segmentio/kafka-go" -) - -type Outbox interface { - AddEvent(data Data) error - GetUnpublishedEvents() ([]Event, error) - MarkEventAsPublished(eventID uint) error - Cleanup() -} - -type KafkaProducer struct { - Writer *kafka.Writer - Outbox Outbox -} - -// NewKafkaProducer initializes a new KafkaProducer. -func NewKafkaProducer(kafkaURL string, outbox Outbox) *KafkaProducer { - writer := &kafka.Writer{ - Addr: kafka.TCP(kafkaURL), - Balancer: &kafka.LeastBytes{}, - AllowAutoTopicCreation: true, - } - - return &KafkaProducer{Writer: writer, Outbox: outbox} -} - -// NewTopic creates a new kafka.TopicConfig if it does not exist. -func (p *KafkaProducer) NewTopic(topic string, partitions, replicationFactor int) error { - conn, err := kafka.Dial("tcp", p.Writer.Addr.String()) - if err != nil { - return err - } - defer conn.Close() - - topicConfig := kafka.TopicConfig{ - Topic: topic, - NumPartitions: partitions, - ReplicationFactor: replicationFactor, - } - - err = conn.CreateTopics(topicConfig) - if err != nil { - return err - } - return nil -} - -// SetTopic changes the topic of the current kafka.Writer -func (p *KafkaProducer) SetTopic(topic string) { - p.Writer.Topic = topic -} - -// Produce fetches for unpublished events, publishes them, and marks them as published. -func (p *KafkaProducer) Produce(ctx context.Context) { - ticker := time.NewTicker(10 * time.Second) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - log.Println("Shutting down worker...") - return - case <-ticker.C: - p.processEvents(p.Outbox) - } - } -} - -// processEvents handles the retrieval and publishing of events. -func (p *KafkaProducer) processEvents(outbox Outbox) { - events, err := outbox.GetUnpublishedEvents() - if err != nil { - log.Println("Error fetching events:", err) - return - } - - for _, event := range events { - if err := p.publishMessage([]byte(event.Data)); err == nil { - if err = outbox.MarkEventAsPublished(event.ID); err != nil { - log.Printf("Failed to mark event %d as published: %v", event.ID, err) - continue - } - } else { - log.Printf("Failed to publish message for event %d: %v", event.ID, err) - } - } - - outbox.Cleanup() -} - -func (p *KafkaProducer) publishMessage(message []byte) error { - // Establishing connection to the leader of the partition - conn, err := kafka.DialLeader( - context.Background(), - "tcp", - p.Writer.Addr.String(), - p.Writer.Topic, - 0) - if err != nil { - log.Printf("Failed to dial leader: %v", err) - return err - } - defer conn.Close() - - // Writing message to the leader - _, err = conn.WriteMessages( - kafka.Message{ - Key: []byte(fmt.Sprintf("Key-%d", time.Now().Unix())), - Value: message, - }, - ) - if err != nil { - log.Printf("Failed to write to leader: %v", err) - return err - } - return nil -} diff --git a/internal/outbox/offset.go b/internal/outbox/offset.go deleted file mode 100644 index 0edf1e1..0000000 --- a/internal/outbox/offset.go +++ /dev/null @@ -1,7 +0,0 @@ -package outbox - -// Offset represents the last published Event for a topic. -type Offset struct { - Topic string `gorm:"primaryKey"` - Offset int64 -} diff --git a/internal/outbox/producer/kafka_producer.go b/internal/outbox/producer/kafka_producer.go new file mode 100644 index 0000000..49f1632 --- /dev/null +++ b/internal/outbox/producer/kafka_producer.go @@ -0,0 +1,169 @@ +package producer + +import ( + "context" + "log" + "strconv" + "time" + + "github.com/pkg/errors" + "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox" + + "gorm.io/gorm" + + "github.com/segmentio/kafka-go" +) + +type Outbox interface { + AddEvent(data outbox.Data) error +} + +type dbConnection interface { + Migrate(models ...any) error + BeginTransaction() (*gorm.DB, error) + GetLastOffset(topic string, partition int) (Offset, error) + FetchUnpublishedEvents(lastOffset uint) ([]outbox.Event, error) + UpdateOffset(offset *Offset) error +} + +type KafkaProducer struct { + db dbConnection + Writer *kafka.Writer + Outbox Outbox +} + +// NewKafkaProducer initializes a new KafkaProducer. +func NewKafkaProducer(kafkaURL string, o Outbox, db dbConnection) (*KafkaProducer, error) { + w := &kafka.Writer{ + Addr: kafka.TCP(kafkaURL), + Balancer: &kafka.LeastBytes{}, + AllowAutoTopicCreation: true, + } + + err := db.Migrate(&Offset{}) + if err != nil { + return nil, errors.Wrap(err, "failed to migrate offset") + } + + return &KafkaProducer{Writer: w, Outbox: o, db: db}, nil +} + +// NewTopic creates a new kafka.TopicConfig if it does not exist. +func (p *KafkaProducer) NewTopic(topic string, partitions, replicationFactor int) error { + conn, err := kafka.Dial("tcp", p.Writer.Addr.String()) + if err != nil { + return err + } + defer conn.Close() + + topicConfig := kafka.TopicConfig{ + Topic: topic, + NumPartitions: partitions, + ReplicationFactor: replicationFactor, + } + + err = conn.CreateTopics(topicConfig) + if err != nil { + return err + } + return nil +} + +// SetTopic changes the topic of the current kafka.Writer +func (p *KafkaProducer) SetTopic(topic string) { + p.Writer.Topic = topic +} + +// Produce fetches for unpublished events, publishes them, and marks them as published. +func (p *KafkaProducer) Produce(ctx context.Context, frequency time.Duration, topic string, partition int) { + ticker := time.NewTicker(frequency) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + log.Println("Shutting down worker...") + return + case <-ticker.C: + p.processEvents(ctx, topic, partition) + } + } +} + +func (p *KafkaProducer) processEvents(ctx context.Context, topic string, partition int) { + // Start a transaction + tx, err := p.db.BeginTransaction() + if err != nil { + log.Printf("Failed to start transaction: %v", err) + return + } + log.Println("Transaction started successfully") + + defer func() { + if r := recover(); r != nil { + tx.Rollback() + log.Printf("Recovered from panic: %v, transaction rolled back", r) + } + }() + + // Retrieve the last processed offset + lastOffset, err := p.db.GetLastOffset(topic, partition) + if err != nil { + if !errors.Is(err, gorm.ErrRecordNotFound) { + tx.Rollback() + log.Printf("Failed to fetch last offset: %v", err) + return + } + + lastOffset = Offset{ + Topic: topic, + Partition: partition, + Offset: 0, + } + } else { + log.Printf("Last offset fetched: %d", lastOffset.Offset) + } + + // Fetch unpublished events based on the last offset + events, err := p.db.FetchUnpublishedEvents(lastOffset.Offset) + if err != nil { + tx.Rollback() + log.Printf("Failed to fetch unpublished events: %v", err) + return + } + log.Printf("Fetched %d unpublished events", len(events)) + + // Process each event + for _, event := range events { + key := []byte(strconv.Itoa(int(event.ID))) + msg := &kafka.Message{ + Key: key, + Value: []byte(event.Data), + Partition: partition, + } + + log.Printf("Preparing to send message with ID: %d", event.ID) + + if err = p.Writer.WriteMessages(ctx, *msg); err != nil { + tx.Rollback() + log.Printf("Failed to send Kafka message: %v", err) + return + } + log.Printf("Message with ID %d sent successfully", event.ID) + + lastOffset.Offset = event.ID + if err = p.db.UpdateOffset(&lastOffset); err != nil { + tx.Rollback() + log.Printf("Failed to update offset after sending message with ID %d: %v", event.ID, err) + return + } + log.Printf("Offset updated successfully for message ID: %d", event.ID) + } + + // Commit the transaction if all events are processed successfully + if err = tx.Commit().Error; err != nil { + log.Printf("Failed to commit transaction: %v", err) + return + } + log.Println("Transaction committed and all events processed successfully") +} diff --git a/internal/outbox/producer/offset.go b/internal/outbox/producer/offset.go new file mode 100644 index 0000000..0bd0d71 --- /dev/null +++ b/internal/outbox/producer/offset.go @@ -0,0 +1,8 @@ +package producer + +// Offset represents the last published event offset for a topic and partition. +type Offset struct { + Topic string `gorm:"primaryKey"` + Partition int `gorm:"primaryKey"` + Offset uint +} diff --git a/internal/storage/gormrepo/consumed_event.go b/internal/storage/gormrepo/consumed_event.go new file mode 100644 index 0000000..eb5fca8 --- /dev/null +++ b/internal/storage/gormrepo/consumed_event.go @@ -0,0 +1,19 @@ +package gormrepo + +import ( + "context" + + "github.com/vladyslavpavlenko/genesis-api-project/internal/email/consumer" +) + +// AddConsumedEvent creates a new consumer.ConsumedEvent record. +func (c *Connection) AddConsumedEvent(event consumer.ConsumedEvent) error { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + result := c.db.WithContext(ctx).Create(event) + if result.Error != nil { + return result.Error + } + return nil +} diff --git a/internal/storage/gormrepo/driver.go b/internal/storage/gormrepo/driver.go index c2802db..55f8130 100644 --- a/internal/storage/gormrepo/driver.go +++ b/internal/storage/gormrepo/driver.go @@ -1,6 +1,7 @@ package gormrepo import ( + "context" "fmt" "log" "time" @@ -9,8 +10,15 @@ import ( "gorm.io/gorm" ) +const timeout = time.Second * 5 + type Connection struct { - DB *gorm.DB + db *gorm.DB +} + +// DB returns a pointer to gorm.DB. +func (c *Connection) DB() *gorm.DB { + return c.db } // Setup sets up a new Connection. @@ -23,7 +31,7 @@ func (c *Connection) Setup(dsn string) error { counts++ } else { log.Println("Connected to Postgres!") - c.DB = db + c.db = db return nil } @@ -48,7 +56,7 @@ func openDB(dsn string) (*gorm.DB, error) { // Close closes a database connection. func (c *Connection) Close() error { - sqlDB, err := c.DB.DB() + sqlDB, err := c.db.DB() if err != nil { return err } @@ -57,10 +65,25 @@ func (c *Connection) Close() error { // Migrate performs a database migration for given models. func (c *Connection) Migrate(models ...any) error { - err := c.DB.AutoMigrate(models...) + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + err := c.db.WithContext(ctx).AutoMigrate(models...) if err != nil { return fmt.Errorf("error migrating models: %w", err) } return nil } + +// BeginTransaction begins a transaction. +func (c *Connection) BeginTransaction() (*gorm.DB, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + tx := c.db.WithContext(ctx).Begin() + if tx.Error != nil { + return nil, tx.Error + } + return tx, nil +} diff --git a/internal/storage/gormrepo/event.go b/internal/storage/gormrepo/event.go new file mode 100644 index 0000000..0b4f747 --- /dev/null +++ b/internal/storage/gormrepo/event.go @@ -0,0 +1,33 @@ +package gormrepo + +import ( + "context" + + "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox" +) + +// AddEvent creates a new outbox.Event record. +func (c *Connection) AddEvent(event *outbox.Event) error { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + result := c.db.WithContext(ctx).Create(event) + if result.Error != nil { + return result.Error + } + return nil +} + +// FetchUnpublishedEvents retrieves all events from the database that have not been +// published after the specified offset. +func (c *Connection) FetchUnpublishedEvents(lastOffset uint) ([]outbox.Event, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + var events []outbox.Event + err := c.db.WithContext(ctx).Where("id > ?", lastOffset).Find(&events).Error + if err != nil { + return nil, err + } + return events, nil +} diff --git a/internal/storage/gormrepo/offset.go b/internal/storage/gormrepo/offset.go new file mode 100644 index 0000000..dbcc22a --- /dev/null +++ b/internal/storage/gormrepo/offset.go @@ -0,0 +1,29 @@ +package gormrepo + +import ( + "context" + + "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox/producer" +) + +// GetLastOffset retrieves the last offset for a given topic from the database. +func (c *Connection) GetLastOffset(topic string, partition int) (producer.Offset, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + var lastOffset producer.Offset + err := c.db.WithContext(ctx).Where("topic = ? AND partition = ?", topic, partition).First(&lastOffset).Error + if err != nil { + return producer.Offset{}, err + } + return lastOffset, nil +} + +// UpdateOffset updates the offset in the database to reflect the latest published +// event's ID. +func (c *Connection) UpdateOffset(offset *producer.Offset) error { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + return c.db.WithContext(ctx).Save(offset).Error +} diff --git a/internal/storage/gormrepo/subscription.go b/internal/storage/gormrepo/subscription.go index 79cf96e..912aa39 100644 --- a/internal/storage/gormrepo/subscription.go +++ b/internal/storage/gormrepo/subscription.go @@ -1,6 +1,7 @@ package gormrepo import ( + "context" "errors" "time" @@ -16,11 +17,14 @@ var ( // AddSubscription creates a new models.Subscription record. func (c *Connection) AddSubscription(email string) error { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + subscription := models.Subscription{ Email: email, CreatedAt: time.Now(), } - result := c.DB.Create(&subscription) + result := c.db.WithContext(ctx).Create(&subscription) if result.Error != nil { var pgErr *pgconn.PgError if errors.As(result.Error, &pgErr) && pgErr.Code == "23505" { @@ -33,7 +37,10 @@ func (c *Connection) AddSubscription(email string) error { // DeleteSubscription deletes a models.Subscription record. func (c *Connection) DeleteSubscription(email string) error { - result := c.DB.Where("email = ?", email).Delete(&models.Subscription{}) + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + result := c.db.WithContext(ctx).Where("email = ?", email).Delete(&models.Subscription{}) if result.Error != nil { return result.Error } @@ -45,12 +52,15 @@ func (c *Connection) DeleteSubscription(email string) error { return nil } -// GetSubscriptions returns a paginated list of subscriptions. Limit specify the number of records to be retrieved +// GetSubscriptions returns a paginated list of subscriptions. Limit specifies the number of records to be retrieved // Limit conditions can be canceled by using `Limit(-1)`. Offset specify the number of records to skip before starting // to return the records. Offset conditions can be canceled by using `Offset(-1)`. func (c *Connection) GetSubscriptions(limit, offset int) ([]models.Subscription, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + var subscriptions []models.Subscription - result := c.DB.Limit(limit).Offset(offset).Find(&subscriptions) + result := c.db.WithContext(ctx).Limit(limit).Offset(offset).Find(&subscriptions) if result.Error != nil { return nil, result.Error } diff --git a/internal/scheduler/cron_scheduler.go b/pkg/scheduler/cron_scheduler.go similarity index 100% rename from internal/scheduler/cron_scheduler.go rename to pkg/scheduler/cron_scheduler.go diff --git a/internal/scheduler/cron_scheduler_test.go b/pkg/scheduler/cron_scheduler_test.go similarity index 93% rename from internal/scheduler/cron_scheduler_test.go rename to pkg/scheduler/cron_scheduler_test.go index 6e794ca..b46bf84 100644 --- a/internal/scheduler/cron_scheduler_test.go +++ b/pkg/scheduler/cron_scheduler_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/vladyslavpavlenko/genesis-api-project/internal/scheduler" + "github.com/vladyslavpavlenko/genesis-api-project/pkg/scheduler" ) func TestNewCronScheduler(t *testing.T) { From f99057e1918364381f1e7fee3d785c30f55a23e5 Mon Sep 17 00:00:00 2001 From: Vladyslav Pavlenko Date: Sun, 7 Jul 2024 00:05:10 +0300 Subject: [PATCH 7/9] refactor: move notifier out of handlers --- internal/app/config/config.go | 4 +- internal/app/run.go | 27 +++------ internal/app/setup.go | 35 ++++++++--- internal/handlers/handlers.go | 2 +- internal/handlers/notifier.go | 65 -------------------- internal/handlers/notifier_test.go | 34 ----------- internal/handlers/repository.go | 4 +- internal/handlers/repository_test.go | 6 -- internal/notifier/notifier.go | 88 ++++++++++++++++++++++++++++ 9 files changed, 129 insertions(+), 136 deletions(-) delete mode 100644 internal/handlers/notifier.go delete mode 100644 internal/handlers/notifier_test.go create mode 100644 internal/notifier/notifier.go diff --git a/internal/app/config/config.go b/internal/app/config/config.go index 3f5b023..121efed 100644 --- a/internal/app/config/config.go +++ b/internal/app/config/config.go @@ -1,12 +1,12 @@ package config import ( - "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox/producer" + "github.com/vladyslavpavlenko/genesis-api-project/internal/notifier" ) // AppConfig holds the application config. type AppConfig struct { - Outbox producer.Outbox + Notifier notifier.Notifier } // NewAppConfig creates a new AppConfig. diff --git a/internal/app/run.go b/internal/app/run.go index 9d59b83..d0ff6fd 100644 --- a/internal/app/run.go +++ b/internal/app/run.go @@ -11,17 +11,16 @@ import ( "syscall" "time" + "github.com/vladyslavpavlenko/genesis-api-project/internal/notifier" schedulerpkg "github.com/vladyslavpavlenko/genesis-api-project/pkg/scheduler" producerpkg "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox/producer" consumerpkg "github.com/vladyslavpavlenko/genesis-api-project/internal/email/consumer" - "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox/gormoutbox" "github.com/robfig/cron/v3" "github.com/vladyslavpavlenko/genesis-api-project/internal/app/config" - "github.com/vladyslavpavlenko/genesis-api-project/internal/handlers" "github.com/vladyslavpavlenko/genesis-api-project/internal/handlers/routes" ) @@ -57,28 +56,20 @@ func Run(appConfig *config.AppConfig) error { } defer appServices.DBConn.Close() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + s := schedulerpkg.NewCronScheduler() - if err = scheduleEmails(s); err != nil { + if err = scheduleEmails(s, appServices.Notifier); err != nil { return fmt.Errorf("failed to schedule emails: %w", err) } s.Start() defer s.Stop() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // Start the event producer for Kafka - outboxService, err := gormoutbox.NewOutbox(appServices.DBConn) - if err != nil { - return fmt.Errorf("failed to create outbox: %w", err) - } - - appConfig.Outbox = outboxService - kafkaURL := os.Getenv("KAFKA_URL") kafkaTopic := "emails-topic" - kafkaProducer, err := producerpkg.NewKafkaProducer(kafkaURL, outboxService, appServices.DBConn) + kafkaProducer, err := producerpkg.NewKafkaProducer(kafkaURL, appServices.Outbox, appServices.DBConn) if err != nil { return fmt.Errorf("failed to create kafka producer: %w", err) } @@ -91,7 +82,6 @@ func Run(appConfig *config.AppConfig) error { kafkaProducer.SetTopic(kafkaTopic) go eventProducer(ctx, kafkaProducer, kafkaTopic, 1) - // Start the event consumer for Kafka kafkaGroupID := "emails-group" kafkaConsumer, err := consumerpkg.NewKafkaConsumer( @@ -114,7 +104,6 @@ func Run(appConfig *config.AppConfig) error { ReadHeaderTimeout: 5 * time.Second, } - // Handle graceful shutdown handleShutdown(srv, cancel) return nil @@ -143,9 +132,9 @@ func handleShutdown(srv *http.Server, cancelFunc context.CancelFunc) { } // scheduleEmails sets up a mailing process. -func scheduleEmails(s scheduler) error { +func scheduleEmails(s scheduler, n *notifier.Notifier) error { _, err := s.Schedule(schedule, func() { - err := handlers.Repo.ProduceMailingEvents() + err := n.ProduceNotificationEvents() if err != nil { log.Printf("Error notifying subscribers: %v", err) } diff --git a/internal/app/setup.go b/internal/app/setup.go index 62f257b..1738251 100644 --- a/internal/app/setup.go +++ b/internal/app/setup.go @@ -5,7 +5,11 @@ import ( "log" "net/http" - "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox" + notifierpkg "github.com/vladyslavpavlenko/genesis-api-project/internal/notifier" + "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox/gormoutbox" + producerpkg "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox/producer" + + outboxpkg "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox" "github.com/vladyslavpavlenko/genesis-api-project/internal/app/config" "github.com/vladyslavpavlenko/genesis-api-project/internal/models" @@ -32,8 +36,11 @@ type envVariables struct { } type services struct { - DBConn *gormrepo.Connection - Sender *email.GomailSender + DBConn *gormrepo.Connection + Sender *email.GomailSender + Fetcher *chain.Node + Notifier *notifierpkg.Notifier + Outbox producerpkg.Outbox } func setup(app *config.AppConfig) (*services, error) { @@ -63,15 +70,27 @@ func setup(app *config.AppConfig) (*services, error) { sender, err := setupSender(&envs) if err != nil { - return nil, fmt.Errorf("error setting up sender: %w", err) + return nil, fmt.Errorf("failed set up sender: %w", err) } - repo := handlers.NewRepo(app, &handlers.Services{Fetcher: fetcher}, dbConn) + outbox, err := gormoutbox.NewOutbox(dbConn) + if err != nil { + return nil, fmt.Errorf("failed to create outbox: %w", err) + } + + notifier := notifierpkg.NewNotifier(dbConn, fetcher, outbox) + + repo := handlers.NewRepo(app, &handlers.Services{ + Fetcher: fetcher, + Notifier: notifier, + }, dbConn) handlers.NewHandlers(repo) return &services{ - DBConn: dbConn, - Sender: sender, + DBConn: dbConn, + Sender: sender, + Fetcher: fetcher, + Outbox: outbox, }, nil } @@ -101,7 +120,7 @@ func connectDB(dsn string) (*gormrepo.Connection, error) { func migrateDB(conn *gormrepo.Connection) error { log.Println("Running migrations...") - err := conn.Migrate(&models.Subscription{}, &outbox.Event{}) + err := conn.Migrate(&models.Subscription{}, &outboxpkg.Event{}) if err != nil { return fmt.Errorf("error running migrations: %w", err) } diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 723b11c..254afaf 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -106,7 +106,7 @@ func (m *Repository) Unsubscribe(w http.ResponseWriter, r *http.Request) { // SendEmails handles the `/sendEmails` request. func (m *Repository) SendEmails(w http.ResponseWriter, _ *http.Request) { // Produce mailing events - err := m.ProduceMailingEvents() + err := m.Services.Notifier.ProduceNotificationEvents() if err != nil { _ = jsonutils.ErrorJSON(w, err, http.StatusInternalServerError) return diff --git a/internal/handlers/notifier.go b/internal/handlers/notifier.go deleted file mode 100644 index 79cc694..0000000 --- a/internal/handlers/notifier.go +++ /dev/null @@ -1,65 +0,0 @@ -package handlers - -import ( - "context" - "fmt" - "strconv" - "sync" - - "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox" - - "github.com/vladyslavpavlenko/genesis-api-project/internal/models" -) - -const batchSize = 100 - -// ProduceMailingEvents handles producing events for currency rate update emails. -func (m *Repository) ProduceMailingEvents() error { - rate, err := m.Services.Fetcher.Fetch(context.Background(), "USD", "UAH") - if err != nil { - return fmt.Errorf("failed to retrieve rate: %w", err) - } - - floatRate, err := strconv.ParseFloat(rate, 64) - if err != nil { - return fmt.Errorf("failed to parse rate: %w", err) - } - - var offset int - errChan := make(chan error, 1) - for { - subscriptions, err := m.DB.GetSubscriptions(batchSize, offset) - if err != nil { - return err - } - if len(subscriptions) == 0 { - break - } - - var wg sync.WaitGroup - for _, sub := range subscriptions { - wg.Add(1) - go func(sub models.Subscription) { - defer wg.Done() - - data := outbox.Data{Email: sub.Email, Rate: floatRate} - if localErr := m.App.Outbox.AddEvent(data); localErr != nil { - select { - case errChan <- localErr: - default: - } - } - }(sub) - } - wg.Wait() - - select { - case err := <-errChan: - return err - default: - } - - offset += batchSize - } - return nil -} diff --git a/internal/handlers/notifier_test.go b/internal/handlers/notifier_test.go deleted file mode 100644 index cc28698..0000000 --- a/internal/handlers/notifier_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package handlers_test - -import ( - "testing" - - "github.com/vladyslavpavlenko/genesis-api-project/internal/storage/gormrepo" - - "github.com/vladyslavpavlenko/genesis-api-project/internal/app/config" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/vladyslavpavlenko/genesis-api-project/internal/handlers" - "github.com/vladyslavpavlenko/genesis-api-project/internal/models" -) - -func TestNotifySubscribers_Success(t *testing.T) { - mockDB := new(MockDB) - mockFetcher := new(MockFetcher) - appConfig := &config.AppConfig{} - dbConn := &gormrepo.Connection{} - services := setupServicesWithMocks(mockFetcher) - repo := handlers.NewRepo(appConfig, services, dbConn) - - subscribers := []models.Subscription{{Email: "user@example.com"}} - mockDB.On("GetSubscriptions").Return(subscribers, nil) - mockFetcher.On("Fetch", mock.Anything, "USD", "UAH").Return("24.5", nil) - - err := repo.ProduceMailingEvents() - - assert.NoError(t, err) - - mockDB.AssertExpectations(t) - mockFetcher.AssertExpectations(t) -} diff --git a/internal/handlers/repository.go b/internal/handlers/repository.go index ff9ed96..9dbdc11 100644 --- a/internal/handlers/repository.go +++ b/internal/handlers/repository.go @@ -3,13 +3,15 @@ package handlers import ( "github.com/vladyslavpavlenko/genesis-api-project/internal/app/config" "github.com/vladyslavpavlenko/genesis-api-project/internal/models" + "github.com/vladyslavpavlenko/genesis-api-project/internal/notifier" "github.com/vladyslavpavlenko/genesis-api-project/internal/rateapi" ) type ( // Services is the repository type. Services struct { - Fetcher rateapi.Fetcher + Fetcher rateapi.Fetcher + Notifier *notifier.Notifier } // Repository is the repository type diff --git a/internal/handlers/repository_test.go b/internal/handlers/repository_test.go index 2781d96..f0fc821 100644 --- a/internal/handlers/repository_test.go +++ b/internal/handlers/repository_test.go @@ -53,12 +53,6 @@ func (m *MockFetcher) Fetch(ctx context.Context, base, target string) (string, e return args.String(0), args.Error(1) } -func setupServicesWithMocks(fetcher *MockFetcher) *handlers.Services { - return &handlers.Services{ - Fetcher: fetcher, - } -} - // TestNewRepo tests the creation of a new repository func TestNewRepo(t *testing.T) { appConfig := &config.AppConfig{} diff --git a/internal/notifier/notifier.go b/internal/notifier/notifier.go new file mode 100644 index 0000000..b4e1965 --- /dev/null +++ b/internal/notifier/notifier.go @@ -0,0 +1,88 @@ +package notifier + +import ( + "context" + "fmt" + "strconv" + "sync" + + "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox/producer" + "github.com/vladyslavpavlenko/genesis-api-project/internal/rateapi" + + "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox" + + "github.com/vladyslavpavlenko/genesis-api-project/internal/models" +) + +const batchSize = 100 + +// dbConnection defines an interface for the database connection. +type dbConnection interface { + GetSubscriptions(limit, offset int) ([]models.Subscription, error) +} + +type Notifier struct { + DB dbConnection + Fetcher rateapi.Fetcher + Outbox producer.Outbox +} + +// NewNotifier creates a new Notifier. +func NewNotifier(db dbConnection, f rateapi.Fetcher, o producer.Outbox) *Notifier { + return &Notifier{ + DB: db, + Fetcher: f, + Outbox: o, + } +} + +// ProduceNotificationEvents handles producing events for currency rate update emails. +func (n *Notifier) ProduceNotificationEvents() error { + rate, err := n.Fetcher.Fetch(context.Background(), "USD", "UAH") + if err != nil { + return fmt.Errorf("failed to retrieve rate: %w", err) + } + + floatRate, err := strconv.ParseFloat(rate, 64) + if err != nil { + return fmt.Errorf("failed to parse rate: %w", err) + } + + var offset int + errChan := make(chan error, 1) + for { + subscriptions, err := n.DB.GetSubscriptions(batchSize, offset) + if err != nil { + return err + } + if len(subscriptions) == 0 { + break + } + + var wg sync.WaitGroup + for _, sub := range subscriptions { + wg.Add(1) + go func(sub models.Subscription) { + defer wg.Done() + + data := outbox.Data{Email: sub.Email, Rate: floatRate} + if localErr := n.Outbox.AddEvent(data); localErr != nil { + select { + case errChan <- localErr: + default: + } + } + }(sub) + } + wg.Wait() + + select { + case err := <-errChan: + return err + default: + } + + offset += batchSize + } + return nil +} From e50142936424fac435e345537dde174894f5e0aa Mon Sep 17 00:00:00 2001 From: Vladyslav Pavlenko Date: Sun, 7 Jul 2024 19:10:01 +0300 Subject: [PATCH 8/9] refactor: change consumed event structure --- internal/email/consumer/consumed_event.go | 15 ++++++++++----- internal/email/consumer/kafka_consumer.go | 15 ++++++++++++--- internal/notifier/notifier.go | 5 ++++- internal/outbox/event.go | 13 +++++++------ internal/outbox/gormoutbox/outbox.go | 5 +---- 5 files changed, 34 insertions(+), 19 deletions(-) diff --git a/internal/email/consumer/consumed_event.go b/internal/email/consumer/consumed_event.go index 7455bce..8db0f43 100644 --- a/internal/email/consumer/consumed_event.go +++ b/internal/email/consumer/consumed_event.go @@ -1,11 +1,16 @@ package consumer -import "time" +import ( + "time" -// ConsumedEvent represents an event consumed by the consumer, including the partition information. + "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox" +) + +// ConsumedEvent represents an event consumed by the consumer. type ConsumedEvent struct { - Topic string `gorm:"primaryKey"` - Partition int `gorm:"primaryKey"` - Offset int64 `gorm:"primaryKey"` + ID uint `gorm:"not null;index"` + Event outbox.Event `gorm:"foreignKey:ID" json:"-"` + Data string ConsumedAt time.Time + UpdatedAt time.Time } diff --git a/internal/email/consumer/kafka_consumer.go b/internal/email/consumer/kafka_consumer.go index 03b6a18..1a59281 100644 --- a/internal/email/consumer/kafka_consumer.go +++ b/internal/email/consumer/kafka_consumer.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log" + "strconv" "time" "github.com/pkg/errors" @@ -48,6 +49,8 @@ func NewKafkaConsumer(kafkaURL, topic string, partition int, groupID string, sen return &KafkaConsumer{Reader: reader, Sender: sender, db: db}, nil } +// Consume is a worker that consumes messages from Kafka and processes them +// to send an email using the Sender interface. func (c *KafkaConsumer) Consume(ctx context.Context) { for { // Attempt to fetch a message from Kafka @@ -68,10 +71,16 @@ func (c *KafkaConsumer) Consume(ctx context.Context) { c.sendMessage(data) // Create a record of the consumed event + keyString := string(m.Key) + eventID, err := strconv.ParseUint(keyString, 10, 64) + if err != nil { + log.Printf("Failed to parse event ID from key: %v", err) + continue + } + consumedEvent := ConsumedEvent{ - Topic: m.Topic, - Partition: m.Partition, - Offset: m.Offset, + ID: uint(eventID), + Data: data.Serialize(), ConsumedAt: time.Now(), } diff --git a/internal/notifier/notifier.go b/internal/notifier/notifier.go index b4e1965..9e3195e 100644 --- a/internal/notifier/notifier.go +++ b/internal/notifier/notifier.go @@ -65,7 +65,10 @@ func (n *Notifier) ProduceNotificationEvents() error { go func(sub models.Subscription) { defer wg.Done() - data := outbox.Data{Email: sub.Email, Rate: floatRate} + data := outbox.Data{ + Email: sub.Email, + Rate: floatRate, + } if localErr := n.Outbox.AddEvent(data); localErr != nil { select { case errChan <- localErr: diff --git a/internal/outbox/event.go b/internal/outbox/event.go index a2f1a86..a8b90ff 100644 --- a/internal/outbox/event.go +++ b/internal/outbox/event.go @@ -2,6 +2,7 @@ package outbox import ( "encoding/json" + "log" "time" ) @@ -18,14 +19,14 @@ type Data struct { Rate float64 `json:"rate"` } -// SerializeData takes a Data struct and serializes it to a JSON string for storage. -func (e *Event) SerializeData(data Data) error { - bytes, err := json.Marshal(data) +// Serialize takes a Data struct and serializes it to a JSON string. +func (d Data) Serialize() string { + bytes, err := json.Marshal(d) if err != nil { - return err + log.Println(err) + return "" } - e.Data = string(bytes) - return nil + return string(bytes) } // DeserializeData deserializes JSON string to Data struct diff --git a/internal/outbox/gormoutbox/outbox.go b/internal/outbox/gormoutbox/outbox.go index 902758a..b83556e 100644 --- a/internal/outbox/gormoutbox/outbox.go +++ b/internal/outbox/gormoutbox/outbox.go @@ -35,10 +35,7 @@ func (o *Outbox) AddEvent(data outbox.Data) error { CreatedAt: time.Now(), } - err := event.SerializeData(data) - if err != nil { - return errors.Wrap(err, "failed to serialize data") - } + event.Data = data.Serialize() return o.db.AddEvent(event) } From fd2dcf709874b1549597fa88f339e01cbf0799d2 Mon Sep 17 00:00:00 2001 From: Vladyslav Pavlenko Date: Sun, 7 Jul 2024 19:42:15 +0300 Subject: [PATCH 9/9] refactor: move interfaces to notifier --- internal/app/run.go | 2 +- internal/handlers/handlers.go | 2 +- internal/notifier/notifier.go | 28 +++++++++++++++++----------- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/internal/app/run.go b/internal/app/run.go index d0ff6fd..8f4ecfc 100644 --- a/internal/app/run.go +++ b/internal/app/run.go @@ -134,7 +134,7 @@ func handleShutdown(srv *http.Server, cancelFunc context.CancelFunc) { // scheduleEmails sets up a mailing process. func scheduleEmails(s scheduler, n *notifier.Notifier) error { _, err := s.Schedule(schedule, func() { - err := n.ProduceNotificationEvents() + err := n.Start() if err != nil { log.Printf("Error notifying subscribers: %v", err) } diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 254afaf..b707d4a 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -106,7 +106,7 @@ func (m *Repository) Unsubscribe(w http.ResponseWriter, r *http.Request) { // SendEmails handles the `/sendEmails` request. func (m *Repository) SendEmails(w http.ResponseWriter, _ *http.Request) { // Produce mailing events - err := m.Services.Notifier.ProduceNotificationEvents() + err := m.Services.Notifier.Start() if err != nil { _ = jsonutils.ErrorJSON(w, err, http.StatusInternalServerError) return diff --git a/internal/notifier/notifier.go b/internal/notifier/notifier.go index 9e3195e..a12ae35 100644 --- a/internal/notifier/notifier.go +++ b/internal/notifier/notifier.go @@ -6,12 +6,8 @@ import ( "strconv" "sync" - "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox/producer" - "github.com/vladyslavpavlenko/genesis-api-project/internal/rateapi" - - "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox" - "github.com/vladyslavpavlenko/genesis-api-project/internal/models" + outboxpkg "github.com/vladyslavpavlenko/genesis-api-project/internal/outbox" ) const batchSize = 100 @@ -21,14 +17,24 @@ type dbConnection interface { GetSubscriptions(limit, offset int) ([]models.Subscription, error) } +// fetcher defines an interface for the fetching data rates. +type fetcher interface { + Fetch(ctx context.Context, base, target string) (string, error) +} + +// outbox defines an interface for writing events to the outbox. +type outbox interface { + AddEvent(data outboxpkg.Data) error +} + type Notifier struct { DB dbConnection - Fetcher rateapi.Fetcher - Outbox producer.Outbox + Fetcher fetcher + Outbox outbox } // NewNotifier creates a new Notifier. -func NewNotifier(db dbConnection, f rateapi.Fetcher, o producer.Outbox) *Notifier { +func NewNotifier(db dbConnection, f fetcher, o outbox) *Notifier { return &Notifier{ DB: db, Fetcher: f, @@ -36,8 +42,8 @@ func NewNotifier(db dbConnection, f rateapi.Fetcher, o producer.Outbox) *Notifie } } -// ProduceNotificationEvents handles producing events for currency rate update emails. -func (n *Notifier) ProduceNotificationEvents() error { +// Start handles producing events for currency rate update emails. +func (n *Notifier) Start() error { rate, err := n.Fetcher.Fetch(context.Background(), "USD", "UAH") if err != nil { return fmt.Errorf("failed to retrieve rate: %w", err) @@ -65,7 +71,7 @@ func (n *Notifier) ProduceNotificationEvents() error { go func(sub models.Subscription) { defer wg.Done() - data := outbox.Data{ + data := outboxpkg.Data{ Email: sub.Email, Rate: floatRate, }