diff --git a/.gitignore b/.gitignore index 63ebfdf26..eb00ee4a3 100644 --- a/.gitignore +++ b/.gitignore @@ -91,3 +91,9 @@ escape_analysis.txt # Compiled test wasm processors pkg/plugin/processor/standalone/test/wasm_processors/*/processor.wasm + +# Test data +**/test/*.txt + +# this one is needed for integration tests +!pkg/provisioning/test/source-file.txt diff --git a/go.mod b/go.mod index fb9f8fabd..e01f30e06 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/Masterminds/sprig/v3 v3.3.0 github.com/NYTimes/gziphandler v1.1.1 github.com/bufbuild/buf v1.41.0 - github.com/conduitio/conduit-commons v0.3.0 + github.com/conduitio/conduit-commons v0.3.1-0.20240913141354-f6dd6c835674 github.com/conduitio/conduit-connector-file v0.7.0 github.com/conduitio/conduit-connector-generator v0.7.0 github.com/conduitio/conduit-connector-kafka v0.9.0 @@ -141,8 +141,8 @@ require ( github.com/daixiang0/gci v0.13.5 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/denis-tingaikin/go-header v0.5.0 // indirect - github.com/dgraph-io/badger/v4 v4.2.0 // indirect - github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dgraph-io/badger/v4 v4.3.0 // indirect + github.com/dgraph-io/ristretto v0.1.2-0.20240116140435-c67e07994f91 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/dlclark/regexp2 v1.11.4 // indirect github.com/docker/cli v27.2.1+incompatible // indirect @@ -180,7 +180,6 @@ require ( github.com/gofrs/flock v0.12.1 // indirect github.com/gofrs/uuid/v5 v5.3.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/glog v1.2.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -200,7 +199,7 @@ require ( github.com/gostaticanalysis/comment v1.4.2 // indirect github.com/gostaticanalysis/forcetypeassert v0.1.0 // indirect github.com/gostaticanalysis/nilerr v0.1.1 // indirect - github.com/hamba/avro/v2 v2.24.0 // indirect + github.com/hamba/avro/v2 v2.25.2 // indirect github.com/hashicorp/go-version v1.7.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.0 // indirect @@ -213,8 +212,8 @@ require ( github.com/jackc/pglogrepl v0.0.0-20240307033717-828fbfe908e9 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.6.0 // indirect - github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/jackc/pgx/v5 v5.7.1 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jdx/go-netrc v1.0.0 // indirect github.com/jgautheron/goconst v1.7.1 // indirect github.com/jingyugao/rowserrcheck v1.1.1 // indirect @@ -375,7 +374,7 @@ require ( modernc.org/libc v1.55.3 // indirect modernc.org/mathutil v1.6.0 // indirect modernc.org/memory v1.8.0 // indirect - modernc.org/sqlite v1.31.1 // indirect + modernc.org/sqlite v1.33.1 // indirect modernc.org/strutil v1.2.0 // indirect modernc.org/token v1.1.0 // indirect mvdan.cc/unparam v0.0.0-20240528143540-8a5130ca722f // indirect diff --git a/go.sum b/go.sum index 32e696896..46d131e8e 100644 --- a/go.sum +++ b/go.sum @@ -198,7 +198,6 @@ github.com/ccojocar/zxcvbn-go v1.0.2/go.mod h1:g1qkXtUSvHP8lhHp5GrSmTz6uWALGRMQd github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charithe/durationcheck v0.0.10 h1:wgw73BiocdBDQPik+zcEoBG/ob8uyBHf2iyoHGPf5w4= @@ -219,8 +218,8 @@ github.com/ckaznocha/intrange v0.2.0/go.mod h1:r5I7nUlAAG56xmkOpw4XVr16BXhwYTUdc github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= -github.com/conduitio/conduit-commons v0.3.0 h1:nxQ++O4dK1p717upkyzbCQu0FLIFyP3OrgHZ9Zxvzvg= -github.com/conduitio/conduit-commons v0.3.0/go.mod h1:roxZ88dv+fpbEjjTzkdGwwbmcpunSuiD8he43y0lAoo= +github.com/conduitio/conduit-commons v0.3.1-0.20240913141354-f6dd6c835674 h1:coVk6aVsbP2u2vheM55GaCtu3+JYdIvOimQ0abUrmLA= +github.com/conduitio/conduit-commons v0.3.1-0.20240913141354-f6dd6c835674/go.mod h1:R/GNsw7iAUy5g2mFUVupAcrfWlGCUACwMvkDDrF39RI= github.com/conduitio/conduit-connector-file v0.7.0 h1:lUfDdpRZleJ/DDXX3NCzHN6VUYKORU/b443mJH6PJU4= github.com/conduitio/conduit-connector-file v0.7.0/go.mod h1:OXmcc1eAXmqmn9XoS/C3TdgZn0W1GMyqfNzUZRFmHNU= github.com/conduitio/conduit-connector-generator v0.7.0 h1:Bqsh/ak7gw6k5E8m0PxXOib0zhNlKbrJcIoLLQ0+S08= @@ -276,12 +275,12 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denis-tingaikin/go-header v0.5.0 h1:SRdnP5ZKvcO9KKRP1KJrhFR3RrlGuD+42t4429eC9k8= github.com/denis-tingaikin/go-header v0.5.0/go.mod h1:mMenU5bWrok6Wl2UsZjy+1okegmwQ3UgWl4V1D8gjlY= -github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= -github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= -github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= -github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgraph-io/badger/v4 v4.3.0 h1:lcsCE1/1qrRhqP+zYx6xDZb8n7U+QlwNicpc676Ub40= +github.com/dgraph-io/badger/v4 v4.3.0/go.mod h1:Sc0T595g8zqAQRDf44n+z3wG4BOqLwceaFntt8KPxUM= +github.com/dgraph-io/ristretto v0.1.2-0.20240116140435-c67e07994f91 h1:Pux6+xANi0I7RRo5E1gflI4EZ2yx3BGZ75JkAIvGEOA= +github.com/dgraph-io/ristretto v0.1.2-0.20240116140435-c67e07994f91/go.mod h1:swkazRqnUf1N62d0Nutz7KIj2UKqsm/H8tD0nBJAXqM= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo= @@ -302,7 +301,6 @@ github.com/dop251/goja v0.0.0-20240806095544-3491d4a58fbe h1:jwFJkgsdelB87ohlXaA github.com/dop251/goja v0.0.0-20240806095544-3491d4a58fbe/go.mod h1:DF+w/nLMIkvRpyhd/0K+Okbh3fVZBtXLwRtS/ccAa5w= github.com/dop251/goja_nodejs v0.0.0-20231122114759-e84d9a924c5c h1:hLoodLRD4KLWIH8eyAQCLcH8EqIrjac7fCkp/fHnvuQ= github.com/dop251/goja_nodejs v0.0.0-20231122114759-e84d9a924c5c/go.mod h1:bhGPmCgCCTSRfiMYWjpS46IDo9EUZXlsuUaPXSWGbv0= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -394,8 +392,6 @@ github.com/gofrs/uuid/v5 v5.3.0/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.2.1 h1:OptwRhECazUx5ix5TTWC3EZhsZEHWcYWY4FQHTIubm4= -github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -497,8 +493,8 @@ github.com/gostaticanalysis/testutil v0.4.0 h1:nhdCmubdmDF6VEatUNjgUZBJKWRqugoIS github.com/gostaticanalysis/testutil v0.4.0/go.mod h1:bLIoPefWXrRi/ssLFWX1dx7Repi5x3CuviD3dgAZaBU= github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= -github.com/hamba/avro/v2 v2.24.0 h1:axTlaYDkcSY0dVekRSy8cdrsj5MG86WqosUQacKCids= -github.com/hamba/avro/v2 v2.24.0/go.mod h1:7vDfy/2+kYCE8WUHoj2et59GTv0ap7ptktMXu0QHePI= +github.com/hamba/avro/v2 v2.25.2 h1:28dqbOCB7wA/3+J1ZN4GQ40tzsFtbtItkTPWgl97el0= +github.com/hamba/avro/v2 v2.25.2/go.mod h1:I8glyswHnpED3Nlx2ZdUe+4LJnCOOyiCzLMno9i/Uu0= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-plugin v1.6.1 h1:P7MR2UP6gNKGPp+y7EZw2kOiq4IR9WiqLvp0XOsVdwI= @@ -534,10 +530,10 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= -github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= -github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= -github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs= +github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jdx/go-netrc v1.0.0 h1:QbLMLyCZGj0NA8glAhxUpf1zDg6cxnWgMBbjq40W0gQ= github.com/jdx/go-netrc v1.0.0/go.mod h1:Gh9eFQJnoTNIRHXl2j5bJXA1u84hQWJWgGh569zF3v8= @@ -1107,7 +1103,6 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1317,8 +1312,8 @@ modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= -modernc.org/sqlite v1.31.1 h1:XVU0VyzxrYHlBhIs1DiEgSl0ZtdnPtbLVy8hSkzxGrs= -modernc.org/sqlite v1.31.1/go.mod h1:UqoylwmTb9F+IqXERT8bW9zzOWN8qwAIcLdzeBZs4hA= +modernc.org/sqlite v1.33.1 h1:trb6Z3YYoeM9eDL1O8do81kP+0ejv+YzgyFo+Gwy0nM= +modernc.org/sqlite v1.33.1/go.mod h1:pXV2xHxhzXZsgT/RtTFAPY6JJDEvOTcTdwADQCCWD4k= modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= diff --git a/pkg/conduit/runtime.go b/pkg/conduit/runtime.go index 05360c1db..c6df8fd15 100644 --- a/pkg/conduit/runtime.go +++ b/pkg/conduit/runtime.go @@ -45,6 +45,7 @@ import ( "github.com/conduitio/conduit/pkg/foundation/metrics" "github.com/conduitio/conduit/pkg/foundation/metrics/measure" "github.com/conduitio/conduit/pkg/foundation/metrics/prometheus" + "github.com/conduitio/conduit/pkg/lifecycle" "github.com/conduitio/conduit/pkg/orchestrator" "github.com/conduitio/conduit/pkg/pipeline" conn_plugin "github.com/conduitio/conduit/pkg/plugin/connector" @@ -63,6 +64,7 @@ import ( "github.com/conduitio/conduit/pkg/web/ui" apiv1 "github.com/conduitio/conduit/proto/api/v1" grpcruntime "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" + "github.com/jpillora/backoff" "github.com/piotrkowalczuk/promgrpc/v4" promclient "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -94,6 +96,7 @@ type Runtime struct { pipelineService *pipeline.Service connectorService *connector.Service processorService *processor.Service + lifecycleService *lifecycle.Service connectorPluginService *conn_plugin.PluginService processorPluginService *proc_plugin.PluginService @@ -201,13 +204,21 @@ func createServices(r *Runtime) error { tokenService, ) + errRecovery := r.Config.Pipelines.ErrorRecovery + backoffCfg := &backoff.Backoff{ + Min: errRecovery.MinDelay, + Max: errRecovery.MaxDelay, + Factor: float64(errRecovery.BackoffFactor), + Jitter: true, + } + plService := pipeline.NewService(r.logger, r.DB) connService := connector.NewService(r.logger, r.DB, r.connectorPersister) procService := processor.NewService(r.logger, r.DB, procPluginService) + lifecycleService := lifecycle.NewService(r.logger, backoffCfg, connService, procService, connPluginService, plService) + provisionService := provisioning.NewService(r.DB, r.logger, plService, connService, procService, connPluginService, lifecycleService, r.Config.Pipelines.Path) - provisionService := provisioning.NewService(r.DB, r.logger, plService, connService, procService, connPluginService, r.Config.Pipelines.Path) - - orc := orchestrator.NewOrchestrator(r.DB, r.logger, plService, connService, procService, connPluginService, procPluginService) + orc := orchestrator.NewOrchestrator(r.DB, r.logger, plService, connService, procService, connPluginService, procPluginService, lifecycleService) r.Orchestrator = orc r.ProvisionService = provisionService @@ -220,6 +231,7 @@ func createServices(r *Runtime) error { r.processorPluginService = procPluginService r.connSchemaService = connSchemaService r.procSchemaService = procSchemaService + r.lifecycleService = lifecycleService return nil } @@ -411,12 +423,12 @@ func (r *Runtime) registerCleanup(t *tomb.Tomb) { // t.Err() can be nil, when we had a call: t.Kill(nil) // t.Err() will be context.Canceled, if the tomb's context was canceled if t.Err() == nil || cerrors.Is(t.Err(), context.Canceled) { - r.pipelineService.StopAll(ctx, pipeline.ErrGracefulShutdown) + r.lifecycleService.StopAll(ctx, pipeline.ErrGracefulShutdown) } else { // tomb died due to a real error - r.pipelineService.StopAll(ctx, cerrors.Errorf("conduit experienced an error: %w", t.Err())) + r.lifecycleService.StopAll(ctx, cerrors.Errorf("conduit experienced an error: %w", t.Err())) } - err := r.pipelineService.Wait(exitTimeout) + err := r.lifecycleService.Wait(exitTimeout) t.Go(func() error { r.connectorPersister.Wait() return r.DB.Close() @@ -758,7 +770,7 @@ func (r *Runtime) initServices(ctx context.Context, t *tomb.Tomb) error { } if r.Config.Pipelines.ExitOnError { - r.pipelineService.OnFailure(func(e pipeline.FailureEvent) { + r.lifecycleService.OnFailure(func(e lifecycle.FailureEvent) { r.logger.Warn(ctx). Err(e.Error). Str(log.PipelineIDField, e.ID). @@ -785,7 +797,7 @@ func (r *Runtime) initServices(ctx context.Context, t *tomb.Tomb) error { } } - err = r.pipelineService.Run(ctx, r.connectorService, r.processorService, r.connectorPluginService) + err = r.lifecycleService.Init(ctx) if err != nil { cerrors.ForEach(err, func(err error) { r.logger.Err(ctx, err).Msg("pipeline failed to be started") diff --git a/pkg/pipeline/dlq.go b/pkg/lifecycle/dlq.go similarity index 97% rename from pkg/pipeline/dlq.go rename to pkg/lifecycle/dlq.go index e602f2d44..8590fffde 100644 --- a/pkg/pipeline/dlq.go +++ b/pkg/lifecycle/dlq.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pipeline +package lifecycle import ( "bytes" @@ -21,7 +21,7 @@ import ( "github.com/conduitio/conduit-commons/opencdc" "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" - "github.com/conduitio/conduit/pkg/pipeline/stream" + "github.com/conduitio/conduit/pkg/lifecycle/stream" ) // DLQDestination is a DLQ handler that forwards DLQ records to a destination diff --git a/pkg/pipeline/dlq_test.go b/pkg/lifecycle/dlq_test.go similarity index 97% rename from pkg/pipeline/dlq_test.go rename to pkg/lifecycle/dlq_test.go index 569cff8e8..815146041 100644 --- a/pkg/pipeline/dlq_test.go +++ b/pkg/lifecycle/dlq_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pipeline +package lifecycle import ( "context" @@ -22,7 +22,7 @@ import ( "github.com/conduitio/conduit/pkg/connector" "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" - streammock "github.com/conduitio/conduit/pkg/pipeline/stream/mock" + streammock "github.com/conduitio/conduit/pkg/lifecycle/stream/mock" "github.com/matryer/is" "go.uber.org/mock/gomock" ) diff --git a/pkg/pipeline/lifecycle.go b/pkg/lifecycle/service.go similarity index 65% rename from pkg/pipeline/lifecycle.go rename to pkg/lifecycle/service.go index a9c35d98d..2849da9fe 100644 --- a/pkg/pipeline/lifecycle.go +++ b/pkg/lifecycle/service.go @@ -1,4 +1,4 @@ -// Copyright © 2022 Meroxa, Inc. +// Copyright © 2024 Meroxa, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pipeline +// Package lifecycle contains the logic to manage the lifecycle of pipelines. +// It is responsible for starting, stopping and managing pipelines. +package lifecycle import ( "context" @@ -23,19 +25,72 @@ import ( "sync/atomic" "time" + "github.com/conduitio/conduit-commons/csync" "github.com/conduitio/conduit/pkg/connector" "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" "github.com/conduitio/conduit/pkg/foundation/metrics" "github.com/conduitio/conduit/pkg/foundation/metrics/measure" - "github.com/conduitio/conduit/pkg/pipeline/stream" + "github.com/conduitio/conduit/pkg/lifecycle/stream" + "github.com/conduitio/conduit/pkg/pipeline" connectorPlugin "github.com/conduitio/conduit/pkg/plugin/connector" "github.com/conduitio/conduit/pkg/processor" + "github.com/jpillora/backoff" "gopkg.in/tomb.v2" ) -// ConnectorFetcher can fetch a connector instance. -type ConnectorFetcher interface { +type FailureEvent struct { + // ID is the ID of the pipeline which failed. + ID string + Error error +} + +type FailureHandler func(FailureEvent) + +// Service manages pipelines. +type Service struct { + logger log.CtxLogger + + backoffCfg *backoff.Backoff + + pipelines PipelineService + connectors ConnectorService + + processors ProcessorService + connectorPlugins ConnectorPluginService + + handlers []FailureHandler + runningPipelines *csync.Map[string, *runnablePipeline] +} + +// NewService initializes and returns a lifecycle.Service. +func NewService( + logger log.CtxLogger, + backoffCfg *backoff.Backoff, + connectors ConnectorService, + processors ProcessorService, + connectorPlugins ConnectorPluginService, + pipelines PipelineService, +) *Service { + return &Service{ + logger: logger.WithComponent("lifecycle.Service"), + backoffCfg: backoffCfg, + connectors: connectors, + processors: processors, + connectorPlugins: connectorPlugins, + pipelines: pipelines, + runningPipelines: csync.NewMap[string, *runnablePipeline](), + } +} + +type runnablePipeline struct { + pipeline *pipeline.Instance + n []stream.Node + t *tomb.Tomb +} + +// ConnectorService can fetch and create a connector instance. +type ConnectorService interface { Get(ctx context.Context, id string) (*connector.Instance, error) Create(ctx context.Context, id string, t connector.Type, plugin string, pipelineID string, cfg connector.Config, p connector.ProvisionType) (*connector.Instance, error) } @@ -46,25 +101,36 @@ type ProcessorService interface { MakeRunnableProcessor(ctx context.Context, i *processor.Instance) (*processor.RunnableProcessor, error) } -// PluginDispenserFetcher can fetch a plugin. -type PluginDispenserFetcher interface { +// ConnectorPluginService can create a connector plugin dispenser. +type ConnectorPluginService interface { NewDispenser(logger log.CtxLogger, name string, connectorID string) (connectorPlugin.Dispenser, error) } -// Run runs pipelines that had the running state in store. -func (s *Service) Run( +// PipelineService can fetch, list and update the status of a pipeline instance. +type PipelineService interface { + Get(ctx context.Context, pipelineID string) (*pipeline.Instance, error) + List(ctx context.Context) map[string]*pipeline.Instance + UpdateStatus(ctx context.Context, pipelineID string, status pipeline.Status, errMsg string) error +} + +// OnFailure registers a handler for a lifecycle.FailureEvent. +// Only errors which happen after a pipeline has been started +// are being sent. +func (s *Service) OnFailure(handler FailureHandler) { + s.handlers = append(s.handlers, handler) +} + +// Init starts all pipelines that have the StatusSystemStopped. +func (s *Service) Init( ctx context.Context, - connFetcher ConnectorFetcher, - procService ProcessorService, - pluginFetcher PluginDispenserFetcher, ) error { var errs []error s.logger.Debug(ctx).Msg("initializing pipelines statuses") - // run pipelines that are in the StatusSystemStopped state - for _, instance := range s.instances { - if instance.GetStatus() == StatusSystemStopped { - err := s.Start(ctx, connFetcher, procService, pluginFetcher, instance.ID) + instances := s.pipelines.List(ctx) + for _, instance := range instances { + if instance.GetStatus() == pipeline.StatusSystemStopped { + err := s.Start(ctx, instance.ID) if err != nil { // try to start remaining pipelines and gather errors errs = append(errs, err) @@ -79,38 +145,34 @@ func (s *Service) Run( // If the pipeline is already running, Start returns ErrPipelineRunning. func (s *Service) Start( ctx context.Context, - connFetcher ConnectorFetcher, - procService ProcessorService, - pluginFetcher PluginDispenserFetcher, pipelineID string, ) error { - pl, err := s.Get(ctx, pipelineID) + pl, err := s.pipelines.Get(ctx, pipelineID) if err != nil { return err } - if pl.GetStatus() == StatusRunning { - return cerrors.Errorf("can't start pipeline %s: %w", pl.ID, ErrPipelineRunning) + + if pl.GetStatus() == pipeline.StatusRunning { + return cerrors.Errorf("can't start pipeline %s: %w", pl.ID, pipeline.ErrPipelineRunning) } s.logger.Debug(ctx).Str(log.PipelineIDField, pl.ID).Msg("starting pipeline") s.logger.Trace(ctx).Str(log.PipelineIDField, pl.ID).Msg("building nodes") - nodes, err := s.buildNodes(ctx, connFetcher, procService, pluginFetcher, pl) + + rp, err := s.buildRunnablePipeline(ctx, pl) if err != nil { return cerrors.Errorf("could not build nodes for pipeline %s: %w", pl.ID, err) } - pl.n = make(map[string]stream.Node) - for _, node := range nodes { - pl.n[node.ID()] = node - } - s.logger.Trace(ctx).Str(log.PipelineIDField, pl.ID).Msg("running nodes") - if err := s.runPipeline(ctx, pl); err != nil { + if err := s.runPipeline(ctx, rp); err != nil { return cerrors.Errorf("failed to run pipeline %s: %w", pl.ID, err) } s.logger.Info(ctx).Str(log.PipelineIDField, pl.ID).Msg("pipeline started") + s.runningPipelines.Set(pl.ID, rp) + return nil } @@ -119,31 +181,32 @@ func (s *Service) Start( // instead the context for all nodes will be canceled which causes them to stop // running as soon as possible. func (s *Service) Stop(ctx context.Context, pipelineID string, force bool) error { - pl, err := s.Get(ctx, pipelineID) - if err != nil { - return err + rp, ok := s.runningPipelines.Get(pipelineID) + + if !ok { + return cerrors.Errorf("pipeline %s is not running: %w", pipelineID, pipeline.ErrPipelineNotRunning) } - if pl.GetStatus() != StatusRunning && pl.GetStatus() != StatusRecovering { - return cerrors.Errorf("can't stop pipeline with status %q: %w", pl.GetStatus(), ErrPipelineNotRunning) + if rp.pipeline.GetStatus() != pipeline.StatusRunning && rp.pipeline.GetStatus() != pipeline.StatusRecovering { + return cerrors.Errorf("can't stop pipeline with status %q: %w", rp.pipeline.GetStatus(), pipeline.ErrPipelineNotRunning) } switch force { case false: - return s.stopGraceful(ctx, pl, nil) + return s.stopGraceful(ctx, rp, nil) case true: - return s.stopForceful(ctx, pl) + return s.stopForceful(ctx, rp) } panic("unreachable code") } -func (s *Service) stopGraceful(ctx context.Context, pl *Instance, reason error) error { +func (s *Service) stopGraceful(ctx context.Context, rp *runnablePipeline, reason error) error { s.logger.Info(ctx). - Str(log.PipelineIDField, pl.ID). - Any(log.PipelineStatusField, pl.GetStatus()). + Str(log.PipelineIDField, rp.pipeline.ID). + Any(log.PipelineStatusField, rp.pipeline.GetStatus()). Msg("gracefully stopping pipeline") var errs []error - for _, n := range pl.n { + for _, n := range rp.n { if node, ok := n.(stream.StoppableNode); ok { // stop all pub nodes s.logger.Trace(ctx).Str(log.NodeIDField, n.ID()).Msg("stopping node") @@ -154,16 +217,17 @@ func (s *Service) stopGraceful(ctx context.Context, pl *Instance, reason error) } } } + return cerrors.Join(errs...) } -func (s *Service) stopForceful(ctx context.Context, pl *Instance) error { +func (s *Service) stopForceful(ctx context.Context, rp *runnablePipeline) error { s.logger.Info(ctx). - Str(log.PipelineIDField, pl.ID). - Any(log.PipelineStatusField, pl.GetStatus()). + Str(log.PipelineIDField, rp.pipeline.ID). + Any(log.PipelineStatusField, rp.pipeline.GetStatus()). Msg("force stopping pipeline") - pl.t.Kill(ErrForceStop) - for _, n := range pl.n { + rp.t.Kill(pipeline.ErrForceStop) + for _, n := range rp.n { if node, ok := n.(stream.ForceStoppableNode); ok { // stop all pub nodes s.logger.Trace(ctx).Str(log.NodeIDField, n.ID()).Msg("force stopping node") @@ -173,18 +237,19 @@ func (s *Service) stopForceful(ctx context.Context, pl *Instance) error { return nil } -// StopAll will ask all the pipelines to stop gracefully +// StopAll will ask all the running pipelines to stop gracefully // (i.e. that existing messages get processed but not new messages get produced). func (s *Service) StopAll(ctx context.Context, reason error) { - for _, pl := range s.instances { - if pl.GetStatus() != StatusRunning && pl.GetStatus() != StatusRecovering { + for _, rp := range s.runningPipelines.All() { + p := rp.pipeline + if p.GetStatus() != pipeline.StatusRunning && p.GetStatus() != pipeline.StatusRecovering { continue } - err := s.stopGraceful(ctx, pl, reason) + err := s.stopGraceful(ctx, rp, reason) if err != nil { s.logger.Warn(ctx). Err(err). - Str(log.PipelineIDField, pl.ID). + Str(log.PipelineIDField, p.ID). Msg("could not stop pipeline") } } @@ -211,7 +276,7 @@ func (s *Service) Wait(timeout time.Duration) error { case <-gracefullyStopped: return err case <-time.After(timeout): - return ErrTimeout + return pipeline.ErrTimeout } } @@ -219,8 +284,15 @@ func (s *Service) Wait(timeout time.Duration) error { // the pipelines failed to stop gracefully. func (s *Service) waitInternal() error { var errs []error - for _, pl := range s.instances { - err := pl.Wait() + + // copy pipelines to keep the map unlocked while we iterate it + pipelines := s.runningPipelines.Copy() + + for _, rp := range pipelines.All() { + if rp.t == nil { + continue + } + err := rp.t.Wait() if err != nil { errs = append(errs, err) } @@ -228,19 +300,25 @@ func (s *Service) waitInternal() error { return cerrors.Join(errs...) } -// buildsNodes will build and connect all nodes configured in the pipeline. -func (s *Service) buildNodes( +// WaitPipeline blocks until the pipeline with the given ID is stopped. +func (s *Service) WaitPipeline(id string) error { + p, ok := s.runningPipelines.Get(id) + if !ok || p.t == nil { + return nil + } + return p.t.Wait() +} + +// buildRunnablePipeline will build and connect all nodes configured in the pipeline. +func (s *Service) buildRunnablePipeline( ctx context.Context, - connFetcher ConnectorFetcher, - procService ProcessorService, - pluginFetcher PluginDispenserFetcher, - pl *Instance, -) ([]stream.Node, error) { + pl *pipeline.Instance, +) (*runnablePipeline, error) { // setup many to many channels fanIn := stream.FaninNode{Name: "fanin"} fanOut := stream.FanoutNode{Name: "fanout"} - sourceNodes, err := s.buildSourceNodes(ctx, connFetcher, procService, pluginFetcher, pl, &fanIn) + sourceNodes, err := s.buildSourceNodes(ctx, pl, &fanIn) if err != nil { return nil, cerrors.Errorf("could not build source nodes: %w", err) } @@ -248,12 +326,12 @@ func (s *Service) buildNodes( return nil, cerrors.New("can't build pipeline without any source connectors") } - processorNodes, err := s.buildProcessorNodes(ctx, procService, pl, pl.ProcessorIDs, &fanIn, &fanOut) + processorNodes, err := s.buildProcessorNodes(ctx, pl, pl.ProcessorIDs, &fanIn, &fanOut) if err != nil { return nil, cerrors.Errorf("could not build processor nodes: %w", err) } - destinationNodes, err := s.buildDestinationNodes(ctx, connFetcher, procService, pluginFetcher, pl, &fanOut) + destinationNodes, err := s.buildDestinationNodes(ctx, pl, &fanOut) if err != nil { return nil, cerrors.Errorf("could not build destination nodes: %w", err) } @@ -276,13 +354,15 @@ func (s *Service) buildNodes( stream.SetLogger(n, nodeLogger) } - return nodes, nil + return &runnablePipeline{ + pipeline: pl, + n: nodes, + }, nil } func (s *Service) buildProcessorNodes( ctx context.Context, - procService ProcessorService, - pl *Instance, + pl *pipeline.Instance, processorIDs []string, first stream.PubNode, last stream.SubNode, @@ -291,12 +371,12 @@ func (s *Service) buildProcessorNodes( prev := first for _, procID := range processorIDs { - instance, err := procService.Get(ctx, procID) + instance, err := s.processors.Get(ctx, procID) if err != nil { return nil, cerrors.Errorf("could not fetch processor: %w", err) } - runnableProc, err := procService.MakeRunnableProcessor(ctx, instance) + runnableProc, err := s.processors.MakeRunnableProcessor(ctx, instance) if err != nil { return nil, err } @@ -319,7 +399,7 @@ func (s *Service) buildProcessorNodes( } func (s *Service) buildParallelProcessorNode( - pl *Instance, + pl *pipeline.Instance, proc *processor.RunnableProcessor, ) *stream.ParallelNode { return &stream.ParallelNode{ @@ -334,7 +414,7 @@ func (s *Service) buildParallelProcessorNode( } func (s *Service) buildProcessorNode( - pl *Instance, + pl *pipeline.Instance, proc *processor.RunnableProcessor, ) *stream.ProcessorNode { return &stream.ProcessorNode{ @@ -346,21 +426,18 @@ func (s *Service) buildProcessorNode( func (s *Service) buildSourceNodes( ctx context.Context, - connFetcher ConnectorFetcher, - procService ProcessorService, - pluginFetcher PluginDispenserFetcher, - pl *Instance, + pl *pipeline.Instance, next stream.SubNode, ) ([]stream.Node, error) { var nodes []stream.Node - dlqHandlerNode, err := s.buildDLQHandlerNode(ctx, connFetcher, pluginFetcher, pl) + dlqHandlerNode, err := s.buildDLQHandlerNode(ctx, pl) if err != nil { return nil, err } for _, connID := range pl.ConnectorIDs { - instance, err := connFetcher.Get(ctx, connID) + instance, err := s.connectors.Get(ctx, connID) if err != nil { return nil, cerrors.Errorf("could not fetch connector: %w", err) } @@ -369,7 +446,7 @@ func (s *Service) buildSourceNodes( continue // skip any connector that's not a source } - src, err := instance.Connector(ctx, pluginFetcher) + src, err := instance.Connector(ctx, s.connectorPlugins) if err != nil { return nil, err } @@ -387,7 +464,7 @@ func (s *Service) buildSourceNodes( metricsNode := s.buildMetricsNode(pl, instance) metricsNode.Sub(ackerNode.Pub()) - procNodes, err := s.buildProcessorNodes(ctx, procService, pl, instance.ProcessorIDs, metricsNode, next) + procNodes, err := s.buildProcessorNodes(ctx, pl, instance.ProcessorIDs, metricsNode, next) if err != nil { return nil, cerrors.Errorf("could not build processor nodes for connector %s: %w", instance.ID, err) } @@ -415,11 +492,9 @@ func (s *Service) buildSourceAckerNode( func (s *Service) buildDLQHandlerNode( ctx context.Context, - connFetcher ConnectorFetcher, - pluginFetcher PluginDispenserFetcher, - pl *Instance, + pl *pipeline.Instance, ) (*stream.DLQHandlerNode, error) { - conn, err := connFetcher.Create( + conn, err := s.connectors.Create( ctx, pl.ID+"-dlq", connector.TypeDestination, @@ -435,7 +510,7 @@ func (s *Service) buildDLQHandlerNode( return nil, cerrors.Errorf("failed to create DLQ destination: %w", err) } - dest, err := conn.Connector(ctx, pluginFetcher) + dest, err := conn.Connector(ctx, s.connectorPlugins) if err != nil { return nil, err } @@ -461,7 +536,7 @@ func (s *Service) buildDLQHandlerNode( } func (s *Service) buildMetricsNode( - pl *Instance, + pl *pipeline.Instance, conn *connector.Instance, ) *stream.MetricsNode { return &stream.MetricsNode{ @@ -487,16 +562,13 @@ func (s *Service) buildDestinationAckerNode( func (s *Service) buildDestinationNodes( ctx context.Context, - connFetcher ConnectorFetcher, - procService ProcessorService, - pluginFetcher PluginDispenserFetcher, - pl *Instance, + pl *pipeline.Instance, prev stream.PubNode, ) ([]stream.Node, error) { var nodes []stream.Node for _, connID := range pl.ConnectorIDs { - instance, err := connFetcher.Get(ctx, connID) + instance, err := s.connectors.Get(ctx, connID) if err != nil { return nil, cerrors.Errorf("could not fetch connector: %w", err) } @@ -505,7 +577,7 @@ func (s *Service) buildDestinationNodes( continue // skip any connector that's not a destination } - dest, err := instance.Connector(ctx, pluginFetcher) + dest, err := instance.Connector(ctx, s.connectorPlugins) if err != nil { return nil, err } @@ -524,7 +596,7 @@ func (s *Service) buildDestinationNodes( destinationNode.Sub(metricsNode.Pub()) ackerNode.Sub(destinationNode.Pub()) - connNodes, err := s.buildProcessorNodes(ctx, procService, pl, instance.ProcessorIDs, prev, metricsNode) + connNodes, err := s.buildProcessorNodes(ctx, pl, instance.ProcessorIDs, prev, metricsNode) if err != nil { return nil, cerrors.Errorf("could not build processor nodes for connector %s: %w", instance.ID, err) } @@ -536,18 +608,18 @@ func (s *Service) buildDestinationNodes( return nodes, nil } -func (s *Service) runPipeline(ctx context.Context, pl *Instance) error { - if pl.t != nil && pl.t.Alive() { - return ErrPipelineRunning +func (s *Service) runPipeline(ctx context.Context, rp *runnablePipeline) error { + if rp.t != nil && rp.t.Alive() { + return pipeline.ErrPipelineRunning } // the tomb is responsible for running goroutines related to the pipeline - pl.t = &tomb.Tomb{} + rp.t = &tomb.Tomb{} // keep tomb alive until the end of this function, this way we guarantee we // can run the cleanup goroutine even if all nodes stop before we get to it keepAlive := make(chan struct{}) - pl.t.Go(func() error { + rp.t.Go(func() error { <-keepAlive return nil }) @@ -556,16 +628,15 @@ func (s *Service) runPipeline(ctx context.Context, pl *Instance) error { // nodesWg is done once all nodes stop running var nodesWg sync.WaitGroup var isGracefulShutdown atomic.Bool - for id := range pl.n { + for _, node := range rp.n { nodesWg.Add(1) - node := pl.n[id] - pl.t.Go(func() (errOut error) { + rp.t.Go(func() (errOut error) { // If any of the nodes stop, the tomb will be put into a dying state // and ctx will be cancelled. // This way, the other nodes will be notified that they need to stop too. //nolint:staticcheck // nil used to use the default (parent provided via WithContext) - ctx := pl.t.Context(nil) + ctx := rp.t.Context(nil) s.logger.Trace(ctx).Str(log.NodeIDField, node.ID()).Msg("running node") defer func() { e := s.logger.Trace(ctx) @@ -577,7 +648,7 @@ func (s *Service) runPipeline(ctx context.Context, pl *Instance) error { defer nodesWg.Done() err := node.Run(ctx) - if cerrors.Is(err, ErrGracefulShutdown) { + if cerrors.Is(err, pipeline.ErrGracefulShutdown) { // This node was shutdown because of ErrGracefulShutdown, we // need to stop this goroutine without returning an error to let // other nodes stop gracefully. We set a boolean that lets the @@ -593,64 +664,73 @@ func (s *Service) runPipeline(ctx context.Context, pl *Instance) error { }) } - s.updateOldStatusMetrics(pl) - - pl.SetStatus(StatusRunning) - pl.Error = "" - - s.updateNewStatusMetrics(pl) - - err := s.store.Set(ctx, pl.ID, pl) + err := s.pipelines.UpdateStatus(ctx, rp.pipeline.ID, pipeline.StatusRunning, "") if err != nil { - return cerrors.Errorf("pipeline not updated: %w", err) + return err } // cleanup function updates the metrics and pipeline status once all nodes // stop running - pl.t.Go(func() error { + rp.t.Go(func() error { // use fresh context for cleanup function, otherwise the updated status // won't be stored ctx := context.Background() nodesWg.Wait() - err := pl.t.Err() - - s.updateOldStatusMetrics(pl) + err := rp.t.Err() switch err { case tomb.ErrStillAlive: // not an actual error, the pipeline stopped gracefully - err = nil if isGracefulShutdown.Load() { // it was triggered by a graceful shutdown of Conduit - pl.SetStatus(StatusSystemStopped) + err = s.pipelines.UpdateStatus(ctx, rp.pipeline.ID, pipeline.StatusSystemStopped, "") } else { // it was manually triggered by a user - pl.SetStatus(StatusUserStopped) + err = s.pipelines.UpdateStatus(ctx, rp.pipeline.ID, pipeline.StatusUserStopped, "") + } + if err != nil { + return err } default: - pl.SetStatus(StatusDegraded) - // we use %+v to get the stack trace too - pl.Error = fmt.Sprintf("%+v", err) + if cerrors.IsFatalError(err) { + // we use %+v to get the stack trace too + err = s.pipelines.UpdateStatus(ctx, rp.pipeline.ID, pipeline.StatusDegraded, fmt.Sprintf("%+v", err)) + if err != nil { + return err + } + } else { + err = s.pipelines.UpdateStatus(ctx, rp.pipeline.ID, pipeline.StatusRecovering, "") + if err != nil { + return err + } + } } s.logger. Err(ctx, err). - Str(log.PipelineIDField, pl.ID). + Str(log.PipelineIDField, rp.pipeline.ID). Msg("pipeline stopped") - s.notify(pl.ID, err) - // It's important to update the metrics before we handle the error from s.Store.Set() (if any), - // since the source of the truth is the actual pipeline (stored in memory). - s.updateNewStatusMetrics(pl) - - storeErr := s.store.Set(ctx, pl.ID, pl) - if storeErr != nil { - return cerrors.Errorf("pipeline not updated: %w", storeErr) - } + // confirmed that all nodes stopped, we can now remove the pipeline from the running pipelines + s.runningPipelines.Delete(rp.pipeline.ID) + s.notify(rp.pipeline.ID, err) return err }) - return nil } + +// notify notifies all registered FailureHandlers about an error. +func (s *Service) notify(pipelineID string, err error) { + if err == nil { + return + } + e := FailureEvent{ + ID: pipelineID, + Error: err, + } + for _, handler := range s.handlers { + handler(e) + } +} diff --git a/pkg/pipeline/lifecycle_test.go b/pkg/lifecycle/service_test.go similarity index 69% rename from pkg/pipeline/lifecycle_test.go rename to pkg/lifecycle/service_test.go index a71ae6bf3..d3bb0a7f0 100644 --- a/pkg/pipeline/lifecycle_test.go +++ b/pkg/lifecycle/service_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pipeline +package lifecycle import ( "context" @@ -29,12 +29,14 @@ import ( "github.com/conduitio/conduit/pkg/connector" "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" - "github.com/conduitio/conduit/pkg/pipeline/stream" + "github.com/conduitio/conduit/pkg/lifecycle/stream" + "github.com/conduitio/conduit/pkg/pipeline" "github.com/conduitio/conduit/pkg/plugin" connectorPlugin "github.com/conduitio/conduit/pkg/plugin/connector" pmock "github.com/conduitio/conduit/pkg/plugin/connector/mock" "github.com/conduitio/conduit/pkg/processor" "github.com/google/uuid" + "github.com/jpillora/backoff" "github.com/matryer/is" "github.com/rs/zerolog" "go.uber.org/mock/gomock" @@ -42,7 +44,7 @@ import ( const testDLQID = "test-dlq" -func TestServiceLifecycle_buildNodes(t *testing.T) { +func TestServiceLifecycle_buildRunnablePipeline(t *testing.T) { is := is.New(t) ctx, killAll := context.WithCancel(context.Background()) defer killAll() @@ -50,17 +52,15 @@ func TestServiceLifecycle_buildNodes(t *testing.T) { logger := log.New(zerolog.Nop()) db := &inmemory.DB{} persister := connector.NewPersister(logger, db, time.Second, 3) - - ps := NewService(logger, db) + b := &backoff.Backoff{} source := dummySource(persister) destination := dummyDestination(persister) dlq := dummyDestination(persister) - pl := &Instance{ + pl := &pipeline.Instance{ ID: uuid.NewString(), - Config: Config{Name: "test-pipeline"}, - status: StatusUserStopped, - DLQ: DLQ{ + Config: pipeline.Config{Name: "test-pipeline"}, + DLQ: pipeline.DLQ{ Plugin: dlq.Plugin, Settings: map[string]string{}, WindowSize: 3, @@ -68,39 +68,51 @@ func TestServiceLifecycle_buildNodes(t *testing.T) { }, ConnectorIDs: []string{source.ID, destination.ID}, } + pl.SetStatus(pipeline.StatusUserStopped) - got, err := ps.buildNodes( - ctx, - testConnectorFetcher{ + ls := NewService( + logger, + b, + testConnectorService{ source.ID: source, destination.ID: destination, testDLQID: dlq, }, - testProcessorFetcher{}, - testPluginFetcher{ + testProcessorService{}, + testConnectorPluginService{ source.Plugin: pmock.NewDispenser(ctrl), destination.Plugin: pmock.NewDispenser(ctrl), dlq.Plugin: pmock.NewDispenser(ctrl), }, + testPipelineService{}, + ) + + got, err := ls.buildRunnablePipeline( + ctx, pl, ) + is.NoErr(err) - want := []stream.Node{ - &stream.SourceNode{}, - &stream.SourceAckerNode{}, - &stream.MetricsNode{}, - &stream.DLQHandlerNode{}, - &stream.FaninNode{}, - &stream.FanoutNode{}, - &stream.MetricsNode{}, - &stream.DestinationNode{}, - &stream.DestinationAckerNode{}, + want := runnablePipeline{ + pipeline: pl, + n: []stream.Node{ + &stream.SourceNode{}, + &stream.SourceAckerNode{}, + &stream.MetricsNode{}, + &stream.DLQHandlerNode{}, + &stream.FaninNode{}, + &stream.FanoutNode{}, + &stream.MetricsNode{}, + &stream.DestinationNode{}, + &stream.DestinationAckerNode{}, + }, } - is.Equal(len(want), len(got)) - for i := range want { - want := want[i] - got := got[i] + + is.Equal(len(want.n), len(got.n)) + for i := range want.n { + want := want.n[i] + got := got.n[i] is.Equal(reflect.TypeOf(want), reflect.TypeOf(got)) // unexpected node type switch got := got.(type) { @@ -125,7 +137,7 @@ func TestServiceLifecycle_buildNodes(t *testing.T) { } } -func TestService_buildNodes_NoSourceNode(t *testing.T) { +func TestService_buildRunnablePipeline_NoSourceNode(t *testing.T) { is := is.New(t) ctx, killAll := context.WithCancel(context.Background()) defer killAll() @@ -133,18 +145,14 @@ func TestService_buildNodes_NoSourceNode(t *testing.T) { logger := log.New(zerolog.Nop()) db := &inmemory.DB{} persister := connector.NewPersister(logger, db, time.Second, 3) - - ps := NewService(logger, db) - - wantErr := "can't build pipeline without any source connectors" + b := &backoff.Backoff{} destination := dummyDestination(persister) dlq := dummyDestination(persister) - pl := &Instance{ + pl := &pipeline.Instance{ ID: uuid.NewString(), - Config: Config{Name: "test-pipeline"}, - status: StatusUserStopped, - DLQ: DLQ{ + Config: pipeline.Config{Name: "test-pipeline"}, + DLQ: pipeline.DLQ{ Plugin: dlq.Plugin, Settings: map[string]string{}, WindowSize: 3, @@ -152,18 +160,21 @@ func TestService_buildNodes_NoSourceNode(t *testing.T) { }, ConnectorIDs: []string{destination.ID}, } + pl.SetStatus(pipeline.StatusUserStopped) - got, err := ps.buildNodes( - ctx, - testConnectorFetcher{ - destination.ID: destination, - testDLQID: dlq, - }, - testProcessorFetcher{}, - testPluginFetcher{ + ls := NewService(logger, b, testConnectorService{ + destination.ID: destination, + testDLQID: dlq, + }, testProcessorService{}, + testConnectorPluginService{ destination.Plugin: pmock.NewDispenser(ctrl), dlq.Plugin: pmock.NewDispenser(ctrl), - }, + }, testPipelineService{}) + + wantErr := "can't build pipeline without any source connectors" + + got, err := ls.buildRunnablePipeline( + ctx, pl, ) @@ -172,7 +183,7 @@ func TestService_buildNodes_NoSourceNode(t *testing.T) { is.Equal(got, nil) } -func TestService_buildNodes_NoDestinationNode(t *testing.T) { +func TestService_buildRunnablePipeline_NoDestinationNode(t *testing.T) { is := is.New(t) ctx, killAll := context.WithCancel(context.Background()) defer killAll() @@ -180,19 +191,27 @@ func TestService_buildNodes_NoDestinationNode(t *testing.T) { logger := log.New(zerolog.Nop()) db := &inmemory.DB{} persister := connector.NewPersister(logger, db, time.Second, 3) - - ps := NewService(logger, db) - - wantErr := "can't build pipeline without any destination connectors" + b := &backoff.Backoff{} source := dummySource(persister) dlq := dummyDestination(persister) - pl := &Instance{ + ls := NewService(logger, b, testConnectorService{ + source.ID: source, + testDLQID: dlq, + }, + testProcessorService{}, + testConnectorPluginService{ + source.Plugin: pmock.NewDispenser(ctrl), + dlq.Plugin: pmock.NewDispenser(ctrl), + }, testPipelineService{}) + + wantErr := "can't build pipeline without any destination connectors" + + pl := &pipeline.Instance{ ID: uuid.NewString(), - Config: Config{Name: "test-pipeline"}, - status: StatusUserStopped, - DLQ: DLQ{ + Config: pipeline.Config{Name: "test-pipeline"}, + DLQ: pipeline.DLQ{ Plugin: dlq.Plugin, Settings: map[string]string{}, WindowSize: 3, @@ -200,18 +219,10 @@ func TestService_buildNodes_NoDestinationNode(t *testing.T) { }, ConnectorIDs: []string{source.ID}, } + pl.SetStatus(pipeline.StatusUserStopped) - got, err := ps.buildNodes( + got, err := ls.buildRunnablePipeline( ctx, - testConnectorFetcher{ - source.ID: source, - testDLQID: dlq, - }, - testProcessorFetcher{}, - testPluginFetcher{ - source.Plugin: pmock.NewDispenser(ctrl), - dlq.Plugin: pmock.NewDispenser(ctrl), - }, pl, ) @@ -228,11 +239,12 @@ func TestServiceLifecycle_PipelineSuccess(t *testing.T) { db := &inmemory.DB{} persister := connector.NewPersister(logger, db, time.Second, 3) defer persister.Wait() + b := &backoff.Backoff{} - ps := NewService(logger, db) + ps := pipeline.NewService(logger, db) // create a host pipeline - pl, err := ps.Create(ctx, uuid.NewString(), Config{Name: "test pipeline"}, ProvisionTypeAPI) + pl, err := ps.Create(ctx, uuid.NewString(), pipeline.Config{Name: "test pipeline"}, pipeline.ProvisionTypeAPI) is.NoErr(err) // create mocked connectors @@ -248,20 +260,21 @@ func TestServiceLifecycle_PipelineSuccess(t *testing.T) { pl, err = ps.AddConnector(ctx, pl.ID, destination.ID) is.NoErr(err) - // start the pipeline now that everything is set up - err = ps.Start( - ctx, - testConnectorFetcher{ - source.ID: source, - destination.ID: destination, - testDLQID: dlq, - }, - testProcessorFetcher{}, - testPluginFetcher{ + ls := NewService(logger, b, testConnectorService{ + source.ID: source, + destination.ID: destination, + testDLQID: dlq, + }, + testProcessorService{}, + testConnectorPluginService{ source.Plugin: sourceDispenser, destination.Plugin: destDispenser, dlq.Plugin: dlqDispenser, - }, + }, ps) + + // start the pipeline now that everything is set up + err = ls.Start( + ctx, pl.ID, ) is.NoErr(err) @@ -269,13 +282,14 @@ func TestServiceLifecycle_PipelineSuccess(t *testing.T) { // wait for pipeline to finish consuming records from the source time.Sleep(100 * time.Millisecond) - is.Equal(StatusRunning, pl.GetStatus()) + is.Equal(pipeline.StatusRunning, pl.GetStatus()) is.Equal("", pl.Error) // stop pipeline before ending test - err = ps.Stop(ctx, pl.ID, false) + err = ls.Stop(ctx, pl.ID, false) is.NoErr(err) - is.NoErr(pl.Wait()) + + is.NoErr(ls.WaitPipeline(pl.ID)) } func TestServiceLifecycle_PipelineError(t *testing.T) { @@ -287,11 +301,12 @@ func TestServiceLifecycle_PipelineError(t *testing.T) { logger := log.Test(t) db := &inmemory.DB{} persister := connector.NewPersister(logger, db, time.Second, 3) + b := &backoff.Backoff{} - ps := NewService(logger, db) + ps := pipeline.NewService(logger, db) // create a host pipeline - pl, err := ps.Create(ctx, uuid.NewString(), Config{Name: "test pipeline"}, ProvisionTypeAPI) + pl, err := ps.Create(ctx, uuid.NewString(), pipeline.Config{Name: "test pipeline"}, pipeline.ProvisionTypeAPI) is.NoErr(err) // create mocked connectors @@ -308,33 +323,35 @@ func TestServiceLifecycle_PipelineError(t *testing.T) { pl, err = ps.AddConnector(ctx, pl.ID, destination.ID) is.NoErr(err) + ls := NewService(logger, b, testConnectorService{ + source.ID: source, + destination.ID: destination, + testDLQID: dlq, + }, + testProcessorService{}, + testConnectorPluginService{ + source.Plugin: sourceDispenser, + destination.Plugin: destDispenser, + dlq.Plugin: dlqDispenser, + }, ps) + events := make(chan FailureEvent, 1) - ps.OnFailure(func(e FailureEvent) { + ls.OnFailure(func(e FailureEvent) { events <- e }) + // start the pipeline now that everything is set up - err = ps.Start( + err = ls.Start( ctx, - testConnectorFetcher{ - source.ID: source, - destination.ID: destination, - testDLQID: dlq, - }, - testProcessorFetcher{}, - testPluginFetcher{ - source.Plugin: sourceDispenser, - destination.Plugin: destDispenser, - dlq.Plugin: dlqDispenser, - }, pl.ID, ) is.NoErr(err) // wait for pipeline to finish - err = pl.Wait() + err = ls.WaitPipeline(pl.ID) is.True(err != nil) - is.Equal(StatusDegraded, pl.GetStatus()) + is.Equal(pipeline.StatusDegraded, pl.GetStatus()) // pipeline errors contain only string messages, so we can only compare the errors by the messages t.Log(pl.Error) @@ -356,11 +373,11 @@ func TestServiceLifecycle_PipelineError(t *testing.T) { func TestServiceLifecycle_StopAll_Recovering(t *testing.T) { type testCase struct { name string - stopFn func(ctx context.Context, is *is.I, pipelineService *Service, pipelineID string) + stopFn func(ctx context.Context, is *is.I, lifecycleService *Service, pipelineID string) // whether we expect the source plugin's Stop() function to be called // (doesn't happen when force-stopping) wantSourceStop bool - want Status + want pipeline.Status wantErr error } @@ -371,11 +388,12 @@ func TestServiceLifecycle_StopAll_Recovering(t *testing.T) { logger := log.New(zerolog.Nop()) db := &inmemory.DB{} persister := connector.NewPersister(logger, db, time.Second, 3) + b := &backoff.Backoff{} - ps := NewService(logger, db) + ps := pipeline.NewService(logger, db) // create a host pipeline - pl, err := ps.Create(ctx, uuid.NewString(), Config{Name: "test pipeline"}, ProvisionTypeAPI) + pl, err := ps.Create(ctx, uuid.NewString(), pipeline.Config{Name: "test pipeline"}, pipeline.ProvisionTypeAPI) is.NoErr(err) // create mocked connectors @@ -393,20 +411,21 @@ func TestServiceLifecycle_StopAll_Recovering(t *testing.T) { pl, err = ps.AddConnector(ctx, pl.ID, destination.ID) is.NoErr(err) - // start the pipeline now that everything is set up - err = ps.Start( - ctx, - testConnectorFetcher{ - source.ID: source, - destination.ID: destination, - testDLQID: dlq, - }, - testProcessorFetcher{}, - testPluginFetcher{ + ls := NewService(logger, b, testConnectorService{ + source.ID: source, + destination.ID: destination, + testDLQID: dlq, + }, + testProcessorService{}, + testConnectorPluginService{ source.Plugin: sourceDispenser, destination.Plugin: destDispenser, dlq.Plugin: dlqDispenser, - }, + }, ps) + + // start the pipeline now that everything is set up + err = ls.Start( + ctx, pl.ID, ) is.NoErr(err) @@ -414,11 +433,11 @@ func TestServiceLifecycle_StopAll_Recovering(t *testing.T) { // wait for pipeline to finish consuming records from the source time.Sleep(100 * time.Millisecond) - pl.SetStatus(StatusRecovering) - tc.stopFn(ctx, is, ps, pl.ID) + pl.SetStatus(pipeline.StatusRecovering) + tc.stopFn(ctx, is, ls, pl.ID) // wait for pipeline to finish - err = pl.Wait() + err = ls.WaitPipeline(pl.ID) if tc.wantErr != nil { is.True(err != nil) } else { @@ -432,29 +451,38 @@ func TestServiceLifecycle_StopAll_Recovering(t *testing.T) { testCases := []testCase{ { name: "system stop (graceful shutdown err)", - stopFn: func(ctx context.Context, is *is.I, ps *Service, pipelineID string) { - ps.StopAll(ctx, ErrGracefulShutdown) + stopFn: func(ctx context.Context, is *is.I, ls *Service, pipelineID string) { + ls.StopAll(ctx, pipeline.ErrGracefulShutdown) }, wantSourceStop: true, - want: StatusSystemStopped, + want: pipeline.StatusSystemStopped, }, { - name: "system stop (terrible err)", - stopFn: func(ctx context.Context, is *is.I, ps *Service, pipelineID string) { - ps.StopAll(ctx, cerrors.New("terrible err")) + name: "system stop (fatal err)", + stopFn: func(ctx context.Context, is *is.I, ls *Service, pipelineID string) { + ls.StopAll(ctx, cerrors.FatalError(cerrors.New("terrible err"))) }, wantSourceStop: true, - want: StatusDegraded, + want: pipeline.StatusDegraded, wantErr: cerrors.New("terrible err"), }, + { + name: "connection error", + stopFn: func(ctx context.Context, is *is.I, ls *Service, pipelineID string) { + ls.StopAll(ctx, cerrors.New("lost connection to database")) + }, + wantSourceStop: true, + want: pipeline.StatusRecovering, + wantErr: cerrors.New("lost connection to database"), + }, { name: "user stop (graceful)", - stopFn: func(ctx context.Context, is *is.I, ps *Service, pipelineID string) { - err := ps.Stop(ctx, pipelineID, false) + stopFn: func(ctx context.Context, is *is.I, ls *Service, pipelineID string) { + err := ls.Stop(ctx, pipelineID, false) is.NoErr(err) }, wantSourceStop: true, - want: StatusUserStopped, + want: pipeline.StatusUserStopped, }, } @@ -472,11 +500,12 @@ func TestServiceLifecycle_PipelineStop(t *testing.T) { logger := log.New(zerolog.Nop()) db := &inmemory.DB{} persister := connector.NewPersister(logger, db, time.Second, 3) + b := &backoff.Backoff{} - ps := NewService(logger, db) + ps := pipeline.NewService(logger, db) // create a host pipeline - pl, err := ps.Create(ctx, uuid.NewString(), Config{Name: "test pipeline"}, ProvisionTypeAPI) + pl, err := ps.Create(ctx, uuid.NewString(), pipeline.Config{Name: "test pipeline"}, pipeline.ProvisionTypeAPI) is.NoErr(err) // create mocked connectors @@ -494,38 +523,39 @@ func TestServiceLifecycle_PipelineStop(t *testing.T) { pl, err = ps.AddConnector(ctx, pl.ID, destination.ID) is.NoErr(err) - // start the pipeline now that everything is set up - err = ps.Start( - ctx, - testConnectorFetcher{ - source.ID: source, - destination.ID: destination, - testDLQID: dlq, - }, - testProcessorFetcher{}, - testPluginFetcher{ + ls := NewService(logger, b, testConnectorService{ + source.ID: source, + destination.ID: destination, + testDLQID: dlq, + }, + testProcessorService{}, + testConnectorPluginService{ source.Plugin: sourceDispenser, destination.Plugin: destDispenser, dlq.Plugin: dlqDispenser, - }, + }, ps) + + // start the pipeline now that everything is set up + err = ls.Start( + ctx, pl.ID, ) is.NoErr(err) // wait for pipeline to finish consuming records from the source time.Sleep(100 * time.Millisecond) - ps.StopAll(ctx, ErrGracefulShutdown) + ls.StopAll(ctx, pipeline.ErrGracefulShutdown) // wait for pipeline to finish - err = pl.Wait() + err = ls.WaitPipeline(pl.ID) is.NoErr(err) - is.Equal(StatusSystemStopped, pl.GetStatus()) + is.Equal(pipeline.StatusSystemStopped, pl.GetStatus()) is.Equal("", pl.Error) } -func TestService_Run_Rerun(t *testing.T) { - runTest := func(t *testing.T, status Status, expected Status) { +func TestServiceLifecycle_Run_Rerun(t *testing.T) { + runTest := func(t *testing.T, status pipeline.Status, expected pipeline.Status) { is := is.New(t) ctx, killAll := context.WithCancel(context.Background()) defer killAll() @@ -533,11 +563,12 @@ func TestService_Run_Rerun(t *testing.T) { logger := log.Test(t) db := &inmemory.DB{} persister := connector.NewPersister(logger, db, time.Second, 3) + b := &backoff.Backoff{} - ps := NewService(logger, db) + ps := pipeline.NewService(logger, db) // create a host pipeline - pl, err := ps.Create(ctx, uuid.NewString(), Config{Name: "test pipeline"}, ProvisionTypeAPI) + pl, err := ps.Create(ctx, uuid.NewString(), pipeline.Config{Name: "test pipeline"}, pipeline.ProvisionTypeAPI) is.NoErr(err) // create mocked connectors @@ -549,7 +580,7 @@ func TestService_Run_Rerun(t *testing.T) { dlq *connector.Instance dlqDispenser *pmock.Dispenser ) - if expected == StatusRunning { + if expected == pipeline.StatusRunning { // mocked connectors that are expected to be started source, sourceDispenser = generatorSource(ctrl, persister, nil, nil, true) destination, destDispenser = asserterDestination(ctrl, persister, nil) @@ -571,23 +602,22 @@ func TestService_Run_Rerun(t *testing.T) { is.NoErr(err) // create a new pipeline service and initialize it - ps = NewService(logger, db) + ps = pipeline.NewService(logger, db) err = ps.Init(ctx) is.NoErr(err) - err = ps.Run( - ctx, - testConnectorFetcher{ - source.ID: source, - destination.ID: destination, - testDLQID: dlq, - }, - testProcessorFetcher{}, - testPluginFetcher{ + + ls := NewService(logger, b, testConnectorService{ + source.ID: source, + destination.ID: destination, + testDLQID: dlq, + }, + testProcessorService{}, + testConnectorPluginService{ source.Plugin: sourceDispenser, destination.Plugin: destDispenser, dlq.Plugin: dlqDispenser, - }, - ) + }, ps) + err = ls.Init(ctx) is.NoErr(err) // give pipeline a chance to start if needed @@ -598,21 +628,22 @@ func TestService_Run_Rerun(t *testing.T) { is.True(got[pl.ID] != nil) is.Equal(got[pl.ID].GetStatus(), expected) - if expected == StatusRunning { + if expected == pipeline.StatusRunning { pl, _ = ps.Get(ctx, pl.ID) - is.NoErr(ps.Stop(ctx, pl.ID, false)) - is.NoErr(pl.Wait()) + + is.NoErr(ls.Stop(ctx, pl.ID, false)) + is.NoErr(ls.WaitPipeline(pl.ID)) } } testCases := []struct { - have Status - want Status + have pipeline.Status + want pipeline.Status }{ - {have: StatusRunning, want: StatusRunning}, - {have: StatusUserStopped, want: StatusUserStopped}, - {have: StatusSystemStopped, want: StatusRunning}, - {have: StatusDegraded, want: StatusDegraded}, + {have: pipeline.StatusRunning, want: pipeline.StatusRunning}, + {have: pipeline.StatusUserStopped, want: pipeline.StatusUserStopped}, + {have: pipeline.StatusSystemStopped, want: pipeline.StatusRunning}, + {have: pipeline.StatusDegraded, want: pipeline.StatusDegraded}, } for _, tt := range testCases { t.Run(fmt.Sprintf("%s->%s", tt.have, tt.want), func(t *testing.T) { @@ -731,43 +762,69 @@ func dummyDestination(persister *connector.Persister) *connector.Instance { return destination } -// testConnectorFetcher fulfills the ConnectorFetcher interface. -type testConnectorFetcher map[string]*connector.Instance +// testConnectorService fulfills the ConnectorService interface. +type testConnectorService map[string]*connector.Instance -func (tcf testConnectorFetcher) Get(_ context.Context, id string) (*connector.Instance, error) { - conn, ok := tcf[id] +func (s testConnectorService) Get(_ context.Context, id string) (*connector.Instance, error) { + conn, ok := s[id] if !ok { return nil, connector.ErrInstanceNotFound } return conn, nil } -func (tcf testConnectorFetcher) Create(context.Context, string, connector.Type, string, string, connector.Config, connector.ProvisionType) (*connector.Instance, error) { - return tcf[testDLQID], nil +func (s testConnectorService) Create(context.Context, string, connector.Type, string, string, connector.Config, connector.ProvisionType) (*connector.Instance, error) { + return s[testDLQID], nil } -// testProcessorFetcher fulfills the ProcessorService interface. -type testProcessorFetcher map[string]*processor.Instance +// testProcessorService fulfills the ProcessorService interface. +type testProcessorService map[string]*processor.Instance -func (tpf testProcessorFetcher) MakeRunnableProcessor(context.Context, *processor.Instance) (*processor.RunnableProcessor, error) { +func (s testProcessorService) MakeRunnableProcessor(context.Context, *processor.Instance) (*processor.RunnableProcessor, error) { return nil, cerrors.New("not implemented") } -func (tpf testProcessorFetcher) Get(_ context.Context, id string) (*processor.Instance, error) { - proc, ok := tpf[id] +func (s testProcessorService) Get(_ context.Context, id string) (*processor.Instance, error) { + proc, ok := s[id] if !ok { return nil, processor.ErrInstanceNotFound } return proc, nil } -// testPluginFetcher fulfills the PluginFetcher interface. -type testPluginFetcher map[string]connectorPlugin.Dispenser +// testConnectorPluginService fulfills the ConnectorPluginService interface. +type testConnectorPluginService map[string]connectorPlugin.Dispenser -func (tpf testPluginFetcher) NewDispenser(_ log.CtxLogger, name string, _ string) (connectorPlugin.Dispenser, error) { - plug, ok := tpf[name] +func (s testConnectorPluginService) NewDispenser(_ log.CtxLogger, name string, _ string) (connectorPlugin.Dispenser, error) { + plug, ok := s[name] if !ok { return nil, plugin.ErrPluginNotFound } return plug, nil } + +// testPipelineService fulfills the PipelineService interface. +type testPipelineService map[string]*pipeline.Instance + +func (s testPipelineService) Get(_ context.Context, pipelineID string) (*pipeline.Instance, error) { + p, ok := s[pipelineID] + if !ok { + return nil, processor.ErrInstanceNotFound + } + return p, nil +} + +func (s testPipelineService) List(_ context.Context) map[string]*pipeline.Instance { + instances := make(map[string]*pipeline.Instance) + return instances +} + +func (s testPipelineService) UpdateStatus(_ context.Context, pipelineID string, status pipeline.Status, errMsg string) error { + p, ok := s[pipelineID] + if !ok { + return processor.ErrInstanceNotFound + } + p.SetStatus(status) + p.Error = errMsg + return nil +} diff --git a/pkg/pipeline/stream/base.go b/pkg/lifecycle/stream/base.go similarity index 100% rename from pkg/pipeline/stream/base.go rename to pkg/lifecycle/stream/base.go diff --git a/pkg/pipeline/stream/base_test.go b/pkg/lifecycle/stream/base_test.go similarity index 100% rename from pkg/pipeline/stream/base_test.go rename to pkg/lifecycle/stream/base_test.go diff --git a/pkg/pipeline/stream/destination.go b/pkg/lifecycle/stream/destination.go similarity index 100% rename from pkg/pipeline/stream/destination.go rename to pkg/lifecycle/stream/destination.go diff --git a/pkg/pipeline/stream/destination_acker.go b/pkg/lifecycle/stream/destination_acker.go similarity index 100% rename from pkg/pipeline/stream/destination_acker.go rename to pkg/lifecycle/stream/destination_acker.go diff --git a/pkg/pipeline/stream/destination_acker_test.go b/pkg/lifecycle/stream/destination_acker_test.go similarity index 99% rename from pkg/pipeline/stream/destination_acker_test.go rename to pkg/lifecycle/stream/destination_acker_test.go index 21a030564..6b7c7188f 100644 --- a/pkg/pipeline/stream/destination_acker_test.go +++ b/pkg/lifecycle/stream/destination_acker_test.go @@ -24,7 +24,7 @@ import ( "github.com/conduitio/conduit-commons/opencdc" "github.com/conduitio/conduit/pkg/connector" "github.com/conduitio/conduit/pkg/foundation/cerrors" - "github.com/conduitio/conduit/pkg/pipeline/stream/mock" + "github.com/conduitio/conduit/pkg/lifecycle/stream/mock" "github.com/matryer/is" "go.uber.org/mock/gomock" ) diff --git a/pkg/pipeline/stream/destination_test.go b/pkg/lifecycle/stream/destination_test.go similarity index 98% rename from pkg/pipeline/stream/destination_test.go rename to pkg/lifecycle/stream/destination_test.go index 77f2d7f02..e4b17126a 100644 --- a/pkg/pipeline/stream/destination_test.go +++ b/pkg/lifecycle/stream/destination_test.go @@ -24,7 +24,7 @@ import ( "github.com/conduitio/conduit-commons/opencdc" "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/metrics/noop" - "github.com/conduitio/conduit/pkg/pipeline/stream/mock" + "github.com/conduitio/conduit/pkg/lifecycle/stream/mock" "github.com/matryer/is" "go.uber.org/mock/gomock" ) diff --git a/pkg/pipeline/stream/dlq.go b/pkg/lifecycle/stream/dlq.go similarity index 100% rename from pkg/pipeline/stream/dlq.go rename to pkg/lifecycle/stream/dlq.go diff --git a/pkg/pipeline/stream/dlq_test.go b/pkg/lifecycle/stream/dlq_test.go similarity index 99% rename from pkg/pipeline/stream/dlq_test.go rename to pkg/lifecycle/stream/dlq_test.go index 2ee6fcbfc..c0e5e25ca 100644 --- a/pkg/pipeline/stream/dlq_test.go +++ b/pkg/lifecycle/stream/dlq_test.go @@ -27,7 +27,7 @@ import ( "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/metrics" "github.com/conduitio/conduit/pkg/foundation/metrics/noop" - "github.com/conduitio/conduit/pkg/pipeline/stream/mock" + "github.com/conduitio/conduit/pkg/lifecycle/stream/mock" "github.com/google/uuid" "github.com/matryer/is" "go.uber.org/mock/gomock" diff --git a/pkg/pipeline/stream/doc.go b/pkg/lifecycle/stream/doc.go similarity index 100% rename from pkg/pipeline/stream/doc.go rename to pkg/lifecycle/stream/doc.go diff --git a/pkg/pipeline/stream/fanin.go b/pkg/lifecycle/stream/fanin.go similarity index 100% rename from pkg/pipeline/stream/fanin.go rename to pkg/lifecycle/stream/fanin.go diff --git a/pkg/pipeline/stream/fanin_select.go b/pkg/lifecycle/stream/fanin_select.go similarity index 100% rename from pkg/pipeline/stream/fanin_select.go rename to pkg/lifecycle/stream/fanin_select.go diff --git a/pkg/pipeline/stream/fanout.go b/pkg/lifecycle/stream/fanout.go similarity index 100% rename from pkg/pipeline/stream/fanout.go rename to pkg/lifecycle/stream/fanout.go diff --git a/pkg/pipeline/stream/fanout_test.go b/pkg/lifecycle/stream/fanout_test.go similarity index 100% rename from pkg/pipeline/stream/fanout_test.go rename to pkg/lifecycle/stream/fanout_test.go diff --git a/pkg/pipeline/stream/logger.go b/pkg/lifecycle/stream/logger.go similarity index 100% rename from pkg/pipeline/stream/logger.go rename to pkg/lifecycle/stream/logger.go diff --git a/pkg/pipeline/stream/message.go b/pkg/lifecycle/stream/message.go similarity index 100% rename from pkg/pipeline/stream/message.go rename to pkg/lifecycle/stream/message.go diff --git a/pkg/pipeline/stream/message_test.go b/pkg/lifecycle/stream/message_test.go similarity index 100% rename from pkg/pipeline/stream/message_test.go rename to pkg/lifecycle/stream/message_test.go diff --git a/pkg/pipeline/stream/messagestatus_string.go b/pkg/lifecycle/stream/messagestatus_string.go similarity index 100% rename from pkg/pipeline/stream/messagestatus_string.go rename to pkg/lifecycle/stream/messagestatus_string.go diff --git a/pkg/pipeline/stream/metrics.go b/pkg/lifecycle/stream/metrics.go similarity index 100% rename from pkg/pipeline/stream/metrics.go rename to pkg/lifecycle/stream/metrics.go diff --git a/pkg/pipeline/stream/mock/destination.go b/pkg/lifecycle/stream/mock/destination.go similarity index 99% rename from pkg/pipeline/stream/mock/destination.go rename to pkg/lifecycle/stream/mock/destination.go index 8efe897bc..ff75a9a58 100644 --- a/pkg/pipeline/stream/mock/destination.go +++ b/pkg/lifecycle/stream/mock/destination.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/conduitio/conduit/pkg/pipeline/stream (interfaces: Destination) +// Source: github.com/conduitio/conduit/pkg/lifecycle/stream (interfaces: Destination) // // Generated by this command: // diff --git a/pkg/pipeline/stream/mock/dlq.go b/pkg/lifecycle/stream/mock/dlq.go similarity index 98% rename from pkg/pipeline/stream/mock/dlq.go rename to pkg/lifecycle/stream/mock/dlq.go index 7de89a3c2..fc753be11 100644 --- a/pkg/pipeline/stream/mock/dlq.go +++ b/pkg/lifecycle/stream/mock/dlq.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/conduitio/conduit/pkg/pipeline/stream (interfaces: DLQHandler) +// Source: github.com/conduitio/conduit/pkg/lifecycle/stream (interfaces: DLQHandler) // // Generated by this command: // diff --git a/pkg/pipeline/stream/mock/processor.go b/pkg/lifecycle/stream/mock/processor.go similarity index 98% rename from pkg/pipeline/stream/mock/processor.go rename to pkg/lifecycle/stream/mock/processor.go index 1b6ed851b..8f2a64e04 100644 --- a/pkg/pipeline/stream/mock/processor.go +++ b/pkg/lifecycle/stream/mock/processor.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/conduitio/conduit/pkg/pipeline/stream (interfaces: Processor) +// Source: github.com/conduitio/conduit/pkg/lifecycle/stream (interfaces: Processor) // // Generated by this command: // diff --git a/pkg/pipeline/stream/mock/source.go b/pkg/lifecycle/stream/mock/source.go similarity index 99% rename from pkg/pipeline/stream/mock/source.go rename to pkg/lifecycle/stream/mock/source.go index f41c864f0..c339f901f 100644 --- a/pkg/pipeline/stream/mock/source.go +++ b/pkg/lifecycle/stream/mock/source.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/conduitio/conduit/pkg/pipeline/stream (interfaces: Source) +// Source: github.com/conduitio/conduit/pkg/lifecycle/stream (interfaces: Source) // // Generated by this command: // diff --git a/pkg/pipeline/stream/node.go b/pkg/lifecycle/stream/node.go similarity index 100% rename from pkg/pipeline/stream/node.go rename to pkg/lifecycle/stream/node.go diff --git a/pkg/pipeline/stream/parallel.go b/pkg/lifecycle/stream/parallel.go similarity index 100% rename from pkg/pipeline/stream/parallel.go rename to pkg/lifecycle/stream/parallel.go diff --git a/pkg/pipeline/stream/parallel_test.go b/pkg/lifecycle/stream/parallel_test.go similarity index 99% rename from pkg/pipeline/stream/parallel_test.go rename to pkg/lifecycle/stream/parallel_test.go index 09db1ebc7..e7c8c7039 100644 --- a/pkg/pipeline/stream/parallel_test.go +++ b/pkg/lifecycle/stream/parallel_test.go @@ -29,7 +29,7 @@ import ( "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" "github.com/conduitio/conduit/pkg/foundation/metrics/noop" - "github.com/conduitio/conduit/pkg/pipeline/stream/mock" + "github.com/conduitio/conduit/pkg/lifecycle/stream/mock" "github.com/matryer/is" "go.uber.org/mock/gomock" ) diff --git a/pkg/pipeline/stream/processor.go b/pkg/lifecycle/stream/processor.go similarity index 100% rename from pkg/pipeline/stream/processor.go rename to pkg/lifecycle/stream/processor.go diff --git a/pkg/pipeline/stream/processor_test.go b/pkg/lifecycle/stream/processor_test.go similarity index 99% rename from pkg/pipeline/stream/processor_test.go rename to pkg/lifecycle/stream/processor_test.go index cdc4168d4..66233e5e6 100644 --- a/pkg/pipeline/stream/processor_test.go +++ b/pkg/lifecycle/stream/processor_test.go @@ -23,7 +23,7 @@ import ( sdk "github.com/conduitio/conduit-processor-sdk" "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/metrics/noop" - "github.com/conduitio/conduit/pkg/pipeline/stream/mock" + "github.com/conduitio/conduit/pkg/lifecycle/stream/mock" "github.com/google/uuid" "github.com/matryer/is" "go.uber.org/mock/gomock" diff --git a/pkg/pipeline/stream/source.go b/pkg/lifecycle/stream/source.go similarity index 100% rename from pkg/pipeline/stream/source.go rename to pkg/lifecycle/stream/source.go diff --git a/pkg/pipeline/stream/source_acker.go b/pkg/lifecycle/stream/source_acker.go similarity index 100% rename from pkg/pipeline/stream/source_acker.go rename to pkg/lifecycle/stream/source_acker.go diff --git a/pkg/pipeline/stream/source_acker_test.go b/pkg/lifecycle/stream/source_acker_test.go similarity index 99% rename from pkg/pipeline/stream/source_acker_test.go rename to pkg/lifecycle/stream/source_acker_test.go index 53d910d1b..151bfc4d2 100644 --- a/pkg/pipeline/stream/source_acker_test.go +++ b/pkg/lifecycle/stream/source_acker_test.go @@ -26,7 +26,7 @@ import ( "github.com/conduitio/conduit-commons/csync" "github.com/conduitio/conduit-commons/opencdc" "github.com/conduitio/conduit/pkg/foundation/cerrors" - "github.com/conduitio/conduit/pkg/pipeline/stream/mock" + "github.com/conduitio/conduit/pkg/lifecycle/stream/mock" "github.com/matryer/is" "go.uber.org/mock/gomock" ) diff --git a/pkg/pipeline/stream/source_test.go b/pkg/lifecycle/stream/source_test.go similarity index 99% rename from pkg/pipeline/stream/source_test.go rename to pkg/lifecycle/stream/source_test.go index a910aea74..e3b8d63e4 100644 --- a/pkg/pipeline/stream/source_test.go +++ b/pkg/lifecycle/stream/source_test.go @@ -26,7 +26,7 @@ import ( "github.com/conduitio/conduit-commons/opencdc" "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/metrics/noop" - "github.com/conduitio/conduit/pkg/pipeline/stream/mock" + "github.com/conduitio/conduit/pkg/lifecycle/stream/mock" connectorPlugin "github.com/conduitio/conduit/pkg/plugin/connector" "github.com/google/uuid" "github.com/matryer/is" diff --git a/pkg/pipeline/stream/stream_test.go b/pkg/lifecycle/stream/stream_test.go similarity index 99% rename from pkg/pipeline/stream/stream_test.go rename to pkg/lifecycle/stream/stream_test.go index 44f3e061e..d04d46621 100644 --- a/pkg/pipeline/stream/stream_test.go +++ b/pkg/lifecycle/stream/stream_test.go @@ -28,8 +28,8 @@ import ( "github.com/conduitio/conduit/pkg/foundation/ctxutil" "github.com/conduitio/conduit/pkg/foundation/log" "github.com/conduitio/conduit/pkg/foundation/metrics/noop" - "github.com/conduitio/conduit/pkg/pipeline/stream" - streammock "github.com/conduitio/conduit/pkg/pipeline/stream/mock" + "github.com/conduitio/conduit/pkg/lifecycle/stream" + streammock "github.com/conduitio/conduit/pkg/lifecycle/stream/mock" connectorPlugin "github.com/conduitio/conduit/pkg/plugin/connector" "github.com/rs/zerolog" "go.uber.org/mock/gomock" diff --git a/pkg/orchestrator/connectors_test.go b/pkg/orchestrator/connectors_test.go index e26a76158..7a448b6da 100644 --- a/pkg/orchestrator/connectors_test.go +++ b/pkg/orchestrator/connectors_test.go @@ -33,7 +33,7 @@ func TestConnectorOrchestrator_Create_Success(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -79,7 +79,7 @@ func TestConnectorOrchestrator_Create_Success(t *testing.T) { AddConnector(gomock.AssignableToTypeOf(ctxType), pl.ID, want.ID). Return(pl, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Connectors.Create(ctx, want.Type, want.Plugin, want.PipelineID, want.Config) is.NoErr(err) is.Equal(want, got) @@ -89,7 +89,7 @@ func TestConnectorOrchestrator_Create_PipelineNotExist(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pipelineID := uuid.NewString() wantErr := pipeline.ErrInstanceNotFound @@ -97,7 +97,7 @@ func TestConnectorOrchestrator_Create_PipelineNotExist(t *testing.T) { Get(gomock.AssignableToTypeOf(ctxType), pipelineID). Return(nil, wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Connectors.Create(ctx, connector.TypeSource, "test-plugin", pipelineID, connector.Config{}) is.True(err != nil) is.True(cerrors.Is(err, wantErr)) @@ -108,7 +108,7 @@ func TestConnectorOrchestrator_Create_PipelineRunning(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -119,7 +119,7 @@ func TestConnectorOrchestrator_Create_PipelineRunning(t *testing.T) { Get(gomock.AssignableToTypeOf(ctxType), pl.ID). Return(pl, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Connectors.Create(ctx, connector.TypeSource, "test-plugin", pl.ID, connector.Config{}) is.True(err != nil) is.True(cerrors.Is(err, pipeline.ErrPipelineRunning)) @@ -130,7 +130,7 @@ func TestConnectorOrchestrator_Create_PipelineProvisionByConfig(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -142,7 +142,7 @@ func TestConnectorOrchestrator_Create_PipelineProvisionByConfig(t *testing.T) { Get(gomock.AssignableToTypeOf(ctxType), pl.ID). Return(pl, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Connectors.Create(ctx, connector.TypeSource, "test-plugin", pl.ID, connector.Config{}) is.Equal(got, nil) is.True(err != nil) @@ -153,7 +153,7 @@ func TestConnectorOrchestrator_Create_CreateConnectorError(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -183,7 +183,7 @@ func TestConnectorOrchestrator_Create_CreateConnectorError(t *testing.T) { ). Return(nil, wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Connectors.Create(ctx, connector.TypeSource, "test-plugin", pl.ID, config) is.True(err != nil) is.True(cerrors.Is(err, wantErr)) @@ -194,7 +194,7 @@ func TestConnectorOrchestrator_Create_AddConnectorError(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -246,7 +246,7 @@ func TestConnectorOrchestrator_Create_AddConnectorError(t *testing.T) { Delete(gomock.AssignableToTypeOf(ctxType), conn.ID, connPluginMock). Return(nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Connectors.Create(ctx, connector.TypeSource, conn.Plugin, pl.ID, conn.Config) is.True(err != nil) is.True(cerrors.Is(err, wantErr)) @@ -257,7 +257,7 @@ func TestConnectorOrchestrator_Delete_Success(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -282,7 +282,7 @@ func TestConnectorOrchestrator_Delete_Success(t *testing.T) { RemoveConnector(gomock.AssignableToTypeOf(ctxType), pl.ID, conn.ID). Return(pl, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) err := orc.Connectors.Delete(ctx, conn.ID) is.NoErr(err) } @@ -291,7 +291,7 @@ func TestConnectorOrchestrator_Delete_ConnectorNotExist(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) id := uuid.NewString() wantErr := cerrors.New("connector doesn't exist") @@ -299,7 +299,7 @@ func TestConnectorOrchestrator_Delete_ConnectorNotExist(t *testing.T) { Get(gomock.AssignableToTypeOf(ctxType), id). Return(nil, wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) err := orc.Connectors.Delete(ctx, id) is.True(err != nil) is.True(cerrors.Is(err, wantErr)) @@ -309,7 +309,7 @@ func TestConnectorOrchestrator_Delete_PipelineRunning(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -328,7 +328,7 @@ func TestConnectorOrchestrator_Delete_PipelineRunning(t *testing.T) { Get(gomock.AssignableToTypeOf(ctxType), pl.ID). Return(pl, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) err := orc.Connectors.Delete(ctx, conn.ID) is.True(err != nil) is.Equal(pipeline.ErrPipelineRunning, err) @@ -338,7 +338,7 @@ func TestConnectorOrchestrator_Delete_ProcessorAttached(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -355,7 +355,7 @@ func TestConnectorOrchestrator_Delete_ProcessorAttached(t *testing.T) { Get(gomock.AssignableToTypeOf(ctxType), conn.ID). Return(conn, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) err := orc.Connectors.Delete(ctx, conn.ID) is.True(err != nil) is.True(cerrors.Is(err, ErrConnectorHasProcessorsAttached)) @@ -365,7 +365,7 @@ func TestConnectorOrchestrator_Delete_Fail(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -388,7 +388,7 @@ func TestConnectorOrchestrator_Delete_Fail(t *testing.T) { Delete(gomock.AssignableToTypeOf(ctxType), conn.ID, connPluginMock). Return(wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) err := orc.Connectors.Delete(ctx, conn.ID) is.True(err != nil) is.True(cerrors.Is(err, wantErr)) @@ -398,7 +398,7 @@ func TestConnectorOrchestrator_Delete_RemoveConnectorFailed(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -437,7 +437,7 @@ func TestConnectorOrchestrator_Delete_RemoveConnectorFailed(t *testing.T) { ). Return(conn, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) err := orc.Connectors.Delete(ctx, conn.ID) is.True(err != nil) is.True(cerrors.Is(err, wantErr)) @@ -447,7 +447,7 @@ func TestConnectorOrchestrator_Update_Success(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -488,7 +488,7 @@ func TestConnectorOrchestrator_Update_Success(t *testing.T) { Update(gomock.AssignableToTypeOf(ctxType), conn.ID, newConfig). Return(want, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Connectors.Update(ctx, conn.ID, newConfig) is.NoErr(err) is.Equal(got, want) @@ -498,7 +498,7 @@ func TestConnectorOrchestrator_Update_ConnectorNotExist(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) id := uuid.NewString() wantErr := cerrors.New("connector doesn't exist") @@ -506,7 +506,7 @@ func TestConnectorOrchestrator_Update_ConnectorNotExist(t *testing.T) { Get(gomock.AssignableToTypeOf(ctxType), id). Return(nil, wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Connectors.Update(ctx, id, connector.Config{}) is.True(got == nil) is.True(err != nil) @@ -517,7 +517,7 @@ func TestConnectorOrchestrator_Update_PipelineRunning(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -536,7 +536,7 @@ func TestConnectorOrchestrator_Update_PipelineRunning(t *testing.T) { Get(gomock.AssignableToTypeOf(ctxType), pl.ID). Return(pl, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Connectors.Update(ctx, conn.ID, connector.Config{}) is.True(got == nil) is.True(err != nil) @@ -547,7 +547,7 @@ func TestConnectorOrchestrator_Update_Fail(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -574,7 +574,7 @@ func TestConnectorOrchestrator_Update_Fail(t *testing.T) { Update(gomock.AssignableToTypeOf(ctxType), conn.ID, connector.Config{}). Return(nil, wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Connectors.Update(ctx, conn.ID, connector.Config{}) is.True(got == nil) is.True(err != nil) diff --git a/pkg/orchestrator/mock/orchestrator.go b/pkg/orchestrator/mock/orchestrator.go index 25a5645f3..78d6f71a5 100644 --- a/pkg/orchestrator/mock/orchestrator.go +++ b/pkg/orchestrator/mock/orchestrator.go @@ -1,9 +1,9 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/conduitio/conduit/pkg/orchestrator (interfaces: PipelineService,ConnectorService,ProcessorService,ConnectorPluginService,ProcessorPluginService) +// Source: github.com/conduitio/conduit/pkg/orchestrator (interfaces: PipelineService,ConnectorService,ProcessorService,ConnectorPluginService,ProcessorPluginService,LifecycleService) // // Generated by this command: // -// mockgen -typed -destination=mock/orchestrator.go -package=mock -mock_names=PipelineService=PipelineService,ConnectorService=ConnectorService,ProcessorService=ProcessorService,ConnectorPluginService=ConnectorPluginService,ProcessorPluginService=ProcessorPluginService . PipelineService,ConnectorService,ProcessorService,ConnectorPluginService,ProcessorPluginService +// mockgen -typed -destination=mock/orchestrator.go -package=mock -mock_names=PipelineService=PipelineService,ConnectorService=ConnectorService,ProcessorService=ProcessorService,ConnectorPluginService=ConnectorPluginService,ProcessorPluginService=ProcessorPluginService,LifecycleService=LifecycleService . PipelineService,ConnectorService,ProcessorService,ConnectorPluginService,ProcessorPluginService,LifecycleService // // Package mock is a generated GoMock package. @@ -356,82 +356,6 @@ func (c *PipelineServiceRemoveProcessorCall) DoAndReturn(f func(context.Context, return c } -// Start mocks base method. -func (m *PipelineService) Start(arg0 context.Context, arg1 pipeline.ConnectorFetcher, arg2 pipeline.ProcessorService, arg3 pipeline.PluginDispenserFetcher, arg4 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Start", arg0, arg1, arg2, arg3, arg4) - ret0, _ := ret[0].(error) - return ret0 -} - -// Start indicates an expected call of Start. -func (mr *PipelineServiceMockRecorder) Start(arg0, arg1, arg2, arg3, arg4 any) *PipelineServiceStartCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*PipelineService)(nil).Start), arg0, arg1, arg2, arg3, arg4) - return &PipelineServiceStartCall{Call: call} -} - -// PipelineServiceStartCall wrap *gomock.Call -type PipelineServiceStartCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *PipelineServiceStartCall) Return(arg0 error) *PipelineServiceStartCall { - c.Call = c.Call.Return(arg0) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *PipelineServiceStartCall) Do(f func(context.Context, pipeline.ConnectorFetcher, pipeline.ProcessorService, pipeline.PluginDispenserFetcher, string) error) *PipelineServiceStartCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *PipelineServiceStartCall) DoAndReturn(f func(context.Context, pipeline.ConnectorFetcher, pipeline.ProcessorService, pipeline.PluginDispenserFetcher, string) error) *PipelineServiceStartCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// Stop mocks base method. -func (m *PipelineService) Stop(arg0 context.Context, arg1 string, arg2 bool) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Stop", arg0, arg1, arg2) - ret0, _ := ret[0].(error) - return ret0 -} - -// Stop indicates an expected call of Stop. -func (mr *PipelineServiceMockRecorder) Stop(arg0, arg1, arg2 any) *PipelineServiceStopCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stop", reflect.TypeOf((*PipelineService)(nil).Stop), arg0, arg1, arg2) - return &PipelineServiceStopCall{Call: call} -} - -// PipelineServiceStopCall wrap *gomock.Call -type PipelineServiceStopCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *PipelineServiceStopCall) Return(arg0 error) *PipelineServiceStopCall { - c.Call = c.Call.Return(arg0) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *PipelineServiceStopCall) Do(f func(context.Context, string, bool) error) *PipelineServiceStopCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *PipelineServiceStopCall) DoAndReturn(f func(context.Context, string, bool) error) *PipelineServiceStopCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - // Update mocks base method. func (m *PipelineService) Update(arg0 context.Context, arg1 string, arg2 pipeline.Config) (*pipeline.Instance, error) { m.ctrl.T.Helper() @@ -1375,3 +1299,102 @@ func (c *ProcessorPluginServiceRegisterStandalonePluginCall) DoAndReturn(f func( c.Call = c.Call.DoAndReturn(f) return c } + +// LifecycleService is a mock of LifecycleService interface. +type LifecycleService struct { + ctrl *gomock.Controller + recorder *LifecycleServiceMockRecorder +} + +// LifecycleServiceMockRecorder is the mock recorder for LifecycleService. +type LifecycleServiceMockRecorder struct { + mock *LifecycleService +} + +// NewLifecycleService creates a new mock instance. +func NewLifecycleService(ctrl *gomock.Controller) *LifecycleService { + mock := &LifecycleService{ctrl: ctrl} + mock.recorder = &LifecycleServiceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *LifecycleService) EXPECT() *LifecycleServiceMockRecorder { + return m.recorder +} + +// Start mocks base method. +func (m *LifecycleService) Start(arg0 context.Context, arg1 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Start", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Start indicates an expected call of Start. +func (mr *LifecycleServiceMockRecorder) Start(arg0, arg1 any) *LifecycleServiceStartCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*LifecycleService)(nil).Start), arg0, arg1) + return &LifecycleServiceStartCall{Call: call} +} + +// LifecycleServiceStartCall wrap *gomock.Call +type LifecycleServiceStartCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *LifecycleServiceStartCall) Return(arg0 error) *LifecycleServiceStartCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *LifecycleServiceStartCall) Do(f func(context.Context, string) error) *LifecycleServiceStartCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *LifecycleServiceStartCall) DoAndReturn(f func(context.Context, string) error) *LifecycleServiceStartCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// Stop mocks base method. +func (m *LifecycleService) Stop(arg0 context.Context, arg1 string, arg2 bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Stop", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// Stop indicates an expected call of Stop. +func (mr *LifecycleServiceMockRecorder) Stop(arg0, arg1, arg2 any) *LifecycleServiceStopCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stop", reflect.TypeOf((*LifecycleService)(nil).Stop), arg0, arg1, arg2) + return &LifecycleServiceStopCall{Call: call} +} + +// LifecycleServiceStopCall wrap *gomock.Call +type LifecycleServiceStopCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *LifecycleServiceStopCall) Return(arg0 error) *LifecycleServiceStopCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *LifecycleServiceStopCall) Do(f func(context.Context, string, bool) error) *LifecycleServiceStopCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *LifecycleServiceStopCall) DoAndReturn(f func(context.Context, string, bool) error) *LifecycleServiceStopCall { + c.Call = c.Call.DoAndReturn(f) + return c +} diff --git a/pkg/orchestrator/orchestrator.go b/pkg/orchestrator/orchestrator.go index d53d8390d..a5477d9ff 100644 --- a/pkg/orchestrator/orchestrator.go +++ b/pkg/orchestrator/orchestrator.go @@ -12,8 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:generate mockgen -typed -destination=mock/orchestrator.go -package=mock -mock_names=PipelineService=PipelineService,ConnectorService=ConnectorService,ProcessorService=ProcessorService,ConnectorPluginService=ConnectorPluginService,ProcessorPluginService=ProcessorPluginService . PipelineService,ConnectorService,ProcessorService,ConnectorPluginService,ProcessorPluginService - +//go:generate mockgen -typed -destination=mock/orchestrator.go -package=mock -mock_names=PipelineService=PipelineService,ConnectorService=ConnectorService,ProcessorService=ProcessorService,ConnectorPluginService=ConnectorPluginService,ProcessorPluginService=ProcessorPluginService,LifecycleService=LifecycleService . PipelineService,ConnectorService,ProcessorService,ConnectorPluginService,ProcessorPluginService,LifecycleService package orchestrator import ( @@ -45,6 +44,7 @@ func NewOrchestrator( processors ProcessorService, connectorPlugins ConnectorPluginService, processorPlugins ProcessorPluginService, + lifecycle LifecycleService, ) *Orchestrator { b := base{ db: db, @@ -54,6 +54,7 @@ func NewOrchestrator( processors: processors, connectorPlugins: connectorPlugins, processorPlugins: processorPlugins, + lifecycle: lifecycle, } return &Orchestrator{ @@ -74,19 +75,10 @@ type base struct { processors ProcessorService connectorPlugins ConnectorPluginService processorPlugins ProcessorPluginService + lifecycle LifecycleService } type PipelineService interface { - Start(ctx context.Context, connFetcher pipeline.ConnectorFetcher, procService pipeline.ProcessorService, pluginFetcher pipeline.PluginDispenserFetcher, pipelineID string) error - // Stop initiates a stop of the given pipeline. The method does not wait for - // the pipeline (and its nodes) to actually stop. - // When force is false the pipeline will try to stop gracefully and drain - // any in-flight messages that have not yet reached the destination. When - // force is true the pipeline will stop without draining in-flight messages. - // It is allowed to execute a force stop even after a graceful stop was - // requested. - Stop(ctx context.Context, pipelineID string, force bool) error - List(ctx context.Context) map[string]*pipeline.Instance Get(ctx context.Context, id string) (*pipeline.Instance, error) Create(ctx context.Context, id string, cfg pipeline.Config, p pipeline.ProvisionType) (*pipeline.Instance, error) @@ -132,3 +124,16 @@ type ProcessorPluginService interface { NewProcessor(ctx context.Context, pluginName string, id string) (processorSdk.Processor, error) RegisterStandalonePlugin(ctx context.Context, path string) (string, error) } + +type LifecycleService interface { + // Start initiates a start of the given pipeline. + Start(ctx context.Context, pipelineID string) error + // Stop initiates a stop of the given pipeline. The method does not wait for + // the pipeline (and its nodes) to actually stop. + // When force is false the pipeline will try to stop gracefully and drain + // any in-flight messages that have not yet reached the destination. When + // force is true the pipeline will stop without draining in-flight messages. + // It is allowed to execute a force stop even after a graceful stop was + // requested. + Stop(ctx context.Context, pipelineID string, force bool) error +} diff --git a/pkg/orchestrator/orchestrator_test.go b/pkg/orchestrator/orchestrator_test.go index e75deb2c9..bcb125b27 100644 --- a/pkg/orchestrator/orchestrator_test.go +++ b/pkg/orchestrator/orchestrator_test.go @@ -27,6 +27,7 @@ import ( "github.com/conduitio/conduit/pkg/connector" "github.com/conduitio/conduit/pkg/foundation/ctxutil" "github.com/conduitio/conduit/pkg/foundation/log" + "github.com/conduitio/conduit/pkg/lifecycle" "github.com/conduitio/conduit/pkg/orchestrator/mock" "github.com/conduitio/conduit/pkg/pipeline" conn_plugin "github.com/conduitio/conduit/pkg/plugin/connector" @@ -37,6 +38,7 @@ import ( proc_builtin "github.com/conduitio/conduit/pkg/plugin/processor/builtin" "github.com/conduitio/conduit/pkg/processor" "github.com/google/go-cmp/cmp" + "github.com/jpillora/backoff" "github.com/matryer/is" "github.com/rs/zerolog" "go.uber.org/mock/gomock" @@ -46,14 +48,15 @@ import ( // a context is passed to a function. var ctxType = reflect.TypeOf((*context.Context)(nil)).Elem() -func newMockServices(t *testing.T) (*mock.PipelineService, *mock.ConnectorService, *mock.ProcessorService, *mock.ConnectorPluginService, *mock.ProcessorPluginService) { +func newMockServices(t *testing.T) (*mock.PipelineService, *mock.ConnectorService, *mock.ProcessorService, *mock.ConnectorPluginService, *mock.ProcessorPluginService, *mock.LifecycleService) { ctrl := gomock.NewController(t) return mock.NewPipelineService(ctrl), mock.NewConnectorService(ctrl), mock.NewProcessorService(ctrl), mock.NewConnectorPluginService(ctrl), - mock.NewProcessorPluginService(ctrl) + mock.NewProcessorPluginService(ctrl), + mock.NewLifecycleService(ctrl) } func TestPipelineSimple(t *testing.T) { @@ -90,14 +93,23 @@ func TestPipelineSimple(t *testing.T) { nil, ) + b := &backoff.Backoff{} + + connectorService := connector.NewService(logger, db, connector.NewPersister(logger, db, time.Second, 3)) + processorService := processor.NewService(logger, db, procPluginService) + pipelineService := pipeline.NewService(logger, db) + + lifecycleService := lifecycle.NewService(logger, b, connectorService, processorService, connPluginService, pipelineService) + orc := NewOrchestrator( db, logger, - pipeline.NewService(logger, db), - connector.NewService(logger, db, connector.NewPersister(logger, db, time.Second, 3)), - processor.NewService(logger, db, procPluginService), + pipelineService, + connectorService, + processorService, connPluginService, procPluginService, + lifecycleService, ) // create a host pipeline @@ -158,7 +170,7 @@ func TestPipelineSimple(t *testing.T) { err = orc.Pipelines.Stop(ctx, pl.ID, false) is.NoErr(err) t.Log("waiting") - err = pl.Wait() + err = lifecycleService.WaitPipeline(pl.ID) is.NoErr(err) t.Log("successfully stopped pipeline") diff --git a/pkg/orchestrator/pipelines.go b/pkg/orchestrator/pipelines.go index 839e0e5bd..e00a3a772 100644 --- a/pkg/orchestrator/pipelines.go +++ b/pkg/orchestrator/pipelines.go @@ -27,12 +27,12 @@ type PipelineOrchestrator base func (po *PipelineOrchestrator) Start(ctx context.Context, id string) error { // TODO lock pipeline - return po.pipelines.Start(ctx, po.connectors, po.processors, po.connectorPlugins, id) + return po.lifecycle.Start(ctx, id) } func (po *PipelineOrchestrator) Stop(ctx context.Context, id string, force bool) error { // TODO lock pipeline - return po.pipelines.Stop(ctx, id, force) + return po.lifecycle.Stop(ctx, id, force) } func (po *PipelineOrchestrator) List(ctx context.Context) map[string]*pipeline.Instance { diff --git a/pkg/orchestrator/pipelines_test.go b/pkg/orchestrator/pipelines_test.go index 19a229317..1ca657047 100644 --- a/pkg/orchestrator/pipelines_test.go +++ b/pkg/orchestrator/pipelines_test.go @@ -31,17 +31,17 @@ func TestPipelineOrchestrator_Start_Success(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), } plBefore.SetStatus(pipeline.StatusSystemStopped) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) - plsMock.EXPECT(). - Start(gomock.AssignableToTypeOf(ctxType), orc.Pipelines.connectors, orc.Pipelines.processors, orc.Pipelines.connectorPlugins, plBefore.ID). + lifecycleMock.EXPECT(). + Start(gomock.AssignableToTypeOf(ctxType), plBefore.ID). Return(nil) err := orc.Pipelines.Start(ctx, plBefore.ID) @@ -52,7 +52,7 @@ func TestPipelineOrchestrator_Start_Fail(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), @@ -60,11 +60,11 @@ func TestPipelineOrchestrator_Start_Fail(t *testing.T) { plBefore.SetStatus(pipeline.StatusSystemStopped) wantErr := cerrors.New("pipeline doesn't exist") - plsMock.EXPECT(). - Start(gomock.AssignableToTypeOf(ctxType), consMock, procsMock, connPluginMock, gomock.AssignableToTypeOf("")). + lifecycleMock.EXPECT(). + Start(gomock.AssignableToTypeOf(ctxType), gomock.AssignableToTypeOf("")). Return(wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) err := orc.Pipelines.Start(ctx, plBefore.ID) is.True(cerrors.Is(err, wantErr)) } @@ -73,15 +73,15 @@ func TestPipelineOrchestrator_Stop_Success(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), } plBefore.SetStatus(pipeline.StatusRunning) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) - plsMock.EXPECT(). + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) + lifecycleMock.EXPECT(). Stop(gomock.AssignableToTypeOf(ctxType), plBefore.ID, false). Return(nil) @@ -93,7 +93,7 @@ func TestPipelineOrchestrator_Stop_Fail(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), @@ -101,11 +101,11 @@ func TestPipelineOrchestrator_Stop_Fail(t *testing.T) { plBefore.SetStatus(pipeline.StatusRunning) wantErr := cerrors.New("pipeline doesn't exist") - plsMock.EXPECT(). + lifecycleMock.EXPECT(). Stop(gomock.AssignableToTypeOf(ctxType), gomock.AssignableToTypeOf(""), true). Return(wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) err := orc.Pipelines.Stop(ctx, plBefore.ID, true) is.True(cerrors.Is(err, wantErr)) } @@ -114,7 +114,7 @@ func TestPipelineOrchestrator_Update_Success(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), @@ -129,7 +129,7 @@ func TestPipelineOrchestrator_Update_Success(t *testing.T) { } want.SetStatus(pipeline.StatusSystemStopped) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), plBefore.ID). Return(plBefore, nil) @@ -146,7 +146,7 @@ func TestPipelineOrchestrator_Update_PipelineRunning(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), @@ -156,7 +156,7 @@ func TestPipelineOrchestrator_Update_PipelineRunning(t *testing.T) { newConfig := pipeline.Config{Name: "new pipeline"} - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), plBefore.ID). Return(plBefore, nil) @@ -170,7 +170,7 @@ func TestPipelineOrchestrator_Update_PipelineProvisionedByConfig(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), @@ -181,7 +181,7 @@ func TestPipelineOrchestrator_Update_PipelineProvisionedByConfig(t *testing.T) { newConfig := pipeline.Config{Name: "new pipeline"} - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), plBefore.ID). Return(plBefore, nil) @@ -195,14 +195,14 @@ func TestPipelineOrchestrator_Delete_Success(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), } plBefore.SetStatus(pipeline.StatusSystemStopped) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), plBefore.ID). Return(plBefore, nil) @@ -218,14 +218,14 @@ func TestPipelineOrchestrator_Delete_PipelineRunning(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), } plBefore.SetStatus(pipeline.StatusRunning) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), plBefore.ID). Return(plBefore, nil) @@ -238,7 +238,7 @@ func TestPipelineOrchestrator_Delete_PipelineProvisionedByConfig(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), @@ -246,7 +246,7 @@ func TestPipelineOrchestrator_Delete_PipelineProvisionedByConfig(t *testing.T) { } plBefore.SetStatus(pipeline.StatusUserStopped) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), plBefore.ID). Return(plBefore, nil) @@ -259,7 +259,7 @@ func TestPipelineOrchestrator_Delete_PipelineHasProcessorsAttached(t *testing.T) is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), @@ -267,7 +267,7 @@ func TestPipelineOrchestrator_Delete_PipelineHasProcessorsAttached(t *testing.T) } plBefore.SetStatus(pipeline.StatusSystemStopped) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), plBefore.ID). Return(plBefore, nil) @@ -280,7 +280,7 @@ func TestPipelineOrchestrator_Delete_PipelineHasConnectorsAttached(t *testing.T) is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), @@ -288,7 +288,7 @@ func TestPipelineOrchestrator_Delete_PipelineHasConnectorsAttached(t *testing.T) } plBefore.SetStatus(pipeline.StatusSystemStopped) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), plBefore.ID). Return(plBefore, nil) @@ -301,10 +301,10 @@ func TestPipelineOrchestrator_Delete_PipelineDoesntExist(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) wantErr := cerrors.New("pipeline doesn't exist") - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), gomock.AssignableToTypeOf("")). Return(nil, wantErr) @@ -317,7 +317,7 @@ func TestPipelineOrchestrator_UpdateDLQ_Success(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), @@ -344,7 +344,7 @@ func TestPipelineOrchestrator_UpdateDLQ_Success(t *testing.T) { } want.SetStatus(plBefore.GetStatus()) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), plBefore.ID). Return(plBefore, nil) @@ -364,14 +364,14 @@ func TestPipelineOrchestrator_UpdateDLQ_PipelineRunning(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), } plBefore.SetStatus(pipeline.StatusRunning) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), plBefore.ID). Return(plBefore, nil) @@ -385,7 +385,7 @@ func TestPipelineOrchestrator_UpdateDLQ_PipelineProvisionedByConfig(t *testing.T is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), @@ -393,7 +393,7 @@ func TestPipelineOrchestrator_UpdateDLQ_PipelineProvisionedByConfig(t *testing.T } plBefore.SetStatus(pipeline.StatusUserStopped) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), plBefore.ID). Return(plBefore, nil) @@ -407,7 +407,7 @@ func TestConnectorOrchestrator_UpdateDLQ_InvalidConfig(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), @@ -423,7 +423,7 @@ func TestConnectorOrchestrator_UpdateDLQ_InvalidConfig(t *testing.T) { } wantErr := cerrors.New("invalid plugin config") - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), plBefore.ID). Return(plBefore, nil) diff --git a/pkg/orchestrator/processors_test.go b/pkg/orchestrator/processors_test.go index fdee6acdd..8fc9868d9 100644 --- a/pkg/orchestrator/processors_test.go +++ b/pkg/orchestrator/processors_test.go @@ -34,7 +34,7 @@ func TestProcessorOrchestrator_CreateOnPipeline_Success(t *testing.T) { ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -72,7 +72,7 @@ func TestProcessorOrchestrator_CreateOnPipeline_Success(t *testing.T) { AddProcessor(gomock.AssignableToTypeOf(ctxType), pl.ID, want.ID). Return(pl, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Processors.Create(ctx, want.Plugin, want.Parent, want.Config, want.Condition) is.NoErr(err) is.Equal(want, got) @@ -83,7 +83,7 @@ func TestProcessorOrchestrator_CreateOnPipeline_PipelineNotExist(t *testing.T) { ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) parent := processor.Parent{ ID: uuid.NewString(), @@ -94,7 +94,7 @@ func TestProcessorOrchestrator_CreateOnPipeline_PipelineNotExist(t *testing.T) { Get(gomock.AssignableToTypeOf(ctxType), parent.ID). Return(nil, wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Processors.Create(ctx, "test-processor", parent, processor.Config{}, "") is.True(err != nil) is.True(cerrors.Is(err, wantErr)) // errors did not match @@ -106,7 +106,7 @@ func TestProcessorOrchestrator_CreateOnPipeline_PipelineRunning(t *testing.T) { ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -121,7 +121,7 @@ func TestProcessorOrchestrator_CreateOnPipeline_PipelineRunning(t *testing.T) { Get(gomock.AssignableToTypeOf(ctxType), pl.ID). Return(pl, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Processors.Create(ctx, "test-processor", parent, processor.Config{}, "") is.True(err != nil) is.Equal(pipeline.ErrPipelineRunning, err) @@ -133,7 +133,7 @@ func TestProcessorOrchestrator_CreateOnPipeline_PipelineProvisionedByConfig(t *t ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -149,7 +149,7 @@ func TestProcessorOrchestrator_CreateOnPipeline_PipelineProvisionedByConfig(t *t Get(gomock.AssignableToTypeOf(ctxType), pl.ID). Return(pl, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Processors.Create(ctx, "test-processor", parent, processor.Config{}, "") is.Equal(got, nil) is.True(err != nil) @@ -161,7 +161,7 @@ func TestProcessorOrchestrator_CreateOnPipeline_CreateProcessorError(t *testing. ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -189,7 +189,7 @@ func TestProcessorOrchestrator_CreateOnPipeline_CreateProcessorError(t *testing. ). Return(nil, wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Processors.Create(ctx, "test-processor", parent, processor.Config{}, "") is.True(err != nil) is.True(cerrors.Is(err, wantErr)) // errors did not match @@ -201,7 +201,7 @@ func TestProcessorOrchestrator_CreateOnPipeline_AddProcessorError(t *testing.T) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -244,7 +244,7 @@ func TestProcessorOrchestrator_CreateOnPipeline_AddProcessorError(t *testing.T) Delete(gomock.AssignableToTypeOf(ctxType), proc.ID). Return(nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Processors.Create(ctx, proc.Plugin, proc.Parent, proc.Config, proc.Condition) is.True(err != nil) is.True(cerrors.Is(err, wantErr)) // errors did not match @@ -256,7 +256,7 @@ func TestProcessorOrchestrator_CreateOnConnector_Success(t *testing.T) { ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -301,7 +301,7 @@ func TestProcessorOrchestrator_CreateOnConnector_Success(t *testing.T) { AddProcessor(gomock.AssignableToTypeOf(ctxType), conn.ID, want.ID). Return(conn, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Processors.Create(ctx, want.Plugin, want.Parent, want.Config, want.Condition) is.NoErr(err) is.Equal(want, got) @@ -312,7 +312,7 @@ func TestProcessorOrchestrator_CreateOnConnector_ConnectorNotExist(t *testing.T) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) parent := processor.Parent{ ID: uuid.NewString(), @@ -323,7 +323,7 @@ func TestProcessorOrchestrator_CreateOnConnector_ConnectorNotExist(t *testing.T) Get(gomock.AssignableToTypeOf(ctxType), parent.ID). Return(nil, wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Processors.Create(ctx, "test-processor", parent, processor.Config{}, "") is.True(err != nil) is.True(cerrors.Is(err, wantErr)) // errors did not match @@ -335,7 +335,7 @@ func TestProcessorOrchestrator_UpdateOnPipeline_Success(t *testing.T) { ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -378,7 +378,7 @@ func TestProcessorOrchestrator_UpdateOnPipeline_Success(t *testing.T) { Update(gomock.AssignableToTypeOf(ctxType), want.ID, want.Config). Return(want, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Processors.Update(ctx, before.ID, newConfig) is.NoErr(err) is.Equal(want, got) @@ -389,7 +389,7 @@ func TestProcessorOrchestrator_UpdateOnPipeline_ProcessorNotExist(t *testing.T) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -416,7 +416,7 @@ func TestProcessorOrchestrator_UpdateOnPipeline_ProcessorNotExist(t *testing.T) Get(gomock.AssignableToTypeOf(ctxType), before.ID). Return(nil, wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Processors.Update(ctx, before.ID, newConfig) is.True(err != nil) is.True(cerrors.Is(err, wantErr)) // errors did not match") @@ -428,7 +428,7 @@ func TestProcessorOrchestrator_UpdateOnPipeline_PipelineRunning(t *testing.T) { ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -457,7 +457,7 @@ func TestProcessorOrchestrator_UpdateOnPipeline_PipelineRunning(t *testing.T) { Get(gomock.AssignableToTypeOf(ctxType), pl.ID). Return(pl, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Processors.Update(ctx, before.ID, newConfig) is.True(err != nil) is.Equal(pipeline.ErrPipelineRunning, err) @@ -469,7 +469,7 @@ func TestProcessorOrchestrator_UpdateOnPipeline_ProcessorProvisionedByConfig(t * ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -497,7 +497,7 @@ func TestProcessorOrchestrator_UpdateOnPipeline_ProcessorProvisionedByConfig(t * Get(gomock.AssignableToTypeOf(ctxType), before.ID). Return(before, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Processors.Update(ctx, before.ID, newConfig) is.Equal(got, nil) is.True(err != nil) @@ -509,7 +509,7 @@ func TestProcessorOrchestrator_UpdateOnPipeline_UpdateFail(t *testing.T) { ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -553,7 +553,7 @@ func TestProcessorOrchestrator_UpdateOnPipeline_UpdateFail(t *testing.T) { Update(gomock.AssignableToTypeOf(ctxType), want.ID, want.Config). Return(nil, wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Processors.Update(ctx, before.ID, newConfig) is.True(err != nil) is.Equal(wantErr, err) @@ -565,7 +565,7 @@ func TestProcessorOrchestrator_UpdateOnConnector_ConnectorNotExist(t *testing.T) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) connID := uuid.NewString() want := &processor.Instance{ @@ -585,7 +585,7 @@ func TestProcessorOrchestrator_UpdateOnConnector_ConnectorNotExist(t *testing.T) Get(gomock.AssignableToTypeOf(ctxType), connID). Return(nil, wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Processors.Update(ctx, want.ID, processor.Config{}) is.True(err != nil) is.True(cerrors.Is(err, wantErr)) // errors did not match @@ -597,7 +597,7 @@ func TestProcessorOrchestrator_DeleteOnPipeline_Success(t *testing.T) { ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -629,7 +629,7 @@ func TestProcessorOrchestrator_DeleteOnPipeline_Success(t *testing.T) { RemoveProcessor(gomock.AssignableToTypeOf(ctxType), pl.ID, want.ID). Return(pl, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) err := orc.Processors.Delete(ctx, want.ID) is.NoErr(err) } @@ -639,7 +639,7 @@ func TestProcessorOrchestrator_DeleteOnPipeline_ProcessorNotExist(t *testing.T) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -663,7 +663,7 @@ func TestProcessorOrchestrator_DeleteOnPipeline_ProcessorNotExist(t *testing.T) Get(gomock.AssignableToTypeOf(ctxType), want.ID). Return(nil, wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) err := orc.Processors.Delete(ctx, want.ID) is.True(err != nil) is.True(cerrors.Is(err, wantErr)) // errors did not match @@ -674,7 +674,7 @@ func TestProcessorOrchestrator_DeleteOnPipeline_PipelineRunning(t *testing.T) { ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -699,7 +699,7 @@ func TestProcessorOrchestrator_DeleteOnPipeline_PipelineRunning(t *testing.T) { Get(gomock.AssignableToTypeOf(ctxType), pl.ID). Return(pl, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) err := orc.Processors.Delete(ctx, want.ID) is.True(err != nil) is.Equal(pipeline.ErrPipelineRunning, err) @@ -710,7 +710,7 @@ func TestProcessorOrchestrator_DeleteOnPipeline_Fail(t *testing.T) { ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -740,7 +740,7 @@ func TestProcessorOrchestrator_DeleteOnPipeline_Fail(t *testing.T) { Delete(gomock.AssignableToTypeOf(ctxType), want.ID). Return(wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) err := orc.Processors.Delete(ctx, want.ID) is.True(err != nil) is.True(cerrors.Is(err, wantErr)) // errors did not match @@ -751,7 +751,7 @@ func TestProcessorOrchestrator_DeleteOnPipeline_RemoveProcessorFail(t *testing.T ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -797,7 +797,7 @@ func TestProcessorOrchestrator_DeleteOnPipeline_RemoveProcessorFail(t *testing.T ). Return(want, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) err := orc.Processors.Delete(ctx, want.ID) is.True(err != nil) } @@ -807,7 +807,7 @@ func TestProcessorOrchestrator_DeleteOnConnector_Fail(t *testing.T) { ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -860,7 +860,7 @@ func TestProcessorOrchestrator_DeleteOnConnector_Fail(t *testing.T) { ). Return(want, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) err := orc.Processors.Delete(ctx, want.ID) is.True(err != nil) } diff --git a/pkg/pipeline/instance.go b/pkg/pipeline/instance.go index 35654eb98..e6b35620e 100644 --- a/pkg/pipeline/instance.go +++ b/pkg/pipeline/instance.go @@ -19,9 +19,6 @@ package pipeline import ( "sync" "time" - - "github.com/conduitio/conduit/pkg/pipeline/stream" - "gopkg.in/tomb.v2" ) const ( @@ -61,9 +58,6 @@ type Instance struct { status Status statusLock sync.RWMutex - - n map[string]stream.Node - t *tomb.Tomb } // encodableInstance is an encodable "view" of Instance @@ -97,13 +91,6 @@ var DefaultDLQ = DLQ{ WindowNackThreshold: 0, } -func (p *Instance) Wait() error { - if p.t == nil { - return nil - } - return p.t.Wait() -} - func (p *Instance) SetStatus(s Status) { p.statusLock.Lock() defer p.statusLock.Unlock() diff --git a/pkg/pipeline/service.go b/pkg/pipeline/service.go index ca8373335..a58f57ab9 100644 --- a/pkg/pipeline/service.go +++ b/pkg/pipeline/service.go @@ -34,14 +34,6 @@ const ( DescriptionLengthLimit = 8192 ) -type FailureEvent struct { - // ID is the ID of the pipeline which failed. - ID string - Error error -} - -type FailureHandler func(FailureEvent) - // Service manages pipelines. type Service struct { logger log.CtxLogger @@ -50,7 +42,6 @@ type Service struct { instances map[string]*Instance instanceNames map[string]bool - handlers []FailureHandler } // NewService initializes and returns a pipeline Service. @@ -119,7 +110,7 @@ func (s *Service) Get(_ context.Context, id string) (*Instance, error) { } // Create will create a new pipeline instance with the given config and return -// it if it was successfully saved to the database. +// if it was successfully saved to the database. func (s *Service) Create(ctx context.Context, id string, cfg Config, p ProvisionType) (*Instance, error) { err := s.validatePipeline(cfg, id) if err != nil { @@ -316,27 +307,6 @@ func (s *Service) Delete(ctx context.Context, pipelineID string) error { return nil } -// OnFailure registers a handler for a pipeline.FailureEvent. -// Only errors which happen after a pipeline has been started -// are being sent. -func (s *Service) OnFailure(handler FailureHandler) { - s.handlers = append(s.handlers, handler) -} - -// notify notifies all registered FailureHandlers about an error. -func (s *Service) notify(pipelineID string, err error) { - if err == nil { - return - } - e := FailureEvent{ - ID: pipelineID, - Error: err, - } - for _, handler := range s.handlers { - handler(e) - } -} - func (s *Service) validatePipeline(cfg Config, id string) error { // contains all the errors occurred while provisioning configuration files. var errs []error @@ -367,6 +337,25 @@ func (s *Service) validatePipeline(cfg Config, id string) error { return cerrors.Join(errs...) } +// UpdateStatus updates the status of a pipeline by the ID. +func (s *Service) UpdateStatus(ctx context.Context, id string, status Status, errMsg string) error { + pipeline, err := s.Get(ctx, id) + if err != nil { + return err + } + s.updateOldStatusMetrics(pipeline) + pipeline.SetStatus(status) + + pipeline.Error = errMsg + s.updateNewStatusMetrics(pipeline) + + err = s.store.Set(ctx, pipeline.ID, pipeline) + if err != nil { + return cerrors.Errorf("pipeline not updated: %w", err) + } + return nil +} + func (s *Service) updateOldStatusMetrics(pl *Instance) { status := strings.ToLower(pl.GetStatus().String()) measure.PipelinesGauge.WithValues(status).Dec() diff --git a/pkg/provisioning/import_test.go b/pkg/provisioning/import_test.go index b32603b8f..bd85ab461 100644 --- a/pkg/provisioning/import_test.go +++ b/pkg/provisioning/import_test.go @@ -35,7 +35,7 @@ func TestService_ExecuteActions_Success(t *testing.T) { ctrl := gomock.NewController(t) ctx := context.Background() - srv, _, _, _, _ := newTestService(ctrl, logger) + srv, _, _, _, _, _ := newTestService(ctrl, logger) var actionCount int countingAction := fakeAction{do: func(ctx context.Context) error { @@ -57,7 +57,7 @@ func TestService_ExecuteActions_Fail(t *testing.T) { ctrl := gomock.NewController(t) ctx := context.Background() - srv, _, _, _, _ := newTestService(ctrl, logger) + srv, _, _, _, _, _ := newTestService(ctrl, logger) var actionCount int countingAction := fakeAction{do: func(ctx context.Context) error { @@ -85,7 +85,7 @@ func TestService_RollbackActions_Success(t *testing.T) { ctrl := gomock.NewController(t) ctx := context.Background() - srv, _, _, _, _ := newTestService(ctrl, logger) + srv, _, _, _, _, _ := newTestService(ctrl, logger) var actionCount int countingAction := fakeAction{rollback: func(ctx context.Context) error { @@ -106,7 +106,7 @@ func TestService_RollbackActions_Fail(t *testing.T) { ctrl := gomock.NewController(t) ctx := context.Background() - srv, _, _, _, _ := newTestService(ctrl, logger) + srv, _, _, _, _, _ := newTestService(ctrl, logger) var actionCount int countingAction := fakeAction{rollback: func(ctx context.Context) error { @@ -131,7 +131,7 @@ func TestActionBuilder_Build(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, pipSrv, connSrv, procSrv, connPlugSrv := newTestService(ctrl, logger) + srv, pipSrv, connSrv, procSrv, connPlugSrv, _ := newTestService(ctrl, logger) oldConfig := config.Pipeline{ ID: "config-id", @@ -353,7 +353,7 @@ func TestActionsBuilder_PreparePipelineActions_Create(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, pipSrv, _, _, _ := newTestService(ctrl, logger) + srv, pipSrv, _, _, _, _ := newTestService(ctrl, logger) oldConfig := config.Pipeline{} newConfig := config.Pipeline{ID: "config-id"} @@ -372,7 +372,7 @@ func TestActionsBuilder_PreparePipelineActions_Delete(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, pipSrv, _, _, _ := newTestService(ctrl, logger) + srv, pipSrv, _, _, _, _ := newTestService(ctrl, logger) oldConfig := config.Pipeline{ID: "config-id"} newConfig := config.Pipeline{} @@ -390,7 +390,7 @@ func TestActionsBuilder_PreparePipelineActions_NoAction(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, _, _, _, _ := newTestService(ctrl, logger) + srv, _, _, _, _, _ := newTestService(ctrl, logger) testCases := []struct { name string @@ -451,7 +451,7 @@ func TestActionsBuilder_PreparePipelineActions_Update(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, pipSrv, _, _, _ := newTestService(ctrl, logger) + srv, pipSrv, _, _, _, _ := newTestService(ctrl, logger) testCases := []struct { name string @@ -503,7 +503,7 @@ func TestActionsBuilder_PrepareConnectorActions_Create(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, _, connSrv, _, connPlugSrv := newTestService(ctrl, logger) + srv, _, connSrv, _, connPlugSrv, _ := newTestService(ctrl, logger) oldConfig := config.Connector{} newConfig := config.Connector{ID: "config-id"} @@ -525,7 +525,7 @@ func TestActionsBuilder_PrepareConnectorActions_Delete(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, _, connSrv, _, connPlugSrv := newTestService(ctrl, logger) + srv, _, connSrv, _, connPlugSrv, _ := newTestService(ctrl, logger) oldConfig := config.Connector{ID: "config-id"} newConfig := config.Connector{} @@ -546,7 +546,7 @@ func TestActionsBuilder_PrepareConnectorActions_NoAction(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, _, _, _, _ := newTestService(ctrl, logger) + srv, _, _, _, _, _ := newTestService(ctrl, logger) testCases := []struct { name string @@ -585,7 +585,7 @@ func TestActionsBuilder_PrepareConnectorActions_Update(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, _, connSrv, _, _ := newTestService(ctrl, logger) + srv, _, connSrv, _, _, _ := newTestService(ctrl, logger) testCases := []struct { name string @@ -623,7 +623,7 @@ func TestActionsBuilder_PrepareConnectorActions_Recreate(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, _, connSrv, _, connPlugSrv := newTestService(ctrl, logger) + srv, _, connSrv, _, connPlugSrv, _ := newTestService(ctrl, logger) pipelineID := uuid.NewString() testCases := []struct { @@ -665,7 +665,7 @@ func TestActionsBuilder_PrepareProcessorActions_Create(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, _, _, procSrv, _ := newTestService(ctrl, logger) + srv, _, _, procSrv, _, _ := newTestService(ctrl, logger) oldConfig := config.Processor{} newConfig := config.Processor{ID: "config-id"} @@ -689,7 +689,7 @@ func TestActionsBuilder_PrepareProcessorActions_Delete(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, _, _, procSrv, _ := newTestService(ctrl, logger) + srv, _, _, procSrv, _, _ := newTestService(ctrl, logger) oldConfig := config.Processor{ID: "config-id"} newConfig := config.Processor{} @@ -713,7 +713,7 @@ func TestActionsBuilder_PrepareProcessorActions_NoAction(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, _, _, _, _ := newTestService(ctrl, logger) + srv, _, _, _, _, _ := newTestService(ctrl, logger) oldConfig := config.Processor{ID: "config-id"} newConfig := config.Processor{ID: "config-id"} @@ -730,7 +730,7 @@ func TestActionsBuilder_PrepareProcessorActions_Update(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, _, _, procSrv, _ := newTestService(ctrl, logger) + srv, _, _, procSrv, _, _ := newTestService(ctrl, logger) parent := processor.Parent{ ID: uuid.NewString(), Type: processor.ParentTypeConnector, @@ -768,7 +768,7 @@ func TestActionsBuilder_PrepareProcessorActions_Recreate(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, _, _, procSrv, _ := newTestService(ctrl, logger) + srv, _, _, procSrv, _, _ := newTestService(ctrl, logger) parent := processor.Parent{ ID: uuid.NewString(), Type: processor.ParentTypePipeline, @@ -808,16 +808,17 @@ func TestActionsBuilder_PrepareProcessorActions_Recreate(t *testing.T) { func intPtr(i int) *int { return &i } -func newTestService(ctrl *gomock.Controller, logger log.CtxLogger) (*Service, *mock.PipelineService, *mock.ConnectorService, *mock.ProcessorService, *mock.ConnectorPluginService) { +func newTestService(ctrl *gomock.Controller, logger log.CtxLogger) (*Service, *mock.PipelineService, *mock.ConnectorService, *mock.ProcessorService, *mock.ConnectorPluginService, *mock.LifecycleService) { db := &inmemory.DB{} pipSrv := mock.NewPipelineService(ctrl) connSrv := mock.NewConnectorService(ctrl) procSrv := mock.NewProcessorService(ctrl) connPlugSrv := mock.NewConnectorPluginService(ctrl) + lifecycleSrv := mock.NewLifecycleService(ctrl) - srv := NewService(db, logger, pipSrv, connSrv, procSrv, connPlugSrv, "") + srv := NewService(db, logger, pipSrv, connSrv, procSrv, connPlugSrv, lifecycleSrv, "") - return srv, pipSrv, connSrv, procSrv, connPlugSrv + return srv, pipSrv, connSrv, procSrv, connPlugSrv, lifecycleSrv } type fakeAction struct { diff --git a/pkg/provisioning/interfaces.go b/pkg/provisioning/interfaces.go index d4e8e6f72..21f2142e2 100644 --- a/pkg/provisioning/interfaces.go +++ b/pkg/provisioning/interfaces.go @@ -24,7 +24,7 @@ import ( "github.com/conduitio/conduit/pkg/processor" ) -//go:generate mockgen -typed -destination=mock/provisioning.go -package=mock -mock_names=PipelineService=PipelineService,ConnectorService=ConnectorService,ProcessorService=ProcessorService,ConnectorPluginService=ConnectorPluginService . PipelineService,ConnectorService,ProcessorService,ConnectorPluginService +//go:generate mockgen -typed -destination=mock/provisioning.go -package=mock -mock_names=PipelineService=PipelineService,ConnectorService=ConnectorService,ProcessorService=ProcessorService,ConnectorPluginService=ConnectorPluginService,LifecycleService=LifecycleService . PipelineService,ConnectorService,ProcessorService,ConnectorPluginService,LifecycleService type PipelineService interface { Get(ctx context.Context, id string) (*pipeline.Instance, error) @@ -34,9 +34,6 @@ type PipelineService interface { Delete(ctx context.Context, pipelineID string) error UpdateDLQ(ctx context.Context, pipelineID string, cfg pipeline.DLQ) (*pipeline.Instance, error) - Start(ctx context.Context, connFetcher pipeline.ConnectorFetcher, procService pipeline.ProcessorService, pluginFetcher pipeline.PluginDispenserFetcher, pipelineID string) error - Stop(ctx context.Context, pipelineID string, force bool) error - AddConnector(ctx context.Context, pipelineID string, connectorID string) (*pipeline.Instance, error) RemoveConnector(ctx context.Context, pipelineID string, connectorID string) (*pipeline.Instance, error) AddProcessor(ctx context.Context, pipelineID string, processorID string) (*pipeline.Instance, error) @@ -72,3 +69,8 @@ type ProcessorService interface { type ConnectorPluginService interface { NewDispenser(ctx log.CtxLogger, name string, connectorID string) (connectorPlugin.Dispenser, error) } + +type LifecycleService interface { + Start(ctx context.Context, pipelineID string) error + Stop(ctx context.Context, pipelineID string, force bool) error +} diff --git a/pkg/provisioning/mock/provisioning.go b/pkg/provisioning/mock/provisioning.go index e7c8ce85c..ae298ff97 100644 --- a/pkg/provisioning/mock/provisioning.go +++ b/pkg/provisioning/mock/provisioning.go @@ -1,9 +1,9 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/conduitio/conduit/pkg/provisioning (interfaces: PipelineService,ConnectorService,ProcessorService,ConnectorPluginService) +// Source: github.com/conduitio/conduit/pkg/provisioning (interfaces: PipelineService,ConnectorService,ProcessorService,ConnectorPluginService,LifecycleService) // // Generated by this command: // -// mockgen -typed -destination=mock/provisioning.go -package=mock -mock_names=PipelineService=PipelineService,ConnectorService=ConnectorService,ProcessorService=ProcessorService,ConnectorPluginService=ConnectorPluginService . PipelineService,ConnectorService,ProcessorService,ConnectorPluginService +// mockgen -typed -destination=mock/provisioning.go -package=mock -mock_names=PipelineService=PipelineService,ConnectorService=ConnectorService,ProcessorService=ProcessorService,ConnectorPluginService=ConnectorPluginService,LifecycleService=LifecycleService . PipelineService,ConnectorService,ProcessorService,ConnectorPluginService,LifecycleService // // Package mock is a generated GoMock package. @@ -354,82 +354,6 @@ func (c *PipelineServiceRemoveProcessorCall) DoAndReturn(f func(context.Context, return c } -// Start mocks base method. -func (m *PipelineService) Start(arg0 context.Context, arg1 pipeline.ConnectorFetcher, arg2 pipeline.ProcessorService, arg3 pipeline.PluginDispenserFetcher, arg4 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Start", arg0, arg1, arg2, arg3, arg4) - ret0, _ := ret[0].(error) - return ret0 -} - -// Start indicates an expected call of Start. -func (mr *PipelineServiceMockRecorder) Start(arg0, arg1, arg2, arg3, arg4 any) *PipelineServiceStartCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*PipelineService)(nil).Start), arg0, arg1, arg2, arg3, arg4) - return &PipelineServiceStartCall{Call: call} -} - -// PipelineServiceStartCall wrap *gomock.Call -type PipelineServiceStartCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *PipelineServiceStartCall) Return(arg0 error) *PipelineServiceStartCall { - c.Call = c.Call.Return(arg0) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *PipelineServiceStartCall) Do(f func(context.Context, pipeline.ConnectorFetcher, pipeline.ProcessorService, pipeline.PluginDispenserFetcher, string) error) *PipelineServiceStartCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *PipelineServiceStartCall) DoAndReturn(f func(context.Context, pipeline.ConnectorFetcher, pipeline.ProcessorService, pipeline.PluginDispenserFetcher, string) error) *PipelineServiceStartCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// Stop mocks base method. -func (m *PipelineService) Stop(arg0 context.Context, arg1 string, arg2 bool) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Stop", arg0, arg1, arg2) - ret0, _ := ret[0].(error) - return ret0 -} - -// Stop indicates an expected call of Stop. -func (mr *PipelineServiceMockRecorder) Stop(arg0, arg1, arg2 any) *PipelineServiceStopCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stop", reflect.TypeOf((*PipelineService)(nil).Stop), arg0, arg1, arg2) - return &PipelineServiceStopCall{Call: call} -} - -// PipelineServiceStopCall wrap *gomock.Call -type PipelineServiceStopCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *PipelineServiceStopCall) Return(arg0 error) *PipelineServiceStopCall { - c.Call = c.Call.Return(arg0) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *PipelineServiceStopCall) Do(f func(context.Context, string, bool) error) *PipelineServiceStopCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *PipelineServiceStopCall) DoAndReturn(f func(context.Context, string, bool) error) *PipelineServiceStopCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - // Update mocks base method. func (m *PipelineService) Update(arg0 context.Context, arg1 string, arg2 pipeline.Config) (*pipeline.Instance, error) { m.ctrl.T.Helper() @@ -1042,3 +966,102 @@ func (c *ConnectorPluginServiceNewDispenserCall) DoAndReturn(f func(log.CtxLogge c.Call = c.Call.DoAndReturn(f) return c } + +// LifecycleService is a mock of LifecycleService interface. +type LifecycleService struct { + ctrl *gomock.Controller + recorder *LifecycleServiceMockRecorder +} + +// LifecycleServiceMockRecorder is the mock recorder for LifecycleService. +type LifecycleServiceMockRecorder struct { + mock *LifecycleService +} + +// NewLifecycleService creates a new mock instance. +func NewLifecycleService(ctrl *gomock.Controller) *LifecycleService { + mock := &LifecycleService{ctrl: ctrl} + mock.recorder = &LifecycleServiceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *LifecycleService) EXPECT() *LifecycleServiceMockRecorder { + return m.recorder +} + +// Start mocks base method. +func (m *LifecycleService) Start(arg0 context.Context, arg1 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Start", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Start indicates an expected call of Start. +func (mr *LifecycleServiceMockRecorder) Start(arg0, arg1 any) *LifecycleServiceStartCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*LifecycleService)(nil).Start), arg0, arg1) + return &LifecycleServiceStartCall{Call: call} +} + +// LifecycleServiceStartCall wrap *gomock.Call +type LifecycleServiceStartCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *LifecycleServiceStartCall) Return(arg0 error) *LifecycleServiceStartCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *LifecycleServiceStartCall) Do(f func(context.Context, string) error) *LifecycleServiceStartCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *LifecycleServiceStartCall) DoAndReturn(f func(context.Context, string) error) *LifecycleServiceStartCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// Stop mocks base method. +func (m *LifecycleService) Stop(arg0 context.Context, arg1 string, arg2 bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Stop", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// Stop indicates an expected call of Stop. +func (mr *LifecycleServiceMockRecorder) Stop(arg0, arg1, arg2 any) *LifecycleServiceStopCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stop", reflect.TypeOf((*LifecycleService)(nil).Stop), arg0, arg1, arg2) + return &LifecycleServiceStopCall{Call: call} +} + +// LifecycleServiceStopCall wrap *gomock.Call +type LifecycleServiceStopCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *LifecycleServiceStopCall) Return(arg0 error) *LifecycleServiceStopCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *LifecycleServiceStopCall) Do(f func(context.Context, string, bool) error) *LifecycleServiceStopCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *LifecycleServiceStopCall) DoAndReturn(f func(context.Context, string, bool) error) *LifecycleServiceStopCall { + c.Call = c.Call.DoAndReturn(f) + return c +} diff --git a/pkg/provisioning/service.go b/pkg/provisioning/service.go index c0b3bcec6..803e50fa7 100644 --- a/pkg/provisioning/service.go +++ b/pkg/provisioning/service.go @@ -40,6 +40,7 @@ type Service struct { connectorService ConnectorService processorService ProcessorService connectorPluginService ConnectorPluginService + lifecycleService LifecycleService pipelinesPath string } @@ -50,6 +51,7 @@ func NewService( connService ConnectorService, procService ProcessorService, connPluginService ConnectorPluginService, + lifecycleService LifecycleService, pipelinesDir string, ) *Service { return &Service{ @@ -60,6 +62,7 @@ func NewService( connectorService: connService, processorService: procService, connectorPluginService: connPluginService, + lifecycleService: lifecycleService, pipelinesPath: pipelinesDir, } } @@ -253,7 +256,8 @@ func (s *Service) provisionPipeline(ctx context.Context, cfg config.Pipeline) er // check if pipeline should be running if cfg.Status == config.StatusRunning { // TODO set status and let the pipeline service start it - err := s.pipelineService.Start(ctx, s.connectorService, s.processorService, s.connectorPluginService, cfg.ID) + + err := s.lifecycleService.Start(ctx, cfg.ID) if err != nil { return cerrors.Errorf("could not start the pipeline %q: %w", cfg.ID, err) } diff --git a/pkg/provisioning/service_test.go b/pkg/provisioning/service_test.go index 2491cd27a..3701d1fa7 100644 --- a/pkg/provisioning/service_test.go +++ b/pkg/provisioning/service_test.go @@ -26,6 +26,7 @@ import ( "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/ctxutil" "github.com/conduitio/conduit/pkg/foundation/log" + "github.com/conduitio/conduit/pkg/lifecycle" "github.com/conduitio/conduit/pkg/pipeline" conn_plugin "github.com/conduitio/conduit/pkg/plugin/connector" "github.com/conduitio/conduit/pkg/plugin/connector/builtin" @@ -40,6 +41,7 @@ import ( p4 "github.com/conduitio/conduit/pkg/provisioning/test/pipelines4-integration-test" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + "github.com/jpillora/backoff" "github.com/matryer/is" "github.com/rs/zerolog" "go.uber.org/mock/gomock" @@ -116,7 +118,7 @@ func TestService_Init_Create(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - service, pipelineService, connService, procService, plugService := newTestService(ctrl, logger) + service, pipelineService, connService, procService, _, lifecycleService := newTestService(ctrl, logger) service.pipelinesPath = "./test/pipelines1" // return a pipeline not provisioned by API @@ -140,7 +142,7 @@ func TestService_Init_Create(t *testing.T) { procService.EXPECT().CreateWithInstance(anyCtx, p1.P1P1) // start pipeline - pipelineService.EXPECT().Start(anyCtx, connService, procService, plugService, p1.P1.ID) + lifecycleService.EXPECT().Start(anyCtx, p1.P1.ID) err := service.Init(context.Background()) is.NoErr(err) @@ -151,7 +153,7 @@ func TestService_Init_Update(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - service, pipelineService, connService, procService, plugService := newTestService(ctrl, logger) + service, pipelineService, connService, procService, _, lifecycleService := newTestService(ctrl, logger) service.pipelinesPath = "./test/pipelines1" pipelineService.EXPECT().List(anyCtx) @@ -175,7 +177,7 @@ func TestService_Init_Update(t *testing.T) { procService.EXPECT().Update(anyCtx, p1.P1P1.ID, p1.P1P1.Config) // start pipeline - pipelineService.EXPECT().Start(anyCtx, connService, procService, plugService, p1.P1.ID) + lifecycleService.EXPECT().Start(anyCtx, p1.P1.ID) err := service.Init(context.Background()) is.NoErr(err) @@ -186,7 +188,7 @@ func TestService_Init_Delete(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - service, pipelineService, connService, procService, plugService := newTestService(ctrl, logger) + service, pipelineService, connService, procService, plugService, _ := newTestService(ctrl, logger) service.pipelinesPath = "" // don't init any pipelines, delete existing ones // old pipeline exists @@ -215,7 +217,7 @@ func TestService_Init_NoRollbackOnFailedStart(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - service, pipelineService, connService, procService, plugService := newTestService(ctrl, logger) + service, pipelineService, connService, procService, _, lifecycleService := newTestService(ctrl, logger) service.pipelinesPath = "./test/pipelines1" // return a pipeline not provisioned by API @@ -240,7 +242,7 @@ func TestService_Init_NoRollbackOnFailedStart(t *testing.T) { // returns an error, no rollback needed wantErr := cerrors.New("error") - pipelineService.EXPECT().Start(anyCtx, connService, procService, plugService, p1.P1.ID).Return(wantErr) + lifecycleService.EXPECT().Start(anyCtx, p1.P1.ID).Return(wantErr) err := service.Init(context.Background()) is.True(cerrors.Is(err, wantErr)) @@ -251,7 +253,7 @@ func TestService_Init_RollbackCreate(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - service, pipelineService, connService, procService, plugService := newTestService(ctrl, logger) + service, pipelineService, connService, procService, plugService, _ := newTestService(ctrl, logger) service.pipelinesPath = "./test/pipelines1" // return a pipeline not provisioned by API @@ -290,7 +292,7 @@ func TestService_Init_RollbackUpdate(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - service, pipelineService, connService, procService, _ := newTestService(ctrl, logger) + service, pipelineService, connService, procService, _, _ := newTestService(ctrl, logger) service.pipelinesPath = "./test/pipelines1" // return a pipeline not provisioned by API @@ -330,7 +332,7 @@ func TestService_Init_MultiplePipelinesDuplicatedPipelineID(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - service, pipelineService, connService, procService, plugService := newTestService(ctrl, logger) + service, pipelineService, connService, _, _, lifecycleService := newTestService(ctrl, logger) service.pipelinesPath = "./test/pipelines2" // return a pipeline not provisioned by API @@ -348,7 +350,7 @@ func TestService_Init_MultiplePipelinesDuplicatedPipelineID(t *testing.T) { connService.EXPECT().CreateWithInstance(anyCtx, p2.P2C1) connService.EXPECT().CreateWithInstance(anyCtx, p2.P2C2) - pipelineService.EXPECT().Start(anyCtx, connService, procService, plugService, p2.P2.ID) + lifecycleService.EXPECT().Start(anyCtx, p2.P2.ID) err := service.Init(context.Background()) is.True(cerrors.Is(err, ErrDuplicatedPipelineID)) // duplicated pipeline id @@ -359,7 +361,7 @@ func TestService_Init_MultiplePipelines(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - service, pipelineService, connService, procService, plugService := newTestService(ctrl, logger) + service, pipelineService, connService, _, _, lifecycleService := newTestService(ctrl, logger) service.pipelinesPath = "./test/pipelines3" // return a pipeline not provisioned by API @@ -379,7 +381,7 @@ func TestService_Init_MultiplePipelines(t *testing.T) { connService.EXPECT().CreateWithInstance(anyCtx, p3.P1C1) connService.EXPECT().CreateWithInstance(anyCtx, p3.P1C2) - pipelineService.EXPECT().Start(anyCtx, connService, procService, plugService, p3.P1.ID) + lifecycleService.EXPECT().Start(anyCtx, p3.P1.ID) // create pipeline2 pipelineService.EXPECT().CreateWithInstance(anyCtx, p3.P2) @@ -390,7 +392,7 @@ func TestService_Init_MultiplePipelines(t *testing.T) { connService.EXPECT().CreateWithInstance(anyCtx, p3.P2C1) connService.EXPECT().CreateWithInstance(anyCtx, p3.P2C2) - pipelineService.EXPECT().Start(anyCtx, connService, procService, plugService, p3.P2.ID) + lifecycleService.EXPECT().Start(anyCtx, p3.P2.ID) err := service.Init(context.Background()) is.NoErr(err) @@ -406,7 +408,7 @@ func TestService_Init_PipelineProvisionedFromAPI(t *testing.T) { ProvisionedBy: pipeline.ProvisionTypeAPI, } - service, pipelineService, _, _, _ := newTestService(ctrl, logger) + service, pipelineService, _, _, _, _ := newTestService(ctrl, logger) service.pipelinesPath = "./test/pipelines1" // pipeline provisioned by API @@ -424,7 +426,7 @@ func TestService_Init_PipelineProvisionedFromAPI_Error(t *testing.T) { ctrl := gomock.NewController(t) otherErr := cerrors.New("GetError") - service, pipelineService, connService, procService, plugService := newTestService(ctrl, logger) + service, pipelineService, connService, procService, _, lifecycleService := newTestService(ctrl, logger) service.pipelinesPath = "./test/pipelines1" // error from calling Get @@ -448,7 +450,7 @@ func TestService_Init_PipelineProvisionedFromAPI_Error(t *testing.T) { procService.EXPECT().CreateWithInstance(anyCtx, p1.P1P1) // start pipeline - pipelineService.EXPECT().Start(anyCtx, connService, procService, plugService, p1.P1.ID) + lifecycleService.EXPECT().Start(anyCtx, p1.P1.ID) err := service.Init(context.Background()) is.True(cerrors.Is(err, otherErr)) @@ -459,7 +461,7 @@ func TestService_Delete(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - service, pipelineService, connService, procService, plugService := newTestService(ctrl, logger) + service, pipelineService, connService, procService, plugService, _ := newTestService(ctrl, logger) // export pipeline pipelineService.EXPECT().Get(anyCtx, oldPipelineInstance.ID).Return(oldPipelineInstance, nil).Times(2) @@ -513,10 +515,16 @@ func TestService_IntegrationTestServices(t *testing.T) { proc_builtin.NewRegistry(logger, proc_builtin.DefaultBuiltinProcessors, schemaRegistry), nil, ) + b := &backoff.Backoff{ + Factor: 2, + Min: time.Millisecond * 100, + Max: time.Second, // 8 tries + } plService := pipeline.NewService(logger, db) connService := connector.NewService(logger, db, connector.NewPersister(logger, db, time.Second, 3)) procService := processor.NewService(logger, db, procPluginService) + lifecycleService := lifecycle.NewService(logger, b, connService, procService, connPluginService, plService) // create destination file destFile := "./test/dest-file.txt" @@ -527,7 +535,7 @@ func TestService_IntegrationTestServices(t *testing.T) { is.NoErr(err) }) - service := NewService(db, logger, plService, connService, procService, connPluginService, "./test/pipelines4-integration-test") + service := NewService(db, logger, plService, connService, procService, connPluginService, lifecycleService, "./test/pipelines4-integration-test") err = service.Init(context.Background()) is.NoErr(err)