diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 63187d674..825dc7aea 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "0.3.0-beta" + ".": "0.4.0-beta" } diff --git a/CHANGELOG.md b/CHANGELOG.md index c1ff528d7..9f762f9e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,101 @@ # Changelog +## [0.4.0-beta](https://github.com/GoogleCloudPlatform/dataproc-templates/compare/v0.3.0-beta...v0.4.0-beta) (2023-08-07) + + +### Features + +* 756 postgre sql to big query ([#805](https://github.com/GoogleCloudPlatform/dataproc-templates/issues/805)) ([121bf93](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/121bf9377af55e9ffed82471d2c0f813699fd4d1)) +* add bq_dataset_region parameter ([a9647ad](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/a9647ad321c7fb9eaf5af49ab440de2b8eeba1b2)) +* Add ORACLE_SCHEMA to parameterize scripts ([969e314](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/969e3141e9c3c8b71f1b97695d84e6c437cd1345)) +* added init file ([3273fe7](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/3273fe7a721ff956a3a8508aa3fdf59f0a2e630a)) +* Added Paramterized script file ([2951e3a](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/2951e3ace58129c691ec9f2ef8d6cc5ff93a1608)) +* HiveToBiqquery parameterize script ([2eec146](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/2eec1464eb00b0ab4ffd0a96a6f255237bcafa83)) +* include logging ([ca1f9be](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/ca1f9bec7c3bc3c824428f4da90150178c283ceb)) +* include logging ([2714b7f](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/2714b7f19fbcc5824b3309d98fc6606378db5c34)) +* include logging ([cf0a81c](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/cf0a81c6421c42c8aaf9e16c08b71a3f613ec378)) +* service account as env variable ([9c93710](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/9c93710310514f630fedaa14974f1fa3da36555f)) +* service account as env variable ([57292f9](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/57292f9130fda81132a5831cdc80ad6999842e7e)) +* update nb constants & script name ([cb07530](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/cb0753042ed1374c019c7ed131e004603ce1515d)) +* update nb with IS_PARAMETERIZED flag ([b4dcd35](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/b4dcd35fc1ea96ba5dc0a1456656c0fbf63411e3)) +* update notebook constants ([5c9bdc2](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/5c9bdc2d076314afe322e40b49c7333f20a2f249)) +* update working dir ([ea6a52b](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/ea6a52b24da8f9f4d24d555e820e12c35e14e3b5)) +* update working dir for parameterization ([5840c88](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/5840c882ae2ae2f96becc268eeba02957dc791f1)) +* Updated notebook to handle parameters ([08dd1c4](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/08dd1c4651bdabbf76bd630da8680a08ae85a7f1)) +* updated readme for parametrized script ([82ff6ad](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/82ff6ad84f46de77ac5a692a280e87cb50337d1e)) +* updated readme for parametrized script ([afac2a6](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/afac2a6f5878be3443bbff330df6c2843c3d960e)) +* updated run_notebook ([c84883f](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/c84883fbafbb9b08d2b4d88c2c70bef70946412a)) + + +### Bug Fixes + +* add log_output ([09dc88a](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/09dc88a7e181ed103181b7640dd71140b2e8b211)) +* black formatting ([fcab888](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/fcab888a01ed569c7c1b5faaf416382262691da5)) +* correct pip install cmd ([f9c17c5](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/f9c17c5f6840aa3299a354f035748f47a250f8ed)) +* correct project id variable ([5c61b94](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/5c61b94d9fbb1f22d4df18533f9f866903acc395)) +* correct sqlachemy execute ([14cb3d3](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/14cb3d35e8a51ef251e476403c7bca582a3a2b60)) +* Deal with upgraded google_cloud_pipeline_components.experimental vs v1 ([4551fab](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/4551fab27cda6dc0e74ee929eddd903b1c8e16f6)) +* Deal with upgraded google_cloud_pipeline_components.experimental vs v1 ([2304179](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/230417971e8ed53340b3825ef9d1c438d6545255)) +* deletes gcs folder created after all tests run ([66e3000](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/66e30006159ab43d5e4938d00dcc8712faf291c5)) +* except statement ([b88758a](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/b88758a1c60c4edac86473f573ed140d0db7b7e5)) +* Fix merge conflict ([deb3a43](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/deb3a434c0f2f4ad98fac02efe32adce63a2e741)) +* Fix merge from main ([5a75ae0](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/5a75ae0c25301bc39fb5b803d136b8954bd52b72)) +* Fix merge from main ([6ad8fe6](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/6ad8fe60d5c4a48a84121f5267644ecd2ae86245)) +* Fix merge from main ([2dd4be5](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/2dd4be5d4681ae896b760fd6f5993900c7074f17)) +* Fix merge from main ([fcc8ca6](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/fcc8ca62b622d6947c46572c878e739db0135084)) +* Fix merge from main ([90a65ec](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/90a65ec38bd316bff49a87204f17366d2222722a)) +* ignored some tests in general template ([ae74321](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/ae7432177f2e29a070dc5fe2141fb41f58cbbf24)) +* Implement de-duplication ([b1c9f8a](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/b1c9f8ae24ae94ecef0694eeb53b7ff711daa8d0)) +* implement de-duplication changes ([1aeef92](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/1aeef924b0e116a7ae547bb1c596c217231210b6)) +* implement de-duplication changes ([02c2508](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/02c25084393a9ca5b1167dd1e42cb92a4ae573de)) +* implement get_common_args function ([6525dc5](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/6525dc508e568c969ccfeb7123bb40b39b642607)) +* implement get_env_vars in base script ([0734fb7](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/0734fb7fd82b269492657653eb598df40400eaa0)) +* integration test - change custom container image ([#810](https://github.com/GoogleCloudPlatform/dataproc-templates/issues/810)) ([8943508](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/894350808c9fcf5b5c80ade4c918f6dc69fc1368)) +* make subnet optional variable ([64c0e68](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/64c0e6858b420f9fda671e2f1b8f36fa5280736f)) +* make subnet optional variable ([294eea5](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/294eea55904f08089c266c26d126c7ad9fa454af)) +* modified except block ([8871b25](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/8871b25bf1207ba1f97d87779088b74c46a5374e)) +* move max_parallelism from common_args ([1f88912](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/1f88912f5426bd56eaf013c3d7169ca0dec433ee)) +* Notebook MYSQLTABLE_LIST to MYSQL_TABLE_LIST ([5abe159](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/5abe15924c455b7903f68f2417887a229ead0781)) +* Notebook MYSQLTABLE_LIST to MYSQL_TABLE_LIST ([e22fb72](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/e22fb72bac5b3a62b6ee79ca06a8cfe0397dbc5a)) +* oracle table list parsing corrected ([5f33fb4](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/5f33fb44544730a006d4043fcf74ad9700dcfc9f)) +* oracle table list parsing corrected ([7f1e913](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/7f1e9139c80a7fce506c1bb62c489d8b4721c2f5)) +* raise exception on job failure ([e5b1a99](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/e5b1a99e0947519cf542431fbf314304508d4268)) +* remove common args from script ([fc125f2](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/fc125f2b20c953bb3fe16a5b96b7e2b548a494e3)) +* remove duplicate get_env_var ([73bad48](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/73bad482f500289be0b5b972a6ca87c138c9ff64)) +* remove log_level from nb_parameters ([634d926](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/634d926b5e5fcf0fbd08903f6edc296258f6ea56)) +* remove log_level from nb_parameters ([fe49799](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/fe497992e326bd3b6525f2707ca6f9ad44523417)) +* remove project id constant ([e9173bb](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/e9173bb3c45174fc9027476f528ae7818926c126)) +* removed duplicate arugment ([e0c3262](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/e0c3262ea14048db888268bd221ef38bc7900bbe)) +* removed pubsub output project parameter ([24a9d18](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/24a9d18cf25f1e90f3f765f49d98b74dd53dda27)) +* removed pubsub output project parameter ([838c23a](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/838c23a09bde4f39c4d9f10ec82ff1259dec7422)) +* removed unused outputProjectID ([5f106c4](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/5f106c4a461723035d87acf7bef352bb7605dfab)) +* removing version parameter while running with cluster ([d9faf34](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/d9faf341267eb0822ee28f937da614eb87f4b353)) +* renamed init file ([aa60431](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/aa60431008e04f69ccb1258f93bf35afc060ab89)) +* Resolve conflicting changes ([8ab7e0d](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/8ab7e0d26a7f058ff365f26f676b030d3c91014f)) +* small fixes in notebook ([155c5c8](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/155c5c8a5b7fd93b133f2269b6c9eab45cc64d65)) +* update project id variable ([97e58f1](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/97e58f106c1e2ecf151a63c0622cf145f87978b6)) +* update PROJECT_ID to PROJECT ([1a06e52](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/1a06e525b475289b0f24819016b949dbb295e258)) +* update scipt_name with oracletopostgres ([5dac5d4](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/5dac5d4b81e2237cd9686b20df2ac2f55969b3e5)) +* update spark to 3.3.2 ([60757dd](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/60757dd5d640b3ed83be666b1e1aba7100ca4a5b)) +* updated indentation ([306e4e5](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/306e4e5853de4d802659356a7b984f8209959111)) +* updated log_output ([0b2a510](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/0b2a5107f652c0a9911868e9d653f09ca7669520)) +* updated notebook with tag parameters ([df007b4](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/df007b42be7157f22a54a1dcdcc28ac7fade4afe)) +* updated notebook_constants ([c4aaaa7](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/c4aaaa702cd8c698a716b5334edd6050911e4fcc)) +* updated pom.xml to avoid commons-cli conflict in cluster mode ([40cc8f7](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/40cc8f7cc48e8d6e8418a915ee0aec3d806dae4c)) +* updated postgres arg in notebook_constants ([5900066](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/590006649f71d927f4cf36f813d719d64d25b5a5)) +* updated sql connection ([8c5680d](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/8c5680d5bda8df147debcc38bce00b39fa14767f)) +* updated sql query error for boolean value ([a71e7f1](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/a71e7f1730de838255a2c44b5a877100809f7aa5)) +* updated to UUID GCS staging location ([a6fef9e](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/a6fef9e4503ed835d7921844041ec6f981cf390d)) + + +### Documentation + +* added Oracle to Postgres notebook blogpost link to README files ([281bde5](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/281bde5819b0324e77991bdc0418c2130b332084)) +* Change Dataflow to Dataproc ([#815](https://github.com/GoogleCloudPlatform/dataproc-templates/issues/815)) ([74cee01](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/74cee0109a521727946e359a05f234ee1890c580)) +* Corrected typos in Snowflake README ([78e9d42](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/78e9d422dc6e16bedd37e8d0f7dd5f36eabd2705)) +* Corrected typos in Snowflake README ([ab60e0e](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/ab60e0edf58335e8a8b5fe173b66ed3ad9db5be5)) +* removed PubSubLiteToGCS from README files ([a5d4c06](https://github.com/GoogleCloudPlatform/dataproc-templates/commit/a5d4c063f16bc433a88ade9c570d0792889856ef)) + ## [0.3.0-beta](https://github.com/GoogleCloudPlatform/dataproc-templates/compare/v0.2.0-beta...v0.3.0-beta) (2023-05-12) diff --git a/python/version.py b/python/version.py index 98e8c3bad..9174f0f57 100644 --- a/python/version.py +++ b/python/version.py @@ -1 +1 @@ -__version__ = "0.3.0-beta" +__version__ = "0.4.0-beta"