From a0fa8a2443bcade1748ddc9ab9ddd604ede2f10a Mon Sep 17 00:00:00 2001 From: Axel Pettersson Date: Thu, 17 Feb 2022 11:49:19 +0100 Subject: [PATCH] Structure for PyPi distribution --- python/README.md | 9 ++++++++ python/ackuq/__init__.py | 0 python/ackuq/pit/__init__.py | 25 ++++++++++++++++++++++ python/{ => ackuq}/pit/context.py | 24 ++++++++++++++++++++++ python/setup.cfg | 2 ++ python/setup.py | 33 ++++++++++++++++++++++++++++++ python/tests/data.py | 24 ++++++++++++++++++++++ python/tests/test_exploding_pit.py | 23 +++++++++++++++++++++ python/tests/test_sort_merge.py | 24 ++++++++++++++++++++++ python/tests/test_union_as_of.py | 24 ++++++++++++++++++++++ python/tests/utils.py | 26 ++++++++++++++++++++++- 11 files changed, 213 insertions(+), 1 deletion(-) create mode 100644 python/README.md create mode 100644 python/ackuq/__init__.py create mode 100644 python/ackuq/pit/__init__.py rename python/{ => ackuq}/pit/context.py (81%) create mode 100644 python/setup.cfg diff --git a/python/README.md b/python/README.md new file mode 100644 index 0000000..75b7589 --- /dev/null +++ b/python/README.md @@ -0,0 +1,9 @@ +# Spark-PIT - Utility library for PIT-joins in PySpark + +## Description + +This projects aims to expose different ways of executing PIT-joins, also called ASOF-joins, in PySpark. This project is created as a part of a research project to evaluate different ways of executing Spark PIT joins. + +## Prerequisite + +In order to run this project in PySpark, you will need to have the JAR file of the Scala implementation be available inside you Spark Session. diff --git a/python/ackuq/__init__.py b/python/ackuq/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python/ackuq/pit/__init__.py b/python/ackuq/pit/__init__.py new file mode 100644 index 0000000..fccc5be --- /dev/null +++ b/python/ackuq/pit/__init__.py @@ -0,0 +1,25 @@ +# +# MIT License +# +# Copyright (c) 2022 Axel Pettersson +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from ackuq.pit.context import PitContext # noqa: F401 diff --git a/python/pit/context.py b/python/ackuq/pit/context.py similarity index 81% rename from python/pit/context.py rename to python/ackuq/pit/context.py index 33fdef9..33cdb59 100644 --- a/python/pit/context.py +++ b/python/ackuq/pit/context.py @@ -1,3 +1,27 @@ +# +# MIT License +# +# Copyright (c) 2022 Axel Pettersson +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + from typing import Any, List, Optional from py4j.java_gateway import JavaPackage diff --git a/python/setup.cfg b/python/setup.cfg new file mode 100644 index 0000000..0f94f37 --- /dev/null +++ b/python/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +description_file = README.md \ No newline at end of file diff --git a/python/setup.py b/python/setup.py index c53ff33..5c8e11a 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,3 +1,27 @@ +# +# MIT License +# +# Copyright (c) 2022 Axel Pettersson +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + import setuptools setuptools.setup( @@ -7,4 +31,13 @@ author_email="axel@pettersson.cc", description="PIT join library for PySpark", url="https://github.com/Ackuq/spark-pit/", + keywords=["PySpark", "point-in-time", "joins", "SparkSQL"], + packages=["ackuq.pit"], + install_requires=[], + classifiers=[ + "Programming Language :: Python :: 3", + "Development Status :: 4 - Beta", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ], ) diff --git a/python/tests/data.py b/python/tests/data.py index b2933a2..d6c98eb 100644 --- a/python/tests/data.py +++ b/python/tests/data.py @@ -1,3 +1,27 @@ +# +# MIT License +# +# Copyright (c) 2022 Axel Pettersson +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + from pyspark.sql import SparkSession from pyspark.sql.types import IntegerType, StringType, StructField, StructType diff --git a/python/tests/test_exploding_pit.py b/python/tests/test_exploding_pit.py index e69de29..199461f 100644 --- a/python/tests/test_exploding_pit.py +++ b/python/tests/test_exploding_pit.py @@ -0,0 +1,23 @@ +# +# MIT License +# +# Copyright (c) 2022 Axel Pettersson +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# diff --git a/python/tests/test_sort_merge.py b/python/tests/test_sort_merge.py index b639594..c8c6bca 100644 --- a/python/tests/test_sort_merge.py +++ b/python/tests/test_sort_merge.py @@ -1,3 +1,27 @@ +# +# MIT License +# +# Copyright (c) 2022 Axel Pettersson +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + from tests.data import SmallDataSortMerge from tests.utils import SparkTests diff --git a/python/tests/test_union_as_of.py b/python/tests/test_union_as_of.py index bc68892..d4133f9 100644 --- a/python/tests/test_union_as_of.py +++ b/python/tests/test_union_as_of.py @@ -1,3 +1,27 @@ +# +# MIT License +# +# Copyright (c) 2022 Axel Pettersson +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + from tests.data import SmallDataUnion from tests.utils import SparkTests diff --git a/python/tests/utils.py b/python/tests/utils.py index bf91196..0387822 100644 --- a/python/tests/utils.py +++ b/python/tests/utils.py @@ -1,3 +1,27 @@ +# +# MIT License +# +# Copyright (c) 2022 Axel Pettersson +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + import os import unittest @@ -5,7 +29,7 @@ from pyspark.sql import SparkSession from pyspark.sql.types import StructField, StructType -from pit.context import PitContext +from ackuq.pit.context import PitContext class SparkTests(unittest.TestCase):