From 413fc2f1d4300d30e1a247ca67f87d23df63ce89 Mon Sep 17 00:00:00 2001 From: Arash Date: Fri, 20 May 2022 18:59:27 -0400 Subject: [PATCH 1/3] make snowflake boto dependency lower --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index c216d5647..c6cbb0eed 100644 --- a/setup.py +++ b/setup.py @@ -157,8 +157,8 @@ def _get_pyarrow_lib_pattern(self, lib_name): install_requires=[ 'azure-common', 'azure-storage-blob', - 'boto3>=1.4.4,<1.10.0', - 'botocore>=1.5.0,<1.13.0', + 'boto3>=1.4.4,<=1.17.0', + 'botocore>=1.5.0,<=1.20.0', 'certifi', 'future', 'six', From 1a26a06448355dc500298374bfed39bda85e2302 Mon Sep 17 00:00:00 2001 From: Steven Uray Date: Sun, 22 May 2022 15:32:11 -0700 Subject: [PATCH 2/3] Adding long_description --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 8fb28c059..189e3e6f2 100644 --- a/setup.py +++ b/setup.py @@ -40,6 +40,7 @@ extensions = None cmd_class = {} +long_description = "our custom snowflake connector fork because we had to pass a security audit without upgrading to sql alchemy 1.4" try: import numpy From 438d4ed069ab261d013d46181c811cf64a78f321 Mon Sep 17 00:00:00 2001 From: Steven Uray Date: Sun, 22 May 2022 19:47:13 -0700 Subject: [PATCH 3/3] Trying diff setup.py from 1.9.0 tag --- setup.py | 225 ++++++++++++++++++++++++------------------------------- 1 file changed, 97 insertions(+), 128 deletions(-) diff --git a/setup.py b/setup.py index 189e3e6f2..c6cbb0eed 100644 --- a/setup.py +++ b/setup.py @@ -1,176 +1,145 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # -# Copyright (c) 2012-2019 Snowflake Computing Inc. All rights reserved. +# Copyright (c) 2012-2019 Snowflake Computing Inc. All right reserved. # - +from codecs import open +from os import path import os import sys -import warnings +from sys import platform from shutil import copy +import glob -from setuptools import Extension, setup +from setuptools import setup, Extension -CONNECTOR_SRC_DIR = os.path.join("src", "snowflake", "connector") +THIS_DIR = path.dirname(path.realpath(__file__)) -VERSION = (1, 1, 1, None) # Default try: - with open( - os.path.join(CONNECTOR_SRC_DIR, "generated_version.py"), encoding="utf-8" - ) as f: - exec(f.read()) -except Exception: - with open(os.path.join(CONNECTOR_SRC_DIR, "version.py"), encoding="utf-8") as f: - exec(f.read()) -version = ".".join([str(v) for v in VERSION if v is not None]) - -# Parse command line flags + from generated_version import VERSION +except: + from version import VERSION +version = '.'.join([str(v) for v in VERSION if v is not None]) -# This list defines the options definitions in a set -options_def = { - "--debug", -} +with open(path.join(THIS_DIR, 'DESCRIPTION.rst'), encoding='utf-8') as f: + long_description = f.read() -# Options is the final parsed command line options -options = {e.lstrip("-"): False for e in options_def} -for flag in options_def: +# Parse command line flags +options = {k: 'OFF' for k in ['--opt', '--debug']} +for flag in options.keys(): if flag in sys.argv: - options[flag.lstrip("-")] = True + options[flag] = 'ON' sys.argv.remove(flag) extensions = None cmd_class = {} -long_description = "our custom snowflake connector fork because we had to pass a security audit without upgrading to sql alchemy 1.4" - -try: - import numpy - import pyarrow - from Cython.Build import cythonize - from Cython.Distutils import build_ext - _ABLE_TO_COMPILE_EXTENSIONS = True -except ImportError: - warnings.warn("Cannot compile native C code, because of a missing build dependency") - _ABLE_TO_COMPILE_EXTENSIONS = False +isBuildExtEnabled = (os.getenv('ENABLE_EXT_MODULES', 'false')).lower() -if _ABLE_TO_COMPILE_EXTENSIONS: +if isBuildExtEnabled == 'true': + from Cython.Distutils import build_ext + from Cython.Build import cythonize + import os + import pyarrow extensions = cythonize( [ - Extension( - name="snowflake.connector.arrow_iterator", - sources=[os.path.join(CONNECTOR_SRC_DIR, "arrow_iterator.pyx")], - ), + Extension(name='snowflake.connector.arrow_iterator', sources=['arrow_iterator.pyx']), + Extension(name='snowflake.connector.arrow_result', sources=['arrow_result.pyx']) ], - ) + build_dir=os.path.join('build', 'cython')) class MyBuildExt(build_ext): - # list of libraries that will be bundled with python connector, - # this list should be carefully examined when pyarrow lib is - # upgraded - arrow_libs_to_copy = { - "linux": ["libarrow.so.600", "libarrow_python.so.600"], - "darwin": ["libarrow.600.dylib", "libarrow_python.600.dylib"], - "win32": ["arrow.dll", "arrow_python.dll"], - } - - arrow_libs_to_link = { - "linux": ["libarrow.so.600", "libarrow_python.so.600"], - "darwin": ["libarrow.600.dylib", "libarrow_python.600.dylib"], - "win32": ["arrow.lib", "arrow_python.lib"], - } - def build_extension(self, ext): - if options["debug"]: - ext.extra_compile_args.append("-g") - ext.extra_link_args.append("-g") current_dir = os.getcwd() - if ext.name == "snowflake.connector.arrow_iterator": - if not os.environ.get("SF_NO_COPY_ARROW_LIB", False): - self._copy_arrow_lib() - CPP_SRC_DIR = os.path.join(CONNECTOR_SRC_DIR, "cpp") - ARROW_ITERATOR_SRC_DIR = os.path.join(CPP_SRC_DIR, "ArrowIterator") - LOGGING_SRC_DIR = os.path.join(CPP_SRC_DIR, "Logging") - - ext.sources += [ - os.path.join(ARROW_ITERATOR_SRC_DIR, "CArrowIterator.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "CArrowChunkIterator.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "CArrowTableIterator.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "SnowflakeType.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "BinaryConverter.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "BooleanConverter.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "DecimalConverter.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "DateConverter.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "FloatConverter.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "IntConverter.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "StringConverter.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "TimeConverter.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "TimeStampConverter.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "Python", "Common.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "Python", "Helpers.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "Util", "time.cpp"), - LOGGING_SRC_DIR + "/logging.cpp", - ] - ext.include_dirs.append(ARROW_ITERATOR_SRC_DIR) - ext.include_dirs.append(LOGGING_SRC_DIR) - - if sys.platform == "win32": - ext.include_dirs.append(pyarrow.get_include()) - ext.include_dirs.append(numpy.get_include()) - elif sys.platform == "linux" or sys.platform == "darwin": - ext.extra_compile_args.append("-isystem" + pyarrow.get_include()) - ext.extra_compile_args.append("-isystem" + numpy.get_include()) - if "std=" not in os.environ.get("CXXFLAGS", ""): - ext.extra_compile_args.append("-std=c++11") - ext.extra_compile_args.append("-D_GLIBCXX_USE_CXX11_ABI=0") - - ext.library_dirs.append( - os.path.join(current_dir, self.build_lib, "snowflake", "connector") - ) + if ext.name == 'snowflake.connector.arrow_iterator': + self._copy_arrow_lib() + + ext.sources += ['cpp/ArrowIterator/CArrowIterator.cpp', + 'cpp/ArrowIterator/CArrowChunkIterator.cpp', + 'cpp/ArrowIterator/CArrowTableIterator.cpp', + 'cpp/ArrowIterator/SnowflakeType.cpp', + 'cpp/ArrowIterator/BinaryConverter.cpp', + 'cpp/ArrowIterator/BooleanConverter.cpp', + 'cpp/ArrowIterator/DecimalConverter.cpp', + 'cpp/ArrowIterator/DateConverter.cpp', + 'cpp/ArrowIterator/FloatConverter.cpp', + 'cpp/ArrowIterator/IntConverter.cpp', + 'cpp/ArrowIterator/StringConverter.cpp', + 'cpp/ArrowIterator/TimeConverter.cpp', + 'cpp/ArrowIterator/TimeStampConverter.cpp', + 'cpp/ArrowIterator/Python/Common.cpp', + 'cpp/ArrowIterator/Python/Helpers.cpp', + 'cpp/ArrowIterator/Util/time.cpp', + 'cpp/Logging/logging.cpp'] + ext.include_dirs.append('cpp/ArrowIterator/') + ext.include_dirs.append('cpp/Logging') + ext.include_dirs.append(pyarrow.get_include()) + + ext.extra_compile_args.append('-std=c++11') + + ext.library_dirs.append(os.path.join(current_dir, self.build_lib, 'snowflake', 'connector')) ext.extra_link_args += self._get_arrow_lib_as_linker_input() - # sys.platform for linux used to return with version suffix, (i.e. linux2, linux3) - # After version 3.3, it will always be just 'linux' - # https://docs.python.org/3/library/sys.html#sys.platform - if sys.platform == "linux": - ext.extra_link_args += ["-Wl,-rpath,$ORIGIN"] - elif sys.platform == "darwin": - # rpath,$ORIGIN only work on linux, did not work on darwin. use @loader_path instead - # fyi, https://medium.com/@donblas/fun-with-rpath-otool-and-install-name-tool-e3e41ae86172 - ext.extra_link_args += ["-rpath", "@loader_path"] + if self._is_unix(): + ext.extra_link_args += ['-Wl,-rpath,$ORIGIN'] build_ext.build_extension(self, ext) + def _is_unix(self): + return platform.startswith('linux') or platform == 'darwin' + def _get_arrow_lib_dir(self): - if "SF_ARROW_LIBDIR" in os.environ: - return os.environ["SF_ARROW_LIBDIR"] return pyarrow.get_library_dirs()[0] def _copy_arrow_lib(self): - libs_to_bundle = self.arrow_libs_to_copy[sys.platform] + arrow_lib = self._get_libs_to_copy() - for lib in libs_to_bundle: - source = f"{self._get_arrow_lib_dir()}/{lib}" - build_dir = os.path.join(self.build_lib, "snowflake", "connector") - copy(source, build_dir) + for lib in arrow_lib: + lib_pattern = self._get_pyarrow_lib_pattern(lib) + source = glob.glob(lib_pattern)[0] + copy(source, os.path.join(self.build_lib, 'snowflake', 'connector')) def _get_arrow_lib_as_linker_input(self): - link_lib = self.arrow_libs_to_link[sys.platform] - ret = [] - - for lib in link_lib: - source = f"{self._get_arrow_lib_dir()}/{lib}" - assert os.path.exists(source) - ret.append(source) - - return ret - - cmd_class = {"build_ext": MyBuildExt} + arrow_lib = pyarrow.get_libraries() + link_lib = [] + for lib in arrow_lib: + lib_pattern = self._get_pyarrow_lib_pattern(lib) + source = glob.glob(lib_pattern)[0] + link_lib.append(source) + + return link_lib + + def _get_libs_to_copy(self): + if self._is_unix(): + return pyarrow.get_libraries() + \ + ['arrow_flight', 'arrow_boost_regex', 'arrow_boost_system', 'arrow_boost_filesystem'] + elif platform == 'win32': + return pyarrow.get_libraries() + ['arrow_flight'] + else: + raise RuntimeError('Building on platform {} is not supported yet.'.format(platform)) + + def _get_pyarrow_lib_pattern(self, lib_name): + if platform.startswith('linux'): + return '{}/lib{}.so*'.format(self._get_arrow_lib_dir(), lib_name) + elif platform == 'darwin': + return '{}/lib{}*dylib'.format(self._get_arrow_lib_dir(), lib_name) + elif platform == 'win32': + return '{}\\{}.lib'.format(self._get_arrow_lib_dir(), lib_name) + else: + raise RuntimeError('Building on platform {} is not supported yet.'.format(platform)) + + cmd_class = { + "build_ext": MyBuildExt + } setup( + name='snowflake-connector-python', version=version, + description=u"Snowflake Connector for Python", ext_modules=extensions, cmdclass=cmd_class, long_description=long_description,