diff --git a/setup.py b/setup.py index 9244cab7c..c6cbb0eed 100644 --- a/setup.py +++ b/setup.py @@ -1,175 +1,241 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # -# Copyright (c) 2012-2019 Snowflake Computing Inc. All rights reserved. +# Copyright (c) 2012-2019 Snowflake Computing Inc. All right reserved. # - +from codecs import open +from os import path import os import sys -import warnings +from sys import platform from shutil import copy +import glob -from setuptools import Extension, setup +from setuptools import setup, Extension -CONNECTOR_SRC_DIR = os.path.join("src", "snowflake", "connector") +THIS_DIR = path.dirname(path.realpath(__file__)) -VERSION = (1, 1, 1, None) # Default try: - with open( - os.path.join(CONNECTOR_SRC_DIR, "generated_version.py"), encoding="utf-8" - ) as f: - exec(f.read()) -except Exception: - with open(os.path.join(CONNECTOR_SRC_DIR, "version.py"), encoding="utf-8") as f: - exec(f.read()) -version = ".".join([str(v) for v in VERSION if v is not None]) - -# Parse command line flags + from generated_version import VERSION +except: + from version import VERSION +version = '.'.join([str(v) for v in VERSION if v is not None]) -# This list defines the options definitions in a set -options_def = { - "--debug", -} +with open(path.join(THIS_DIR, 'DESCRIPTION.rst'), encoding='utf-8') as f: + long_description = f.read() -# Options is the final parsed command line options -options = {e.lstrip("-"): False for e in options_def} -for flag in options_def: +# Parse command line flags +options = {k: 'OFF' for k in ['--opt', '--debug']} +for flag in options.keys(): if flag in sys.argv: - options[flag.lstrip("-")] = True + options[flag] = 'ON' sys.argv.remove(flag) extensions = None cmd_class = {} -try: - import numpy - import pyarrow - from Cython.Build import cythonize - from Cython.Distutils import build_ext - - _ABLE_TO_COMPILE_EXTENSIONS = True -except ImportError: - warnings.warn("Cannot compile native C code, because of a missing build dependency") - _ABLE_TO_COMPILE_EXTENSIONS = False +isBuildExtEnabled = (os.getenv('ENABLE_EXT_MODULES', 'false')).lower() -if _ABLE_TO_COMPILE_EXTENSIONS: +if isBuildExtEnabled == 'true': + from Cython.Distutils import build_ext + from Cython.Build import cythonize + import os + import pyarrow extensions = cythonize( [ - Extension( - name="snowflake.connector.arrow_iterator", - sources=[os.path.join(CONNECTOR_SRC_DIR, "arrow_iterator.pyx")], - ), + Extension(name='snowflake.connector.arrow_iterator', sources=['arrow_iterator.pyx']), + Extension(name='snowflake.connector.arrow_result', sources=['arrow_result.pyx']) ], - ) + build_dir=os.path.join('build', 'cython')) class MyBuildExt(build_ext): - # list of libraries that will be bundled with python connector, - # this list should be carefully examined when pyarrow lib is - # upgraded - arrow_libs_to_copy = { - "linux": ["libarrow.so.600", "libarrow_python.so.600"], - "darwin": ["libarrow.600.dylib", "libarrow_python.600.dylib"], - "win32": ["arrow.dll", "arrow_python.dll"], - } - - arrow_libs_to_link = { - "linux": ["libarrow.so.600", "libarrow_python.so.600"], - "darwin": ["libarrow.600.dylib", "libarrow_python.600.dylib"], - "win32": ["arrow.lib", "arrow_python.lib"], - } - def build_extension(self, ext): - if options["debug"]: - ext.extra_compile_args.append("-g") - ext.extra_link_args.append("-g") current_dir = os.getcwd() - if ext.name == "snowflake.connector.arrow_iterator": - if not os.environ.get("SF_NO_COPY_ARROW_LIB", False): - self._copy_arrow_lib() - CPP_SRC_DIR = os.path.join(CONNECTOR_SRC_DIR, "cpp") - ARROW_ITERATOR_SRC_DIR = os.path.join(CPP_SRC_DIR, "ArrowIterator") - LOGGING_SRC_DIR = os.path.join(CPP_SRC_DIR, "Logging") - - ext.sources += [ - os.path.join(ARROW_ITERATOR_SRC_DIR, "CArrowIterator.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "CArrowChunkIterator.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "CArrowTableIterator.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "SnowflakeType.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "BinaryConverter.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "BooleanConverter.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "DecimalConverter.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "DateConverter.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "FloatConverter.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "IntConverter.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "StringConverter.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "TimeConverter.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "TimeStampConverter.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "Python", "Common.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "Python", "Helpers.cpp"), - os.path.join(ARROW_ITERATOR_SRC_DIR, "Util", "time.cpp"), - LOGGING_SRC_DIR + "/logging.cpp", - ] - ext.include_dirs.append(ARROW_ITERATOR_SRC_DIR) - ext.include_dirs.append(LOGGING_SRC_DIR) - - if sys.platform == "win32": - ext.include_dirs.append(pyarrow.get_include()) - ext.include_dirs.append(numpy.get_include()) - elif sys.platform == "linux" or sys.platform == "darwin": - ext.extra_compile_args.append("-isystem" + pyarrow.get_include()) - ext.extra_compile_args.append("-isystem" + numpy.get_include()) - if "std=" not in os.environ.get("CXXFLAGS", ""): - ext.extra_compile_args.append("-std=c++11") - ext.extra_compile_args.append("-D_GLIBCXX_USE_CXX11_ABI=0") - - ext.library_dirs.append( - os.path.join(current_dir, self.build_lib, "snowflake", "connector") - ) + if ext.name == 'snowflake.connector.arrow_iterator': + self._copy_arrow_lib() + + ext.sources += ['cpp/ArrowIterator/CArrowIterator.cpp', + 'cpp/ArrowIterator/CArrowChunkIterator.cpp', + 'cpp/ArrowIterator/CArrowTableIterator.cpp', + 'cpp/ArrowIterator/SnowflakeType.cpp', + 'cpp/ArrowIterator/BinaryConverter.cpp', + 'cpp/ArrowIterator/BooleanConverter.cpp', + 'cpp/ArrowIterator/DecimalConverter.cpp', + 'cpp/ArrowIterator/DateConverter.cpp', + 'cpp/ArrowIterator/FloatConverter.cpp', + 'cpp/ArrowIterator/IntConverter.cpp', + 'cpp/ArrowIterator/StringConverter.cpp', + 'cpp/ArrowIterator/TimeConverter.cpp', + 'cpp/ArrowIterator/TimeStampConverter.cpp', + 'cpp/ArrowIterator/Python/Common.cpp', + 'cpp/ArrowIterator/Python/Helpers.cpp', + 'cpp/ArrowIterator/Util/time.cpp', + 'cpp/Logging/logging.cpp'] + ext.include_dirs.append('cpp/ArrowIterator/') + ext.include_dirs.append('cpp/Logging') + ext.include_dirs.append(pyarrow.get_include()) + + ext.extra_compile_args.append('-std=c++11') + + ext.library_dirs.append(os.path.join(current_dir, self.build_lib, 'snowflake', 'connector')) ext.extra_link_args += self._get_arrow_lib_as_linker_input() - # sys.platform for linux used to return with version suffix, (i.e. linux2, linux3) - # After version 3.3, it will always be just 'linux' - # https://docs.python.org/3/library/sys.html#sys.platform - if sys.platform == "linux": - ext.extra_link_args += ["-Wl,-rpath,$ORIGIN"] - elif sys.platform == "darwin": - # rpath,$ORIGIN only work on linux, did not work on darwin. use @loader_path instead - # fyi, https://medium.com/@donblas/fun-with-rpath-otool-and-install-name-tool-e3e41ae86172 - ext.extra_link_args += ["-rpath", "@loader_path"] + if self._is_unix(): + ext.extra_link_args += ['-Wl,-rpath,$ORIGIN'] build_ext.build_extension(self, ext) + def _is_unix(self): + return platform.startswith('linux') or platform == 'darwin' + def _get_arrow_lib_dir(self): - if "SF_ARROW_LIBDIR" in os.environ: - return os.environ["SF_ARROW_LIBDIR"] return pyarrow.get_library_dirs()[0] def _copy_arrow_lib(self): - libs_to_bundle = self.arrow_libs_to_copy[sys.platform] + arrow_lib = self._get_libs_to_copy() - for lib in libs_to_bundle: - source = f"{self._get_arrow_lib_dir()}/{lib}" - build_dir = os.path.join(self.build_lib, "snowflake", "connector") - copy(source, build_dir) + for lib in arrow_lib: + lib_pattern = self._get_pyarrow_lib_pattern(lib) + source = glob.glob(lib_pattern)[0] + copy(source, os.path.join(self.build_lib, 'snowflake', 'connector')) def _get_arrow_lib_as_linker_input(self): - link_lib = self.arrow_libs_to_link[sys.platform] - ret = [] - - for lib in link_lib: - source = f"{self._get_arrow_lib_dir()}/{lib}" - assert os.path.exists(source) - ret.append(source) - - return ret - - cmd_class = {"build_ext": MyBuildExt} + arrow_lib = pyarrow.get_libraries() + link_lib = [] + for lib in arrow_lib: + lib_pattern = self._get_pyarrow_lib_pattern(lib) + source = glob.glob(lib_pattern)[0] + link_lib.append(source) + + return link_lib + + def _get_libs_to_copy(self): + if self._is_unix(): + return pyarrow.get_libraries() + \ + ['arrow_flight', 'arrow_boost_regex', 'arrow_boost_system', 'arrow_boost_filesystem'] + elif platform == 'win32': + return pyarrow.get_libraries() + ['arrow_flight'] + else: + raise RuntimeError('Building on platform {} is not supported yet.'.format(platform)) + + def _get_pyarrow_lib_pattern(self, lib_name): + if platform.startswith('linux'): + return '{}/lib{}.so*'.format(self._get_arrow_lib_dir(), lib_name) + elif platform == 'darwin': + return '{}/lib{}*dylib'.format(self._get_arrow_lib_dir(), lib_name) + elif platform == 'win32': + return '{}\\{}.lib'.format(self._get_arrow_lib_dir(), lib_name) + else: + raise RuntimeError('Building on platform {} is not supported yet.'.format(platform)) + + cmd_class = { + "build_ext": MyBuildExt + } setup( + name='snowflake-connector-python', version=version, + description=u"Snowflake Connector for Python", ext_modules=extensions, cmdclass=cmd_class, + long_description=long_description, + author='Snowflake, Inc', + author_email='support@snowflake.com', + license='Apache License, Version 2.0', + keywords="Snowflake db database cloud analytics warehouse", + url='https://www.snowflake.com/', + download_url='https://www.snowflake.com/', + use_2to3=False, + + # NOTE: Python 3.4 will be dropped within one month. + python_requires='>=2.7.9,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*', + + install_requires=[ + 'azure-common', + 'azure-storage-blob', + 'boto3>=1.4.4,<=1.17.0', + 'botocore>=1.5.0,<=1.20.0', + 'certifi', + 'future', + 'six', + 'pytz', + 'pycryptodomex>=3.2,!=3.5.0', + 'pyOpenSSL>=16.2.0', + 'cffi>=1.9', + 'cryptography>=1.8.2', + 'ijson', + 'pyjwt', + 'idna', + 'pyasn1>=0.4.0,<0.5.0;python_version<"3.0"', + 'pyasn1-modules>=0.2.0,<0.3.0;python_version<"3.0"', + 'enum34;python_version<"3.4"', + ], + + namespace_packages=['snowflake'], + packages=[ + 'snowflake.connector', + 'snowflake.connector.tool', + ], + package_dir={ + 'snowflake.connector': '.', + 'snowflake.connector.tool': 'tool', + }, + package_data={ + 'snowflake.connector': ['*.pem', '*.json', '*.rst', 'LICENSE.txt'], + }, + + entry_points={ + 'console_scripts': [ + 'snowflake-dump-ocsp-response = ' + 'snowflake.connector.tool.dump_ocsp_response:main', + 'snowflake-dump-ocsp-response-cache = ' + 'snowflake.connector.tool.dump_ocsp_response_cache:main', + 'snowflake-dump-certs = ' + 'snowflake.connector.tool.dump_certs:main', + 'snowflake-export-certs = ' + 'snowflake.connector.tool.export_certs:main', + ], + }, + extras_require={ + "secure-local-storage": [ + 'keyring!=16.1.0' + ], + "arrow-result": [ + 'pyarrow>=0.14.0;python_version>"3.4"', + 'pyarrow>=0.14.0;python_version<"3.0"' + ] + }, + + classifiers=[ + 'Development Status :: 5 - Production/Stable', + + 'Environment :: Console', + 'Environment :: Other Environment', + + 'Intended Audience :: Developers', + 'Intended Audience :: Education', + 'Intended Audience :: Information Technology', + 'Intended Audience :: System Administrators', + + 'License :: OSI Approved :: Apache Software License', + + 'Operating System :: OS Independent', + + 'Programming Language :: SQL', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + + 'Topic :: Database', + 'Topic :: Software Development', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Application Frameworks', + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Topic :: Scientific/Engineering :: Information Analysis', + ], )