Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

make snowflake boto dependency lower #1

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open

make snowflake boto dependency lower #1

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
413fc2f
make snowflake boto dependency lower
AAfghahi May 20, 2022
d2cc961
Merge branch 'main' into preset/v1.9.0
AAfghahi May 20, 2022
1a26a06
Adding long_description
stevenuray May 22, 2022
438d4ed
Trying diff setup.py from 1.9.0 tag
May 23, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
320 changes: 193 additions & 127 deletions setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,175 +1,241 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (c) 2012-2019 Snowflake Computing Inc. All rights reserved.
# Copyright (c) 2012-2019 Snowflake Computing Inc. All right reserved.
#

from codecs import open
from os import path
import os
import sys
import warnings
from sys import platform
from shutil import copy
import glob

from setuptools import Extension, setup
from setuptools import setup, Extension

CONNECTOR_SRC_DIR = os.path.join("src", "snowflake", "connector")
THIS_DIR = path.dirname(path.realpath(__file__))

VERSION = (1, 1, 1, None) # Default
try:
with open(
os.path.join(CONNECTOR_SRC_DIR, "generated_version.py"), encoding="utf-8"
) as f:
exec(f.read())
except Exception:
with open(os.path.join(CONNECTOR_SRC_DIR, "version.py"), encoding="utf-8") as f:
exec(f.read())
version = ".".join([str(v) for v in VERSION if v is not None])

# Parse command line flags
from generated_version import VERSION
except:
from version import VERSION
version = '.'.join([str(v) for v in VERSION if v is not None])

# This list defines the options definitions in a set
options_def = {
"--debug",
}
with open(path.join(THIS_DIR, 'DESCRIPTION.rst'), encoding='utf-8') as f:
long_description = f.read()

# Options is the final parsed command line options
options = {e.lstrip("-"): False for e in options_def}

for flag in options_def:
# Parse command line flags
options = {k: 'OFF' for k in ['--opt', '--debug']}
for flag in options.keys():
if flag in sys.argv:
options[flag.lstrip("-")] = True
options[flag] = 'ON'
sys.argv.remove(flag)

extensions = None
cmd_class = {}

try:
import numpy
import pyarrow
from Cython.Build import cythonize
from Cython.Distutils import build_ext

_ABLE_TO_COMPILE_EXTENSIONS = True
except ImportError:
warnings.warn("Cannot compile native C code, because of a missing build dependency")
_ABLE_TO_COMPILE_EXTENSIONS = False
isBuildExtEnabled = (os.getenv('ENABLE_EXT_MODULES', 'false')).lower()

if _ABLE_TO_COMPILE_EXTENSIONS:
if isBuildExtEnabled == 'true':
from Cython.Distutils import build_ext
from Cython.Build import cythonize
import os
import pyarrow

extensions = cythonize(
[
Extension(
name="snowflake.connector.arrow_iterator",
sources=[os.path.join(CONNECTOR_SRC_DIR, "arrow_iterator.pyx")],
),
Extension(name='snowflake.connector.arrow_iterator', sources=['arrow_iterator.pyx']),
Extension(name='snowflake.connector.arrow_result', sources=['arrow_result.pyx'])
],
)
build_dir=os.path.join('build', 'cython'))

class MyBuildExt(build_ext):

# list of libraries that will be bundled with python connector,
# this list should be carefully examined when pyarrow lib is
# upgraded
arrow_libs_to_copy = {
"linux": ["libarrow.so.600", "libarrow_python.so.600"],
"darwin": ["libarrow.600.dylib", "libarrow_python.600.dylib"],
"win32": ["arrow.dll", "arrow_python.dll"],
}

arrow_libs_to_link = {
"linux": ["libarrow.so.600", "libarrow_python.so.600"],
"darwin": ["libarrow.600.dylib", "libarrow_python.600.dylib"],
"win32": ["arrow.lib", "arrow_python.lib"],
}

def build_extension(self, ext):
if options["debug"]:
ext.extra_compile_args.append("-g")
ext.extra_link_args.append("-g")
current_dir = os.getcwd()

if ext.name == "snowflake.connector.arrow_iterator":
if not os.environ.get("SF_NO_COPY_ARROW_LIB", False):
self._copy_arrow_lib()
CPP_SRC_DIR = os.path.join(CONNECTOR_SRC_DIR, "cpp")
ARROW_ITERATOR_SRC_DIR = os.path.join(CPP_SRC_DIR, "ArrowIterator")
LOGGING_SRC_DIR = os.path.join(CPP_SRC_DIR, "Logging")

ext.sources += [
os.path.join(ARROW_ITERATOR_SRC_DIR, "CArrowIterator.cpp"),
os.path.join(ARROW_ITERATOR_SRC_DIR, "CArrowChunkIterator.cpp"),
os.path.join(ARROW_ITERATOR_SRC_DIR, "CArrowTableIterator.cpp"),
os.path.join(ARROW_ITERATOR_SRC_DIR, "SnowflakeType.cpp"),
os.path.join(ARROW_ITERATOR_SRC_DIR, "BinaryConverter.cpp"),
os.path.join(ARROW_ITERATOR_SRC_DIR, "BooleanConverter.cpp"),
os.path.join(ARROW_ITERATOR_SRC_DIR, "DecimalConverter.cpp"),
os.path.join(ARROW_ITERATOR_SRC_DIR, "DateConverter.cpp"),
os.path.join(ARROW_ITERATOR_SRC_DIR, "FloatConverter.cpp"),
os.path.join(ARROW_ITERATOR_SRC_DIR, "IntConverter.cpp"),
os.path.join(ARROW_ITERATOR_SRC_DIR, "StringConverter.cpp"),
os.path.join(ARROW_ITERATOR_SRC_DIR, "TimeConverter.cpp"),
os.path.join(ARROW_ITERATOR_SRC_DIR, "TimeStampConverter.cpp"),
os.path.join(ARROW_ITERATOR_SRC_DIR, "Python", "Common.cpp"),
os.path.join(ARROW_ITERATOR_SRC_DIR, "Python", "Helpers.cpp"),
os.path.join(ARROW_ITERATOR_SRC_DIR, "Util", "time.cpp"),
LOGGING_SRC_DIR + "/logging.cpp",
]
ext.include_dirs.append(ARROW_ITERATOR_SRC_DIR)
ext.include_dirs.append(LOGGING_SRC_DIR)

if sys.platform == "win32":
ext.include_dirs.append(pyarrow.get_include())
ext.include_dirs.append(numpy.get_include())
elif sys.platform == "linux" or sys.platform == "darwin":
ext.extra_compile_args.append("-isystem" + pyarrow.get_include())
ext.extra_compile_args.append("-isystem" + numpy.get_include())
if "std=" not in os.environ.get("CXXFLAGS", ""):
ext.extra_compile_args.append("-std=c++11")
ext.extra_compile_args.append("-D_GLIBCXX_USE_CXX11_ABI=0")

ext.library_dirs.append(
os.path.join(current_dir, self.build_lib, "snowflake", "connector")
)
if ext.name == 'snowflake.connector.arrow_iterator':
self._copy_arrow_lib()

ext.sources += ['cpp/ArrowIterator/CArrowIterator.cpp',
'cpp/ArrowIterator/CArrowChunkIterator.cpp',
'cpp/ArrowIterator/CArrowTableIterator.cpp',
'cpp/ArrowIterator/SnowflakeType.cpp',
'cpp/ArrowIterator/BinaryConverter.cpp',
'cpp/ArrowIterator/BooleanConverter.cpp',
'cpp/ArrowIterator/DecimalConverter.cpp',
'cpp/ArrowIterator/DateConverter.cpp',
'cpp/ArrowIterator/FloatConverter.cpp',
'cpp/ArrowIterator/IntConverter.cpp',
'cpp/ArrowIterator/StringConverter.cpp',
'cpp/ArrowIterator/TimeConverter.cpp',
'cpp/ArrowIterator/TimeStampConverter.cpp',
'cpp/ArrowIterator/Python/Common.cpp',
'cpp/ArrowIterator/Python/Helpers.cpp',
'cpp/ArrowIterator/Util/time.cpp',
'cpp/Logging/logging.cpp']
ext.include_dirs.append('cpp/ArrowIterator/')
ext.include_dirs.append('cpp/Logging')
ext.include_dirs.append(pyarrow.get_include())

ext.extra_compile_args.append('-std=c++11')

ext.library_dirs.append(os.path.join(current_dir, self.build_lib, 'snowflake', 'connector'))
ext.extra_link_args += self._get_arrow_lib_as_linker_input()

# sys.platform for linux used to return with version suffix, (i.e. linux2, linux3)
# After version 3.3, it will always be just 'linux'
# https://docs.python.org/3/library/sys.html#sys.platform
if sys.platform == "linux":
ext.extra_link_args += ["-Wl,-rpath,$ORIGIN"]
elif sys.platform == "darwin":
# rpath,$ORIGIN only work on linux, did not work on darwin. use @loader_path instead
# fyi, https://medium.com/@donblas/fun-with-rpath-otool-and-install-name-tool-e3e41ae86172
ext.extra_link_args += ["-rpath", "@loader_path"]
if self._is_unix():
ext.extra_link_args += ['-Wl,-rpath,$ORIGIN']

build_ext.build_extension(self, ext)

def _is_unix(self):
return platform.startswith('linux') or platform == 'darwin'

def _get_arrow_lib_dir(self):
if "SF_ARROW_LIBDIR" in os.environ:
return os.environ["SF_ARROW_LIBDIR"]
return pyarrow.get_library_dirs()[0]

def _copy_arrow_lib(self):
libs_to_bundle = self.arrow_libs_to_copy[sys.platform]
arrow_lib = self._get_libs_to_copy()

for lib in libs_to_bundle:
source = f"{self._get_arrow_lib_dir()}/{lib}"
build_dir = os.path.join(self.build_lib, "snowflake", "connector")
copy(source, build_dir)
for lib in arrow_lib:
lib_pattern = self._get_pyarrow_lib_pattern(lib)
source = glob.glob(lib_pattern)[0]
copy(source, os.path.join(self.build_lib, 'snowflake', 'connector'))

def _get_arrow_lib_as_linker_input(self):
link_lib = self.arrow_libs_to_link[sys.platform]
ret = []

for lib in link_lib:
source = f"{self._get_arrow_lib_dir()}/{lib}"
assert os.path.exists(source)
ret.append(source)

return ret

cmd_class = {"build_ext": MyBuildExt}
arrow_lib = pyarrow.get_libraries()
link_lib = []
for lib in arrow_lib:
lib_pattern = self._get_pyarrow_lib_pattern(lib)
source = glob.glob(lib_pattern)[0]
link_lib.append(source)

return link_lib

def _get_libs_to_copy(self):
if self._is_unix():
return pyarrow.get_libraries() + \
['arrow_flight', 'arrow_boost_regex', 'arrow_boost_system', 'arrow_boost_filesystem']
elif platform == 'win32':
return pyarrow.get_libraries() + ['arrow_flight']
else:
raise RuntimeError('Building on platform {} is not supported yet.'.format(platform))

def _get_pyarrow_lib_pattern(self, lib_name):
if platform.startswith('linux'):
return '{}/lib{}.so*'.format(self._get_arrow_lib_dir(), lib_name)
elif platform == 'darwin':
return '{}/lib{}*dylib'.format(self._get_arrow_lib_dir(), lib_name)
elif platform == 'win32':
return '{}\\{}.lib'.format(self._get_arrow_lib_dir(), lib_name)
else:
raise RuntimeError('Building on platform {} is not supported yet.'.format(platform))

cmd_class = {
"build_ext": MyBuildExt
}

setup(
name='snowflake-connector-python',
version=version,
description=u"Snowflake Connector for Python",
ext_modules=extensions,
cmdclass=cmd_class,
long_description=long_description,
author='Snowflake, Inc',
author_email='[email protected]',
license='Apache License, Version 2.0',
keywords="Snowflake db database cloud analytics warehouse",
url='https://www.snowflake.com/',
download_url='https://www.snowflake.com/',
use_2to3=False,

# NOTE: Python 3.4 will be dropped within one month.
python_requires='>=2.7.9,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*',

install_requires=[
'azure-common',
'azure-storage-blob',
'boto3>=1.4.4,<=1.17.0',
'botocore>=1.5.0,<=1.20.0',
'certifi',
'future',
'six',
'pytz',
'pycryptodomex>=3.2,!=3.5.0',
'pyOpenSSL>=16.2.0',
'cffi>=1.9',
'cryptography>=1.8.2',
'ijson',
'pyjwt',
'idna',
'pyasn1>=0.4.0,<0.5.0;python_version<"3.0"',
'pyasn1-modules>=0.2.0,<0.3.0;python_version<"3.0"',
'enum34;python_version<"3.4"',
],

namespace_packages=['snowflake'],
packages=[
'snowflake.connector',
'snowflake.connector.tool',
],
package_dir={
'snowflake.connector': '.',
'snowflake.connector.tool': 'tool',
},
package_data={
'snowflake.connector': ['*.pem', '*.json', '*.rst', 'LICENSE.txt'],
},

entry_points={
'console_scripts': [
'snowflake-dump-ocsp-response = '
'snowflake.connector.tool.dump_ocsp_response:main',
'snowflake-dump-ocsp-response-cache = '
'snowflake.connector.tool.dump_ocsp_response_cache:main',
'snowflake-dump-certs = '
'snowflake.connector.tool.dump_certs:main',
'snowflake-export-certs = '
'snowflake.connector.tool.export_certs:main',
],
},
extras_require={
"secure-local-storage": [
'keyring!=16.1.0'
],
"arrow-result": [
'pyarrow>=0.14.0;python_version>"3.4"',
'pyarrow>=0.14.0;python_version<"3.0"'
]
},

classifiers=[
'Development Status :: 5 - Production/Stable',

'Environment :: Console',
'Environment :: Other Environment',

'Intended Audience :: Developers',
'Intended Audience :: Education',
'Intended Audience :: Information Technology',
'Intended Audience :: System Administrators',

'License :: OSI Approved :: Apache Software License',

'Operating System :: OS Independent',

'Programming Language :: SQL',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',

'Topic :: Database',
'Topic :: Software Development',
'Topic :: Software Development :: Libraries',
'Topic :: Software Development :: Libraries :: Application Frameworks',
'Topic :: Software Development :: Libraries :: Python Modules',
'Topic :: Scientific/Engineering :: Information Analysis',
],
)