diff --git a/Makefile.PL b/Makefile.PL index 2636133..7d1a1b4 100644 --- a/Makefile.PL +++ b/Makefile.PL @@ -3,6 +3,7 @@ use Config; use Cwd qw(abs_path); use ExtUtils::MakeMaker 6.64; use Getopt::Long; +use File::Spec; use File::Basename; GetOptions( @@ -26,7 +27,7 @@ unless ($sel) { for $p (split /$sep/, $ENV{PATH}) { $p =~ s/^~/$ENV{HOME}/; for $exe_version ('3', '') { - my $py = "$p/python$exe_version$exe"; + my $py = File::Spec->catfile($p, "python$exe_version$exe"); next unless -f $py and -x $py; next if $pythons{abs_path($py)}++; # filter symlinked duplicates my $version = get_python_major_version($py); @@ -73,7 +74,6 @@ unless ($sel) { $sel = { path => $sel } unless ref $sel eq 'HASH'; # in case the user entered a path print "Using $sel->{path}\n"; - my $py_major_version = get_python_major_version($sel->{path}); #============================================================================ @@ -82,7 +82,9 @@ my $py_major_version = get_python_major_version($sel->{path}); interrogate($sel); # Fix up the libpath and libpython -die "Could not find Python.h in include path. make will not work" unless -e "$sel->{incpath}/Python.h"; +die "Could not find Python.h in include path. make will not work" + unless -e File::Spec->catfile($sel->{incpath}, "Python.h"); + substr($sel->{incpath}, 0, 0) = "-I"; substr($sel->{libpath}, 0, 0) = "-L"; $sel->{libpython} =~ s/lib(.*)(?:\.\Q$Config{dlext}\E|\Q$Config{_a}\E)/-l$1/; @@ -92,6 +94,7 @@ push @flags, debug_flag() if defined $gdb; push @flags, '-DI_PY_DEBUG' if $debug; push @flags, "-DPY_MAJOR_VERSION=$py_major_version"; push @flags, 'none (perl Makefile.PL --help for details)' unless @flags; +$sel->{syslib} = '' if $sel->{syslib} eq "None"; print <{syslibs} @@ -120,6 +123,7 @@ WriteMakefile( 'Inline' => 0.46, 'Digest::MD5' => 2.50, 'Data::Dumper' => 0, + 'File::Spec' => 0, }, TEST_REQUIRES => { 'Test' => 0, @@ -140,11 +144,114 @@ WriteMakefile( }, }, clean => {FILES => 'blib_test/'}, + dynamic_lib => { + OTHERLDFLAGS => ($sel->{rpath} ? "-Wl,-rpath,$sel->{rpath}" : ''), + }, + ); #============================================================================ # Tries to ask the python interpreter what libraries we need, where its # include directories are, etc. +# Typical values of the Python sysconfig variables: +# +# - VERSION +# +# - Ubuntu : 3.10 +# - Windows: 310 +# - macOS : 3.10 +# +# - BINDIR +# +# - Ubuntu : +# - system version: /usr/bin +# - pyenv version : /home/username/.pyenv/versions/3.9.4/bin +# - Windows : C:\Python\Python310 (NOTE: missing trailing "bin") +# - macOS : +# - system version : /Applications/Xcode.app/Contents/Developer/Library +# /Frameworks/Python3.framework/Versions/3.8/bin +# - pyenv version : /Users/username/.pyenv/versions/3.10.0-debug/bin +# +# - LIBS: +# +# - Ubuntu : -lcrypt -lpthread -ldl -lutil -lm +# - Windows: [None] +# - macOS : +# - system version : -ldl -lSystem -framework CoreFoundation +# - pyenv version : -ldl -framework CoreFoundation +# +# - INCLUDEPY +# +# - Ubuntu : /usr/include/python3.9 +# - Windows: C:\Python\Python310\Include +# - macOS +# - system version : /Applications/Xcode.app/Contents/Developer/Library +# /Frameworks/Python3.framework/Versions/3.8/Headers +# - pyenv version : /Users/username/.pyenv/versions/3.10.0-debug/include/python3.10d +# +# - LIBPL +# +# - Ubuntu : +# - system version : /usr/lib/python3.9/config-3.9-x86_64-linux-gnu +# - pyenv version : /home/username/.pyenv/versions/3.8.9 +# /lib/python3.8/config-3.8-x86_64-linux-gnu (NOTE: this folder does +# not contain a shared library even if python was built with +# --enable-shared. However, the folder LIBDIR, see below, does) +# - Windows: [None] (NOTE: this should be "$BINDIR/libs" on windows) +# - macOS : +# - system version : /Applications/Xcode.app/Contents/Developer/Library/Frameworks +# /Python3.framework/Versions/3.8/lib/python3.8/config-3.8-darwin +# - pyenv version : /Users/username/.pyenv/versions/3.10.0-debug/lib +# /python3.10/config-3.10d-darwin +# +# - LDLIBRARY +# +# - Ubuntu : +# - if python was built with --enable-shared : libpython3.9.so +# - else : libpython3.9.a +# - Windows: [None] (NOTE: this should be "python310.lib" on windows, +# where a .lib file is a so-called import-library on Windows. The +# import library reference a .dll library in $BINDIR. +# On Windows there is also a stable-across-versions-subset library +# called "python3.lib" (which references "python3.dll" in $BINDIR, +# see https://www.python.org/dev/peps/pep-0384/ for more information.) +# - macOS : +# - system version: Python3.framework/Versions/3.8/Python3 (NOTE: this is a .dylib +# i.e. a shared library) +# - pyenv version : +# - if built with --enable-shared : libpython3.10d.dylib +# - else : libpython3.10d.a +# +# - LIBRARY +# +# - Ubuntu : libpython3.9.a +# - Windows: [None] (NOTE: static library is not available but import library exists +# see note on LDLIBRARY above) +# - macOS : libpython3.10d.a +# +# - LIBDEST +# +# - Ubuntu : /home/username/.pyenv/versions/3.9.4/lib/python3.9 +# - Windows: C:\Python310\Lib (NOTE: this folder does not contain anything interesting +# to us. No static or shared libraries here, but the import +# library is in the libs folder C:\Python310\libs and the +# .dll library is in the BINDIR C:\Python310) +# - macOS : +# - system version : /Applications/Xcode.app/Contents/Developer/Library +# /Frameworks/Python3.framework/Versions/3.8/lib/python3.8 +# - pyenv version : /Users/username/.pyenv/versions/3.10.0-debug/lib/python3.10 +# +# - LIBDIR +# +# - Ubuntu : +# - system version : /usr/lib/x86_64-linux-gnu +# - pyenv version : /home/username/.pyenv/versions/3.9.4/lib +# - Windows : [None] +# - macOS : +# - system version: /Applications/Xcode.app/Contents/Developer/Library +# /Frameworks/Python3.framework/Versions/3.8/lib +# - pyenv version : /Users/username/.pyenv/versions/3.10.0-debug/lib +# #============================================================================ sub interrogate { my $ref = shift; @@ -154,36 +261,190 @@ sub interrogate { $ref->{libpath} = get_config_var($ref, "LIBPL"); $ref->{ldlib} = get_config_var($ref, "LDLIBRARY"); $ref->{libpython} = get_config_var($ref, "LIBRARY"); - my $tmp = rindex($ref->{libpython}, '/') + 1; - $ref->{libpython} = substr($ref->{libpython}, $tmp); + my $tmp = File::Spec->canonpath($ref->{libpython}); + my @dirs = File::Spec->splitdir( $tmp ); + $ref->{libpython} = $dirs[-1]; + # On Windows, Python config var "LIBRARY" is not defined if ($ref->{libpython} eq 'None') { special_get_libpath($ref); } - $ref->{libpath} = join '/', (get_config_var($ref, "LIBDEST"), - 'config') - if ($ref->{libpath} eq 'None'); + $ref->{libpath} = File::Spec->catfile(get_config_var($ref, "LIBDEST"), 'config') + if ($ref->{libpath} eq 'None'); + $ref->{rpath} = ''; # only used if we are linking with a shared library, see below. + $ref->{cflags} = get_config_var($ref, 'CFLAGS'); + $ref->{config_args} = get_config_var($ref, 'CONFIG_ARGS'); + $ref->{enable_shared} = (get_config_var($ref, 'Py_ENABLE_SHARED') eq "1"); + if (using_macos_system_python($ref)) { + add_rpath_for_macos_system_python($ref); + } + elsif ($^O ne "MSWin32") { # we use the import library on Windows, + # see special_get_libpath() below + special_non_windows_check_shared_static_libs($ref); + } return query_options($ref) unless sanity_check($ref); } +# on macOS using the system python, the path to the shared and static libraries +# is given by "libpath" (LIBPL). In this directory there exists two files: +# - libpythonxxx.a +# - libpythonxxx.dylib +# the first name (the static library libpythonxxx.a) is given by the python sysconfig +# variable "libpython" (LIBRARY). However, both these files are symlinks to a shared +# library called "Python" which is located relative to LIBPL with path: +# ../../../Python3. This file "Python3" (or "Python2" ??), is a dylib with an embedded +# @rpath magic search path given by: @rpath/Python3.framework/Versions/3.8/Python3 which +# is also the ID of the library (LC_ID_DYLIB) which means that Python.so (the Perl +# generated interface) must include an rpath to the directory 3 levels above the +# location of "Python3" (which is 6 levels above libpythonxxx.dylib in LIBPL). +# This directory is fortunately given by the config variable +# PYTHONFRAMEWORKPREFIX. +# +sub add_rpath_for_macos_system_python { + my ($ref) = @_; + + $ref->{rpath} = get_config_var($ref, 'PYTHONFRAMEWORKPREFIX'); +}; + +sub special_non_windows_check_shared_static_libs { + my ($ref) = @_; + + if (shared_lib_priority($ref)) { + $ref->{libpython} = $ref->{ldlib}; + my $shared_lib = File::Spec->catfile($ref->{libpath}, $ref->{libpython}); + if (!-f $shared_lib) { + # Special case for pyenv. The shared library exists in $LIBDIR instead of in + # $LIBPL + $ref->{libpath} = get_config_var($ref, "LIBDIR"); + } + $ref->{rpath} = $ref->{libpath}; + } + else { + if (!check_static_library_ok($ref)) { + # In this case we may find a shared library to link with in + # $LIBDIR instead of in $LIBPL, this happens if you install Python + # with pyenv (on the other hand for the system python there will + # a shared library in both $LIBDIR and $LIBPL, see issue #29 for more information. + # TODO: However, this still does not work for pyenv (tested on Ubuntu). For some + # reason this shared library does not behave well unless python was also compiled + # with -fPIC option. + if (( $ref->{cflags} !~ /\Q-fPIC\E/i) && ($ref->{path} !~ m{^/(?:usr/)?bin/python})) { + if ($ref->{enable_shared}) { + warn "WARNING: This python's shared library was compiled with --enable-shared but not " + ." with -fPIC option, this might lead to strange runtime behavior.\n"; + } + else { + # TODO: strangely this seems to work fine on macOS. + # More investigation is needed here... + warn "WARNING: This python was not compiled with --enable-shared and not " + . "with -fPIC.\n" + . "WARNING: This is known to not work on linux.\n"; + } + } + $ref->{libpath} = get_config_var($ref, "LIBDIR"); + $ref->{libpython} = $ref->{ldlib}; + $ref->{rpath} = $ref->{libpath}; + } + } +} + +sub shared_lib_priority { + my ($ref) = @_; + + if ($ref->{libpython} ne $ref->{ldlib}) { + # This should happen if python was compiled with --enable-shared + # In this case the linker will prefer the shared library + my $static_lib = File::Spec->catfile($ref->{libpath}, $ref->{libpython}); + my $shared_lib = File::Spec->catfile($ref->{libpath}, $ref->{ldlib}); + return 1 if (-f $static_lib) && (-f $shared_lib); + } + return 0; +} + +sub check_static_library_ok { + my ($ref) = @_; + + # We should check if the static library was compiled with -fPIC, or else + # we cannot create a shared Python.so from it. + # TODO: It seem like it is possible to build python with --enable-shared + # and without CFLAGS=-fPIC, and this will make both libpythonxx.so and + # libpythonxx.a position independent, but in this case strange things + # may happen at runtime (i.e. when running "make test"), see issue #29 + # for more information. This should be investigated futher to determine what + # is actually going on. + # This seems to not be a problem on macOS though. + return 1 if $ref->{cflags} =~ /\Q-fPIC\E/i; + warn "WARNING: The static python library seems not to be position indepenent.\n" + . "WARNING: If this does not work you should try again with a " + . "python version that was compiled with CFLAGS=-fPIC\n"; + return 0; +} + +sub using_macos_system_python { + my ($ref) = @_; + + return ($^O eq "darwin") && ($ref->{path} =~ m{^/usr/bin/python}); +} + +sub check_shared_lib_support() { + my ($ref) = @_; + + # Windows python always have a shared lib + return 1 if $^O eq "MSWin32"; + # The system python always have a shared lib on macOS + return 1 if using_macos_system_python($ref); + # See https://stackoverflow.com/a/23202055/2173773 + return $ref->{enable_shared}; +} + +sub get_python_version { + my $ref = shift; + my $major = `$ref->{path} -c "import sys; print(sys.version_info[0])"`; + my $minor = `$ref->{path} -c "import sys; print(sys.version_info[1])"`; + return ($major, $minor); +} + +# On Windows, Python config var "LIBRARY" is not defined, so we try another method +# to obtain the library path name sub special_get_libpath { - # For when sysconfig does not work (i.e. on Windows) my $ref = shift; - my $val = `$ref->{path} -c "import distutils.command.build_ext; d=distutils.core.Distribution(); b=distutils.command.build_ext.build_ext(d);b.finalize_options();print(b.library_dirs[0])" 2>&1`; + my ($major, $minor) = get_python_version($ref); + my $cmd; + if (($major == 3 && $minor >=10) || $major > 3 ) { + $cmd = 'import setuptools.command.build_ext; d=setuptools.dist.Distribution();' + .'b=setuptools.command.build_ext.build_ext(d)'; + } + else { + $cmd = 'import distutils.command.build_ext; d=distutils.core.Distribution();' + . 'b=distutils.command.build_ext.build_ext(d)'; + } + my @lines = `$ref->{path} -c "$cmd;b.finalize_options();print(b.library_dirs[0])" 2>&1`; + my $val = $lines[-1]; chomp $val; + return '' if !$val; + # On Windows, $val should now be equal to $BINDIR/libs + my $pyscript = "import sysconfig; " + . "print(sysconfig.get_config_var('VERSION'))"; + my $version = `$ref->{path} -c "$pyscript"`; + chomp $version; $ref->{libpath} = $val; - $ref->{libpython} = basename((glob("$val/libpython*.a"))[0]); + $ref->{libpython} = "python${version}.lib"; # Note: on Windows this is an import library, + # that referes to a shared library + # (not a static library) + # The above file should always exist, alternatively we could + # set libpath to $BINDIR and libpython to python$version.dll and use the shared library + # directly instead of using the import library. return $val; } sub test_interrogate { my $ref = shift; - `$ref->{path} -c "import distutils.sysconfig; distutils.sysconfig.get_config_var" 2>&1`; + `$ref->{path} -c "import sysconfig; sysconfig.get_config_var" 2>&1`; print <{libpython} = $ref->{ldlib} - if not -f join '/', $ref->{libpath}, $ref->{libpython} - and -f join '/', $ref->{libpath}, $ref->{ldlib}; - + if not -f File::Spec->catfile($ref->{libpath}, $ref->{libpython}) + and -f File::Spec->catfile($ref->{libpath}, $ref->{ldlib}); + my $libpath = File::Spec->catfile($ref->{libpath}, $ref->{libpython}); unless (-d $ref->{libpath} && -d $ref->{incpath} && - (-f join '/', $ref->{libpath}, $ref->{libpython}) + (-f File::Spec->catfile($ref->{libpath}, $ref->{libpython})) ) { print <{syslibs} - Python Library: $sel->{libpath}/$sel->{libpython} - Include Path: $sel->{incpath} + Extra Libs: $ref->{syslibs} + Python Library: $libpath + Include Path: $ref->{incpath} END # ' stupid vim. } @@ -227,7 +488,7 @@ sub get_config_var { my $ref = shift; my $key = shift; my $exe = $ref->{path}; - my $val = `$exe -c "import distutils.sysconfig; print(distutils.sysconfig.get_config_var('$key'))"`; + my $val = `$exe -c "import sysconfig; print(sysconfig.get_config_var('$key'))"`; chomp $val; return $val; } @@ -236,21 +497,21 @@ sub query_options { my $ref = shift; # Every python I've seen needs pthreads. Obviously not on windows. - my $libs_guess = $ref->{syslibs} ? $ref->{syslibs} : + my $libs_guess = $ref->{syslibs} ? $ref->{syslibs} : $^O eq 'MSWin32' ? '' : '-lpthread'; print <{syslibs} = prompt("Enter extra libraries (e.g. -lfoo -lbar)", + $ref->{syslibs} = prompt("Enter extra libraries (e.g. -lfoo -lbar)", $libs_guess); print <{libpath} = substr($lib, 0, rindex($lib, '/')); - $ref->{libpython} = substr($lib, rindex($lib, '/')+1); + my ($volume, $directories, $file) = File::Spec->splitpath( $lib ); + $ref->{libpath} = File::Spec->canonpath(File::Spec->catpath($volume, $directories)); + $ref->{libpython} = $file; print <splitpath($path); + $path = File::Spec->canonpath(File::Spec->catpath(@parts[0..1])); + $file = $parts[-1]; } $path = abs_path($path); - return defined $file ? join '/', $path, $file : $path; + return defined $file ? File::Spec->catfile($path, $file) : $path; } sub debug_flag {