Merge pull request #140 from johnhaddon/publicVariables

Python 3 foundational work

Appleseed/config.py

@@ -25,15 +25,6 @@
 
 	},
 
-	"variables" : {
-
-		# Make sure we pick up the python headers from {buildDir},
-		# rather than any system level headers. We refer to this
-		# variable in "commands" below.
-		"pythonIncludeDir" : "{buildDir}/include/python2.7",
-
-	},
-
 	"commands" : [
 
 		"mkdir build",
@@ -77,15 +68,4 @@
 
 	],
 
-	"platform:osx" : {
-
-		"variables" : {
-
-			# Python headers have a different location on OSX.
-			"pythonIncludeDir" : "{buildDir}/lib/Python.framework/Headers",
-
-		},
-
-	},
-
 }

Boost/config.py

@@ -18,7 +18,7 @@
 		"MACOSX_DEPLOYMENT_TARGET" : "10.9",
 		# Give a helping hand to find the python headers, since the bootstrap
 		# below doesn't always seem to get it right.
-		"CPLUS_INCLUDE_PATH" : "{buildDir}/include/python2.7",
+		"CPLUS_INCLUDE_PATH" : "{pythonIncludeDir}",
 
 	},
 

OpenImageIO/config.py

@@ -19,6 +19,7 @@
 			" -D CMAKE_INSTALL_LIBDIR={buildDir}/lib"
 			" -D CMAKE_PREFIX_PATH={buildDir}"
 			" -D USE_FFMPEG=NO"
+			" -D USE_PYTHON=NO"
 			" ..",
 		"cd gafferBuild && make install -j {jobs} VERBOSE=1",
 		"cp {buildDir}/share/doc/OpenImageIO/openimageio.pdf {buildDir}/doc",

OpenVDB/config.py

@@ -10,12 +10,6 @@
 
 	"dependencies" : [ "Blosc", "TBB", "OpenEXR", "Python" ],
 
-	"variables" : {
-
-		"pythonVersion" : "2.7",
-
-	},
-
 	"commands" : [
 
 		"cd openvdb && make install"
@@ -54,26 +48,4 @@
 
 	],
 
-	"platform:linux" : {
-
-		"variables" : {
-
-			"pythonIncludeDir" : "{buildDir}/include/python{pythonVersion}",
-			"pythonLibDir" : "{buildDir}/lib",
-
-		},
-
-	},
-
-	"platform:osx" : {
-
-		"variables" : {
-
-			"pythonIncludeDir" : "{buildDir}/lib/Python.framework/Headers",
-			"pythonLibDir" : "{buildDir}/lib/Python.framework/Versions/{pythonVersion}",
-
-		},
-
-	},
-
 }

Python/config.py

@@ -1,5 +1,14 @@
 {
 
+	"publicVariables" : {
+
+		"pythonVersion" : "2.7",
+		"pythonMajorVersion" : "2",
+		"pythonIncludeDir" : "{buildDir}/include/python{pythonVersion}",
+		"pythonLibDir" : "{buildDir}/lib",
+
+	},
+
 	"downloads" : [
 
 		"https://www.python.org/ftp/python/2.7.15/Python-2.7.15.tgz",
@@ -27,7 +36,7 @@
 
 		"lib/libpython*{sharedLibraryExtension}*",
 		"lib/Python.framework*",
-		"lib/python2.7",
+		"lib/python{pythonVersion}",
 
 	],
 
@@ -45,11 +54,18 @@
 
 		},
 
+		"publicVariables" : {
+
+			"pythonIncludeDir" : "{buildDir}/lib/Python.framework/Headers",
+			"pythonLibDir" : "{buildDir}/lib/Python.framework/Versions/{pythonVersion}/lib",
+
+		},
+
 		"symbolicLinks" : [
 
 			( "{buildDir}/bin/python", "../lib/Python.framework/Versions/Current/bin/python" ),
-			( "{buildDir}/bin/python2", "../lib/Python.framework/Versions/Current/bin/python2" ),
-			( "{buildDir}/bin/python2.7", "../lib/Python.framework/Versions/Current/bin/python2.7" ),
+			( "{buildDir}/bin/python{pythonMajorVersion}", "../lib/Python.framework/Versions/Current/bin/python{pythonMajorVersion}" ),
+			( "{buildDir}/bin/python{pythonVersion}", "../lib/Python.framework/Versions/Current/bin/python{pythonVersion}" ),
 
 		],
 

USD/config.py

@@ -24,7 +24,7 @@
 			" -D ALEMBIC_DIR={buildDir}/lib"
 			" -D OPENEXR_LOCATION={buildDir}/lib"
 			# Needed to prevent CMake picking up system python libraries on Mac.
-			" -D CMAKE_FRAMEWORK_PATH={buildDir}/lib/Python.framework/Versions/2.7/lib"
+			" -D CMAKE_FRAMEWORK_PATH={pythonLibDir}"
 			" ."
 		,
 

build.py

@@ -16,17 +16,6 @@
 
 __version = "1.1.0"
 
-def __globalVariables( buildDir ) :
-
-	return {
-		"buildDir" : buildDir,
-		"jobs" : multiprocessing.cpu_count(),
-		"path" : os.environ["PATH"],
-		"version" : __version,
-		"platform" : "osx" if sys.platform == "darwin" else "linux",
-		"sharedLibraryExtension" : ".dylib" if sys.platform == "darwin" else ".so",
-	}
-
 def __projects() :
 
 	configFiles = glob.glob( "*/config.py" )
@@ -62,7 +51,7 @@ def __decompress( archive ) :
 		# Badly behaved archive
 		return "./"
 
-def __loadConfig( project, buildDir ) :
+def __loadConfig( project ) :
 
 	# Load file. Really we want to use JSON to
 	# enforce a "pure data" methodology, but JSON
@@ -87,23 +76,24 @@ def __loadConfig( project, buildDir ) :
 		else :
 			config[key] = value
 
-	# Apply variable substitutions.
+	return config
+
+def __substitute( config, variables, forDigest = False ) :
 
-	variables = config.get( "variables", {} ).copy()
-	variables.update( __globalVariables( buildDir ) )
-	variables.update( {
-		"buildDir" : "{buildDir}", # Don't substitute these yet,
-		"jobs" : "{jobs}",         # because they shouldn't affect the digest.
-	} )
+	if not forDigest :
+		# These shouldn't affect the output of the build, so
+		# hold them constant when computing a digest.
+		variables = variables.copy()
+		variables.update( { "buildDir" : "{buildDir}", "jobs" : "{jobs}" } )
 
-	def __substitute( o ) :
+	def substituteWalk( o ) :
 
 		if isinstance( o, dict ) :
-			return { k : __substitute( v ) for k, v in o.items() }
+			return { k : substituteWalk( v ) for k, v in o.items() }
 		elif isinstance( o, list ) :
-			return [ __substitute( x ) for x in o ]
+			return [ substituteWalk( x ) for x in o ]
 		elif isinstance( o, tuple ) :
-			return tuple( __substitute( x ) for x in o )
+			return tuple( substituteWalk( x ) for x in o )
 		elif isinstance( o, str ) :
 			while True :
 				s = o.format( **variables )
@@ -114,12 +104,13 @@ def __substitute( o ) :
 		else :
 			return o
 
-	config = __substitute( config )
+	return substituteWalk( config )
 
-	# Compute digest. This needs to account for everything that
+def __updateDigest( project, config ) :
+
+	# This needs to account for everything that
 	# `__buildProject()` is sensitive to.
 
-	config["digest"] = hashlib.md5()
 	config["digest"].update( str( config["downloads"] ) )
 	config["digest"].update( str( config.get( "license" ) ) )
 	config["digest"].update( str( config.get( "environment" ) ) )
@@ -132,33 +123,43 @@ def __substitute( o ) :
 		with open( patch ) as f :
 			config["digest"].update( f.read() )
 
-	# Apply the substitutions we avoided earlier.
-
-	variables.update( __globalVariables( buildDir ) )
-	config = __substitute( config )
-
-	return config
-
-def __loadConfigs( buildDir ) :
+def __loadConfigs( variables ) :
 
 	# Load configs
 
 	configs = {}
 	for project in __projects() :
-		configs[project] = __loadConfig( project, buildDir )
+		configs[project] = __loadConfig( project )
 
-	# Update digests of each project with the digests
-	# of all its dependencies.
+	# Walk dependency tree to compute digests and
+	# apply substitutions.
 
 	visited = set()
 	def walk( project, configs ) :
 
 		if project in visited :
 			return
 
-		for dependency in configs[project].get( "dependencies", [] ) :
+		projectConfig = configs[project]
+		projectConfig["digest"] = hashlib.md5()
+
+		# Visit dependencies to gather their public variables
+		# and apply their digest to this project.
+
+		projectVariables = variables.copy()
+		for dependency in projectConfig.get( "dependencies", [] ) :
 			walk( dependency, configs )
-			configs[project]["digest"].update( configs[dependency]["digest"].hexdigest() )
+			projectConfig["digest"].update( configs[dependency]["digest"].hexdigest() )
+			projectVariables.update( configs[dependency].get( "publicVariables", {} ) )
+
+		# Apply substitutions and update digest.
+
+		projectVariables.update( projectConfig.get( "publicVariables", {} ) )
+		projectVariables.update( projectConfig.get( "variables", {} ) )
+
+		projectConfig = __substitute( projectConfig, projectVariables, forDigest = True )
+		__updateDigest( project, projectConfig )
+		configs[project] = __substitute( projectConfig, projectVariables, forDigest = False )
 
 		visited.add( project )
 
@@ -231,6 +232,7 @@ def __buildProject( project, config, buildDir ) :
 		subprocess.check_call( command, shell = True, env = environment )
 
 	for link in config.get( "symbolicLinks", [] ) :
+		sys.stderr.write( "Linking {} to {}\n".format( link[0], link[1] ) )
 		if os.path.exists( link[0] ) :
 			os.remove( link[0] )
 		os.symlink( link[1], link[0] )
@@ -331,10 +333,19 @@ def walk( project, configs, buildDir ) :
 
 args = parser.parse_args()
 
-configs = __loadConfigs( args.buildDir )
+variables = {
+	"buildDir" : args.buildDir,
+	"jobs" : multiprocessing.cpu_count(),
+	"path" : os.environ["PATH"],
+	"version" : __version,
+	"platform" : "osx" if sys.platform == "darwin" else "linux",
+	"sharedLibraryExtension" : ".dylib" if sys.platform == "darwin" else ".so",
+}
+
+configs = __loadConfigs( variables )
 
 __checkEnvironment( args.projects, configs )
 __buildProjects( args.projects, configs, args.buildDir )
 
 if args.package :
-	__buildPackage( args.projects, configs, args.buildDir, args.package.format( **__globalVariables( args.buildDir ) ) )
+	__buildPackage( args.projects, configs, args.buildDir, args.package.format( **variables ) )