From a8f481382a1b2d3d3a53fbf3dc951cd61db74f1f Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Fri, 21 May 2021 01:03:58 +0300 Subject: [PATCH] resolves #164 un-hardcode list of Windows versions Partially resolves #138 (support for insider base image is still missing) This commit: 1. Drops the list of valid Windows versions. Instead, ue4-docker now trusts user input and will fail during image download if user enters something nonexistent 2. Drops 1603->ltsc2016, 1809->ltsc2019 and 2009->20H2 renaming. Instead, ue4-docker directly maps host OS release to Windows Server Core image tag 3. However, ue4-docker now uses advanced logic to determine host OS release. It tries to use DisplayName registry key and fallbacks to ReleaseId if DisplayName doesn't exist. This allows to properly detect 20H2 and 21H1 releases. On the negative side, some checks are lost: 1. It is no longer possible to check that container version is newer that host OS. Though is should still be rejected by Hyper-V 2. ue4-docker no longer prevents user from using suffix that collides with Windows Server Core image tags --- ue4docker/build.py | 26 ++- ue4docker/diagnostics/base.py | 3 +- ue4docker/diagnostics/diagnostic_maxsize.py | 2 +- .../windows/Dockerfile | 4 +- .../windows/verify-host-dlls.py | 15 +- ue4docker/info.py | 2 +- .../infrastructure/BuildConfiguration.py | 19 +-- ue4docker/infrastructure/WindowsUtils.py | 154 +++++------------- ue4docker/main.py | 7 +- 9 files changed, 71 insertions(+), 161 deletions(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index 97060c36..f63bf40c 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -112,22 +112,23 @@ def build(): # Provide the user with feedback so they are aware of the Windows-specific values being used logger.info('WINDOWS CONTAINER SETTINGS', False) logger.info('Isolation mode: {}'.format(config.isolation), False) - logger.info('Base OS image tag: {} (host OS is {})'.format(config.basetag, WindowsUtils.systemStringShort()), False) + logger.info('Base OS image tag: {}'.format(config.basetag), False) + logger.info('Host OS: {}'.format(WindowsUtils.systemString()), False) logger.info('Memory limit: {}'.format('No limit' if config.memLimit is None else '{:.2f}GB'.format(config.memLimit)), False) logger.info('Detected max image size: {:.0f}GB'.format(DockerUtils.maxsize()), False) logger.info('Directory to copy DLLs from: {}\n'.format(config.dlldir), False) - + # Verify that the specified base image tag is not a release that has reached End Of Life (EOL) - if WindowsUtils.isEndOfLifeWindowsVersion(config.basetag) == True: + if not config.ignoreEOL and WindowsUtils.isEndOfLifeWindowsVersion(config.basetag): logger.error('Error: detected EOL base OS image tag: {}'.format(config.basetag), False) logger.error('This version of Windows has reached End Of Life (EOL), which means', False) logger.error('Microsoft no longer supports or maintains container base images for it.', False) logger.error('You will need to use a base image tag for a supported version of Windows.', False) sys.exit(1) - + # Verify that the host OS is not a release that is blacklisted due to critical bugs - if config.ignoreBlacklist == False and WindowsUtils.isBlacklistedWindowsVersion() == True: - logger.error('Error: detected blacklisted host OS version: {}'.format(WindowsUtils.systemStringShort()), False) + if not config.ignoreBlacklist and WindowsUtils.isBlacklistedWindowsVersion(): + logger.error('Error: detected blacklisted host OS version: {}'.format(WindowsUtils.systemString()), False) logger.error('', False) logger.error('This version of Windows contains one or more critical bugs that', False) logger.error('render it incapable of successfully building UE4 container images.', False) @@ -136,15 +137,10 @@ def build(): logger.error('For more information, see:', False) logger.error('https://unrealcontainers.com/docs/concepts/windows-containers', False) sys.exit(1) - - # Verify that the user is not attempting to build images with a newer kernel version than the host OS - if WindowsUtils.isNewerBaseTag(config.hostBasetag, config.basetag): - logger.error('Error: cannot build container images with a newer kernel version than that of the host OS!') - sys.exit(1) - + # Check if the user is building a different kernel version to the host OS but is still copying DLLs from System32 - differentKernels = WindowsUtils.isInsiderPreview() or config.basetag != config.hostBasetag - if config.pullPrerequisites == False and differentKernels == True and config.dlldir == config.defaultDllDir: + differentKernels = config.basetag != config.hostBasetag + if not config.pullPrerequisites and differentKernels and config.dlldir == config.defaultDllDir: logger.error('Error: building images with a different kernel version than the host,', False) logger.error('but a custom DLL directory has not specified via the `-dlldir=DIR` arg.', False) logger.error('The DLL files will be the incorrect version and the container OS will', False) @@ -239,7 +235,7 @@ def build(): # (This is the only image that does not use any user-supplied tag suffix, since the tag always reflects any customisations) prereqsArgs = ['--build-arg', 'BASEIMAGE=' + config.baseImage] if config.containerPlatform == 'windows': - prereqsArgs = prereqsArgs + ['--build-arg', 'HOST_VERSION=' + WindowsUtils.getWindowsBuild()] + prereqsArgs = prereqsArgs + ['--build-arg', 'HOST_BUILD=' + str(WindowsUtils.getWindowsBuild())] # Build or pull the UE4 build prerequisites image (don't pull it if we're copying Dockerfiles to an output directory) if config.layoutDir is None and config.pullPrerequisites == True: diff --git a/ue4docker/diagnostics/base.py b/ue4docker/diagnostics/base.py index d974118f..bd726f2a 100644 --- a/ue4docker/diagnostics/base.py +++ b/ue4docker/diagnostics/base.py @@ -60,8 +60,7 @@ def _generateWindowsBuildArgs(self, logger, basetagOverride=None, isolationOverr ''' # Determine the appropriate container image base tag for the host system release unless the user specified a base tag - buildArgs = [] - defaultBaseTag = WindowsUtils.getReleaseBaseTag(WindowsUtils.getWindowsRelease()) + defaultBaseTag = WindowsUtils.getWindowsRelease() baseTag = basetagOverride if basetagOverride is not None else defaultBaseTag buildArgs = ['--build-arg', 'BASETAG={}'.format(baseTag)] diff --git a/ue4docker/diagnostics/diagnostic_maxsize.py b/ue4docker/diagnostics/diagnostic_maxsize.py index c7e30fe5..0b831eb7 100644 --- a/ue4docker/diagnostics/diagnostic_maxsize.py +++ b/ue4docker/diagnostics/diagnostic_maxsize.py @@ -47,7 +47,7 @@ def run(self, logger, args=[]): return False # Verify that we are running Windows Server or Windows 10 version 1903 or newer - if WindowsUtils.getWindowsVersion()['patch'] < 18362: + if WindowsUtils.getWindowsBuild() < 18362: logger.info('[maxsize] This diagnostic only applies to Windows Server and Windows 10 version 1903 and newer.', False) return True diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile index f8fa73c4..e3e51967 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile @@ -20,11 +20,11 @@ RUN choco install -y 7zip curl && choco install -y python --version=3.7.5 COPY *.dll C:\GatheredDlls\ # Verify that the DLL files copied from the host can be loaded by the container OS -ARG HOST_VERSION +ARG HOST_BUILD RUN pip install pywin32 COPY copy.py verify-host-dlls.py C:\ RUN C:\copy.py "C:\GatheredDlls\*.dll" C:\Windows\System32\ -RUN python C:\verify-host-dlls.py %HOST_VERSION% C:\GatheredDlls +RUN python C:\verify-host-dlls.py %HOST_BUILD% C:\GatheredDlls # Gather the required DirectX runtime files, since Windows Server Core does not include them RUN curl --progress-bar -L "https://download.microsoft.com/download/8/4/A/84A35BF1-DAFE-4AE8-82AF-AD2AE20B6B14/directx_Jun2010_redist.exe" --output %TEMP%\directx_redist.exe diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/verify-host-dlls.py b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/verify-host-dlls.py index f58c84a2..757a8442 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/verify-host-dlls.py +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/verify-host-dlls.py @@ -1,4 +1,5 @@ -import glob, os, platform, sys, win32api, winreg +import glob, os, sys, win32api + # Adapted from the code in this SO answer: def getDllVersion(dllPath): @@ -10,20 +11,10 @@ def getDllVersion(dllPath): info['FileVersionLS'] % 65536 ) -def getVersionRegKey(subkey): - key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion') - value = winreg.QueryValueEx(key, subkey) - winreg.CloseKey(key) - return value[0] - -def getOsVersion(): - version = platform.win32_ver()[1] - build = getVersionRegKey('BuildLabEx').split('.')[1] - return '{}.{}'.format(version, build) # Print the host and container OS build numbers print('Host OS build number: {}'.format(sys.argv[1])) -print('Container OS build number: {}'.format(getOsVersion())) +print('Container OS build number: {}'.format(sys.getwindowsversion().build)) sys.stdout.flush() # Verify each DLL file in the directory specified by our command-line argument diff --git a/ue4docker/info.py b/ue4docker/info.py index 2db9935f..03f95df3 100644 --- a/ue4docker/info.py +++ b/ue4docker/info.py @@ -4,7 +4,7 @@ def _osName(dockerInfo): if platform.system() == 'Windows': - return WindowsUtils.systemStringLong() + return WindowsUtils.systemString() elif platform.system() == 'Darwin': return DarwinUtils.systemString() else: diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 29284e8a..23df47aa 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -92,6 +92,7 @@ def addArguments(parser): parser.add_argument('--combine', action='store_true', help='Combine generated Dockerfiles into a single multi-stage build Dockerfile') parser.add_argument('--monitor', action='store_true', help='Monitor resource usage during builds (useful for debugging)') parser.add_argument('-interval', type=float, default=20.0, help='Sampling interval in seconds when resource monitoring has been enabled using --monitor (default is 20 seconds)') + parser.add_argument('--ignore-eol', action='store_true', help='Run builds even on EOL versions of Windows (advanced use only)') parser.add_argument('--ignore-blacklist', action='store_true', help='Run builds even on blacklisted versions of Windows (advanced use only)') parser.add_argument('-v', '--verbose', action='store_true', help='Enable verbose output during builds (useful for debugging)') @@ -150,6 +151,7 @@ def __init__(self, parser, argv): self.excludedComponents = set(self.args.exclude) self.baseImage = None self.prereqsTag = None + self.ignoreEOL = self.args.ignore_eol self.ignoreBlacklist = self.args.ignore_blacklist self.verbose = self.args.verbose self.layoutDir = self.args.layout @@ -226,30 +228,21 @@ def _generateWindowsConfig(self): self.dlldir = self.args.dlldir if self.args.dlldir is not None else self.defaultDllDir # Determine base tag for the Windows release of the host system - self.hostRelease = WindowsUtils.getWindowsRelease() - self.hostBasetag = WindowsUtils.getReleaseBaseTag(self.hostRelease) + self.hostBasetag = WindowsUtils.getWindowsRelease() # Store the tag for the base Windows Server Core image self.basetag = self.args.basetag if self.args.basetag is not None else self.hostBasetag self.baseImage = 'mcr.microsoft.com/windows/servercore:' + self.basetag self.prereqsTag = self.basetag - - # Verify that any user-specified base tag is valid - if WindowsUtils.isValidBaseTag(self.basetag) == False: - raise RuntimeError('unrecognised Windows Server Core base image tag "{}", supported tags are {}'.format(self.basetag, WindowsUtils.getValidBaseTags())) - - # Verify that any user-specified tag suffix does not collide with our base tags - if WindowsUtils.isValidBaseTag(self.suffix) == True: - raise RuntimeError('tag suffix cannot be any of the Windows Server Core base image tags: {}'.format(WindowsUtils.getValidBaseTags())) - + # If the user has explicitly specified an isolation mode then use it, otherwise auto-detect if self.args.isolation is not None: self.isolation = self.args.isolation else: # If we are able to use process isolation mode then use it, otherwise fallback to the Docker daemon's default isolation mode - differentKernels = WindowsUtils.isInsiderPreview() or self.basetag != self.hostBasetag - hostSupportsProcess = WindowsUtils.isWindowsServer() or int(self.hostRelease) >= 1809 + differentKernels = self.basetag != self.hostBasetag + hostSupportsProcess = WindowsUtils.supportsProcessIsolation() dockerSupportsProcess = parse_version(DockerUtils.version()['Version']) >= parse_version('18.09.0') if not differentKernels and hostSupportsProcess and dockerSupportsProcess: self.isolation = 'process' diff --git a/ue4docker/infrastructure/WindowsUtils.py b/ue4docker/infrastructure/WindowsUtils.py index 720eec50..29de7f9e 100644 --- a/ue4docker/infrastructure/WindowsUtils.py +++ b/ue4docker/infrastructure/WindowsUtils.py @@ -1,111 +1,87 @@ from .DockerUtils import DockerUtils -from .PackageUtils import PackageUtils from pkg_resources import parse_version -import os, platform +import platform, sys if platform.system() == 'Windows': import winreg -# Import the `semver` package even when the conflicting `node-semver` package is present -semver = PackageUtils.importFile('semver', os.path.join(PackageUtils.getPackageLocation('semver'), 'semver.py')) - class WindowsUtils(object): - - # The latest Windows build version we recognise as a non-Insider build - _latestReleaseBuild = 19042 - - # The list of Windows Server Core base image tags that we recognise, in ascending version number order - _validTags = ['ltsc2016', '1709', '1803', 'ltsc2019', '1903', '1909', '2004', '20H2'] - + # The list of Windows Server and Windows 10 host OS releases that are blacklisted due to critical bugs # (See: ) _blacklistedReleases = ['1903', '1909'] - + # The list of Windows Server Core container image releases that are unsupported due to having reached EOL _eolReleases = ['1709'] - + @staticmethod - def _getVersionRegKey(subkey): + def _getVersionRegKey(subkey : str) -> str: ''' Retrieves the specified Windows version key from the registry + + @raises FileNotFoundError if registry key doesn't exist ''' key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion') value = winreg.QueryValueEx(key, subkey) winreg.CloseKey(key) return value[0] - + @staticmethod - def requiredHostDlls(basetag): + def requiredHostDlls(basetag: str) -> [str]: ''' Returns the list of required host DLL files for the specified container image base tag ''' - + # `ddraw.dll` is only required under Windows Server 2016 version 1607 common = ['dsound.dll', 'opengl32.dll', 'glu32.dll'] return ['ddraw.dll'] + common if basetag == 'ltsc2016' else common - + @staticmethod - def requiredSizeLimit(): + def requiredSizeLimit() -> float: ''' Returns the minimum required image size limit (in GB) for Windows containers ''' return 400.0 - + @staticmethod - def minimumRequiredVersion(): + def minimumRequiredBuild() -> int: ''' Returns the minimum required version of Windows 10 / Windows Server, which is release 1607 - + (1607 is the first build to support Windows containers, as per: ) ''' - return '10.0.14393' - - @staticmethod - def systemStringShort(): - ''' - Generates a concise human-readable version string for the Windows host system - ''' - return 'Windows {} version {}'.format( - 'Server' if WindowsUtils.isWindowsServer() else '10', - WindowsUtils.getWindowsRelease() - ) - + return 14393 + @staticmethod - def systemStringLong(): + def systemString() -> str: ''' Generates a verbose human-readable version string for the Windows host system ''' - return '{} Version {} (OS Build {}.{})'.format( + return '{} Version {} (Build {}.{})'.format( WindowsUtils._getVersionRegKey('ProductName'), WindowsUtils.getWindowsRelease(), - WindowsUtils.getWindowsVersion()['patch'], + WindowsUtils.getWindowsBuild(), WindowsUtils._getVersionRegKey('UBR') ) - - @staticmethod - def getWindowsVersion(): - ''' - Returns the version information for the Windows host system as a semver instance - ''' - return semver.parse(platform.win32_ver()[1]) - + @staticmethod - def getWindowsRelease(): + def getWindowsRelease() -> str: ''' Determines the Windows 10 / Windows Server release (1607, 1709, 1803, etc.) of the Windows host system ''' - return WindowsUtils._getVersionRegKey('ReleaseId') - + try: + return WindowsUtils._getVersionRegKey('DisplayVersion') + except FileNotFoundError: + return WindowsUtils._getVersionRegKey('ReleaseId') + @staticmethod - def getWindowsBuild(): + def getWindowsBuild() -> int: ''' - Returns the full Windows version number as a string, including the build number + Returns build number for the Windows host system ''' - version = platform.win32_ver()[1] - build = WindowsUtils._getVersionRegKey('BuildLabEx').split('.')[1] - return '{}.{}'.format(version, build) - + return sys.getwindowsversion().build + @staticmethod def isBlacklistedWindowsVersion(release=None): ''' @@ -115,7 +91,7 @@ def isBlacklistedWindowsVersion(release=None): dockerVersion = parse_version(DockerUtils.version()['Version']) release = WindowsUtils.getWindowsRelease() if release is None else release return release in WindowsUtils._blacklistedReleases and dockerVersion < parse_version('19.03.6') - + @staticmethod def isEndOfLifeWindowsVersion(release=None): ''' @@ -124,68 +100,20 @@ def isEndOfLifeWindowsVersion(release=None): ''' release = WindowsUtils.getWindowsRelease() if release is None else release return release in WindowsUtils._eolReleases - - @staticmethod - def isSupportedWindowsVersion(): - ''' - Verifies that the Windows host system meets our minimum Windows version requirements - ''' - return semver.compare(platform.win32_ver()[1], WindowsUtils.minimumRequiredVersion()) >= 0 - + @staticmethod - def isWindowsServer(): + def isWindowsServer() -> bool: ''' Determines if the Windows host system is Windows Server ''' + # TODO: Replace this with something more reliable return 'Windows Server' in WindowsUtils._getVersionRegKey('ProductName') - - @staticmethod - def isInsiderPreview(): - ''' - Determines if the Windows host system is a Windows Insider preview build - ''' - version = WindowsUtils.getWindowsVersion() - return version['patch'] > WindowsUtils._latestReleaseBuild - - @staticmethod - def getReleaseBaseTag(release): - ''' - Retrieves the tag for the Windows Server Core base image matching the specified Windows 10 / Windows Server release - ''' - - # For Windows Insider preview builds, build the latest release tag - if WindowsUtils.isInsiderPreview(): - return WindowsUtils._validTags[-1] - - # This lookup table is based on the list of valid tags from - return { - '1709': '1709', - '1803': '1803', - '1809': 'ltsc2019', - '1903': '1903', - '1909': '1909', - '2004': '2004', - '2009': '20H2', - '20H2': '20H2' - }.get(release, 'ltsc2016') - - @staticmethod - def getValidBaseTags(): - ''' - Returns the list of valid tags for the Windows Server Core base image, in ascending chronological release order - ''' - return WindowsUtils._validTags - - @staticmethod - def isValidBaseTag(tag): - ''' - Determines if the specified tag is a valid Windows Server Core base image tag - ''' - return tag in WindowsUtils._validTags - + @staticmethod - def isNewerBaseTag(older, newer): + def supportsProcessIsolation() -> bool: ''' - Determines if the base tag `newer` is chronologically newer than the base tag `older` + Determines whether the Windows host system supports process isolation for containers + + @see https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/hyperv-container ''' - return WindowsUtils._validTags.index(newer) > WindowsUtils._validTags.index(older) + return WindowsUtils.isWindowsServer() or WindowsUtils.getWindowsBuild() >= 17763 diff --git a/ue4docker/main.py b/ue4docker/main.py index 75f47b2c..4a4bc3d7 100644 --- a/ue4docker/main.py +++ b/ue4docker/main.py @@ -32,8 +32,11 @@ def main(): _exitWithError('Error: could not detect Docker daemon version. Please ensure Docker is installed.\n\nError details: {}'.format(error)) # Under Windows, verify that the host is a supported version - if platform.system() == 'Windows' and WindowsUtils.isSupportedWindowsVersion() == False: - _exitWithError('Error: the detected version of Windows ({}) is not supported. Windows 10 / Windows Server version 1607 or newer is required.'.format(platform.win32_ver()[1])) + if platform.system() == 'Windows': + host_build = WindowsUtils.getWindowsBuild() + min_build = WindowsUtils.minimumRequiredBuild() + if host_build < min_build: + _exitWithError('Error: the detected build of Windows ({}) is not supported. {} or newer is required.'.format(host_build, min_build)) # Under macOS, verify that the host is a supported version if platform.system() == 'Darwin' and DarwinUtils.isSupportedMacOsVersion() == False: