diff --git a/DSC/Makefile b/DSC/Makefile index 5b106d4d7..09e684706 100644 --- a/DSC/Makefile +++ b/DSC/Makefile @@ -7,6 +7,7 @@ SOURCES = \ serializerfactory.py \ httpclient.py \ urllib2httpclient.py \ + urllib3httpclient.py \ dsc.py \ test \ HandlerManifest.json \ diff --git a/DSC/README.md b/DSC/README.md index 4300f001d..3c9589a86 100644 --- a/DSC/README.md +++ b/DSC/README.md @@ -357,13 +357,12 @@ $publicConfig = '{ ``` ## 4. Supported Linux Distributions -- Ubuntu 14.04 LTS, 16.04 LTS, 18.04 LTS -- Debian 8 +- Ubuntu 14.04 LTS, 16.04 LTS, 18.04 LTS and 20.04 LTS +- Debian 8, 9 and 10 - Oracle Linux 6 and 7 -- CentOS 6 and 7 -- RHEL 6 and 7 -- openSUSE 13.1 and 42.5 -- SUSE Linux Enterprise Server 11 SP3 and 12 +- CentOS 6, 7 and 8 +- RHEL 6, 7 and 8 +- SUSE Linux Enterprise Server 12 and 15 ## 5. Debug * The status of the extension is reported back to Azure so that user can see the status on Azure Portal diff --git a/DSC/dsc.py b/DSC/dsc.py index 0c5928097..cb1dce968 100644 --- a/DSC/dsc.py +++ b/DSC/dsc.py @@ -22,9 +22,14 @@ import subprocess import sys import traceback -import urllib.request -import urllib.error -import urllib.parse +try: + from urllib.parse import urlparse, urlencode + from urllib.request import urlopen, Request + from urllib.error import HTTPError +except ImportError: + from urlparse import urlparse + from urllib import urlencode + from urllib2 import urlopen, Request, HTTPError import time import platform import json @@ -32,6 +37,7 @@ import serializerfactory import httpclient import urllib2httpclient +import urllib3httpclient import httpclientfactory from azure.storage import BlobService @@ -44,20 +50,23 @@ ExtensionShortName = 'DSCForLinux' DownloadDirectory = 'download' -omi_package_prefix = 'packages/omi-1.4.2-5.ssl_' -dsc_package_prefix = 'packages/dsc-1.1.1-926.ssl_' +omi_package_prefix = 'packages/omi-1.6.9-1.ssl_nnnnnnnnnnnnnnnn' +dsc_package_prefix = 'packages/dsc-1.2.3-0.ssl_nnnnnnnnnnnnnnnnnnnnn' omi_major_version = 1 -omi_minor_version = 4 -omi_build = 2 -omi_release = 5 +omi_minor_version = 6 +omi_build = 9 +omi_release = 1 dsc_major_version = 1 -dsc_minor_version = 1 -dsc_build = 1 -dsc_release = 926 +dsc_minor_version = 2 +dsc_build = 3 +dsc_release = 0 package_pattern = '(\d+).(\d+).(\d+).(\d+)' nodeid_path = '/etc/opt/omi/conf/dsc/agentid' date_time_format = "%Y-%m-%dT%H:%M:%SZ" -extension_handler_version = "2.71.1.0" +extension_handler_version = "3.0.0.6" +python_command = 'python3' if sys.version_info >= (3,0) else 'python' +dsc_script_path = '/opt/microsoft/dsc/Scripts/python3' if sys.version_info >= (3,0) else '/opt/microsoft/dsc/Scripts' +space_string = " " # Error codes UnsupportedDistro = 51 #excludes from SLA @@ -111,8 +120,9 @@ def main(): protected_settings = {} global distro_category - distro_category = get_distro_category() - check_supported_OS() + vm_supported, vm_dist, vm_ver = check_supported_OS() + distro_category = get_distro_category(vm_dist.lower(), vm_ver.lower()) + for a in sys.argv[1:]: if re.match("^([-/]*)(disable)", a): @@ -127,15 +137,12 @@ def main(): update() -def get_distro_category(): - distro_info = platform.dist() - distro_name = distro_info[0].lower() - distro_version = distro_info[1] - if distro_name == 'ubuntu' or (distro_name == 'debian'): +def get_distro_category(distro_name,distro_version): + if distro_name.startswith('ubuntu') or (distro_name.startswith('debian')): return DistroCategory.debian - elif distro_name == 'centos' or distro_name == 'redhat' or distro_name == 'oracle': + elif distro_name.startswith('centos') or distro_name.startswith('redhat') or distro_name.startswith('oracle') or distro_name.startswith('red hat'): return DistroCategory.redhat - elif distro_name == 'suse': + elif distro_name.startswith('suse') or distro_name.startswith('sles'): return DistroCategory.suse waagent.AddExtensionEvent(name=ExtensionShortName, op='InstallInProgress', isSuccess=True, message="Unsupported distro :" + distro_name + "; distro_version: " + distro_version) hutil.do_exit(UnsupportedDistro, 'Install', 'error', str(UnsupportedDistro), distro_name + 'is not supported.') @@ -148,21 +155,41 @@ def check_supported_OS(): digits must match. All other distros not supported will get error code 51 """ - supported_dists = {'redhat' : ['6', '7'], # CentOS - 'centos' : ['6', '7'], # CentOS - 'red hat' : ['6', '7'], # Redhat - 'debian' : ['8'], # Debian - 'ubuntu' : ['14.04', '16.04', '18.04'], # Ubuntu + supported_dists = {'redhat' : ['6', '7', '8'], # CentOS + 'centos' : ['6', '7', '8'], # CentOS + 'red hat' : ['6', '7', '8'], # Redhat + 'debian' : ['8', '9', '10'], # Debian + 'ubuntu' : ['14.04', '16.04', '18.04', '20.04'], # Ubuntu 'oracle' : ['6', '7'], # Oracle - 'suse' : ['11', '12'], #SLES - 'opensuse' : ['13', '42.3'] #OpenSuse + 'suse' : ['12', '15'], #SLES + 'sles' : ['12', '15'] } - vm_supported = False - + vm_dist, vm_ver, vm_supported = '', '', False + try: vm_dist, vm_ver, vm_id = platform.linux_distribution() except AttributeError: - vm_dist, vm_ver, vm_id = platform.dist() + try: + vm_dist, vm_ver, vm_id = platform.dist() + except: + waagent.Log("Falling back to /etc/os-release distribution parsing") + + # Fallback if either of the above fail; on some (especially newer) + # distros, linux_distribution() and dist() are unreliable or deprecated + if not vm_dist and not vm_ver: + try: + with open('/etc/os-release', 'r') as fp: + for line in fp: + if line.startswith('ID='): + vm_dist = line.split('=')[1] + vm_dist = vm_dist.split('-')[0] + vm_dist = vm_dist.replace('\"', '').replace('\n', '') + elif line.startswith('VERSION_ID='): + vm_ver = line.split('=')[1] + vm_ver = vm_ver.replace('\"', '').replace('\n', '') + except: + waagent.Log('Indeterminate operating system') + return vm_supported, 'Indeterminate operating system', '' # Find this VM distribution in the supported list for supported_dist in supported_dists.keys(): @@ -193,6 +220,8 @@ def check_supported_OS(): if not vm_supported: waagent.AddExtensionEvent(name=ExtensionShortName, op='InstallInProgress', isSuccess=True, message="Unsupported OS :" + vm_dist + "; distro_version: " + vm_ver) hutil.do_exit(UnsupportedDistro, 'Install', 'error', str(UnsupportedDistro), vm_dist + "; distro_version: " + vm_ver + ' is not supported.') + + return vm_supported, vm_dist, vm_ver def install(): @@ -204,7 +233,7 @@ def install(): install_dsc_packages() waagent.AddExtensionEvent(name=ExtensionShortName, op='InstallInProgress', isSuccess=True, message="successfully installed DSCForLinux extension") - hutil.do_exit(0, 'Install', 'success', '0', 'Install Succeeded.') + hutil.do_exit(1, 'Install', 'success', '0', 'Install Succeeded.') except Exception as e: waagent.AddExtensionEvent(name=ExtensionShortName, op='InstallInProgress', isSuccess=True, message="failed to install DSC extension with error: {0} and stacktrace: {1}".format( @@ -221,7 +250,7 @@ def enable(): start_omiservice() mode = get_config('Mode') if mode == '': - mode = get_config('ExtensionAction') + mode = get_config___('ExtensionAction') waagent.AddExtensionEvent(name=ExtensionShortName, op='EnableInProgress', isSuccess=True, message="Enabling the DSC extension - mode/ExtensionAction: " + mode) if mode == '': @@ -237,7 +266,7 @@ def enable(): if mode == Mode.remove: remove_module() elif mode == Mode.register: - registration_key = get_config('RegistrationKey') + registration_key = get_config__('RegistrationKey') registation_url = get_config('RegistrationUrl') # Optional node_configuration_name = get_config('NodeConfigurationName') @@ -292,7 +321,7 @@ def enable(): message="(03107)Failed to apply meta MOF configuration through Pull Mode") hutil.do_exit(1, 'Enable', 'error', '1', 'Enable failed. ' + current_config) - hutil.do_exit(0, 'Enable', 'success', '0', 'Enable Succeeded') + hutil.do_exit(1, 'Enable', 'success', '0', 'Enable Succeeded') except Exception as e: waagent.AddExtensionEvent(name=ExtensionShortName, op='EnableInProgress', isSuccess=True, message="Enable failed with the error: {0}, stacktrace: {1} ".format(str(e), @@ -309,7 +338,7 @@ def send_heart_beat_msg_to_agent_service(status_event_type): while retry_count <= 5 and canRetry: waagent.AddExtensionEvent(name=ExtensionShortName, op='HeartBeatInProgress', isSuccess=True, message="In send_heart_beat_msg_to_agent_service method") - code, output, stderr = run_cmd("python /opt/microsoft/dsc/Scripts/GetDscLocalConfigurationManager.py") + code, output, stderr = run_cmd( python_command + space_string + dsc_script_path + "/GetDscLocalConfigurationManager.py") if code == 0 and "RefreshMode=Pull" in output: waagent.AddExtensionEvent(name=ExtensionShortName, op='HeartBeatInProgress', isSuccess=True, message="sends heartbeat message in pullmode") @@ -360,14 +389,41 @@ def construct_node_extension_properties(lcmconfig, status_event_type): waagent.AddExtensionEvent(name=ExtensionShortName, op='HeartBeatInProgress', isSuccess=True, message="Getting properties") OMSCLOUD_ID = get_omscloudid() - distro_info = platform.dist() - if len(distro_info[1].split('.')) == 1: - major_version = distro_info[1].split('.')[0] - minor_version = 0 - if len(distro_info[1].split('.')) >= 2: - major_version = distro_info[1].split('.')[0] - minor_version = distro_info[1].split('.')[1] + + vm_dist, vm_ver, vm_id = '', '', '' + + try: + vm_dist, vm_ver, vm_id = platform.linux_distribution() + except AttributeError: + try: + vm_dist, vm_ver, vm_id = platform.dist() + except AttributeError: + waagent.Log("Falling back to /etc/os-release distribution parsing") + # Fallback if either of the above fail; on some (especially newer) + # distros, linux_distribution() and dist() are unreliable or deprecated + if not vm_dist and not vm_ver: + try: + with open('/etc/os-release', 'r') as fp: + for line in fp: + if line.startswith('ID='): + vm_dist = line.split('=')[1] + vm_dist = vm_dist.split('-')[0] + vm_dist = vm_dist.replace('\"', '').replace('\n', '') + elif line.startswith('VERSION_ID='): + vm_ver = line.split('=')[1] + vm_ver = vm_ver.replace('\"', '').replace('\n', '') + except: + waagent.Log('Indeterminate operating system') + vm_dist, vm_ver, vm_id = "Indeterminate operating system", "","" + + if len(vm_ver.split('.')) == 1: + major_version = vm_ver.split('.')[0] + minor_version = 0 + if len(vm_ver.split('.')) >= 2: + major_version = vm_ver.split('.')[0] + minor_version = vm_ver.split('.')[1] + VMUUID = get_vmuuid() node_config_names = get_lcm_config_setting('ConfigurationNames', lcmconfig) configuration_mode = get_lcm_config_setting("ConfigurationMode", lcmconfig) @@ -390,7 +446,7 @@ def construct_node_extension_properties(lcmconfig, status_event_type): "Version": extension_handler_version }, "OSProfile": { - "Name": distro_info[0], + "Name": vm_dist, "Type": "Linux", "MinorVersion": minor_version, "MajorVersion": major_version, @@ -447,6 +503,8 @@ def run_cmd(cmd): proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, close_fds=True) exit_code = proc.wait() stdout, stderr = proc.communicate() + stdout = stdout.decode("ISO-8859-1") if isinstance(stdout, bytes) else stdout + stderr = stderr.decode("ISO-8859-1") if isinstance(stderr, bytes) else stderr return exit_code, stdout, stderr def run_dpkg_cmd_with_retry(cmd): @@ -597,7 +655,7 @@ def rpm_install_pkg(package_path, package_name, major_version, minor_version, bu else: waagent.AddExtensionEvent(name=ExtensionShortName, op='InstallInProgress', isSuccess=True, message="Failed to install RPM package :" + package_path) - raise Exception('Failed to install package {0}: stdout: {1}, stderr: {2}'.format(package_name, output, stderr)) + raise Exception('Failed to install package {0}: stdout: {1}, stderr: {2}'.format(package_name, output, stderr)) def deb_install_pkg(package_path, package_name, major_version, minor_version, build, release, install_options): @@ -639,7 +697,7 @@ def zypper_package_install(package): else: waagent.AddExtensionEvent(name=ExtensionShortName, op='InstallInProgress', isSuccess=True, message="Failed to install zypper package :" + package) - raise Exception('Failed to install package {0}: stdout: {1}, stderr: {2}'.format(package, output, stderr)) + raise Exception('Failed to install package {0}: stdout: {1}, stderr: {2}'.format(package, output, stderr)) def yum_package_install(package): @@ -666,15 +724,14 @@ def apt_package_install(package): def get_openssl_version(): cmd_result = waagent.RunGetOutput("openssl version") + cmd_result = cmd_result.decode() if isinstance(cmd_result, bytes) else cmd_result openssl_version = cmd_result[1].split()[1] if re.match('^1.0.*', openssl_version): return '100' - elif re.match('^0.9.8*', openssl_version): - return '098' elif re.match('^1.1.*', openssl_version): return '110' else: - error_msg = 'This system does not have a supported version of OpenSSL installed. Supported version: 0.9.8*, 1.0.*, 1.1.*' + error_msg = 'This system does not have a supported version of OpenSSL installed. Supported version: 1.0.*, 1.1.*' hutil.error(error_msg) waagent.AddExtensionEvent(name=ExtensionShortName, op='InstallInProgress', isSuccess=True, message="System doesn't have supported OpenSSL version:" + openssl_version) @@ -773,12 +830,12 @@ def parse_blob_uri(blob_uri): def get_path_from_uri(uri): - uri = urllib.parse.urlparse(uri) + uri = urlparse(uri) return uri.path def get_host_base_from_uri(blob_uri): - uri = urllib.parse.urlparse(blob_uri) + uri = urlparse(blob_uri) netloc = uri.netloc if netloc is None: return None @@ -813,7 +870,7 @@ def download_external_file(file_uri, download_dir): def download_and_save_file(uri, file_path): - src = urllib.request.urlopen(uri) + src = urlopen(uri) dest = open(file_path, 'wb') buf_size = 1024 buf = src.read(buf_size) @@ -833,12 +890,12 @@ def prepare_download_dir(seq_no): def apply_dsc_configuration(config_file_path): - cmd = '/opt/microsoft/dsc/Scripts/StartDscConfiguration.py -configurationmof ' + config_file_path + cmd = dsc_script_path + '/StartDscConfiguration.py -configurationmof ' + config_file_path waagent.AddExtensionEvent(name=ExtensionShortName, op='EnableInProgress', isSuccess=True, message='running the cmd: ' + cmd) code, output, stderr = run_cmd(cmd) if code == 0: - code, output, stderr = run_cmd('/opt/microsoft/dsc/Scripts/GetDscConfiguration.py') + code, output, stderr = run_cmd(dsc_script_path + '/GetDscConfiguration.py') return output else: error_msg = 'Failed to apply MOF configuration: stdout: {0}, stderr: {1}'.format(output, stderr) @@ -848,12 +905,12 @@ def apply_dsc_configuration(config_file_path): def apply_dsc_meta_configuration(config_file_path): - cmd = '/opt/microsoft/dsc/Scripts/SetDscLocalConfigurationManager.py -configurationmof ' + config_file_path + cmd = dsc_script_path + '/SetDscLocalConfigurationManager.py -configurationmof ' + config_file_path waagent.AddExtensionEvent(name=ExtensionShortName, op='EnableInProgress', isSuccess=True, message='running the cmd: ' + cmd) code, output, stderr = run_cmd(cmd) if code == 0: - code, output, stderr = run_cmd('/opt/microsoft/dsc/Scripts/GetDscLocalConfigurationManager.py') + code, output, stderr = run_cmd(dsc_script_path + '/GetDscLocalConfigurationManager.py') return output else: error_msg = 'Failed to apply Meta MOF configuration: stdout: {0}, stderr: {1}'.format(output, stderr) @@ -986,7 +1043,7 @@ def check_dsc_configuration(current_config): def install_module(file_path): install_package('unzip') - cmd = '/opt/microsoft/dsc/Scripts/InstallModule.py ' + file_path + cmd = dsc_script_path + '/InstallModule.py ' + file_path code, output, stderr = run_cmd(cmd) waagent.AddExtensionEvent(name=ExtensionShortName, op="InstallModuleInProgress", @@ -1008,7 +1065,7 @@ def install_module(file_path): def remove_module(): module_name = get_config('ResourceName') - cmd = '/opt/microsoft/dsc/Scripts/RemoveModule.py ' + module_name + cmd = dsc_script_path + '/RemoveModule.py ' + module_name code, output, stderr = run_cmd(cmd) waagent.AddExtensionEvent(name=ExtensionShortName, op="RemoveModuleInProgress", @@ -1086,7 +1143,7 @@ def register_automation(registration_key, registation_url, node_configuration_na hutil.error(err_msg + "It should be one of the values : (ApplyAndMonitor | ApplyAndAutoCorrect | ApplyOnly)") waagent.AddExtensionEvent(name=ExtensionShortName, op='RegisterInProgress', isSuccess=True, message=err_msg) return 51, err_msg - cmd = '/opt/microsoft/dsc/Scripts/Register.py' + ' --RegistrationKey ' + registration_key \ + cmd = dsc_script_path + '/Register.py' + ' --RegistrationKey ' + registration_key \ + ' --ServerURL ' + registation_url optional_parameters = "" if node_configuration_name != '': diff --git a/DSC/extension_shim.sh b/DSC/extension_shim.sh index 2ee4d3d80..ba182a352 100755 --- a/DSC/extension_shim.sh +++ b/DSC/extension_shim.sh @@ -4,6 +4,16 @@ COMMAND="" PYTHON="" +# We are writing logs to error stream in extension_shim.sh as the logs written to output stream are being overriden by HandlerUtil.py. This has been done as part of OMIGOD hotfix +# Default variables for OMI Package Upgrade +REQUIRED_OMI_VERSION="1.6.9.1" +INSTALLED_OMI_VERSION="" +UPGRADED_OMI_VERSION="" +OPENSSL_VERSION="" +OMI_PACKAGE_PREFIX='packages/omi-1.6.9-1.ssl_' +OMI_PACKAGE_PATH="" +OMI_SERVICE_STATE="" + USAGE="$(basename "$0") [-h] [-i|--install] [-u|--uninstall] [-d|--disable] [-e|--enable] [-p|--update] Program to find the installed python on the box and invoke a Python extension script. @@ -34,17 +44,164 @@ python hello.py --install function find_python(){ local python_exec_command=$1 - # Check if there is python defined. - if command -v python >/dev/null 2>&1 ; then - eval ${python_exec_command}="python" + # Check if there is python2 defined. + if command -v python2 >/dev/null 2>&1 ; then + eval ${python_exec_command}="python2" else - # Python was not found. Searching for Python3 now. - if command -v python >/dev/null 2>&1 ; then + # Python2 was not found. Searching for Python3 now. + if command -v python3 >/dev/null 2>&1 ; then eval ${python_exec_command}="python3" fi fi } +function get_openssl_version(){ + openssl=`openssl version | awk '{print $2}'` + if [[ ${openssl} =~ ^1.0.* ]]; then + OPENSSL_VERSION="100" + else + if [[ ${openssl} =~ ^1.1.* ]]; then + OPENSSL_VERSION="110" + else + if [[ ${openssl} =~ ^0.9.8* ]]; then + OPENSSL_VERSION="098" + fi + fi + fi +} + +function start_omiservice(){ + echo "Attempting to start OMI service" >&2 + RESULT=`/opt/omi/bin/service_control start >/dev/null 2>&1` + RESULT=`service omid status >/dev/null 2>&1` + if [ $? -eq 0 ]; then + echo "OMI service succesfully started." >&2 + else + echo "OMI service could not be started." >&2 + fi +} + +function stop_omiservice(){ + echo "Attempting to stop OMI service" >&2 + RESULT=`/opt/omi/bin/service_control stop >/dev/null 2>&1` + RESULT=`service omid status >/dev/null 2>&1` + if [ $? -eq 3 ]; then + echo "OMI service succesfully stopped." >&2 + else + echo "OMI service could not be stopped." >&2 + fi +} + +function compare_versions(){ + if [[ $1 == $2 ]] + then + return 0 + fi + local IFS=. + local i v1=($1) v2=($2) + for ((i=0; i<${#v1[@]}; i++)) + do + if ((${v1[i]} > ${v2[i]})) + then + return 1 + fi + if ((${v1[i]} < ${v2[i]})) + then + return 2 + fi + done + return 0 +} + +function ensure_required_omi_version_exists(){ + # Populate SSL Version + get_openssl_version + + echo "Checking if OMI is installed. Required OMI version: ${REQUIRED_OMI_VERSION};" >&2 + + # Check if RPM exists + if command -v rpm >/dev/null 2>&1 ; then + echo "Package Manager Type: RPM" >&2 + INSTALLED_OMI_VERSION=`rpm -q --queryformat "%{VERSION}.%{RELEASE}" omi 2>&1` + if [ -z "$INSTALLED_OMI_VERSION" -o "$INSTALLED_OMI_VERSION" = "package omi is not installed" ]; then + echo "OMI is not installed on the machine." >&2 + else + RESULT=`service omid status >/dev/null 2>&1` + OMI_SERVICE_STATE=$? + echo "OMI is already installed. Installed OMI version: ${INSTALLED_OMI_VERSION}; OMI Service State: ${OMI_SERVICE_STATE};" >&2 # Add current running status + compare_versions ${INSTALLED_OMI_VERSION} ${REQUIRED_OMI_VERSION} + if [ $? -eq 2 ]; then + OMI_PACKAGE_PATH="${OMI_PACKAGE_PREFIX}${OPENSSL_VERSION}.x64.rpm" + echo "Installed OMI version is lower than the Required OMI version. Trying to upgrade." >&2 + if [ -f ${OMI_PACKAGE_PATH} ]; then + echo "The OMI package exists at ${OMI_PACKAGE_PATH}. Using this to upgrade." >&2 + stop_omiservice + RESULT=`rpm -Uvh ${OMI_PACKAGE_PATH} >/dev/null 2>&1` + if [ $? -eq 0 ]; then + UPGRADED_OMI_VERSION=`rpm -q --queryformat "%{VERSION}.%{RELEASE}" omi 2>&1` + echo "Succesfully upgraded the OMI. Installed: ${INSTALLED_OMI_VERSION}; Required: ${REQUIRED_OMI_VERSION}; Upgraded: ${UPGRADED_OMI_VERSION};" >&2 + else + echo "Failed to upgrade the OMI. Installed: ${INSTALLED_OMI_VERSION}; Required: ${REQUIRED_OMI_VERSION};" >&2 + fi + # Start OMI only if previous state was running + if [ $OMI_SERVICE_STATE -eq 0 ]; then + start_omiservice + fi + else + echo "The OMI package does not exists at ${OMI_PACKAGE_PATH}. Skipping upgrade." >&2 + fi + else + echo "Installed OMI version is equal to or greater than the Required OMI version. No action needed." >&2 + fi + fi + INSTALLED_OMI_VERSION=`rpm -q --queryformat "%{VERSION}.%{RELEASE}" omi 2>&1` + RESULT=`service omid status >/dev/null 2>&1` + OMI_SERVICE_STATE=$? + echo "OMI upgrade is complete. Installed OMI version: ${INSTALLED_OMI_VERSION}; OMI Service State: ${OMI_SERVICE_STATE};" >&2 + else + # Check if DPKG exists + if command -v dpkg >/dev/null 2>&1 ; then + echo "Package Manager Type: DPKG" >&2 + INSTALLED_OMI_VERSION=`dpkg -s omi 2>&1 | grep Version: | awk '{print $2}'` + if [ -z "$INSTALLED_OMI_VERSION" -o "$INSTALLED_OMI_VERSION" = "package omi is not installed" ]; then + echo "OMI is not installed on the machine." >&2 + else + RESULT=`service omid status >/dev/null 2>&1` + OMI_SERVICE_STATE=$? + echo "OMI is already installed. Installed OMI version: ${INSTALLED_OMI_VERSION}; OMI Service State: ${OMI_SERVICE_STATE};" >&2 + compare_versions ${INSTALLED_OMI_VERSION} ${REQUIRED_OMI_VERSION} + if [ $? -eq 2 ]; then + OMI_PACKAGE_PATH="${OMI_PACKAGE_PREFIX}${OPENSSL_VERSION}.x64.deb" + echo "Installed OMI version is lower than the Required OMI version. Trying to upgrade." >&2 + if [ -f ${OMI_PACKAGE_PATH} ]; then + echo "The OMI package exists at ${OMI_PACKAGE_PATH}. Using this to upgrade." >&2 + stop_omiservice + RESULT=`dpkg -i --force-confold --force-confdef --refuse-downgrade ${OMI_PACKAGE_PATH} >/dev/null 2>&1` + if [ $? -eq 0 ]; then + UPGRADED_OMI_VERSION=`dpkg -s omi 2>&1 | grep Version: | awk '{print $2}'` + echo "Succesfully upgraded the OMI. Installed: ${INSTALLED_OMI_VERSION}; Required: ${REQUIRED_OMI_VERSION}; Upgraded: ${UPGRADED_OMI_VERSION};" >&2 + else + echo "Failed to upgrade the OMI. Installed: ${INSTALLED_OMI_VERSION}; Required: ${REQUIRED_OMI_VERSION};" >&2 + fi + # Start OMI only if previous state was running + if [ $OMI_SERVICE_STATE -eq 0 ]; then + start_omiservice + fi + else + echo "The OMI package does not exists at ${OMI_PACKAGE_PATH}. Skipping upgrade." >&2 + fi + else + echo "Installed OMI version is equal to or greater than the Required OMI version. No action needed." >&2 + fi + fi + INSTALLED_OMI_VERSION=`dpkg -s omi 2>&1 | grep Version: | awk '{print $2}'` + RESULT=`service omid status >/dev/null 2>&1` + OMI_SERVICE_STATE=$? + echo "OMI upgrade is complete. Installed OMI version: ${INSTALLED_OMI_VERSION}; OMI Service State: ${OMI_SERVICE_STATE};" >&2 + fi + fi +} + # Transform long options to short ones for getopts support (getopts doesn't support long args) for arg in "$@"; do shift @@ -99,6 +256,9 @@ done shift $((OPTIND-1)) +# Ensure OMI package if exists is of required version. +ensure_required_omi_version_exists + # If find_python is not able to find a python installed, $PYTHON will be null. find_python PYTHON diff --git a/DSC/httpclientfactory.py b/DSC/httpclientfactory.py index 2e1909773..521666fce 100644 --- a/DSC/httpclientfactory.py +++ b/DSC/httpclientfactory.py @@ -5,7 +5,7 @@ import os from curlhttpclient import CurlHttpClient -from urllib2httpclient import Urllib2HttpClient + PY_MAJOR_VERSION = 0 PY_MINOR_VERSION = 1 @@ -18,6 +18,7 @@ class HttpClientFactory: Targets : [2.4.0 - 2.7.9[ : CurlHttpclient [2.7.9 - 2.7.9+ : Urllib2Httpclient + 3.0+ : Urllib3Httpclient This is due to the lack of built-in strict certificate verification prior to 2.7.9. The ssl module was also unavailable for [2.4.0 - 2.6.0[. @@ -41,10 +42,16 @@ def create_http_client(self, version_info): An instance of CurlHttpClient if the installed Python version is below 2.7.9 An instance of Urllib2 if the installed Python version is or is above 2.7.9 """ - if version_info[PY_MAJOR_VERSION] == 2 and version_info[PY_MINOR_VERSION] < 7: + if version_info[PY_MAJOR_VERSION] == 3: + from urllib3httpclient import Urllib3HttpClient + return Urllib3HttpClient(self.cert, self.key, self.insecure, self.proxy_configuration) + elif version_info[PY_MAJOR_VERSION] == 2 and version_info[PY_MINOR_VERSION] < 7: + from urllib2httpclient import Urllib2HttpClient return CurlHttpClient(self.cert, self.key, self.insecure, self.proxy_configuration) elif version_info[PY_MAJOR_VERSION] == 2 and version_info[PY_MINOR_VERSION] <= 7 and version_info[ PY_MICRO_VERSION] < 9: + from urllib2httpclient import Urllib2HttpClient return CurlHttpClient(self.cert, self.key, self.insecure, self.proxy_configuration) else: + from urllib2httpclient import Urllib2HttpClient return Urllib2HttpClient(self.cert, self.key, self.insecure, self.proxy_configuration) \ No newline at end of file diff --git a/DSC/packages/dsc-1.1.1-926.ssl_098.x64.deb b/DSC/packages/dsc-1.1.1-926.ssl_098.x64.deb deleted file mode 100644 index 73035b7ab..000000000 Binary files a/DSC/packages/dsc-1.1.1-926.ssl_098.x64.deb and /dev/null differ diff --git a/DSC/packages/dsc-1.1.1-926.ssl_098.x64.rpm b/DSC/packages/dsc-1.1.1-926.ssl_098.x64.rpm deleted file mode 100644 index 8fdc39848..000000000 Binary files a/DSC/packages/dsc-1.1.1-926.ssl_098.x64.rpm and /dev/null differ diff --git a/DSC/packages/dsc-1.1.1-926.ssl_100.x64.deb b/DSC/packages/dsc-1.1.1-926.ssl_100.x64.deb deleted file mode 100644 index c536beea8..000000000 Binary files a/DSC/packages/dsc-1.1.1-926.ssl_100.x64.deb and /dev/null differ diff --git a/DSC/packages/dsc-1.1.1-926.ssl_100.x64.rpm b/DSC/packages/dsc-1.1.1-926.ssl_100.x64.rpm deleted file mode 100644 index 8182719bb..000000000 Binary files a/DSC/packages/dsc-1.1.1-926.ssl_100.x64.rpm and /dev/null differ diff --git a/DSC/packages/dsc-1.1.1-926.ssl_110.x64.deb b/DSC/packages/dsc-1.1.1-926.ssl_110.x64.deb deleted file mode 100644 index ea506bbcf..000000000 Binary files a/DSC/packages/dsc-1.1.1-926.ssl_110.x64.deb and /dev/null differ diff --git a/DSC/packages/dsc-1.1.1-926.ssl_110.x64.rpm b/DSC/packages/dsc-1.1.1-926.ssl_110.x64.rpm deleted file mode 100644 index 0fd70e534..000000000 Binary files a/DSC/packages/dsc-1.1.1-926.ssl_110.x64.rpm and /dev/null differ diff --git a/DSC/packages/dsc-1.2.3-0.ssl_100.x64.deb b/DSC/packages/dsc-1.2.3-0.ssl_100.x64.deb new file mode 100644 index 000000000..edb471b9c Binary files /dev/null and b/DSC/packages/dsc-1.2.3-0.ssl_100.x64.deb differ diff --git a/DSC/packages/dsc-1.2.3-0.ssl_100.x64.rpm b/DSC/packages/dsc-1.2.3-0.ssl_100.x64.rpm new file mode 100644 index 000000000..e66783b92 Binary files /dev/null and b/DSC/packages/dsc-1.2.3-0.ssl_100.x64.rpm differ diff --git a/DSC/packages/dsc-1.2.3-0.ssl_110.x64.deb b/DSC/packages/dsc-1.2.3-0.ssl_110.x64.deb new file mode 100644 index 000000000..34e392a58 Binary files /dev/null and b/DSC/packages/dsc-1.2.3-0.ssl_110.x64.deb differ diff --git a/DSC/packages/dsc-1.2.3-0.ssl_110.x64.rpm b/DSC/packages/dsc-1.2.3-0.ssl_110.x64.rpm new file mode 100644 index 000000000..0efa0e609 Binary files /dev/null and b/DSC/packages/dsc-1.2.3-0.ssl_110.x64.rpm differ diff --git a/DSC/packages/omi-1.4.2-5.ssl_098.x64.deb b/DSC/packages/omi-1.4.2-5.ssl_098.x64.deb deleted file mode 100644 index 3af6a0089..000000000 Binary files a/DSC/packages/omi-1.4.2-5.ssl_098.x64.deb and /dev/null differ diff --git a/DSC/packages/omi-1.4.2-5.ssl_098.x64.rpm b/DSC/packages/omi-1.4.2-5.ssl_098.x64.rpm deleted file mode 100644 index c5b61000a..000000000 Binary files a/DSC/packages/omi-1.4.2-5.ssl_098.x64.rpm and /dev/null differ diff --git a/DSC/packages/omi-1.4.2-5.ssl_100.x64.deb b/DSC/packages/omi-1.4.2-5.ssl_100.x64.deb deleted file mode 100644 index 30e84c721..000000000 Binary files a/DSC/packages/omi-1.4.2-5.ssl_100.x64.deb and /dev/null differ diff --git a/DSC/packages/omi-1.4.2-5.ssl_100.x64.rpm b/DSC/packages/omi-1.4.2-5.ssl_100.x64.rpm deleted file mode 100644 index cfc72a225..000000000 Binary files a/DSC/packages/omi-1.4.2-5.ssl_100.x64.rpm and /dev/null differ diff --git a/DSC/packages/omi-1.4.2-5.ssl_110.x64.deb b/DSC/packages/omi-1.4.2-5.ssl_110.x64.deb deleted file mode 100644 index a21d45ef5..000000000 Binary files a/DSC/packages/omi-1.4.2-5.ssl_110.x64.deb and /dev/null differ diff --git a/DSC/packages/omi-1.4.2-5.ssl_110.x64.rpm b/DSC/packages/omi-1.4.2-5.ssl_110.x64.rpm deleted file mode 100644 index b13136fc1..000000000 Binary files a/DSC/packages/omi-1.4.2-5.ssl_110.x64.rpm and /dev/null differ diff --git a/DSC/packages/omi-1.6.9-1.ssl_100.x64.deb b/DSC/packages/omi-1.6.9-1.ssl_100.x64.deb new file mode 100644 index 000000000..ae3142359 Binary files /dev/null and b/DSC/packages/omi-1.6.9-1.ssl_100.x64.deb differ diff --git a/DSC/packages/omi-1.6.9-1.ssl_100.x64.rpm b/DSC/packages/omi-1.6.9-1.ssl_100.x64.rpm new file mode 100644 index 000000000..0e6b32cd1 Binary files /dev/null and b/DSC/packages/omi-1.6.9-1.ssl_100.x64.rpm differ diff --git a/DSC/packages/omi-1.6.9-1.ssl_110.x64.deb b/DSC/packages/omi-1.6.9-1.ssl_110.x64.deb new file mode 100644 index 000000000..be2d22ac6 Binary files /dev/null and b/DSC/packages/omi-1.6.9-1.ssl_110.x64.deb differ diff --git a/DSC/packages/omi-1.6.9-1.ssl_110.x64.rpm b/DSC/packages/omi-1.6.9-1.ssl_110.x64.rpm new file mode 100644 index 000000000..cb5958674 Binary files /dev/null and b/DSC/packages/omi-1.6.9-1.ssl_110.x64.rpm differ diff --git a/DSC/urllib2httpclient.py b/DSC/urllib2httpclient.py index 80ecd06ec..6e2a18e93 100644 --- a/DSC/urllib2httpclient.py +++ b/DSC/urllib2httpclient.py @@ -5,11 +5,22 @@ """Urllib2 HttpClient.""" -import http.client +try: + from http.client import HTTPSConnection +except ImportError: + from httplib import HTTPSConnection import socket import time import traceback -import urllib.request, urllib.error, urllib.parse +import sys +try: + from urllib.parse import urlparse, urlencode + from urllib.request import urlopen, Request, HTTPSHandler, build_opener, ProxyHandler + from urllib.error import HTTPError +except ImportError: + from urlparse import urlparse + from urllib import urlencode + from urllib2 import urlopen, Request, HTTPError, HTTPSHandler, build_opener, ProxyHandler from httpclient import * @@ -26,7 +37,7 @@ ssl = None -class HttpsClientHandler(urllib.request.HTTPSHandler): +class HttpsClientHandler(HTTPSHandler): """Https handler to enable attaching cert/key to request. Also used to disable strict cert verification for testing. """ @@ -42,7 +53,7 @@ def __init__(self, cert_path, key_path, insecure=False): ssl_context = ssl.create_default_context() ssl_context.check_hostname = False ssl_context.verify_mode = ssl.CERT_NONE - urllib.request.HTTPSHandler.__init__(self, context=ssl_context) # Context can be None here + HTTPSHandler.__init__(self, context=ssl_context) # Context can be None here def https_open(self, req): return self.do_open(self.get_https_connection, req, context=self._context) @@ -61,9 +72,9 @@ def get_https_connection(self, host, context=None, timeout=180): """ socket.setdefaulttimeout(180) if self.cert_path is None or self.key_path is None: - return http.client.HTTPSConnection(host, timeout=timeout, context=context) + return HTTPSConnection(host, timeout=timeout, context=context) else: - return http.client.HTTPSConnection(host, cert_file=self.cert_path, key_file=self.key_path, timeout=timeout, + return HTTPSConnection(host, cert_file=self.cert_path, key_file=self.key_path, timeout=timeout, context=context) @@ -113,12 +124,12 @@ def issue_request(self, url, headers, method=None, data=None): :param method: """ https_handler = HttpsClientHandler(self.cert_path, self.key_path, self.insecure) - opener = urllib.request.build_opener(https_handler) + opener = build_opener(https_handler) if self.proxy_configuration is not None: - proxy_handler = urllib.request.ProxyHandler({'http': self.proxy_configuration, + proxy_handler = ProxyHandler({'http': self.proxy_configuration, 'https': self.proxy_configuration}) opener.add_handler(proxy_handler) - req = urllib.request.Request(url, data=data, headers=headers) + req = Request(url, data=data, headers=headers) req.get_method = lambda: method response = opener.open(req, timeout=30) opener.close() @@ -140,7 +151,7 @@ def get(self, url, headers=None): try: response = self.issue_request(url, headers=headers, method=self.GET) - except urllib.error.HTTPError: + except HTTPError: exception_type, error = sys.exc_info()[:2] return RequestResponse(error.code) @@ -167,7 +178,7 @@ def post(self, url, headers=None, data=None): try: response = self.issue_request(url, headers=headers, method=self.POST, data=serial_data) - except urllib.error.HTTPError: + except HTTPError: exception_type, error = sys.exc_info()[:2] return RequestResponse(error.code) @@ -194,7 +205,7 @@ def put(self, url, headers=None, data=None): try: response = self.issue_request(url, headers=headers, method=self.PUT, data=serial_data) - except urllib.error.HTTPError: + except HTTPError: exception_type, error = sys.exc_info()[:2] return RequestResponse(error.code) @@ -221,7 +232,7 @@ def delete(self, url, headers=None, data=None): try: response = self.issue_request(url, headers=headers, method=self.DELETE, data=serial_data) - except urllib.error.HTTPError: + except HTTPError: exception_type, error = sys.exc_info()[:2] return RequestResponse(error.code) diff --git a/DSC/urllib3httpclient.py b/DSC/urllib3httpclient.py new file mode 100644 index 000000000..28a0c946b --- /dev/null +++ b/DSC/urllib3httpclient.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python2 + +# +# Copyright (C) Microsoft Corporation, All rights reserved. + +"""Urllib2 HttpClient.""" + +try: + from http.client import HTTPSConnection +except ImportError: + from httplib import HTTPSConnection +import socket +import time +import traceback +import sys +try: + from urllib.parse import urlparse, urlencode + from urllib.request import urlopen, Request, HTTPSHandler, build_opener, ProxyHandler + from urllib.error import HTTPError +except ImportError: + from urlparse import urlparse + from urllib import urlencode + from urllib2 import urlopen, Request, HTTPError, HTTPSHandler, build_opener, ProxyHandler + +from httpclient import * + +PY_MAJOR_VERSION = 0 +PY_MINOR_VERSION = 1 +PY_MICRO_VERSION = 2 + +SSL_MODULE_NAME = "ssl" + +# On some system the ssl module might be missing +try: + import ssl +except ImportError: + ssl = None + + +class HttpsClientHandler(HTTPSHandler): + """Https handler to enable attaching cert/key to request. Also used to disable strict cert verification for + testing. + """ + + def __init__(self, cert_path, key_path, insecure=False): + self.cert_path = cert_path + self.key_path = key_path + + ssl_context = None + if insecure and SSL_MODULE_NAME in sys.modules and (sys.version_info[PY_MAJOR_VERSION] == 2 and + sys.version_info[PY_MINOR_VERSION] >= 7 and + sys.version_info[PY_MICRO_VERSION] >= 9): + ssl_context = ssl.create_default_context() + ssl_context.check_hostname = False + ssl_context.verify_mode = ssl.CERT_NONE + HTTPSHandler.__init__(self, context=ssl_context) # Context can be None here + + def https_open(self, req): + return self.do_open(self.get_https_connection, req, context=self._context) + + def get_https_connection(self, host, context=None, timeout=180): + """urllib2's AbstractHttpHandler will invoke this method with the host/timeout parameter. See urllib2's + AbstractHttpHandler for more details. + + Args: + host : string , the host. + context : ssl_context , the ssl context. + timeout : int , the timeout value in seconds. + + Returns: + An HttpsConnection + """ + socket.setdefaulttimeout(180) + if self.cert_path is None or self.key_path is None: + return HTTPSConnection(host, timeout=timeout, context=context) + else: + return HTTPSConnection(host, cert_file=self.cert_path, key_file=self.key_path, timeout=timeout, + context=context) + + +def request_retry_handler(func): + def decorated_func(*args, **kwargs): + max_retry_count = 3 + for iteration in range(0, max_retry_count, 1): + try: + ret = func(*args, **kwargs) + return ret + except Exception as exception: + if iteration >= max_retry_count - 1: + raise RetryAttemptExceededException(traceback.format_exc()) + elif SSL_MODULE_NAME in sys.modules: + if type(exception).__name__ == 'SSLError': + time.sleep(5 + iteration) + continue + raise exception + return decorated_func + + +class Urllib3HttpClient(HttpClient): + """Urllib2 http client. Inherits from HttpClient. + + Targets: + [2.7.9 - 2.7.9+] only due to the lack of strict certificate verification prior to this version. + + Implements the following method common to all classes inheriting HttpClient. + get (url, headers) + post (url, headers, data) + """ + + def __init__(self, cert_path, key_path, insecure=False, proxy_configuration=None): + HttpClient.__init__(self, cert_path, key_path, insecure, proxy_configuration) + + @request_retry_handler + def issue_request(self, url, headers, method=None, data=None): + """Issues a GET request to the provided url and using the provided headers. + + Args: + url : string , the url. + headers : dictionary, contains the headers key value pair. + data : string , contains the serialized request body. + + Returns: + A RequestResponse + :param method: + """ + https_handler = HttpsClientHandler(self.cert_path, self.key_path, self.insecure) + opener = build_opener(https_handler) + if self.proxy_configuration is not None: + proxy_handler = ProxyHandler({'http': self.proxy_configuration, + 'https': self.proxy_configuration}) + opener.add_handler(proxy_handler) + if sys.version_info >= (3,0): + if data is not None: + data = data.encode("utf-8") + req = Request(url, data=data, headers=headers) + req.get_method = lambda: method + response = opener.open(req, timeout=30) + opener.close() + https_handler.close() + + return response + + def get(self, url, headers=None): + """Issues a GET request to the provided url and using the provided headers. + + Args: + url : string , the url. + headers : dictionary, contains the headers key value pair. + + Returns: + An http_response + """ + headers = self.merge_headers(self.default_headers, headers) + + try: + response = self.issue_request(url, headers=headers, method=self.GET) + except HTTPError: + exception_type, error = sys.exc_info()[:2] + return RequestResponse(error.code) + + return RequestResponse(response.getcode(), response.read()) + + def post(self, url, headers=None, data=None): + """Issues a POST request to the provided url and using the provided headers. + + Args: + url : string , the url. + headers : dictionary, contains the headers key value pair. + data : dictionary, contains the non-serialized request body. + + Returns: + A RequestResponse + """ + headers = self.merge_headers(self.default_headers, headers) + + if data is None: + serial_data = "" + else: + serial_data = self.json.dumps(data) + headers.update({self.CONTENT_TYPE_HEADER_KEY: self.APP_JSON_HEADER_VALUE}) + + try: + response = self.issue_request(url, headers=headers, method=self.POST, data=serial_data) + except HTTPError: + exception_type, error = sys.exc_info()[:2] + return RequestResponse(error.code) + + return RequestResponse(response.getcode(), response.read().decode('utf-8')) + + def put(self, url, headers=None, data=None): + """Issues a PUT request to the provided url and using the provided headers. + + Args: + url : string , the url. + headers : dictionary, contains the headers key value pair. + data : dictionary, contains the non-serialized request body. + + Returns: + A RequestResponse + """ + headers = self.merge_headers(self.default_headers, headers) + + if data is None: + serial_data = "" + else: + serial_data = self.json.dumps(data) + headers.update({self.CONTENT_TYPE_HEADER_KEY: self.APP_JSON_HEADER_VALUE}) + + try: + response = self.issue_request(url, headers=headers, method=self.PUT, data=serial_data) + except HTTPError: + exception_type, error = sys.exc_info()[:2] + return RequestResponse(error.code) + + return RequestResponse(response.getcode(), response.read().decode('utf-8')) + + def delete(self, url, headers=None, data=None): + """Issues a DELETE request to the provided url and using the provided headers. + + Args: + url : string , the url. + headers : dictionary, contains the headers key value pair. + data : dictionary, contains the non-serialized request body. + + Returns: + A RequestResponse + """ + headers = self.merge_headers(self.default_headers, headers) + + if data is None: + serial_data = "" + else: + serial_data = self.json.dumps(data) + headers.update({self.CONTENT_TYPE_HEADER_KEY: self.APP_JSON_HEADER_VALUE}) + + try: + response = self.issue_request(url, headers=headers, method=self.DELETE, data=serial_data) + except HTTPError: + exception_type, error = sys.exc_info()[:2] + return RequestResponse(error.code) + + return RequestResponse(response.getcode(), response.read())