packaging: Adding Centos8, Ubuntu 20.04, XCPNG8.1 Support (apache#4068)

* DB : Add support for MySQL 8

- Splits commands to create user and grant access on database, the old
statement is no longer supported by MySQL 8.x
- `NO_AUTO_CREATE_USER` is no longer supported by MySQL 8.x so remove
that from db.properties conn parameters

For mysql-server 8.x setup the following changes were added/tested to
make it work with CloudStack in /etc/mysql/mysql.conf.d/mysqld.cnf and
then restart the mysql-server process:

    server_id = 1
    sql-mode="STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION,ERROR_FOR_DIVISION_BY_ZERO,NO_ZERO_DATE,NO_ZERO_IN_DATE,NO_ENGINE_SUBSTITUTION"
    innodb_rollback_on_timeout=1
    innodb_lock_wait_timeout=600
    max_connections=1000
    log-bin=mysql-bin
    binlog-format = 'ROW'

    default-authentication-plugin=mysql_native_password

Notice the last line above, this is to reset the old password based
authentication used by MySQL 5.x.

Developers can set empty password as follows:

    > sudo mysql -u root
    ALTER USER 'root'@'localhost' IDENTIFIED BY '';

In libvirt repository, there are two related commits

2019-08-23 13:13 Daniel P. Berrangé            ● rpm: don't enable socket activation in upgrade if --listen present
2019-08-22 14:52 Daniel P. Berrangé            ● remote: forbid the --listen arg when systemd socket activation

In libvirt.spec.in

        /bin/systemctl mask libvirtd.socket >/dev/null 2>&1 || :
        /bin/systemctl mask libvirtd-ro.socket >/dev/null 2>&1 || :
        /bin/systemctl mask libvirtd-admin.socket >/dev/null 2>&1 || :
        /bin/systemctl mask libvirtd-tls.socket >/dev/null 2>&1 || :
        /bin/systemctl mask libvirtd-tcp.socket >/dev/null 2>&1 || :

Co-authored-by: Wei Zhou <[email protected]>
Co-authored-by: Abhishek Kumar <[email protected]>
Co-authored-by: Rohit Yadav <[email protected]>

agent/bindir/cloud-guest-tool.in

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -78,7 +78,7 @@ class GuestCommand:
         info['network'] = 'guest-network-get-interfaces'
 
         result = dict()
-        for key, cmd in info.items():
+        for key, cmd in list(info.items()):
             result[key] = self.virt.agent_command(self.dom, cmd)
 
         return result, 0

agent/bindir/cloud-setup-agent.in

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -31,56 +31,56 @@ from cloudutils.serviceConfig import configureLibvirtConfig
 from optparse import OptionParser
 
 def getUserInputs():
-    print "Welcome to the CloudStack Agent Setup:"
+    print("Welcome to the CloudStack Agent Setup:")
 
     cfo = configFileOps("@AGENTSYSCONFDIR@/agent.properties")
     oldMgt = cfo.getEntry("host")
 
-    mgtSvr = raw_input("Please input the Management Server Hostname/IP-Address:[%s]"%oldMgt)
+    mgtSvr = input("Please input the Management Server Hostname/IP-Address:[%s]"%oldMgt)
     if mgtSvr == "":
         mgtSvr = oldMgt
     try:
         socket.getaddrinfo(mgtSvr, 443)
     except:
-        print "Failed to resolve %s. Please input a valid hostname or IP-Address."%mgtSvr
+        print("Failed to resolve %s. Please input a valid hostname or IP-Address."%mgtSvr)
         exit(1)
 
     oldToken = cfo.getEntry("zone")
-    zoneToken = raw_input("Please input the Zone Id:[%s]"%oldToken)
+    zoneToken = input("Please input the Zone Id:[%s]"%oldToken)
 
     if zoneToken == "":
         zoneToken = oldToken
 
     oldPod = cfo.getEntry("pod")
-    podId = raw_input("Please input the Pod Id:[%s]"%oldPod)
+    podId = input("Please input the Pod Id:[%s]"%oldPod)
 
     if podId == "":
        podId  = oldToken
 
     oldCluster = cfo.getEntry("cluster")
-    clusterId = raw_input("Please input the Cluster Id:[%s]"%oldCluster)
+    clusterId = input("Please input the Cluster Id:[%s]"%oldCluster)
     if clusterId == "":
         clusterId = oldCluster
 
     oldHypervisor = cfo.getEntry("hypervisor")
     if oldHypervisor == "":
         oldHypervisor = "kvm"
 
-    hypervisor = raw_input("Please input the Hypervisor type kvm/lxc:[%s]"%oldHypervisor)
+    hypervisor = input("Please input the Hypervisor type kvm/lxc:[%s]"%oldHypervisor)
     if hypervisor == "":
         hypervisor = oldHypervisor
 
     try:
         defaultNic = networkConfig.getDefaultNetwork()
     except:
-        print "Failed to get default route. Please configure your network to have a default route"
+        print("Failed to get default route. Please configure your network to have a default route")
         exit(1)
 
     defNic = defaultNic.name
-    network = raw_input("Please choose which network used to create VM:[%s]"%defNic)
+    network = input("Please choose which network used to create VM:[%s]"%defNic)
     if network == "":
         if defNic == "":
-            print "You need to specifiy one of Nic or bridge on your system"
+            print("You need to specifiy one of Nic or bridge on your system")
             exit(1)
         elif network == "":
             network = defNic
@@ -115,7 +115,7 @@ if __name__ == '__main__':
 
     if not options.auto and options.secure:
         configureLibvirtConfig(True)
-        print "Libvirtd with TLS configured"
+        print("Libvirtd with TLS configured")
         sys.exit(0)
 
     if options.auto is None:
@@ -131,10 +131,10 @@ if __name__ == '__main__':
         if glbEnv.uuid == "":
             glbEnv.uuid = bash("uuidgen").getStdout()
     else:
-        for para, value in options.__dict__.items():
+        for para, value in list(options.__dict__.items()):
             if value is None:
-                print "Missing operand:%s"%para
-                print "Try %s --help for more information"%sys.argv[0]
+                print("Missing operand:%s"%para)
+                print("Try %s --help for more information"%sys.argv[0])
                 sys.exit(1)
 
         glbEnv.uuid = options.guid
@@ -149,14 +149,14 @@ if __name__ == '__main__':
 
     glbEnv.secure = options.secure
 
-    print "Starting to configure your system:"
+    print("Starting to configure your system:")
     syscfg = sysConfigFactory.getSysConfigFactory(glbEnv)
     try:
         syscfg.config()
-        print "CloudStack Agent setup is done!"
-    except (CloudRuntimeException,CloudInternalException), e:
-        print e
-        print "Try to restore your system:"
+        print("CloudStack Agent setup is done!")
+    except (CloudRuntimeException,CloudInternalException) as e:
+        print(e)
+        print("Try to restore your system:")
         try:
             syscfg.restore()
         except:

agent/bindir/cloudstack-agent-upgrade.in

@@ -1,14 +1,14 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
 # regarding copyright ownership.  The ASF licenses this file
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -32,7 +32,7 @@ def upgradeBridgeName(brName, enslavedDev):
     print("find physical device %s"%phyDev)
     newBrName = "br" + phyDev + "-" + vlanId
     print("new bridge name %s"%newBrName)
-    bash("ip link set %s down"%brName) 
+    bash("ip link set %s down"%brName)
     bash("ip link set %s name %s"%(brName, newBrName))
     bash("ip link set %s up" %newBrName)
     cmd = "iptables-save | grep FORWARD | grep -w " +  brName
@@ -47,16 +47,16 @@ def upgradeBridgeName(brName, enslavedDev):
         except:
             logging.exception("Ignoring failure to update rules for rule " + rule + " on bridge " + brName)
 if __name__ == '__main__':
-   netlib = networkConfig() 
+   netlib = networkConfig()
    bridges = netlib.listNetworks()
-   bridges = filter(isOldStyleBridge, bridges)
+   bridges = list(filter(isOldStyleBridge, bridges))
    for br in bridges:
        enslavedDev = netlib.getEnslavedDev(br, 1)
        if enslavedDev is not None:
            upgradeBridgeName(br, enslavedDev)
-   
+
    bridges = netlib.listNetworks()
-   bridges = filter(isOldStyleBridge, bridges)
+   bridges = list(filter(isOldStyleBridge, bridges))
    if len(bridges) > 0:
        print("Warning: upgrade is not finished, still some bridges have the old style name:" + str(bridges))
    else:

agent/bindir/libvirtqemuhook.in

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -32,7 +32,7 @@ logging.basicConfig(filename='/var/log/libvirt/qemu-hook.log',
 logger = logging.getLogger('qemu-hook')
 
 customDir = "/etc/libvirt/hooks/custom"
-customDirPermissions = 0744
+customDirPermissions = 0o744
 timeoutSeconds = 10 * 60
 validQemuActions = ['prepare', 'start', 'started', 'stopped', 'release', 'migrate', 'restore', 'reconnect', 'attach']
 
@@ -128,9 +128,8 @@ def terminateProcess(process, scriptName):
 
 
 def getCustomScriptsFromDirectory():
-    return sorted(filter(lambda fileName: (fileName is not None) & (fileName != "") & ('_' in fileName) &
-                                          (fileName.startswith((action + '_')) | fileName.startswith(('all' + '_'))),
-                         os.listdir(customDir)), key=lambda fileName: substringAfter(fileName, '_'))
+    return sorted([fileName for fileName in os.listdir(customDir) if (fileName is not None) & (fileName != "") & ('_' in fileName) &
+                                          (fileName.startswith((action + '_')) | fileName.startswith(('all' + '_')))], key=lambda fileName: substringAfter(fileName, '_'))
 
 
 def substringAfter(s, delimiter):

agent/bindir/rolling-maintenance.in

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -43,7 +43,7 @@ def execute_script(stage, script, payload, timeout):
 
         success = True if exitStatus == 0 or exitStatus == AVOID_MAINTENANCE_EXIT_STATUS else False
         avoid_maintenance = True if exitStatus == AVOID_MAINTENANCE_EXIT_STATUS else False
-        return {"success": success, "message": stdout.strip(), "avoidmaintenance": avoid_maintenance}
+        return {"success": success, "message": stdout.decode('utf-8').strip(), "avoidmaintenance": avoid_maintenance}
     except Exception as e:
         logger.error("Error in stage %s: %s" % (script, e))
         sys.exit(1)

client/bindir/cloud-setup-management.in

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -35,26 +35,26 @@ if __name__ == '__main__':
     if options.https:
         glbEnv.svrMode = "HttpsServer"
     if options.tomcat7:
-        print "The --tomcat7 option is deprecated, CloudStack now uses embedded Jetty server."
+        print("The --tomcat7 option is deprecated, CloudStack now uses embedded Jetty server.")
     if options.nostart:
         glbEnv.noStart = True
 
     glbEnv.mode = "Server"
 
-    print "Starting to configure CloudStack Management Server:"
+    print("Starting to configure CloudStack Management Server:")
     try:
         syscfg = sysConfigFactory.getSysConfigFactory(glbEnv)
     except UnknownSystemException:
-        print >>sys.stderr, ("Error: CloudStack failed to detect your "
-                "operating system. Exiting.")
+        print(("Error: CloudStack failed to detect your "
+                "operating system. Exiting."), file=sys.stderr)
         sys.exit(1)
     try:
-    	syscfg.registerService(cloudManagementConfig)
+        syscfg.registerService(cloudManagementConfig)
         syscfg.config()
-        print "CloudStack Management Server setup is Done!"
-    except (CloudRuntimeException, CloudInternalException), e:
-        print e
-        print "Try to restore your system:"
+        print("CloudStack Management Server setup is Done!")
+    except (CloudRuntimeException, CloudInternalException) as e:
+        print(e)
+        print("Try to restore your system:")
         try:
             syscfg.restore()
         except:

client/bindir/cloud-update-xenserver-licenses.in

@@ -1,4 +1,4 @@
-#!/usr/bin/python -W ignore::DeprecationWarning
+#!/usr/bin/python3 -W ignore::DeprecationWarning
 # -*- coding: utf-8 -*-
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -65,7 +65,7 @@ def getknownhosts(host,username,password):
 	usernames = dict(cur.fetchall())
 	cur.execute("SELECT h.private_ip_address,d.value FROM cloud.host h inner join cloud.host_details d on (h.id = d.host_id) where d.name = 'password' and setup = 1")
 	passwords = dict(cur.fetchall())
-	creds = dict( [ [x,(usernames[x],passwords[x])] for x in usernames.keys() ] )
+	creds = dict( [ [x,(usernames[x],passwords[x])] for x in list(usernames.keys()) ] )
 	cur.close()
 	conn.close()
 	return creds
@@ -121,7 +121,7 @@ class XenServerConfigurator(Thread):
 			if self.retval != 0: self.state = 'failed'
 			else: self.state = 'finished'
 
-		except Exception,e:
+		except Exception as e:
 			self.state = 'failed'
 			self.retval = e
 			#raise
@@ -144,39 +144,39 @@ if options.all:
 	if len(args) != 0: e("IP addresses cannot be specified if -a is specified")
 	config = read_properties(cfg)
 	creds = getknownhosts(config["db.cloud.host"],config["db.cloud.username"],config["db.cloud.password"])
-	hosts = creds.keys()
+	hosts = list(creds.keys())
 else:
 	if not args: e("You must specify at least one IP address, or -a")
 	hosts = args
 	creds = parseuserpwfromhosts(hosts)
 
 try:
 	keyfiledata = file(licensefile).read(-1)
-except OSError,e:
+except OSError as e:
 	sys.stderr.write("The file %s cannot be opened"%licensefile)
 	sys.exit(1)
 
 configurators = []
-for host,(user,password) in creds.items():
+for host,(user,password) in list(creds.items()):
 	configurators.append ( XenServerConfigurator(host,user,password,keyfiledata ) )
 
 
 for c in configurators: c.start()
 
 for c in configurators:
-	print c.host + "...",
+	print(c.host + "...", end=' ')
 	c.join()
 	if c.state == 'failed':
 		if c.retval:
 			msg = "failed with return code %s: %s%s"%(c.retval,c.stdout,c.stderr)
 			msg = msg.strip()
-			print msg
-		else: print "failed: %s"%c.retval
+			print(msg)
+		else: print("failed: %s"%c.retval)
 	else:
-		print "done"
+		print("done")
 
 successes = len( [ a for a in configurators if not a.state == 'failed' ] )
 failures = len( [ a for a in configurators if a.state == 'failed' ] )
 
-print "%3s successes"%successes
-print "%3s failures"%failures
+print("%3s successes"%successes)
+print("%3s failures"%failures)

client/conf/db.properties.in

@@ -39,7 +39,7 @@ db.cloud.testWhileIdle=true
 db.cloud.timeBetweenEvictionRunsMillis=40000
 db.cloud.minEvictableIdleTimeMillis=240000
 db.cloud.poolPreparedStatements=false
-db.cloud.url.params=prepStmtCacheSize=517&cachePrepStmts=true&sessionVariables=sql_mode='STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION'&serverTimezone=UTC
+db.cloud.url.params=prepStmtCacheSize=517&cachePrepStmts=true&sessionVariables=sql_mode='STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION'&serverTimezone=UTC
 
 # CloudStack database SSL settings
 db.cloud.useSSL=false

cloud-cli/bindir/cloud-tool

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file