Merge branch 'master' into python2to3vr

Author: GabrielBrascher

.travis.yml

@@ -27,10 +27,14 @@ jdk:
 python:
   - "2.7"
 
+node_js:
+  - 12
+
 cache:
   directories:
     - $HOME/.m2
   timeout: 500
+  npm: false
 
 notifications:
   email: false

INSTALL.md

@@ -28,6 +28,12 @@ Set up Maven (3.6.0):
     # echo export PATH=/usr/local/maven/bin:${PATH} >> ~/.bashrc # or .zshrc or .profile
     # source ~/.bashrc
 
+Setup up NodeJS (LTS):
+
+    # curl -sL https://rpm.nodesource.com/setup_12.x | sudo bash -
+    # sudo yum install nodejs
+    # sudo npm install -g @vue/cli npm-check-updates
+
 Start the MySQL service:
 
     $ service mysqld start
@@ -130,7 +136,7 @@ To create rpms, install the following extra packages:
 
     # yum -y install rpm-build
     # yum -y install ws-commons-util
-    # yum -y instal gcc
+    # yum -y install gcc
     # yum -y install glibc-devel
     # yum -y install MySQL-python
 

PULL_REQUEST_TEMPLATE.md

@@ -1,5 +1,7 @@
-## Description
-<!--- Describe your changes in detail -->
+### Description
+
+This PR...
+<!--- Describe your changes in DETAIL - And how has behaviour functionally changed. -->
 
 <!-- For new features, provide link to FS, dev ML discussion etc. -->
 <!-- In case of bug fix, the expected and actual behaviours, steps to reproduce. -->
@@ -8,17 +10,39 @@
 <!-- For addressing multiple issues/PRs, use multiple "Fixes: #<id>" -->
 <!-- Fixes: # -->
 
-## Types of changes
-<!--- What types of changes does your code introduce? Put an `x` in all the boxes that apply: -->
+<!--- ********************************************************************************* -->
+<!--- NOTE: AUTOMATATION USES THE DESCRIPTIONS TO SET LABELS AND PRODUCE DOCUMENTATION. -->
+<!--- PLEASE PUT AN 'X' in only **ONE** box -->
+<!--- ********************************************************************************* -->
+
+### Types of changes
+
 - [ ] Breaking change (fix or feature that would cause existing functionality to change)
 - [ ] New feature (non-breaking change which adds functionality)
 - [ ] Bug fix (non-breaking change which fixes an issue)
 - [ ] Enhancement (improves an existing feature and functionality)
 - [ ] Cleanup (Code refactoring and cleanup, that may add test cases)
 
-## Screenshots (if appropriate):
+### Feature/Enhancement Scale or Bug Severity
+
+#### Feature/Enhancement Scale
+
+- [ ] Major
+- [ ] Minor
+
+#### Bug Severity
+
+- [ ] BLOCKER
+- [ ] Critical
+- [ ] Major
+- [ ] Minor
+- [ ] Trivial
+
+
+### Screenshots (if appropriate):
+
 
-## How Has This Been Tested?
+### How Has This Been Tested?
 <!-- Please describe in detail how you tested your changes. -->
 <!-- Include details of your testing environment, and the tests you ran to -->
 <!-- see how your change affects other areas of the code, etc. -->

agent/bindir/cloud-guest-tool.in

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -78,7 +78,7 @@ class GuestCommand:
         info['network'] = 'guest-network-get-interfaces'
 
         result = dict()
-        for key, cmd in info.items():
+        for key, cmd in list(info.items()):
             result[key] = self.virt.agent_command(self.dom, cmd)
 
         return result, 0

agent/bindir/cloud-setup-agent.in

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -31,56 +31,56 @@ from cloudutils.serviceConfig import configureLibvirtConfig
 from optparse import OptionParser
 
 def getUserInputs():
-    print "Welcome to the CloudStack Agent Setup:"
+    print("Welcome to the CloudStack Agent Setup:")
 
     cfo = configFileOps("@AGENTSYSCONFDIR@/agent.properties")
     oldMgt = cfo.getEntry("host")
 
-    mgtSvr = raw_input("Please input the Management Server Hostname/IP-Address:[%s]"%oldMgt)
+    mgtSvr = input("Please input the Management Server Hostname/IP-Address:[%s]"%oldMgt)
     if mgtSvr == "":
         mgtSvr = oldMgt
     try:
         socket.getaddrinfo(mgtSvr, 443)
     except:
-        print "Failed to resolve %s. Please input a valid hostname or IP-Address."%mgtSvr
+        print("Failed to resolve %s. Please input a valid hostname or IP-Address."%mgtSvr)
         exit(1)
 
     oldToken = cfo.getEntry("zone")
-    zoneToken = raw_input("Please input the Zone Id:[%s]"%oldToken)
+    zoneToken = input("Please input the Zone Id:[%s]"%oldToken)
 
     if zoneToken == "":
         zoneToken = oldToken
 
     oldPod = cfo.getEntry("pod")
-    podId = raw_input("Please input the Pod Id:[%s]"%oldPod)
+    podId = input("Please input the Pod Id:[%s]"%oldPod)
 
     if podId == "":
        podId  = oldToken
 
     oldCluster = cfo.getEntry("cluster")
-    clusterId = raw_input("Please input the Cluster Id:[%s]"%oldCluster)
+    clusterId = input("Please input the Cluster Id:[%s]"%oldCluster)
     if clusterId == "":
         clusterId = oldCluster
 
     oldHypervisor = cfo.getEntry("hypervisor")
     if oldHypervisor == "":
         oldHypervisor = "kvm"
 
-    hypervisor = raw_input("Please input the Hypervisor type kvm/lxc:[%s]"%oldHypervisor)
+    hypervisor = input("Please input the Hypervisor type kvm/lxc:[%s]"%oldHypervisor)
     if hypervisor == "":
         hypervisor = oldHypervisor
 
     try:
         defaultNic = networkConfig.getDefaultNetwork()
     except:
-        print "Failed to get default route. Please configure your network to have a default route"
+        print("Failed to get default route. Please configure your network to have a default route")
         exit(1)
 
     defNic = defaultNic.name
-    network = raw_input("Please choose which network used to create VM:[%s]"%defNic)
+    network = input("Please choose which network used to create VM:[%s]"%defNic)
     if network == "":
         if defNic == "":
-            print "You need to specifiy one of Nic or bridge on your system"
+            print("You need to specifiy one of Nic or bridge on your system")
             exit(1)
         elif network == "":
             network = defNic
@@ -115,7 +115,7 @@ if __name__ == '__main__':
 
     if not options.auto and options.secure:
         configureLibvirtConfig(True)
-        print "Libvirtd with TLS configured"
+        print("Libvirtd with TLS configured")
         sys.exit(0)
 
     if options.auto is None:
@@ -131,10 +131,10 @@ if __name__ == '__main__':
         if glbEnv.uuid == "":
             glbEnv.uuid = bash("uuidgen").getStdout()
     else:
-        for para, value in options.__dict__.items():
+        for para, value in list(options.__dict__.items()):
             if value is None:
-                print "Missing operand:%s"%para
-                print "Try %s --help for more information"%sys.argv[0]
+                print("Missing operand:%s"%para)
+                print("Try %s --help for more information"%sys.argv[0])
                 sys.exit(1)
 
         glbEnv.uuid = options.guid
@@ -149,14 +149,14 @@ if __name__ == '__main__':
 
     glbEnv.secure = options.secure
 
-    print "Starting to configure your system:"
+    print("Starting to configure your system:")
     syscfg = sysConfigFactory.getSysConfigFactory(glbEnv)
     try:
         syscfg.config()
-        print "CloudStack Agent setup is done!"
-    except (CloudRuntimeException,CloudInternalException), e:
-        print e
-        print "Try to restore your system:"
+        print("CloudStack Agent setup is done!")
+    except (CloudRuntimeException,CloudInternalException) as e:
+        print(e)
+        print("Try to restore your system:")
         try:
             syscfg.restore()
         except:

agent/bindir/cloudstack-agent-upgrade.in

@@ -1,14 +1,14 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
 # regarding copyright ownership.  The ASF licenses this file
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -32,7 +32,7 @@ def upgradeBridgeName(brName, enslavedDev):
     print("find physical device %s"%phyDev)
     newBrName = "br" + phyDev + "-" + vlanId
     print("new bridge name %s"%newBrName)
-    bash("ip link set %s down"%brName) 
+    bash("ip link set %s down"%brName)
     bash("ip link set %s name %s"%(brName, newBrName))
     bash("ip link set %s up" %newBrName)
     cmd = "iptables-save | grep FORWARD | grep -w " +  brName
@@ -47,16 +47,16 @@ def upgradeBridgeName(brName, enslavedDev):
         except:
             logging.exception("Ignoring failure to update rules for rule " + rule + " on bridge " + brName)
 if __name__ == '__main__':
-   netlib = networkConfig() 
+   netlib = networkConfig()
    bridges = netlib.listNetworks()
-   bridges = filter(isOldStyleBridge, bridges)
+   bridges = list(filter(isOldStyleBridge, bridges))
    for br in bridges:
        enslavedDev = netlib.getEnslavedDev(br, 1)
        if enslavedDev is not None:
            upgradeBridgeName(br, enslavedDev)
-   
+
    bridges = netlib.listNetworks()
-   bridges = filter(isOldStyleBridge, bridges)
+   bridges = list(filter(isOldStyleBridge, bridges))
    if len(bridges) > 0:
        print("Warning: upgrade is not finished, still some bridges have the old style name:" + str(bridges))
    else:

agent/bindir/libvirtqemuhook.in

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -32,7 +32,7 @@ logging.basicConfig(filename='/var/log/libvirt/qemu-hook.log',
 logger = logging.getLogger('qemu-hook')
 
 customDir = "/etc/libvirt/hooks/custom"
-customDirPermissions = 0744
+customDirPermissions = 0o744
 timeoutSeconds = 10 * 60
 validQemuActions = ['prepare', 'start', 'started', 'stopped', 'release', 'migrate', 'restore', 'reconnect', 'attach']
 
@@ -128,9 +128,8 @@ def terminateProcess(process, scriptName):
 
 
 def getCustomScriptsFromDirectory():
-    return sorted(filter(lambda fileName: (fileName is not None) & (fileName != "") & ('_' in fileName) &
-                                          (fileName.startswith((action + '_')) | fileName.startswith(('all' + '_'))),
-                         os.listdir(customDir)), key=lambda fileName: substringAfter(fileName, '_'))
+    return sorted([fileName for fileName in os.listdir(customDir) if (fileName is not None) & (fileName != "") & ('_' in fileName) &
+                                          (fileName.startswith((action + '_')) | fileName.startswith(('all' + '_')))], key=lambda fileName: substringAfter(fileName, '_'))
 
 
 def substringAfter(s, delimiter):

agent/bindir/rolling-maintenance.in

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -43,7 +43,7 @@ def execute_script(stage, script, payload, timeout):
 
         success = True if exitStatus == 0 or exitStatus == AVOID_MAINTENANCE_EXIT_STATUS else False
         avoid_maintenance = True if exitStatus == AVOID_MAINTENANCE_EXIT_STATUS else False
-        return {"success": success, "message": stdout.strip(), "avoidmaintenance": avoid_maintenance}
+        return {"success": success, "message": stdout.decode('utf-8').strip(), "avoidmaintenance": avoid_maintenance}
     except Exception as e:
         logger.error("Error in stage %s: %s" % (script, e))
         sys.exit(1)

agent/conf/agent.properties

@@ -258,3 +258,8 @@ hypervisor.type=kvm
 # timer.
 # For all actions refer to the libvirt documentation.
 # Recommended values are: none, reset and poweroff.
+#
+iscsi.session.cleanup.enabled=false
+# Automatically clean up iscsi sessions not attached to any VM.
+# Should be enabled for users using managed storage for example solidfire.
+# Should be disabled for users with unmanaged iscsi connections on their hosts

agent/pom.xml

@@ -24,7 +24,7 @@
     <parent>
         <groupId>org.apache.cloudstack</groupId>
         <artifactId>cloudstack</artifactId>
-        <version>4.15.0.0-SNAPSHOT</version>
+        <version>4.16.0.0-SNAPSHOT</version>
     </parent>
     <dependencies>
         <dependency>