Clearlinux master (GoogleCloudPlatform#119)

* Initial working fio micro on Clear Linux.

* Found a cleaner way to install packages that didn't require a linux_packages/<pkgname.py> and also clarified the metadata collected about Clear Linux.

* Removed the fio package installation. It is not needed since fio is pulled from its git repo and compiled.

* First commit of a functional code for fio on the AWS Clear Linux image.

* Bugfix: mkdir fails on Clear when /etc/sudoers.d already exists.

* Removed some librdkafka methods for Clear that were misplaced, and not applicable to branch.

* Removed some commented code, more Clear stuff previously overlooked.

* Removed some refactoring that wasn't required.

* Removing a tab in linux_virtual_machine.py, and a space + an import statement in collectd.py.

perfkitbenchmarker/linux_packages/fio.py

@@ -75,6 +75,22 @@ def _Install(vm):
              patch=FIO_HIST_LOG_PARSER_PATCH))
 
 
+def _InstallClear(vm):
+  """Installs the fio package on the Clear VM."""
+  vm.RemoteCommand('sudo pip2 install pandas numpy')
+  vm.RemoteCommand('git clone {0} {1}'.format(GIT_REPO, FIO_DIR))
+  vm.RemoteCommand('cd {0} && git checkout {1}'.format(FIO_DIR, GIT_TAG))
+  vm.RemoteCommand('cd {0} && ./configure && make'.format(FIO_DIR))
+  if flags.FLAGS.fio_hist_log:
+    vm.PushDataFile(FIO_HIST_LOG_PARSER_PATCH)
+    vm.RemoteCommand(
+        ('cp {log_parser_path}/{log_parser} ./; '
+         'patch {log_parser} {patch}').format(
+             log_parser_path=FIO_HIST_LOG_PARSER_PATH,
+             log_parser=FIO_HIST_LOG_PARSER,
+             patch=FIO_HIST_LOG_PARSER_PATCH))
+
+
 def YumInstall(vm):
   """Installs the fio package on the VM."""
   vm.InstallPackages('libaio-devel libaio bc zlib-devel')
@@ -86,6 +102,10 @@ def AptInstall(vm):
   vm.InstallPackages('libaio-dev libaio1 bc zlib1g-dev')
   _Install(vm)
 
+def SwupdInstall(vm):
+  """Installs the dependencies for the fio package on the VM."""
+  vm.InstallPackages('os-testsuite-phoronix-server python2-basic which')
+  _InstallClear(vm)
 
 def ParseJobFile(job_file):
   """Parse fio job file as dictionaries of sample metadata.

perfkitbenchmarker/linux_virtual_machine.py

@@ -55,6 +55,7 @@
              '7/x86_64/Packages/e/epel-release-7-11.noarch.rpm')
 
 LSB_DESCRIPTION_REGEXP = r'Description:\s*(.*)\s*'
+CLEAR_BUILD_REGEXP = r'Installed version:\s*(.*)\s*'
 UPDATE_RETRIES = 5
 SSH_RETRIES = 10
 DEFAULT_SSH_PORT = 22
@@ -290,7 +291,8 @@ def PrepareVMEnvironment(self):
       if self.is_static:
         self.SnapshotPackages()
       self.SetupPackageManager()
-      self.InstallPackages('python')
+      if (self.OS_TYPE != 'clear'):
+        self.InstallPackages('python')
     self.SetFiles()
     self.DoSysctls()
     self.DoConfigureNetworkForBBR()
@@ -414,9 +416,13 @@ def os_info(self):
     if self.os_metadata.get('os_info'):
       return self.os_metadata['os_info']
     else:
-      self.Install('lsb_release')
-      stdout, _ = self.RemoteCommand('lsb_release -d')
-      return regex_util.ExtractGroup(LSB_DESCRIPTION_REGEXP, stdout)
+      if (self.OS_TYPE == 'clear'):
+        stdout, _ = self.RemoteCommand('swupd info|grep Installed')
+        return "Clear Linux build: %s" % regex_util.ExtractGroup(CLEAR_BUILD_REGEXP, stdout) 
+      else:
+        self.Install('lsb_release')
+        stdout, _ = self.RemoteCommand('lsb_release -d')
+        return regex_util.ExtractGroup(LSB_DESCRIPTION_REGEXP, stdout)
 
   @property
   def kernel_release(self):
@@ -981,6 +987,107 @@ def _GetSmbService(self):
       raise errors.Resource.CreationError('No SMB Service created')
     return smb
 
+class ClearMixin(BaseLinuxMixin):
+  """Class holding Clear Linux specific VM methods and attributes."""
+
+  OS_TYPE = os_types.CLEAR
+  BASE_OS_TYPE = os_types.CLEAR
+
+  def OnStartup(self):
+    """Eliminates the need to have a tty to run sudo commands."""
+    super(ClearMixin, self).OnStartup()
+    self.RemoteHostCommand('sudo swupd autoupdate --disable')
+    self.RemoteHostCommand('sudo mkdir -p /etc/sudoers.d')
+    self.RemoteHostCommand('echo \'Defaults:%s !requiretty\' | '
+                           'sudo tee /etc/sudoers.d/pkb' % self.user_name,
+                           login_shell=True)
+
+  def PackageCleanup(self):
+    """Cleans up all installed packages.
+
+    Performs the normal package cleanup, then deletes the file
+    added to the /etc/sudoers.d directory during startup.
+    """
+    super(ClearMixin, self).PackageCleanup()
+    self.RemoteCommand('sudo rm /etc/sudoers.d/pkb')
+
+  def SnapshotPackages(self):
+    """Grabs a snapshot of the currently installed packages."""
+    self.RemoteCommand('sudo swupd bundle-list > %s/bundle_list'
+                       % linux_packages.INSTALL_DIR)
+
+  def RestorePackages(self):
+    """Restores the currently installed bundles to those snapshotted."""
+    self.RemoteCommand(
+        'sudo swupd bundle-list | grep --fixed-strings --line-regexp --invert-match --file '
+        '%s/bundle_list | xargs --no-run-if-empty sudo swupd bundle-remove' %
+        linux_packages.INSTALL_DIR,
+        ignore_failure=True)
+
+  def HasPackage(self, package):
+    """Returns True iff the package is available for installation."""
+    return self.TryRemoteCommand('sudo swupd bundle-list --all | grep %s' % package,
+                                 suppress_warning=True)
+
+  def InstallPackages(self, packages):
+    """Installs packages using the swupd bundle manager."""
+    self.RemoteCommand('sudo swupd bundle-add %s' % packages)
+
+  def InstallPackageGroup(self, package_group):
+    """Installs a 'package group' using the yum package manager."""
+    self.RemoteCommand('sudo swupd bundle-add "%s"' % package_group)
+
+  def Install(self, package_name):
+    """Installs a PerfKit package on the VM."""
+    if not self.install_packages:
+      return
+    if package_name not in self._installed_packages:
+      package = linux_packages.PACKAGES[package_name]
+      if hasattr(package, 'SwupdInstall'):
+        package.SwupdInstall(self)
+      elif hasattr(package, 'Install'):
+        package.Install(self)
+      else:
+        raise KeyError('Package %s has no install method for Clear Linux.' %
+                       package_name)
+      self._installed_packages.add(package_name)
+
+  def Uninstall(self, package_name):
+    """Uninstalls a PerfKit package on the VM."""
+    package = linux_packages.PACKAGES[package_name]
+    if hasattr(package, 'SwupdUninstall'):
+      package.YumUninstall(self)
+    elif hasattr(package, 'Uninstall'):
+      package.Uninstall(self)
+
+  def GetPathToConfig(self, package_name):
+    """Returns the path to the config file for PerfKit packages.
+
+    This function is mostly useful when config files locations
+    don't match across distributions (such as mysql). Packages don't
+    need to implement it if this is not the case.
+    """
+    package = linux_packages.PACKAGES[package_name]
+    return package.YumGetPathToConfig(self)
+
+  def GetServiceName(self, package_name):
+    """Returns the service name of a PerfKit package.
+
+    This function is mostly useful when service names don't
+    match across distributions (such as mongodb). Packages don't
+    need to implement it if this is not the case.
+    """
+    package = linux_packages.PACKAGES[package_name]
+    return package.YumGetServiceName(self)
+
+  def SetupProxy(self):
+    """Sets up proxy configuration variables for the cloud environment."""
+    super(ClearMixin, self).SetupProxy()
+    env_proxy_file = '/etc/environment'
+
+    if FLAGS.http_proxy:
+      self.RemoteCommand("echo -e 'http_proxy= %s' | sudo tee -a %s" % (
+          FLAGS.http_proxy, env_proxy_file))
 
 class RhelMixin(BaseLinuxMixin):
   """Class holding RHEL specific VM methods and attributes."""

perfkitbenchmarker/os_types.py

@@ -16,6 +16,7 @@
 from perfkitbenchmarker import flags
 
 CENTOS7 = 'centos7'
+CLEAR = 'clear'
 DEBIAN = 'debian'
 DEBIAN9 = 'debian9'
 JUJU = 'juju'
@@ -33,6 +34,7 @@
 
 LINUX_OS_TYPES = [
     CENTOS7,
+    CLEAR,
     DEBIAN,
     DEBIAN9,
     JUJU,
@@ -51,7 +53,7 @@
     WINDOWS2019,
 ]
 ALL = LINUX_OS_TYPES + WINDOWS_OS_TYPES
-BASE_OS_TYPES = [DEBIAN, RHEL, WINDOWS]
+BASE_OS_TYPES = [CLEAR, DEBIAN, RHEL, WINDOWS]
 
 flags.DEFINE_enum(
     'os_type', UBUNTU1604, ALL,

perfkitbenchmarker/providers/aws/aws_virtual_machine.py

@@ -846,6 +846,16 @@ def GetVmStatusCode(self):
     return self.spot_status_code
 
 
+class ClearBasedAwsVirtualMachine(AwsVirtualMachine,
+                                       linux_virtual_machine.ClearMixin):
+  IMAGE_NAME_FILTER = 'clear/images/*/clear-*'
+  IMAGE_OWNER = '679593333241'  # For marketplace images.
+
+  def __init__(self, vm_spec):
+    super(ClearBasedAwsVirtualMachine, self).__init__(vm_spec)
+    user_name_set = FLAGS['aws_user_name'].present
+    self.user_name = FLAGS.aws_user_name if user_name_set else 'clear'
+
 class DebianBasedAwsVirtualMachine(AwsVirtualMachine,
                                    linux_virtual_machine.DebianMixin):
   """Class with configuration for AWS Debian virtual machines."""

perfkitbenchmarker/static_virtual_machine.py

@@ -199,6 +199,7 @@ def ReadStaticVirtualMachineFile(cls, file_obj):
         os_types.WINDOWS: required_keys | frozenset(['password']),
         os_types.DEBIAN: linux_required_keys,
         os_types.RHEL: linux_required_keys,
+        os_types.CLEAR: linux_required_keys,
         os_types.UBUNTU_CONTAINER: linux_required_keys,
     }
 
@@ -324,6 +325,9 @@ class DebianBasedStaticVirtualMachine(StaticVirtualMachine,
                                       linux_virtual_machine.DebianMixin):
   pass
 
+class ClearBasedStaticVirtualMachine(StaticVirtualMachine,
+                                      linux_virtual_machine.ClearMixin):
+  pass
 
 
 class Centos7BasedStaticVirtualMachine(StaticVirtualMachine,