Merge pull request #75 from gvallee/mpi_manifest

Finish support for Manifest and base images from the library

Author: gvallee

README.syvalidate.md

@@ -55,8 +55,8 @@ The tool achieves this by installing a specific version of MPI on the host and a
 (currently based on Ubuntu) with a specific version of MPI that will run certain MPI programs within it to test the comptibility. 
 
 The version of a given MPI implementation to be used throughout an experiment is defined in a configuration file. For example, a 
-default configuration file for Open MPI is available in `etc/openmpi.conf` and a default configuration file for MPICH is available 
-in `etc/mpich.conf`. Users *must* specify the configuration file on the command line when running the tool (see examples). 
+default configuration file for Open MPI is available in `etc/sympi_openmpi.conf` and a default configuration file for MPICH is available 
+in `etc/sympi_mpich.conf`. Users *must* specify the configuration file on the command line when running the tool (see examples). 
 
 Once the tool has completed, view the ``openmpi-results.txt``/``mpich-results.txt`` to view results of various combinations of the 
 versions and pick the host-container version combination most suitable to you.
@@ -71,39 +71,39 @@ At the moment, we support two tests:
 
 ## Run the tool with the default Open MPI versions and a simple helloworld test
 
-``syvalidate -configfile `pwd`/etc/openmpi.conf``
+``syvalidate -configfile `pwd`/etc/sympi_openmpi.conf``
 
 ## Run the tool with the default Open MPI versions and Netpipe
 
-``syvalidate -configfile `pwd`/etc/openmpi.conf -netpipe``
+``syvalidate -configfile `pwd`/etc/sympi_openmpi.conf -netpipe``
 
 ## Run the tool with the default Open MPI versions and IMB
 
-``syvalidate -configfile `pwd`/etc/openmpi.conf -imb``
+``syvalidate -configfile `pwd`/etc/sympi_openmpi.conf -imb``
 
 ## Run the tool with the default MPICH versions and a simple helloworld test
 
-``syvalidate -configfile `pwd`/etc/mpich.conf``
+``syvalidate -configfile `pwd`/etc/sympi_mpich.conf``
 
 ## Run the tool with the default MPICH versions and Netpipe
 
-``syvalidate -configfile `pwd`/etc/mpich.conf -netpipe``
+``syvalidate -configfile `pwd`/etc/sympi_mpich.conf -netpipe``
 
 ## Run the tool with the default MPICH versions and IMB
 
-``syvalidate -configfile `pwd`/etc/mpich.conf -imb``
+``syvalidate -configfile `pwd`/etc/sympi_mpich.conf -imb``
 
 ## Run the tool with the default Intel MPI versions and a simple helloworld test
 
-``syvalidate -configfile `pwd`/etc/intel.conf``
+``syvalidate -configfile `pwd`/etc/sympi_intel.conf``
 
 ## Run the tool with the default Intel MPI versions and Netpipe
 
-``syvalidate -configfile `pwd`/etc/intel.conf -netpipe``
+``syvalidate -configfile `pwd`/etc/sympi_intel.conf -netpipe``
 
 ## Run the tool with the default Intel MPI versions and collective operation test
 
-``syvalidate -configfile `pwd`/etc/intel.conf -imb``
+``syvalidate -configfile `pwd`/etc/sympi_intel.conf -imb``
 
 These commands will run various MPI programs to test the compatibility between different versions:
 - a basic HelloWorld test,

cmd/sycontainerize/sycontainerize.go

@@ -13,13 +13,13 @@ import (
 	"os"
 	"strconv"
 
+	"github.com/gvallee/go_util/pkg/util"
+	"github.com/gvallee/kv/pkg/kv"
 	"github.com/sylabs/singularity-mpi/internal/pkg/checker"
-	"github.com/sylabs/singularity-mpi/internal/pkg/kv"
 	"github.com/sylabs/singularity-mpi/internal/pkg/launcher"
 	"github.com/sylabs/singularity-mpi/internal/pkg/sy"
 	"github.com/sylabs/singularity-mpi/internal/pkg/sys"
-	util "github.com/sylabs/singularity-mpi/internal/pkg/util/file"
-	"github.com/sylabs/singularity-mpi/pkg/containizer"
+	"github.com/sylabs/singularity-mpi/pkg/containerizer"
 )
 
 func main() {
@@ -92,7 +92,7 @@ func main() {
 	}
 
 	log.Println("* Creating container for your application...")
-	_, err = containizer.ContainerizeApp(&sysCfg)
+	_, err = containerizer.ContainerizeApp(&sysCfg)
 	if err != nil {
 		log.Fatalf("failed to create container for app: %s", err)
 	}

cmd/sympi/sympi.go

@@ -16,16 +16,16 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/gvallee/go_util/pkg/util"
+	"github.com/gvallee/kv/pkg/kv"
 	"github.com/sylabs/singularity-mpi/internal/pkg/buildenv"
 	"github.com/sylabs/singularity-mpi/internal/pkg/builder"
 	"github.com/sylabs/singularity-mpi/internal/pkg/checker"
 	"github.com/sylabs/singularity-mpi/internal/pkg/implem"
-	"github.com/sylabs/singularity-mpi/internal/pkg/kv"
 	"github.com/sylabs/singularity-mpi/internal/pkg/manifest"
 	"github.com/sylabs/singularity-mpi/internal/pkg/sy"
 	"github.com/sylabs/singularity-mpi/internal/pkg/sympierr"
 	"github.com/sylabs/singularity-mpi/internal/pkg/sys"
-	util "github.com/sylabs/singularity-mpi/internal/pkg/util/file"
 	"github.com/sylabs/singularity-mpi/pkg/sympi"
 )
 
@@ -367,7 +367,7 @@ func installSingularity(id string, params []string, sysCfg *sys.Config) error {
 
 func listAvail(sysCfg *sys.Config) error {
 	fmt.Println("The following versions of Singularity can be installed:")
-	cfgFile := filepath.Join(sysCfg.EtcDir, "singularity.conf")
+	cfgFile := filepath.Join(sysCfg.EtcDir, "sympi_singularity.conf")
 	kvs, err := kv.LoadKeyValueConfig(cfgFile)
 	if err != nil {
 		return fmt.Errorf("failed to load configuration from %s: %s", cfgFile, err)
@@ -377,7 +377,7 @@ func listAvail(sysCfg *sys.Config) error {
 	}
 
 	fmt.Println("The following versions of Open MPI can be installed:")
-	cfgFile = filepath.Join(sysCfg.EtcDir, "openmpi.conf")
+	cfgFile = filepath.Join(sysCfg.EtcDir, sys.GetMPIConfigFileName("openmpi"))
 	kvs, err = kv.LoadKeyValueConfig(cfgFile)
 	if err != nil {
 		return fmt.Errorf("failed to load configuration from %s: %s", cfgFile, err)
@@ -387,7 +387,7 @@ func listAvail(sysCfg *sys.Config) error {
 	}
 
 	fmt.Println("The following versions of MPICH can be installed:")
-	cfgFile = filepath.Join(sysCfg.EtcDir, "mpich.conf")
+	cfgFile = filepath.Join(sysCfg.EtcDir, sys.GetMPIConfigFileName("mpich"))
 	kvs, err = kv.LoadKeyValueConfig(cfgFile)
 	if err != nil {
 		return fmt.Errorf("failed to load configuration from %s: %s", cfgFile, err)

cmd/syrun/syrun.go

@@ -10,7 +10,7 @@ import (
 	"log"
 	"os"
 
-	util "github.com/sylabs/singularity-mpi/internal/pkg/util/file"
+	"github.com/gvallee/go_util/pkg/util"
 	"github.com/sylabs/singularity-mpi/pkg/sympi"
 )
 

cmd/syvalidate/syvalidate.go

@@ -15,19 +15,19 @@ import (
 	"path/filepath"
 	"strconv"
 
+	"github.com/gvallee/go_util/pkg/util"
+	"github.com/gvallee/kv/pkg/kv"
 	"github.com/sylabs/singularity-mpi/internal/pkg/app"
 	"github.com/sylabs/singularity-mpi/internal/pkg/buildenv"
 	"github.com/sylabs/singularity-mpi/internal/pkg/checker"
 	"github.com/sylabs/singularity-mpi/internal/pkg/configparser"
 	cfg "github.com/sylabs/singularity-mpi/internal/pkg/configparser"
 	"github.com/sylabs/singularity-mpi/internal/pkg/container"
-	"github.com/sylabs/singularity-mpi/internal/pkg/kv"
 	"github.com/sylabs/singularity-mpi/internal/pkg/launcher"
 	"github.com/sylabs/singularity-mpi/internal/pkg/results"
 	"github.com/sylabs/singularity-mpi/internal/pkg/sy"
 	"github.com/sylabs/singularity-mpi/internal/pkg/syexec"
 	"github.com/sylabs/singularity-mpi/internal/pkg/sys"
-	util "github.com/sylabs/singularity-mpi/internal/pkg/util/file"
 	exp "github.com/sylabs/singularity-mpi/pkg/experiments"
 )
 
@@ -161,7 +161,7 @@ func run(experiments []exp.Config, sysCfg *sys.Config, syConfig *sy.MPIToolConfi
 			log.Printf("Running experiment %d/%d with host MPI %s and container MPI %s\n", i+1, sysCfg.Nrun, e.HostMPI.Version, e.ContainerMPI.Version)
 			newRes, err = runExperiment(e, sysCfg, syConfig)
 			if err != nil {
-				log.Fatalf("failure during the execution of experiment: %s", err)
+				log.Printf("[ERROR] failure during the execution of experiment: %s", err)
 			}
 			newResults = append(newResults, newRes)
 
@@ -261,7 +261,7 @@ func main() {
 	}
 
 	/* Argument parsing */
-	configFile := flag.String("configfile", sysCfg.EtcDir+"/openmpi.conf", "Path to the configuration file specifying which versions of a given implementation of MPI to test")
+	configFile := flag.String("configfile", sysCfg.EtcDir+"/sympi_openmpi.conf", "Path to the configuration file specifying which versions of a given implementation of MPI to test")
 	outputFile := flag.String("outputFile", "", "Full path to the output file")
 	verbose := flag.Bool("v", false, "Enable verbose mode")
 	netpipe := flag.Bool("netpipe", false, "Run NetPipe as test")

etc/intelmpi.conf renamed to etc/sympi_intelmpi.conf

@@ -0,0 +1,4 @@
+eoan	library://vallee/ubuntu/19.10:latest
+disco	library://vallee/ubuntu/19.04:latest
+bionic	library://vallee/ubuntu/18.04:latest
+xenial	library://vallee/ubuntu/16.04:latest

etc/mpich.conf renamed to etc/sympi_mpich.conf

@@ -1,3 +1,8 @@
 module github.com/sylabs/singularity-mpi
 
-go 1.13
+go 1.11
+
+require (
+	github.com/gvallee/go_util v1.0.0
+	github.com/gvallee/kv v1.0.0
+)

etc/ofi.conf renamed to etc/sympi_ofi.conf

@@ -0,0 +1,4 @@
+github.com/gvallee/go_util v1.0.0 h1:N1Op0Nscjv0mht2NJfGfMDe+2+ptfH1lKseb52jQkbY=
+github.com/gvallee/go_util v1.0.0/go.mod h1:fTexpwdH/n05Ziu0TXJIQsr7E+46QpBxNdeOOsyC0/s=
+github.com/gvallee/kv v1.0.0 h1:QE3Ua8JewroqJqc+J9RWtL7KUu7rQmfLfxlBVY5t1ko=
+github.com/gvallee/kv v1.0.0/go.mod h1:sfSclfFfLV+Y+9e9FayIbBUOtvbt1779S6q52bSSU5E=

etc/openmpi-images.conf renamed to etc/sympi_openmpi-images.conf

@@ -21,4 +21,18 @@ type Info struct {
 
 	// InstallCmd is the command to use to install the application
 	InstallCmd string
+
+	// ExpectedRankOutput specifies what is the expected output from EACH rank
+	// A few keyword can be used for runtime-specific parameters
+	// Use '#NP' to specify the job size
+	// Use '#RANK' to specify the rank number
+	ExpectedRankOutput string
+
+	// ExpectedNote specifies what is the expected note from an application
+	//
+	// A note is the result of an application-specific parsing/analysis of the
+	// application's output that is specific to this framework. For instance,
+	// for netpipe, the expected note is something like 'max bandwidth: 44.773 Gbps; latency: 50.609 nsecs'
+	// todo: should support regexp here
+	ExpectedNote string
 }

etc/openmpi.conf renamed to etc/sympi_openmpi.conf

@@ -19,5 +19,6 @@ func GetHelloworld(sysCfg *sys.Config) Info {
 	hw.Name = "helloworld"
 	hw.BinPath = "/opt/mpitest"
 	hw.Source = "file://" + filepath.Join(sysCfg.TemplateDir, "mpitest.c")
+	hw.ExpectedRankOutput = "Hello, I am rank #RANK/#NP"
 	return hw
 }

etc/singularity.conf renamed to etc/sympi_singularity.conf

@@ -17,5 +17,6 @@ func GetNetpipe(sysCfg *sys.Config) Info {
 	netpipe.BinPath = "/opt/NetPIPE-5.1.4/NPmpi"
 	netpipe.Source = "http://netpipe.cs.ksu.edu/download/NetPIPE-5.1.4.tar.gz"
 	netpipe.InstallCmd = "make mpi"
+	netpipe.ExpectedNote = "max bandwidth: "
 	return netpipe
 }

etc/sympi_ubuntu.conf

@@ -6,13 +6,13 @@
 package autotools
 
 import (
-	"bytes"
 	"fmt"
 	"log"
-	"os/exec"
 	"path/filepath"
+	"strings"
 
-	util "github.com/sylabs/singularity-mpi/internal/pkg/util/file"
+	"github.com/gvallee/go_util/pkg/util"
+	"github.com/sylabs/singularity-mpi/internal/pkg/syexec"
 )
 
 // Config represents the configuration of the autotools-compliant software to configure/compile/install
@@ -45,17 +45,18 @@ func Configure(cfg *Config) error {
 	}
 
 	log.Printf("-> Running 'configure': %s %s\n", configurePath, cmdArgs)
-	var stdout, stderr bytes.Buffer
-	cmd := exec.Command(configurePath)
+	var cmd syexec.SyCmd
+	cmd.BinPath = "./configure"
+	cmd.ManifestName = "configure"
+	cmd.ManifestDir = cfg.Install
 	if len(cmdArgs) > 0 {
-		cmd = exec.Command(configurePath, cmdArgs...)
+		cmd.ManifestData = []string{strings.Join(cmdArgs, " ")}
+		cmd.CmdArgs = cmdArgs
 	}
-	cmd.Dir = cfg.Source
-	cmd.Stderr = &stderr
-	cmd.Stdout = &stdout
-	err := cmd.Run()
-	if err != nil {
-		return fmt.Errorf("command failed: %s - stdout: %s - stderr: %s", err, stdout.String(), stderr.String())
+	cmd.ExecDir = cfg.Source
+	res := cmd.Run()
+	if res.Err != nil {
+		return fmt.Errorf("command failed: %s - stdout: %s - stderr: %s", res.Err, res.Stdout, res.Stderr)
 	}
 
 	return nil