validation: use t.Fail when checking for main test errors

So far `util.Fatal()`, which calls `os.Exit(1)`, has been widely used to
exit immediately when a critical error happened. Its downside is though
that no further function can called after that moment, even defer
functions cannot be called. That's fine if there's nothing to clean
up or the test is simple enough. Though it could be an issue, for
example, when a test should print out TAP output by calling
`t.AutoPlan()`, or when it should clean up something.

So let's try to use `util.Fatal()` only for errors from critical cases
like errors from `util.GetDefaultGenerator()` or `util.PrepareBundle()`.
In case of main test errors, use instead `t.Fail(err.Error())` to print
out errors in TAP outputs.

Partly addresses opencontainers#582

/cc @liangchenye

Signed-off-by: Dongsu Park <[email protected]>

validation/config_updates_without_affect.go

@@ -82,7 +82,7 @@ func main() {
 		err = r.Delete()
 	}
 	if err != nil {
-		util.Fatal(err)
+		t.Fail(err.Error())
 	}
 
 	t.AutoPlan()

validation/delete.go

@@ -85,7 +85,7 @@ func main() {
 			util.WaitingForStatus(testRuntime, util.LifecycleStatusStopped, time.Second*10, time.Second*1)
 			err = testRuntime.Delete()
 			if err != nil {
-				util.Fatal(err)
+				t.Fail(err.Error())
 			}
 		}
 	}

validation/hooks_stdin.go

@@ -140,7 +140,7 @@ func main() {
 
 	err = util.RuntimeLifecycleValidate(config)
 	if err != nil {
-		util.Fatal(err)
+		t.Fail(err.Error())
 	}
 
 	expectedState := rspecs.State{

validation/hostname.go

@@ -40,7 +40,7 @@ func main() {
 
 	for _, h := range hostnames {
 		if err := testHostname(t, h); err != nil {
-			util.Fatal(err)
+			t.Fail(err.Error())
 		}
 	}
 }

validation/kill_no_effect.go

@@ -57,7 +57,7 @@ func main() {
 	}
 	err = util.RuntimeLifecycleValidate(config)
 	if err != nil && err != targetErr {
-		util.Fatal(err)
+		t.Fail(err.Error())
 	} else {
 		util.SpecErrorOK(t, err == nil, targetErr, nil)
 	}

validation/killsig.go

@@ -29,6 +29,7 @@ func main() {
 	containerID := uuid.NewV4().String()
 	sigConfig, err := util.GetDefaultGenerator()
 	if err != nil {
+		os.RemoveAll(bundleDir)
 		util.Fatal(err)
 	}
 	rootDir := filepath.Join(bundleDir, sigConfig.Spec().Root.Path)

validation/linux_ns_itype.go

@@ -124,7 +124,7 @@ func main() {
 
 	err := testNamespaceInheritType(t)
 	if err != nil {
-		util.Fatal(err)
+		t.Fail(err.Error())
 	}
 
 	t.AutoPlan()

validation/linux_ns_nopath.go

@@ -125,7 +125,7 @@ func main() {
 
 	err := testNamespaceNoPath(t)
 	if err != nil {
-		util.Fatal(err)
+		t.Fail(err.Error())
 	}
 
 	t.AutoPlan()