implement start operation tests

Signed-off-by: Liang Chenye <liangchenye@huawei.com>

Author: liangchenye

specerror/runtime.go

@@ -68,10 +68,10 @@ const (
 	PropApplyFailNotCreate
 	// StartWithoutIDGenError represents "`start` operation MUST generate an error if it is not provided the container ID."
 	StartWithoutIDGenError
-	// StartNonCreateHaveNoEffect represents "Attempting to `start` a container that is not `created` MUST have no effect on the container."
-	StartNonCreateHaveNoEffect
-	// StartNonCreateGenError represents "Attempting to `start` a container that is not `created` MUST generate an error."
-	StartNonCreateGenError
+	// StartNotCreatedHaveNoEffect represents "Attempting to `start` a container that is not `created` MUST have no effect on the container."
+	StartNotCreatedHaveNoEffect
+	// StartNotCreatedGenError represents "Attempting to `start` a container that is not `created` MUST generate an error."
+	StartNotCreatedGenError
 	// StartProcImplement represents "`start` operation MUST run the user-specified program as specified by `process`."
 	StartProcImplement
 	// StartWithProcUnsetGenError represents "`start` operation MUST generate an error if `process` was not set."
@@ -163,8 +163,8 @@ func init() {
 	register(PropApplyFailGenError, rfc2119.Must, createRef)
 	register(PropApplyFailNotCreate, rfc2119.Must, createRef)
 	register(StartWithoutIDGenError, rfc2119.Must, startRef)
-	register(StartNonCreateHaveNoEffect, rfc2119.Must, startRef)
-	register(StartNonCreateGenError, rfc2119.Must, startRef)
+	register(StartNotCreatedHaveNoEffect, rfc2119.Must, startRef)
+	register(StartNotCreatedGenError, rfc2119.Must, startRef)
 	register(StartProcImplement, rfc2119.Must, startRef)
 	register(StartWithProcUnsetGenError, rfc2119.Must, startRef)
 	register(KillWithoutIDGenError, rfc2119.Must, killRef)

validation/start.go

@@ -0,0 +1,116 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"time"
+
+	"github.com/mndrix/tap-go"
+	rspecs "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/runtime-tools/specerror"
+	"github.com/opencontainers/runtime-tools/validation/util"
+	uuid "github.com/satori/go.uuid"
+)
+
+func main() {
+	t := tap.New()
+	t.Header(0)
+
+	bundleDir, err := util.PrepareBundle()
+	if err != nil {
+		util.Fatal(err)
+	}
+	defer os.RemoveAll(bundleDir)
+
+	containerID := uuid.NewV4().String()
+
+	r, err := util.NewRuntime(util.RuntimeCommand, bundleDir)
+	if err != nil {
+		util.Fatal(err)
+	}
+	g := util.GetDefaultGenerator()
+	g.SetProcessArgs([]string{"sh", "-c", fmt.Sprintf("echo 'process called' >> %s", "/output")})
+	r.SetConfig(g)
+	output := filepath.Join(bundleDir, g.Spec().Root.Path, "output")
+
+	// start without id
+	err = r.Start()
+	util.SpecErrorOK(t, err != nil, specerror.NewError(specerror.StartWithoutIDGenError, fmt.Errorf("start` operation MUST generate an error if it is not provided the container ID"), rspecs.Version), err)
+
+	// set id for the remaining tests
+	r.SetID(containerID)
+
+	// start a not `created` container - case one: non-exist container
+	err = r.Start()
+	util.SpecErrorOK(t, err != nil, specerror.NewError(specerror.StartNotCreatedGenError, fmt.Errorf("attempting to `start` a container that is not `created` MUST generate an error"), rspecs.Version), err)
+
+	err = r.Create()
+	if err != nil {
+		util.Fatal(err)
+	}
+	// start a `created` container
+	err = r.Start()
+	if err != nil {
+		util.SpecErrorOK(t, false, specerror.NewError(specerror.StartProcImplement, fmt.Errorf("`start` operation MUST run the user-specified program as specified by `process`"), rspecs.Version), err)
+	} else {
+		err := util.WaitingForStatus(r, util.LifecycleStatusStopped, time.Second*10, time.Second*1)
+		if err != nil {
+			util.Fatal(err)
+		}
+		outputData, outputErr := ioutil.ReadFile(output)
+		// check the output
+		util.SpecErrorOK(t, outputErr == nil && string(outputData) == "process called\n", specerror.NewError(specerror.StartProcImplement, fmt.Errorf("`start` operation MUST run the user-specified program as specified by `process`"), rspecs.Version), outputErr)
+	}
+
+	// start a not `created` container - case two: exist and `stopped`
+	err = r.Start()
+	// must generate an error
+	util.SpecErrorOK(t, err != nil, specerror.NewError(specerror.StartNotCreatedGenError, fmt.Errorf("attempting to `start` a container that is not `created` MUST generate an error"), rspecs.Version), err)
+
+	err = util.WaitingForStatus(r, util.LifecycleStatusStopped, time.Second*10, time.Second*1)
+	if err != nil {
+		util.Fatal(err)
+	}
+
+	outputData, outputErr := ioutil.ReadFile(output)
+	// must have no effect, it will not be something like 'process called\nprocess called\n'
+	util.SpecErrorOK(t, outputErr == nil && string(outputData) == "process called\n", specerror.NewError(specerror.StartNotCreatedHaveNoEffect, fmt.Errorf("attempting to `start` a container that is not `created` MUST have no effect on the container"), rspecs.Version), outputErr)
+
+	err = r.Delete()
+	if err != nil {
+		util.Fatal(err)
+	}
+
+	g.Spec().Process = nil
+	r.SetConfig(g)
+	err = r.Create()
+	// 'create' could fail according to the spec when the process is nil
+	// only do the test when a container is created
+	if err == nil {
+		err = r.Start()
+		util.SpecErrorOK(t, err == nil, specerror.NewError(specerror.StartWithProcUnsetGenError, fmt.Errorf("`start` operation MUST generate an error if `process` was not set"), rspecs.Version), err)
+		err = util.WaitingForStatus(r, util.LifecycleStatusStopped, time.Second*10, time.Second*1)
+		if err == nil {
+			err = r.Delete()
+		}
+		if err != nil {
+			util.Fatal(err)
+		}
+	} else {
+		diagnostic := map[string]string{
+			"error": err.Error(),
+		}
+		if e, ok := err.(*exec.ExitError); ok {
+			if len(e.Stderr) > 0 {
+				diagnostic["stderr"] = string(e.Stderr)
+			}
+		}
+		t.Skip(1, "`start` operation MUST generate an error if `process` was not set")
+		t.YAML(diagnostic)
+	}
+
+	t.AutoPlan()
+}

validation/util/test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/mrunalp/fileutils"
 	rspec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/opencontainers/runtime-tools/generate"
+	"github.com/opencontainers/runtime-tools/specerror"
 	"github.com/satori/go.uuid"
 )
 
@@ -102,6 +103,24 @@ func Skip(message string, diagnostic interface{}) {
 	}
 }
 
+// SpecErrorOK generates TAP output indicating whether a spec code test passed or failed.
+func SpecErrorOK(t *tap.T, expected bool, specErr error, detailedErr error) {
+	t.Ok(expected, specErr.(*specerror.Error).Err.Err.Error())
+	diagnostic := map[string]string{
+		"reference": specErr.(*specerror.Error).Err.Reference,
+	}
+
+	if detailedErr != nil {
+		diagnostic["error"] = detailedErr.Error()
+		if e, ok := detailedErr.(*exec.ExitError); ok {
+			if len(e.Stderr) > 0 {
+				diagnostic["stderr"] = string(e.Stderr)
+			}
+		}
+	}
+	t.YAML(diagnostic)
+}
+
 // PrepareBundle creates a test bundle in a temporary directory.
 func PrepareBundle() (string, error) {
 	bundleDir, err := ioutil.TempDir("", "ocitest")