Merge pull request #1955 from michelleN/test-failure-hook

feat(*): add support for test-failure hook

_proto/hapi/release/hook.proto

@@ -33,6 +33,7 @@ message Hook {
         PRE_ROLLBACK = 7;
         POST_ROLLBACK = 8;
         RELEASE_TEST_SUCCESS = 9;
+        RELEASE_TEST_FAILURE = 10;
 	}
 	string name = 1;
 	// Kind is the Kubernetes kind.

pkg/hooks/hooks.go

+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hooks
+
+import (
+	"errors"
+
+	"k8s.io/helm/pkg/proto/hapi/release"
+)
+
+// HookAnno is the label name for a hook
+const HookAnno = "helm.sh/hook"
+
+// Types of hooks
+const (
+	PreInstall         = "pre-install"
+	PostInstall        = "post-install"
+	PreDelete          = "pre-delete"
+	PostDelete         = "post-delete"
+	PreUpgrade         = "pre-upgrade"
+	PostUpgrade        = "post-upgrade"
+	PreRollback        = "pre-rollback"
+	PostRollback       = "post-rollback"
+	ReleaseTestSuccess = "test-success"
+	ReleaseTestFailure = "test-failure"
+)
+
+func FilterTestHooks(hooks []*release.Hook) ([]*release.Hook, error) {
+	testHooks := []*release.Hook{}
+	notFoundErr := errors.New("no tests found")
+
+	if len(hooks) == 0 {
+		return nil, notFoundErr
+	}
+
+	for _, h := range hooks {
+		for _, e := range h.Events {
+			if e == release.Hook_RELEASE_TEST_SUCCESS || e == release.Hook_RELEASE_TEST_FAILURE {
+				testHooks = append(testHooks, h)
+				continue
+			}
+		}
+	}
+
+	if len(testHooks) == 0 {
+		return nil, notFoundErr
+	}
+
+	return testHooks, nil
+}

pkg/proto/hapi/release/hook.pb.go

@@ -52,19 +52,21 @@ const (
 	Hook_PRE_ROLLBACK         Hook_Event = 7
 	Hook_POST_ROLLBACK        Hook_Event = 8
 	Hook_RELEASE_TEST_SUCCESS Hook_Event = 9
+	Hook_RELEASE_TEST_FAILURE Hook_Event = 10
 )
 
 var Hook_Event_name = map[int32]string{
-	0: "UNKNOWN",
-	1: "PRE_INSTALL",
-	2: "POST_INSTALL",
-	3: "PRE_DELETE",
-	4: "POST_DELETE",
-	5: "PRE_UPGRADE",
-	6: "POST_UPGRADE",
-	7: "PRE_ROLLBACK",
-	8: "POST_ROLLBACK",
-	9: "RELEASE_TEST_SUCCESS",
+	0:  "UNKNOWN",
+	1:  "PRE_INSTALL",
+	2:  "POST_INSTALL",
+	3:  "PRE_DELETE",
+	4:  "POST_DELETE",
+	5:  "PRE_UPGRADE",
+	6:  "POST_UPGRADE",
+	7:  "PRE_ROLLBACK",
+	8:  "POST_ROLLBACK",
+	9:  "RELEASE_TEST_SUCCESS",
+	10: "RELEASE_TEST_FAILURE",
 }
 var Hook_Event_value = map[string]int32{
 	"UNKNOWN":              0,
@@ -77,6 +79,7 @@ var Hook_Event_value = map[string]int32{
 	"PRE_ROLLBACK":         7,
 	"POST_ROLLBACK":        8,
 	"RELEASE_TEST_SUCCESS": 9,
+	"RELEASE_TEST_FAILURE": 10,
 }
 
 func (x Hook_Event) String() string {
@@ -119,27 +122,28 @@ func init() {
 func init() { proto.RegisterFile("hapi/release/hook.proto", fileDescriptor0) }
 
 var fileDescriptor0 = []byte{
-	// 343 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x4c, 0x90, 0xdf, 0x6e, 0xa2, 0x40,
-	0x14, 0xc6, 0x17, 0x41, 0xd0, 0xa3, 0xeb, 0xb2, 0x93, 0x4d, 0x76, 0xe2, 0x4d, 0x8d, 0x57, 0x5e,
-	0x0d, 0x8d, 0x4d, 0x1f, 0x00, 0x75, 0xd2, 0x36, 0x12, 0x34, 0x03, 0xa6, 0x49, 0x6f, 0x08, 0xa6,
-	0xa3, 0x12, 0x85, 0x21, 0x82, 0x7d, 0x82, 0x3e, 0x55, 0x9f, 0xae, 0x99, 0xe1, 0x4f, 0x7a, 0x77,
-	0xf8, 0x9d, 0x1f, 0xdf, 0xcc, 0x37, 0xf0, 0xff, 0x14, 0xe7, 0x89, 0x73, 0xe5, 0x17, 0x1e, 0x17,
-	0xdc, 0x39, 0x09, 0x71, 0x26, 0xf9, 0x55, 0x94, 0x02, 0x0d, 0xe5, 0x82, 0xd4, 0x8b, 0xf1, 0xdd,
-	0x51, 0x88, 0xe3, 0x85, 0x3b, 0x6a, 0xb7, 0xbf, 0x1d, 0x9c, 0x32, 0x49, 0x79, 0x51, 0xc6, 0x69,
-	0x5e, 0xe9, 0xd3, 0x4f, 0x1d, 0x8c, 0x67, 0x21, 0xce, 0x08, 0x81, 0x91, 0xc5, 0x29, 0xc7, 0xda,
-	0x44, 0x9b, 0xf5, 0x99, 0x9a, 0x25, 0x3b, 0x27, 0xd9, 0x3b, 0xee, 0x54, 0x4c, 0xce, 0x92, 0xe5,
-	0x71, 0x79, 0xc2, 0x7a, 0xc5, 0xe4, 0x8c, 0xc6, 0xd0, 0x4b, 0xe3, 0x2c, 0x39, 0xf0, 0xa2, 0xc4,
-	0x86, 0xe2, 0xed, 0x37, 0xba, 0x07, 0x93, 0x7f, 0xf0, 0xac, 0x2c, 0x70, 0x77, 0xa2, 0xcf, 0x46,
-	0x73, 0x4c, 0x7e, 0x5e, 0x90, 0xc8, 0xb3, 0x09, 0x95, 0x02, 0xab, 0x3d, 0xf4, 0x08, 0xbd, 0x4b,
-	0x5c, 0x94, 0xd1, 0xf5, 0x96, 0x61, 0x73, 0xa2, 0xcd, 0x06, 0xf3, 0x31, 0xa9, 0x6a, 0x90, 0xa6,
-	0x06, 0x09, 0x9b, 0x1a, 0xcc, 0x92, 0x2e, 0xbb, 0x65, 0xd3, 0x2f, 0x0d, 0xba, 0x2a, 0x08, 0x0d,
-	0xc0, 0xda, 0xf9, 0x6b, 0x7f, 0xf3, 0xea, 0xdb, 0xbf, 0xd0, 0x1f, 0x18, 0x6c, 0x19, 0x8d, 0x5e,
-	0xfc, 0x20, 0x74, 0x3d, 0xcf, 0xd6, 0x90, 0x0d, 0xc3, 0xed, 0x26, 0x08, 0x5b, 0xd2, 0x41, 0x23,
-	0x00, 0xa9, 0xac, 0xa8, 0x47, 0x43, 0x6a, 0xeb, 0xea, 0x17, 0x69, 0xd4, 0xc0, 0x68, 0x32, 0x76,
-	0xdb, 0x27, 0xe6, 0xae, 0xa8, 0xdd, 0x6d, 0x33, 0x1a, 0x62, 0x2a, 0xc2, 0x68, 0xc4, 0x36, 0x9e,
-	0xb7, 0x70, 0x97, 0x6b, 0xdb, 0x42, 0x7f, 0xe1, 0xb7, 0x72, 0x5a, 0xd4, 0x43, 0x18, 0xfe, 0x31,
-	0xea, 0x51, 0x37, 0xa0, 0x51, 0x48, 0x83, 0x30, 0x0a, 0x76, 0xcb, 0x25, 0x0d, 0x02, 0xbb, 0xbf,
-	0xe8, 0xbf, 0x59, 0xf5, 0x8b, 0xec, 0x4d, 0x55, 0xf2, 0xe1, 0x3b, 0x00, 0x00, 0xff, 0xff, 0xdf,
-	0xef, 0x1c, 0xfd, 0xe2, 0x01, 0x00, 0x00,
+	// 354 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x64, 0x90, 0xdd, 0x6e, 0xa2, 0x40,
+	0x18, 0x86, 0x17, 0x41, 0xd0, 0xd1, 0x75, 0x67, 0x27, 0x9b, 0xec, 0xc4, 0x93, 0x35, 0x1e, 0x79,
+	0x34, 0x6c, 0x6c, 0x7a, 0x01, 0xa8, 0xd3, 0xd6, 0x48, 0xd0, 0x0c, 0x90, 0x26, 0x3d, 0x21, 0x98,
+	0x8e, 0x4a, 0x14, 0x86, 0x08, 0xf6, 0x72, 0x7a, 0x55, 0xbd, 0xa0, 0x66, 0x86, 0x9f, 0x34, 0xe9,
+	0xd9, 0xc7, 0xf3, 0x3e, 0x7c, 0x33, 0xef, 0x80, 0xbf, 0xa7, 0x38, 0x4f, 0xec, 0x2b, 0xbf, 0xf0,
+	0xb8, 0xe0, 0xf6, 0x49, 0x88, 0x33, 0xc9, 0xaf, 0xa2, 0x14, 0x68, 0x28, 0x03, 0x52, 0x07, 0xe3,
+	0x7f, 0x47, 0x21, 0x8e, 0x17, 0x6e, 0xab, 0x6c, 0x7f, 0x3b, 0xd8, 0x65, 0x92, 0xf2, 0xa2, 0x8c,
+	0xd3, 0xbc, 0xd2, 0xa7, 0xef, 0x3a, 0x30, 0x9e, 0x84, 0x38, 0x23, 0x04, 0x8c, 0x2c, 0x4e, 0x39,
+	0xd6, 0x26, 0xda, 0xac, 0xcf, 0xd4, 0x2c, 0xd9, 0x39, 0xc9, 0x5e, 0x71, 0xa7, 0x62, 0x72, 0x96,
+	0x2c, 0x8f, 0xcb, 0x13, 0xd6, 0x2b, 0x26, 0x67, 0x34, 0x06, 0xbd, 0x34, 0xce, 0x92, 0x03, 0x2f,
+	0x4a, 0x6c, 0x28, 0xde, 0x7e, 0xa3, 0xff, 0xc0, 0xe4, 0x6f, 0x3c, 0x2b, 0x0b, 0xdc, 0x9d, 0xe8,
+	0xb3, 0xd1, 0x1c, 0x93, 0xaf, 0x17, 0x24, 0xf2, 0x6c, 0x42, 0xa5, 0xc0, 0x6a, 0x0f, 0xdd, 0x83,
+	0xde, 0x25, 0x2e, 0xca, 0xe8, 0x7a, 0xcb, 0xb0, 0x39, 0xd1, 0x66, 0x83, 0xf9, 0x98, 0x54, 0x35,
+	0x48, 0x53, 0x83, 0x04, 0x4d, 0x0d, 0x66, 0x49, 0x97, 0xdd, 0xb2, 0xe9, 0x87, 0x06, 0xba, 0x6a,
+	0x11, 0x1a, 0x00, 0x2b, 0xf4, 0x36, 0xde, 0xf6, 0xd9, 0x83, 0x3f, 0xd0, 0x2f, 0x30, 0xd8, 0x31,
+	0x1a, 0xad, 0x3d, 0x3f, 0x70, 0x5c, 0x17, 0x6a, 0x08, 0x82, 0xe1, 0x6e, 0xeb, 0x07, 0x2d, 0xe9,
+	0xa0, 0x11, 0x00, 0x52, 0x59, 0x51, 0x97, 0x06, 0x14, 0xea, 0xea, 0x17, 0x69, 0xd4, 0xc0, 0x68,
+	0x76, 0x84, 0xbb, 0x47, 0xe6, 0xac, 0x28, 0xec, 0xb6, 0x3b, 0x1a, 0x62, 0x2a, 0xc2, 0x68, 0xc4,
+	0xb6, 0xae, 0xbb, 0x70, 0x96, 0x1b, 0x68, 0xa1, 0xdf, 0xe0, 0xa7, 0x72, 0x5a, 0xd4, 0x43, 0x18,
+	0xfc, 0x61, 0xd4, 0xa5, 0x8e, 0x4f, 0xa3, 0x80, 0xfa, 0x41, 0xe4, 0x87, 0xcb, 0x25, 0xf5, 0x7d,
+	0xd8, 0xff, 0x96, 0x3c, 0x38, 0x6b, 0x37, 0x64, 0x14, 0x82, 0x45, 0xff, 0xc5, 0xaa, 0xdf, 0x6a,
+	0x6f, 0xaa, 0xfa, 0x77, 0x9f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x69, 0x41, 0x62, 0x57, 0xfc, 0x01,
+	0x00, 0x00,
 }

pkg/releasetesting/environment.go

@@ -19,7 +19,12 @@ package releasetesting
 import (
 	"bytes"
 	"fmt"
+	"log"
+	"time"
 
+	"k8s.io/kubernetes/pkg/api"
+
+	"k8s.io/helm/pkg/proto/hapi/release"
 	"k8s.io/helm/pkg/proto/hapi/services"
 	"k8s.io/helm/pkg/tiller/environment"
 )
@@ -32,6 +37,50 @@ type Environment struct {
 	Timeout    int64
 }
 
+func (env *Environment) createTestPod(test *test) error {
+	b := bytes.NewBufferString(test.manifest)
+	if err := env.KubeClient.Create(env.Namespace, b, env.Timeout, false); err != nil {
+		log.Printf(err.Error())
+		test.result.Info = err.Error()
+		test.result.Status = release.TestRun_FAILURE
+		return err
+	}
+
+	return nil
+}
+
+func (env *Environment) getTestPodStatus(test *test) (api.PodPhase, error) {
+	b := bytes.NewBufferString(test.manifest)
+	status, err := env.KubeClient.WaitAndGetCompletedPodPhase(env.Namespace, b, time.Duration(env.Timeout)*time.Second)
+	if err != nil {
+		log.Printf("Error getting status for pod %s: %s", test.result.Name, err)
+		test.result.Info = err.Error()
+		test.result.Status = release.TestRun_UNKNOWN
+		return status, err
+	}
+
+	return status, err
+}
+
+func (env *Environment) streamResult(r *release.TestRun) error {
+	switch r.Status {
+	case release.TestRun_SUCCESS:
+		if err := env.streamSuccess(r.Name); err != nil {
+			return err
+		}
+	case release.TestRun_FAILURE:
+		if err := env.streamFailed(r.Name); err != nil {
+			return err
+		}
+
+	default:
+		if err := env.streamUnknown(r.Name, r.Info); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (env *Environment) streamRunning(name string) error {
 	msg := "RUNNING: " + name
 	return env.streamMessage(msg)

pkg/releasetesting/environment_test.go

@@ -23,11 +23,41 @@ import (
 	"os"
 	"testing"
 
+	"k8s.io/helm/pkg/proto/hapi/release"
 	tillerEnv "k8s.io/helm/pkg/tiller/environment"
 )
 
+func TestCreateTestPodSuccess(t *testing.T) {
+	env := testEnvFixture()
+	test := testFixture()
+
+	err := env.createTestPod(test)
+	if err != nil {
+		t.Errorf("Expected no error, got an error: %s", err)
+	}
+}
+
+func TestCreateTestPodFailure(t *testing.T) {
+	env := testEnvFixture()
+	env.KubeClient = newCreateFailingKubeClient()
+	test := testFixture()
+
+	err := env.createTestPod(test)
+	if err == nil {
+		t.Errorf("Expected error, got no error")
+	}
+
+	if test.result.Info == "" {
+		t.Errorf("Expected error to be saved in test result info but found empty string")
+	}
+
+	if test.result.Status != release.TestRun_FAILURE {
+		t.Errorf("Expected test result status to be failure but got: %v", test.result.Status)
+	}
+}
+
 func TestDeleteTestPods(t *testing.T) {
-	mockTestSuite := testSuiteFixture()
+	mockTestSuite := testSuiteFixture([]string{manifestWithTestSuccessHook})
 	mockTestEnv := newMockTestingEnvironment()
 	mockTestEnv.KubeClient = newGetFailingKubeClient()
 
@@ -46,7 +76,7 @@ func TestDeleteTestPods(t *testing.T) {
 }
 
 func TestDeleteTestPodsFailingDelete(t *testing.T) {
-	mockTestSuite := testSuiteFixture()
+	mockTestSuite := testSuiteFixture([]string{manifestWithTestSuccessHook})
 	mockTestEnv := newMockTestingEnvironment()
 	mockTestEnv.KubeClient = newDeleteFailingKubeClient()
 
@@ -82,18 +112,22 @@ func (mte MockTestingEnvironment) streamSuccess(name string) error       { retur
 func (mte MockTestingEnvironment) streamUnknown(name, info string) error { return nil }
 func (mte MockTestingEnvironment) streamMessage(msg string) error        { return nil }
 
+type getFailingKubeClient struct {
+	tillerEnv.PrintingKubeClient
+}
+
 func newGetFailingKubeClient() *getFailingKubeClient {
 	return &getFailingKubeClient{
 		PrintingKubeClient: tillerEnv.PrintingKubeClient{Out: os.Stdout},
 	}
 }
 
-type getFailingKubeClient struct {
-	tillerEnv.PrintingKubeClient
+func (p *getFailingKubeClient) Get(ns string, r io.Reader) (string, error) {
+	return "", errors.New("In the end, they did not find Nemo.")
 }
 
-func (p *getFailingKubeClient) Get(ns string, r io.Reader) (string, error) {
-	return "", errors.New("Get failed")
+type deleteFailingKubeClient struct {
+	tillerEnv.PrintingKubeClient
 }
 
 func newDeleteFailingKubeClient() *deleteFailingKubeClient {
@@ -102,10 +136,20 @@ func newDeleteFailingKubeClient() *deleteFailingKubeClient {
 	}
 }
 
-type deleteFailingKubeClient struct {
+func (p *deleteFailingKubeClient) Delete(ns string, r io.Reader) error {
+	return errors.New("delete failed")
+}
+
+type createFailingKubeClient struct {
 	tillerEnv.PrintingKubeClient
 }
 
-func (p *deleteFailingKubeClient) Delete(ns string, r io.Reader) error {
-	return errors.New("In the end, they did not find Nemo.")
+func newCreateFailingKubeClient() *createFailingKubeClient {
+	return &createFailingKubeClient{
+		PrintingKubeClient: tillerEnv.PrintingKubeClient{Out: os.Stdout},
+	}
+}
+
+func (p *createFailingKubeClient) Create(ns string, r io.Reader, t int64, shouldWait bool) error {
+	return errors.New("We ran out of budget and couldn't create finding-nemo")
 }

pkg/releasetesting/test_suite.go

@@ -17,16 +17,14 @@ limitations under the License.
 package releasetesting
 
 import (
-	"bytes"
 	"fmt"
-	"log"
 	"strings"
-	"time"
 
 	"github.com/ghodss/yaml"
 	"github.com/golang/protobuf/ptypes/timestamp"
 	"k8s.io/kubernetes/pkg/api"
 
+	"k8s.io/helm/pkg/hooks"
 	"k8s.io/helm/pkg/proto/hapi/release"
 	util "k8s.io/helm/pkg/releaseutil"
 	"k8s.io/helm/pkg/timeconv"
@@ -41,14 +39,15 @@ type TestSuite struct {
 }
 
 type test struct {
-	manifest string
-	result   *release.TestRun
+	manifest        string
+	expectedSuccess bool
+	result          *release.TestRun
 }
 
 // NewTestSuite takes a release object and returns a TestSuite object with test definitions
 //  extracted from the release
 func NewTestSuite(rel *release.Release) (*TestSuite, error) {
-	testManifests, err := extractTestManifestsFromHooks(rel.Hooks, rel.Name)
+	testManifests, err := extractTestManifestsFromHooks(rel.Hooks)
 	if err != nil {
 		return nil, err
 	}
@@ -61,11 +60,11 @@ func NewTestSuite(rel *release.Release) (*TestSuite, error) {
 	}, nil
 }
 
-// Run executes tests in a test suite and stores a result within the context of a given environment
-func (t *TestSuite) Run(env *Environment) error {
-	t.StartedAt = timeconv.Now()
+// Run executes tests in a test suite and stores a result within a given environment
+func (ts *TestSuite) Run(env *Environment) error {
+	ts.StartedAt = timeconv.Now()
 
-	for _, testManifest := range t.TestManifests {
+	for _, testManifest := range ts.TestManifests {
 		test, err := newTest(testManifest)
 		if err != nil {
 			return err
@@ -77,7 +76,7 @@ func (t *TestSuite) Run(env *Environment) error {
 		}
 
 		resourceCreated := true
-		if err := t.createTestPod(test, env); err != nil {
+		if err := env.createTestPod(test); err != nil {
 			resourceCreated = false
 			if streamErr := env.streamError(test.result.Info); streamErr != nil {
 				return err
@@ -87,7 +86,7 @@ func (t *TestSuite) Run(env *Environment) error {
 		resourceCleanExit := true
 		status := api.PodUnknown
 		if resourceCreated {
-			status, err = t.getTestPodStatus(test, env)
+			status, err = env.getTestPodStatus(test)
 			if err != nil {
 				resourceCleanExit = false
 				if streamErr := env.streamUnknown(test.result.Name, test.result.Info); streamErr != nil {
@@ -96,54 +95,59 @@ func (t *TestSuite) Run(env *Environment) error {
 			}
 		}
 
-		if resourceCreated && resourceCleanExit && status == api.PodSucceeded {
-			test.result.Status = release.TestRun_SUCCESS
-			if streamErr := env.streamSuccess(test.result.Name); streamErr != nil {
-				return streamErr
+		if resourceCreated && resourceCleanExit {
+			if err := test.assignTestResult(status); err != nil {
+				return err
 			}
-		} else if resourceCreated && resourceCleanExit && status == api.PodFailed {
-			test.result.Status = release.TestRun_FAILURE
-			if streamErr := env.streamFailed(test.result.Name); streamErr != nil {
+
+			if err := env.streamResult(test.result); err != nil {
 				return err
 			}
 		}
 
 		test.result.CompletedAt = timeconv.Now()
-		t.Results = append(t.Results, test.result)
+		ts.Results = append(ts.Results, test.result)
 	}
 
-	t.CompletedAt = timeconv.Now()
+	ts.CompletedAt = timeconv.Now()
 	return nil
 }
 
-// NOTE: may want to move this function to pkg/tiller in the future
-func filterHooksForTestHooks(hooks []*release.Hook, releaseName string) ([]*release.Hook, error) {
-	testHooks := []*release.Hook{}
-	notFoundErr := fmt.Errorf("no tests found for release %s", releaseName)
-
-	if len(hooks) == 0 {
-		return nil, notFoundErr
-	}
-
-	for _, h := range hooks {
-		for _, e := range h.Events {
-			if e == release.Hook_RELEASE_TEST_SUCCESS {
-				testHooks = append(testHooks, h)
-				continue
-			}
+func (t *test) assignTestResult(podStatus api.PodPhase) error {
+	switch podStatus {
+	case api.PodSucceeded:
+		if t.expectedSuccess {
+			t.result.Status = release.TestRun_SUCCESS
+		} else {
+			t.result.Status = release.TestRun_FAILURE
+		}
+	case api.PodFailed:
+		if !t.expectedSuccess {
+			t.result.Status = release.TestRun_SUCCESS
+		} else {
+			t.result.Status = release.TestRun_FAILURE
 		}
+	default:
+		t.result.Status = release.TestRun_UNKNOWN
 	}
 
-	if len(testHooks) == 0 {
-		return nil, notFoundErr
-	}
+	return nil
+}
 
-	return testHooks, nil
+func expectedSuccess(hookTypes []string) (bool, error) {
+	for _, hookType := range hookTypes {
+		hookType = strings.ToLower(strings.TrimSpace(hookType))
+		if hookType == hooks.ReleaseTestSuccess {
+			return true, nil
+		} else if hookType == hooks.ReleaseTestFailure {
+			return false, nil
+		}
+	}
+	return false, fmt.Errorf("No %s or %s hook found", hooks.ReleaseTestSuccess, hooks.ReleaseTestFailure)
 }
 
-// NOTE: may want to move this function to pkg/tiller in the future
-func extractTestManifestsFromHooks(hooks []*release.Hook, releaseName string) ([]string, error) {
-	testHooks, err := filterHooksForTestHooks(hooks, releaseName)
+func extractTestManifestsFromHooks(h []*release.Hook) ([]string, error) {
+	testHooks, err := hooks.FilterTestHooks(h)
 	if err != nil {
 		return nil, err
 	}
@@ -169,36 +173,18 @@ func newTest(testManifest string) (*test, error) {
 		return nil, fmt.Errorf("%s is not a pod", sh.Metadata.Name)
 	}
 
+	hookTypes := sh.Metadata.Annotations[hooks.HookAnno]
+	expected, err := expectedSuccess(strings.Split(hookTypes, ","))
+	if err != nil {
+		return nil, err
+	}
+
 	name := strings.TrimSuffix(sh.Metadata.Name, ",")
 	return &test{
-		manifest: testManifest,
+		manifest:        testManifest,
+		expectedSuccess: expected,
 		result: &release.TestRun{
 			Name: name,
 		},
 	}, nil
 }
-
-func (t *TestSuite) createTestPod(test *test, env *Environment) error {
-	b := bytes.NewBufferString(test.manifest)
-	if err := env.KubeClient.Create(env.Namespace, b, env.Timeout, false); err != nil {
-		log.Printf(err.Error())
-		test.result.Info = err.Error()
-		test.result.Status = release.TestRun_FAILURE
-		return err
-	}
-
-	return nil
-}
-
-func (t *TestSuite) getTestPodStatus(test *test, env *Environment) (api.PodPhase, error) {
-	b := bytes.NewBufferString(test.manifest)
-	status, err := env.KubeClient.WaitAndGetCompletedPodPhase(env.Namespace, b, time.Duration(env.Timeout)*time.Second)
-	if err != nil {
-		log.Printf("Error getting status for pod %s: %s", test.result.Name, err)
-		test.result.Info = err.Error()
-		test.result.Status = release.TestRun_UNKNOWN
-		return status, err
-	}
-
-	return status, err
-}

pkg/releasetesting/test_suite_test.go

@@ -37,6 +37,43 @@ import (
 	tillerEnv "k8s.io/helm/pkg/tiller/environment"
 )
 
+const manifestWithTestSuccessHook = `
+apiVersion: v1
+kind: Pod
+metadata:
+  name: finding-nemo,
+  annotations:
+    "helm.sh/hook": test-success
+spec:
+  containers:
+  - name: nemo-test
+    image: fake-image
+    cmd: fake-command
+`
+
+const manifestWithTestFailureHook = `
+apiVersion: v1
+kind: Pod
+metadata:
+  name: gold-rush,
+  annotations:
+    "helm.sh/hook": test-failure
+spec:
+  containers:
+  - name: gold-finding-test
+    image: fake-gold-finding-image
+    cmd: fake-gold-finding-command
+`
+const manifestWithInstallHooks = `apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: test-cm
+  annotations:
+    "helm.sh/hook": post-install,pre-delete
+data:
+  name: value
+`
+
 func TestNewTestSuite(t *testing.T) {
 	rel := releaseStub()
 
@@ -48,7 +85,8 @@ func TestNewTestSuite(t *testing.T) {
 
 func TestRun(t *testing.T) {
 
-	ts := testSuiteFixture()
+	testManifests := []string{manifestWithTestSuccessHook, manifestWithTestFailureHook}
+	ts := testSuiteFixture(testManifests)
 	if err := ts.Run(testEnvFixture()); err != nil {
 		t.Errorf("%s", err)
 	}
@@ -61,8 +99,8 @@ func TestRun(t *testing.T) {
 		t.Errorf("Expected CompletedAt to not be nil. Got: %v", ts.CompletedAt)
 	}
 
-	if len(ts.Results) != 1 {
-		t.Errorf("Expected 1 test result. Got %v", len(ts.Results))
+	if len(ts.Results) != 2 {
+		t.Errorf("Expected 2 test result. Got %v", len(ts.Results))
 	}
 
 	result := ts.Results[0]
@@ -82,25 +120,66 @@ func TestRun(t *testing.T) {
 		t.Errorf("Expected test result to be successful, got: %v", result.Status)
 	}
 
+	result2 := ts.Results[1]
+	if result2.StartedAt == nil {
+		t.Errorf("Expected test StartedAt to not be nil. Got: %v", result2.StartedAt)
+	}
+
+	if result2.CompletedAt == nil {
+		t.Errorf("Expected test CompletedAt to not be nil. Got: %v", result2.CompletedAt)
+	}
+
+	if result2.Name != "gold-rush" {
+		t.Errorf("Expected test name to be gold-rush, Got: %v", result2.Name)
+	}
+
+	if result2.Status != release.TestRun_FAILURE {
+		t.Errorf("Expected test result to be successful, got: %v", result2.Status)
+	}
+
 }
 
-func TestGetTestPodStatus(t *testing.T) {
-	ts := testSuiteFixture()
+func TestRunSuccessWithTestFailureHook(t *testing.T) {
+	ts := testSuiteFixture([]string{manifestWithTestFailureHook})
+	env := testEnvFixture()
+	env.KubeClient = newPodFailedKubeClient()
+	if err := ts.Run(env); err != nil {
+		t.Errorf("%s", err)
+	}
 
-	status, err := ts.getTestPodStatus(testFixture(), testEnvFixture())
-	if err != nil {
-		t.Errorf("Expected getTestPodStatus not to return err, Got: %s", err)
+	if ts.StartedAt == nil {
+		t.Errorf("Expected StartedAt to not be nil. Got: %v", ts.StartedAt)
+	}
+
+	if ts.CompletedAt == nil {
+		t.Errorf("Expected CompletedAt to not be nil. Got: %v", ts.CompletedAt)
 	}
 
-	if status != api.PodSucceeded {
-		t.Errorf("Expected pod status to be succeeded, Got: %s ", status)
+	if len(ts.Results) != 1 {
+		t.Errorf("Expected 1 test result. Got %v", len(ts.Results))
 	}
 
+	result := ts.Results[0]
+	if result.StartedAt == nil {
+		t.Errorf("Expected test StartedAt to not be nil. Got: %v", result.StartedAt)
+	}
+
+	if result.CompletedAt == nil {
+		t.Errorf("Expected test CompletedAt to not be nil. Got: %v", result.CompletedAt)
+	}
+
+	if result.Name != "gold-rush" {
+		t.Errorf("Expected test name to be gold-rush, Got: %v", result.Name)
+	}
+
+	if result.Status != release.TestRun_SUCCESS {
+		t.Errorf("Expected test result to be successful, got: %v", result.Status)
+	}
 }
 
 func TestExtractTestManifestsFromHooks(t *testing.T) {
 	rel := releaseStub()
-	testManifests, err := extractTestManifestsFromHooks(rel.Hooks, rel.Name)
+	testManifests, err := extractTestManifestsFromHooks(rel.Hooks)
 	if err != nil {
 		t.Errorf("Expected no error, Got: %s", err)
 	}
@@ -117,34 +196,11 @@ func chartStub() *chart.Chart {
 		},
 		Templates: []*chart.Template{
 			{Name: "templates/hello", Data: []byte("hello: world")},
-			{Name: "templates/hooks", Data: []byte(manifestWithTestHook)},
+			{Name: "templates/hooks", Data: []byte(manifestWithTestSuccessHook)},
 		},
 	}
 }
 
-var manifestWithTestHook = `
-apiVersion: v1
-kind: Pod
-metadata:
-  name: finding-nemo,
-  annotations:
-    "helm.sh/hook": test-success
-spec:
-  containers:
-  - name: nemo-test
-    image: fake-image
-    cmd: fake-command
-`
-var manifestWithInstallHooks = `apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: test-cm
-  annotations:
-    "helm.sh/hook": post-install,pre-delete
-data:
-  name: value
-`
-
 func releaseStub() *release.Release {
 	date := timestamp.Timestamp{Seconds: 242085845, Nanos: 0}
 	return &release.Release{
@@ -163,7 +219,7 @@ func releaseStub() *release.Release {
 				Name:     "finding-nemo",
 				Kind:     "Pod",
 				Path:     "finding-nemo",
-				Manifest: manifestWithTestHook,
+				Manifest: manifestWithTestSuccessHook,
 				Events: []release.Hook_Event{
 					release.Hook_RELEASE_TEST_SUCCESS,
 				},
@@ -184,13 +240,15 @@ func releaseStub() *release.Release {
 
 func testFixture() *test {
 	return &test{
-		manifest: manifestWithTestHook,
+		manifest: manifestWithTestSuccessHook,
 		result:   &release.TestRun{},
 	}
 }
 
-func testSuiteFixture() *TestSuite {
-	testManifests := []string{manifestWithTestHook}
+func testSuiteFixture(testManifests []string) *TestSuite {
+	if len(testManifests) == 0 {
+		testManifests = []string{manifestWithTestSuccessHook, manifestWithTestFailureHook}
+	}
 	testResults := []*release.TestRun{}
 	ts := &TestSuite{
 		TestManifests: testManifests,
@@ -227,16 +285,30 @@ func (rs mockStream) SendMsg(v interface{}) error    { return nil }
 func (rs mockStream) RecvMsg(v interface{}) error    { return nil }
 func (rs mockStream) Context() context.Context       { return helm.NewContext() }
 
+type podSucceededKubeClient struct {
+	tillerEnv.PrintingKubeClient
+}
+
 func newPodSucceededKubeClient() *podSucceededKubeClient {
 	return &podSucceededKubeClient{
 		PrintingKubeClient: tillerEnv.PrintingKubeClient{Out: os.Stdout},
 	}
 }
 
-type podSucceededKubeClient struct {
+func (p *podSucceededKubeClient) WaitAndGetCompletedPodPhase(ns string, r io.Reader, timeout time.Duration) (api.PodPhase, error) {
+	return api.PodSucceeded, nil
+}
+
+type podFailedKubeClient struct {
 	tillerEnv.PrintingKubeClient
 }
 
-func (p *podSucceededKubeClient) WaitAndGetCompletedPodPhase(ns string, r io.Reader, timeout time.Duration) (api.PodPhase, error) {
-	return api.PodSucceeded, nil
+func newPodFailedKubeClient() *podFailedKubeClient {
+	return &podFailedKubeClient{
+		PrintingKubeClient: tillerEnv.PrintingKubeClient{Out: os.Stdout},
+	}
+}
+
+func (p *podFailedKubeClient) WaitAndGetCompletedPodPhase(ns string, r io.Reader, timeout time.Duration) (api.PodPhase, error) {
+	return api.PodFailed, nil
 }

pkg/tiller/hooks.go

@@ -25,35 +25,22 @@ import (
 	"github.com/ghodss/yaml"
 
 	"k8s.io/helm/pkg/chartutil"
+	"k8s.io/helm/pkg/hooks"
 	"k8s.io/helm/pkg/proto/hapi/release"
 	util "k8s.io/helm/pkg/releaseutil"
 )
 
-// hookAnno is the label name for a hook
-const hookAnno = "helm.sh/hook"
-
-const (
-	preInstall         = "pre-install"
-	postInstall        = "post-install"
-	preDelete          = "pre-delete"
-	postDelete         = "post-delete"
-	preUpgrade         = "pre-upgrade"
-	postUpgrade        = "post-upgrade"
-	preRollback        = "pre-rollback"
-	postRollback       = "post-rollback"
-	releaseTestSuccess = "test-success"
-)
-
 var events = map[string]release.Hook_Event{
-	preInstall:         release.Hook_PRE_INSTALL,
-	postInstall:        release.Hook_POST_INSTALL,
-	preDelete:          release.Hook_PRE_DELETE,
-	postDelete:         release.Hook_POST_DELETE,
-	preUpgrade:         release.Hook_PRE_UPGRADE,
-	postUpgrade:        release.Hook_POST_UPGRADE,
-	preRollback:        release.Hook_PRE_ROLLBACK,
-	postRollback:       release.Hook_POST_ROLLBACK,
-	releaseTestSuccess: release.Hook_RELEASE_TEST_SUCCESS,
+	hooks.PreInstall:         release.Hook_PRE_INSTALL,
+	hooks.PostInstall:        release.Hook_POST_INSTALL,
+	hooks.PreDelete:          release.Hook_PRE_DELETE,
+	hooks.PostDelete:         release.Hook_POST_DELETE,
+	hooks.PreUpgrade:         release.Hook_PRE_UPGRADE,
+	hooks.PostUpgrade:        release.Hook_POST_UPGRADE,
+	hooks.PreRollback:        release.Hook_PRE_ROLLBACK,
+	hooks.PostRollback:       release.Hook_POST_ROLLBACK,
+	hooks.ReleaseTestSuccess: release.Hook_RELEASE_TEST_SUCCESS,
+	hooks.ReleaseTestFailure: release.Hook_RELEASE_TEST_FAILURE,
 }
 
 // manifest represents a manifest file, which has a name and some content.
@@ -117,7 +104,7 @@ func sortManifests(files map[string]string, apis chartutil.VersionSet, sort Sort
 			continue
 		}
 
-		hookTypes, ok := sh.Metadata.Annotations[hookAnno]
+		hookTypes, ok := sh.Metadata.Annotations[hooks.HookAnno]
 		if !ok {
 			generic = append(generic, manifest{name: n, content: c, head: &sh})
 			continue

pkg/tiller/release_server.go

@@ -32,6 +32,7 @@ import (
 	"k8s.io/kubernetes/pkg/client/typed/discovery"
 
 	"k8s.io/helm/pkg/chartutil"
+	"k8s.io/helm/pkg/hooks"
 	"k8s.io/helm/pkg/kube"
 	"k8s.io/helm/pkg/proto/hapi/chart"
 	"k8s.io/helm/pkg/proto/hapi/release"
@@ -313,7 +314,7 @@ func (s *ReleaseServer) performUpdate(originalRelease, updatedRelease *release.R
 
 	// pre-upgrade hooks
 	if !req.DisableHooks {
-		if err := s.execHook(updatedRelease.Hooks, updatedRelease.Name, updatedRelease.Namespace, preUpgrade, req.Timeout); err != nil {
+		if err := s.execHook(updatedRelease.Hooks, updatedRelease.Name, updatedRelease.Namespace, hooks.PreUpgrade, req.Timeout); err != nil {
 			return res, err
 		}
 	}
@@ -331,7 +332,7 @@ func (s *ReleaseServer) performUpdate(originalRelease, updatedRelease *release.R
 
 	// post-upgrade hooks
 	if !req.DisableHooks {
-		if err := s.execHook(updatedRelease.Hooks, updatedRelease.Name, updatedRelease.Namespace, postUpgrade, req.Timeout); err != nil {
+		if err := s.execHook(updatedRelease.Hooks, updatedRelease.Name, updatedRelease.Namespace, hooks.PostUpgrade, req.Timeout); err != nil {
 			return res, err
 		}
 	}
@@ -471,7 +472,7 @@ func (s *ReleaseServer) performRollback(currentRelease, targetRelease *release.R
 
 	// pre-rollback hooks
 	if !req.DisableHooks {
-		if err := s.execHook(targetRelease.Hooks, targetRelease.Name, targetRelease.Namespace, preRollback, req.Timeout); err != nil {
+		if err := s.execHook(targetRelease.Hooks, targetRelease.Name, targetRelease.Namespace, hooks.PreRollback, req.Timeout); err != nil {
 			return res, err
 		}
 	}
@@ -489,7 +490,7 @@ func (s *ReleaseServer) performRollback(currentRelease, targetRelease *release.R
 
 	// post-rollback hooks
 	if !req.DisableHooks {
-		if err := s.execHook(targetRelease.Hooks, targetRelease.Name, targetRelease.Namespace, postRollback, req.Timeout); err != nil {
+		if err := s.execHook(targetRelease.Hooks, targetRelease.Name, targetRelease.Namespace, hooks.PostRollback, req.Timeout); err != nil {
 			return res, err
 		}
 	}
@@ -829,7 +830,7 @@ func (s *ReleaseServer) performRelease(r *release.Release, req *services.Install
 
 	// pre-install hooks
 	if !req.DisableHooks {
-		if err := s.execHook(r.Hooks, r.Name, r.Namespace, preInstall, req.Timeout); err != nil {
+		if err := s.execHook(r.Hooks, r.Name, r.Namespace, hooks.PreInstall, req.Timeout); err != nil {
 			return res, err
 		}
 	}
@@ -878,7 +879,7 @@ func (s *ReleaseServer) performRelease(r *release.Release, req *services.Install
 
 	// post-install hooks
 	if !req.DisableHooks {
-		if err := s.execHook(r.Hooks, r.Name, r.Namespace, postInstall, req.Timeout); err != nil {
+		if err := s.execHook(r.Hooks, r.Name, r.Namespace, hooks.PostInstall, req.Timeout); err != nil {
 			msg := fmt.Sprintf("Release %q failed post-install: %s", r.Name, err)
 			log.Printf("warning: %s", msg)
 			r.Info.Status.Code = release.Status_FAILED
@@ -988,7 +989,7 @@ func (s *ReleaseServer) UninstallRelease(c ctx.Context, req *services.UninstallR
 	res := &services.UninstallReleaseResponse{Release: rel}
 
 	if !req.DisableHooks {
-		if err := s.execHook(rel.Hooks, rel.Name, rel.Namespace, preDelete, req.Timeout); err != nil {
+		if err := s.execHook(rel.Hooks, rel.Name, rel.Namespace, hooks.PreDelete, req.Timeout); err != nil {
 			return res, err
 		}
 	}
@@ -1034,7 +1035,7 @@ func (s *ReleaseServer) UninstallRelease(c ctx.Context, req *services.UninstallR
 	}
 
 	if !req.DisableHooks {
-		if err := s.execHook(rel.Hooks, rel.Name, rel.Namespace, postDelete, req.Timeout); err != nil {
+		if err := s.execHook(rel.Hooks, rel.Name, rel.Namespace, hooks.PostDelete, req.Timeout); err != nil {
 			es = append(es, err.Error())
 		}
 	}