Merge pull request #3540 from flant/hook_recreate_delete_policy

Implement before-hook-creation delete policy

_proto/hapi/release/hook.proto

@@ -38,6 +38,7 @@ message Hook {
 	enum DeletePolicy {
 	    SUCCEEDED = 0;
 	    FAILED = 1;
+	    BEFORE_HOOK_CREATION = 2;
 	}
 	string name = 1;
 	// Kind is the Kubernetes kind.

docs/charts_hooks.md

@@ -180,4 +180,19 @@ It is also possible to define policies that determine when to delete correspondi
     "helm.sh/hook-delete-policy": hook-succeeded
 ```
 
-When using `"helm.sh/hook-delete-policy"` annotation, you can choose its value from `"hook-succeeded"` and `"hook-failed"`. The value `"hook-succeeded"` specifies Tiller should delete the hook after the hook is successfully executed, while the value `"hook-failed"`specifies Tiller should delete the hook if the hook failed during execution.
+You can choose one or more defined annotation values:
+* `"hook-succeeded"` specifies Tiller should delete the hook after the hook is successfully executed.
+* `"hook-failed"` specifies Tiller should delete the hook if the hook failed during execution.
+* `"before-hook-creation"` specifies Tiller should delete the previous hook before the new hook is launched.
+
+### Automatically delete hook from previous release
+
+When helm release being updated it is possible, that hook resource already exists in cluster. By default helm will try to create resource and fail with `"... already exists"` error.
+
+One might choose `"helm.sh/hook-delete-policy": "before-hook-creation"` over `"helm.sh/hook-delete-policy": "hook-succeeded,hook-failed"` because:
+
+* It is convinient to keep failed hook job resource in kubernetes for example for manual debug.
+* It may be necessary to keep succeeded hook resource in kubernetes for some reason.
+* At the same time it is not desireable to do manual resource deletion before helm release upgrade.
+
+`"helm.sh/hook-delete-policy": "before-hook-creation"` annotation on hook causes tiller to remove the hook from previous release if there is one before the new hook is launched and can be used with another policies.

pkg/hooks/hooks.go

@@ -47,6 +47,7 @@ const (
 const (
 	HookSucceeded      = "hook-succeeded"
 	HookFailed         = "hook-failed"
+	BeforeHookCreation = "before-hook-creation"
 )
 
 // FilterTestHooks filters the list of hooks are returns only testing hooks.

pkg/proto/hapi/release/hook.pb.go

@@ -91,15 +91,18 @@ type Hook_DeletePolicy int32
 const (
 	Hook_SUCCEEDED            Hook_DeletePolicy = 0
 	Hook_FAILED               Hook_DeletePolicy = 1
+	Hook_BEFORE_HOOK_CREATION Hook_DeletePolicy = 2
 )
 
 var Hook_DeletePolicy_name = map[int32]string{
 	0: "SUCCEEDED",
 	1: "FAILED",
+	2: "BEFORE_HOOK_CREATION",
 }
 var Hook_DeletePolicy_value = map[string]int32{
 	"SUCCEEDED":            0,
 	"FAILED":               1,
+	"BEFORE_HOOK_CREATION": 2,
 }
 
 func (x Hook_DeletePolicy) String() string {
@@ -196,32 +199,33 @@ func init() {
 func init() { proto.RegisterFile("hapi/release/hook.proto", fileDescriptor0) }
 
 var fileDescriptor0 = []byte{
-	// 428 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0xd1, 0x6e, 0xda, 0x30,
-	0x14, 0x86, 0x9b, 0x12, 0x12, 0x38, 0x50, 0xea, 0x59, 0xd3, 0x66, 0x71, 0x53, 0xc4, 0x15, 0xbb,
-	0x09, 0x53, 0xa7, 0x3d, 0x40, 0x4a, 0xce, 0xd6, 0xaa, 0x11, 0x20, 0x27, 0x68, 0xd2, 0x6e, 0xa2,
-	0x74, 0xb8, 0x10, 0x11, 0xe2, 0x88, 0x98, 0x4d, 0x7b, 0xa6, 0xbd, 0xce, 0x1e, 0x68, 0xb2, 0x09,
-	0x59, 0xa5, 0xed, 0xee, 0x9c, 0xef, 0x7c, 0x76, 0xce, 0x1f, 0xc3, 0xdb, 0x6d, 0x5a, 0x66, 0xd3,
-	0x83, 0xc8, 0x45, 0x5a, 0x89, 0xe9, 0x56, 0xca, 0x9d, 0x57, 0x1e, 0xa4, 0x92, 0xb4, 0xaf, 0x07,
-	0x5e, 0x3d, 0x18, 0xde, 0x6c, 0xa4, 0xdc, 0xe4, 0x62, 0x6a, 0x66, 0x4f, 0xc7, 0xe7, 0xa9, 0xca,
-	0xf6, 0xa2, 0x52, 0xe9, 0xbe, 0x3c, 0xe9, 0xe3, 0x5f, 0x36, 0xd8, 0xf7, 0x52, 0xee, 0x28, 0x05,
-	0xbb, 0x48, 0xf7, 0x82, 0x59, 0x23, 0x6b, 0xd2, 0xe5, 0xa6, 0xd6, 0x6c, 0x97, 0x15, 0x6b, 0x76,
-	0x79, 0x62, 0xba, 0xd6, 0xac, 0x4c, 0xd5, 0x96, 0xb5, 0x4e, 0x4c, 0xd7, 0x74, 0x08, 0x9d, 0x7d,
-	0x5a, 0x64, 0xcf, 0xa2, 0x52, 0xcc, 0x36, 0xbc, 0xe9, 0xe9, 0x7b, 0x70, 0xc4, 0x77, 0x51, 0xa8,
-	0x8a, 0xb5, 0x47, 0xad, 0xc9, 0xe0, 0x96, 0x79, 0x2f, 0x17, 0xf4, 0xf4, 0xb7, 0x3d, 0xd4, 0x02,
-	0xaf, 0x3d, 0xfa, 0x11, 0x3a, 0x79, 0x5a, 0xa9, 0xe4, 0x70, 0x2c, 0x98, 0x33, 0xb2, 0x26, 0xbd,
-	0xdb, 0xa1, 0x77, 0x8a, 0xe1, 0x9d, 0x63, 0x78, 0xf1, 0x39, 0x06, 0x77, 0xb5, 0xcb, 0x8f, 0x05,
-	0x7d, 0x03, 0xce, 0x0f, 0x91, 0x6d, 0xb6, 0x8a, 0xb9, 0x23, 0x6b, 0xd2, 0xe6, 0x75, 0x47, 0xef,
-	0xe1, 0x7a, 0x2d, 0x72, 0xa1, 0x44, 0x52, 0xca, 0x3c, 0xfb, 0x96, 0x89, 0x8a, 0x75, 0xcc, 0x26,
-	0x37, 0xff, 0xd9, 0x24, 0x30, 0xe6, 0x52, 0x8b, 0x3f, 0xf9, 0x60, 0xfd, 0xb7, 0xcb, 0x44, 0x35,
-	0xfe, 0x6d, 0x41, 0xdb, 0xac, 0x4a, 0x7b, 0xe0, 0xae, 0xe6, 0x8f, 0xf3, 0xc5, 0x97, 0x39, 0xb9,
-	0xa0, 0xd7, 0xd0, 0x5b, 0x72, 0x4c, 0x1e, 0xe6, 0x51, 0xec, 0x87, 0x21, 0xb1, 0x28, 0x81, 0xfe,
-	0x72, 0x11, 0xc5, 0x0d, 0xb9, 0xa4, 0x03, 0x00, 0xad, 0x04, 0x18, 0x62, 0x8c, 0xa4, 0x65, 0x8e,
-	0x68, 0xa3, 0x06, 0xf6, 0xf9, 0x8e, 0xd5, 0xf2, 0x33, 0xf7, 0x03, 0x24, 0xed, 0xe6, 0x8e, 0x33,
-	0x71, 0x0c, 0xe1, 0x98, 0xf0, 0x45, 0x18, 0xde, 0xf9, 0xb3, 0x47, 0xe2, 0xd2, 0x57, 0x70, 0x65,
-	0x9c, 0x06, 0x75, 0x28, 0x83, 0xd7, 0x1c, 0x43, 0xf4, 0x23, 0x4c, 0x62, 0x8c, 0xe2, 0x24, 0x5a,
-	0xcd, 0x66, 0x18, 0x45, 0xa4, 0xfb, 0xcf, 0xe4, 0x93, 0xff, 0x10, 0xae, 0x38, 0x12, 0x18, 0xbf,
-	0x83, 0xfe, 0xcb, 0xd8, 0xf4, 0x0a, 0xba, 0xe6, 0x18, 0x06, 0x18, 0x90, 0x0b, 0x0a, 0xe0, 0x68,
-	0x17, 0x03, 0x62, 0xdd, 0x75, 0xbf, 0xba, 0xf5, 0xef, 0x7a, 0x72, 0xcc, 0x5b, 0x7c, 0xf8, 0x13,
-	0x00, 0x00, 0xff, 0xff, 0xb9, 0x8a, 0xe1, 0xaf, 0x89, 0x02, 0x00, 0x00,
+	// 445 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0x51, 0x8f, 0x9a, 0x40,
+	0x10, 0x80, 0x8f, 0x13, 0x41, 0x47, 0xcf, 0xdb, 0x6e, 0x9a, 0x76, 0xe3, 0xcb, 0x19, 0x9f, 0x7c,
+	0xc2, 0xe6, 0x9a, 0xfe, 0x00, 0x84, 0xb9, 0x6a, 0x24, 0x60, 0x16, 0x4c, 0x93, 0xbe, 0x10, 0xae,
+	0xee, 0x29, 0x11, 0x81, 0x08, 0xb6, 0xe9, 0x0f, 0xec, 0x3f, 0xe8, 0x0f, 0x6a, 0x76, 0x45, 0x7b,
+	0x49, 0xfb, 0x36, 0xf3, 0xcd, 0x37, 0xc3, 0x0c, 0x0b, 0xef, 0x77, 0x49, 0x99, 0x4e, 0x8f, 0x22,
+	0x13, 0x49, 0x25, 0xa6, 0xbb, 0xa2, 0xd8, 0x5b, 0xe5, 0xb1, 0xa8, 0x0b, 0xda, 0x97, 0x05, 0xab,
+	0x29, 0x0c, 0x1f, 0xb6, 0x45, 0xb1, 0xcd, 0xc4, 0x54, 0xd5, 0x9e, 0x4f, 0x2f, 0xd3, 0x3a, 0x3d,
+	0x88, 0xaa, 0x4e, 0x0e, 0xe5, 0x59, 0x1f, 0xff, 0xd2, 0x41, 0x9f, 0x17, 0xc5, 0x9e, 0x52, 0xd0,
+	0xf3, 0xe4, 0x20, 0x98, 0x36, 0xd2, 0x26, 0x5d, 0xae, 0x62, 0xc9, 0xf6, 0x69, 0xbe, 0x61, 0xb7,
+	0x67, 0x26, 0x63, 0xc9, 0xca, 0xa4, 0xde, 0xb1, 0xd6, 0x99, 0xc9, 0x98, 0x0e, 0xa1, 0x73, 0x48,
+	0xf2, 0xf4, 0x45, 0x54, 0x35, 0xd3, 0x15, 0xbf, 0xe6, 0xf4, 0x03, 0x18, 0xe2, 0xbb, 0xc8, 0xeb,
+	0x8a, 0xb5, 0x47, 0xad, 0xc9, 0xe0, 0x91, 0x59, 0xaf, 0x17, 0xb4, 0xe4, 0xb7, 0x2d, 0x94, 0x02,
+	0x6f, 0x3c, 0xfa, 0x09, 0x3a, 0x59, 0x52, 0xd5, 0xf1, 0xf1, 0x94, 0x33, 0x63, 0xa4, 0x4d, 0x7a,
+	0x8f, 0x43, 0xeb, 0x7c, 0x86, 0x75, 0x39, 0xc3, 0x8a, 0x2e, 0x67, 0x70, 0x53, 0xba, 0xfc, 0x94,
+	0xd3, 0x77, 0x60, 0xfc, 0x10, 0xe9, 0x76, 0x57, 0x33, 0x73, 0xa4, 0x4d, 0xda, 0xbc, 0xc9, 0xe8,
+	0x1c, 0xee, 0x37, 0x22, 0x13, 0xb5, 0x88, 0xcb, 0x22, 0x4b, 0xbf, 0xa5, 0xa2, 0x62, 0x1d, 0xb5,
+	0xc9, 0xc3, 0x7f, 0x36, 0x71, 0x95, 0xb9, 0x92, 0xe2, 0x4f, 0x3e, 0xd8, 0xfc, 0xcd, 0x52, 0x51,
+	0x8d, 0x7f, 0x6b, 0xd0, 0x56, 0xab, 0xd2, 0x1e, 0x98, 0x6b, 0x7f, 0xe9, 0x07, 0x5f, 0x7c, 0x72,
+	0x43, 0xef, 0xa1, 0xb7, 0xe2, 0x18, 0x2f, 0xfc, 0x30, 0xb2, 0x3d, 0x8f, 0x68, 0x94, 0x40, 0x7f,
+	0x15, 0x84, 0xd1, 0x95, 0xdc, 0xd2, 0x01, 0x80, 0x54, 0x5c, 0xf4, 0x30, 0x42, 0xd2, 0x52, 0x2d,
+	0xd2, 0x68, 0x80, 0x7e, 0x99, 0xb1, 0x5e, 0x7d, 0xe6, 0xb6, 0x8b, 0xa4, 0x7d, 0x9d, 0x71, 0x21,
+	0x86, 0x22, 0x1c, 0x63, 0x1e, 0x78, 0xde, 0xcc, 0x76, 0x96, 0xc4, 0xa4, 0x6f, 0xe0, 0x4e, 0x39,
+	0x57, 0xd4, 0xa1, 0x0c, 0xde, 0x72, 0xf4, 0xd0, 0x0e, 0x31, 0x8e, 0x30, 0x8c, 0xe2, 0x70, 0xed,
+	0x38, 0x18, 0x86, 0xa4, 0xfb, 0x4f, 0xe5, 0xc9, 0x5e, 0x78, 0x6b, 0x8e, 0x04, 0xc6, 0x0e, 0xf4,
+	0x5f, 0x9f, 0x4d, 0xef, 0xa0, 0xab, 0xda, 0xd0, 0x45, 0x97, 0xdc, 0x50, 0x00, 0x43, 0xba, 0xe8,
+	0x12, 0x4d, 0x0e, 0x99, 0xe1, 0x53, 0xc0, 0x31, 0x9e, 0x07, 0xc1, 0x32, 0x76, 0x38, 0xda, 0xd1,
+	0x22, 0xf0, 0xc9, 0xed, 0xac, 0xfb, 0xd5, 0x6c, 0x7e, 0xe4, 0xb3, 0xa1, 0x5e, 0xe9, 0xe3, 0x9f,
+	0x00, 0x00, 0x00, 0xff, 0xff, 0x13, 0x64, 0x75, 0x6c, 0xa3, 0x02, 0x00, 0x00,
 }

pkg/tiller/hooks.go

@@ -48,6 +48,7 @@ var events = map[string]release.Hook_Event{
 var deletePolices = map[string]release.Hook_DeletePolicy{
 	hooks.HookSucceeded:      release.Hook_SUCCEEDED,
 	hooks.HookFailed:         release.Hook_FAILED,
+	hooks.BeforeHookCreation: release.Hook_BEFORE_HOOK_CREATION,
 }
 
 // Manifest represents a manifest file, which has a name and some content.
@@ -189,21 +190,14 @@ func (file *manifestFile) sort(result *result) error {
 
 		result.hooks = append(result.hooks, h)
 
-		isKnownDeletePolices := false
-		dps, ok := entry.Metadata.Annotations[hooks.HookDeleteAnno]
-		if ok {
-			for _, dp := range strings.Split(dps, ",") {
-				dp = strings.ToLower(strings.TrimSpace(dp))
-				p, exist := deletePolices[dp]
+		operateAnnotationValues(entry, hooks.HookDeleteAnno, func(value string) {
+			policy, exist := deletePolices[value]
 			if exist {
-					isKnownDeletePolices = true
-					h.DeletePolicies = append(h.DeletePolicies, p)
-				}
-			}
-			if !isKnownDeletePolices {
-				log.Printf("info: skipping unknown hook delete policy: %q", dps)
-			}
+				h.DeletePolicies = append(h.DeletePolicies, policy)
+			} else {
+				log.Printf("info: skipping unknown hook delete policy: %q", value)
 			}
+		})
 	}
 
 	return nil
@@ -228,3 +222,12 @@ func calculateHookWeight(entry util.SimpleHead) int32 {
 
 	return int32(hw)
 }
+
+func operateAnnotationValues(entry util.SimpleHead, annotation string, operate func(p string)) {
+	if dps, ok := entry.Metadata.Annotations[annotation]; ok {
+		for _, dp := range strings.Split(dps, ",") {
+			dp = strings.ToLower(strings.TrimSpace(dp))
+			operate(dp)
+		}
+	}
+}

pkg/tiller/release_server.go

@@ -347,6 +347,9 @@ func (s *ReleaseServer) execHook(hs []*release.Hook, name, namespace, hook strin
 	executingHooks = sortByHookWeight(executingHooks)
 
 	for _, h := range executingHooks {
+		if err := s.deleteHookIfShouldBeDeletedByDeletePolicy(h, hooks.BeforeHookCreation, name, namespace, hook, kubeCli); err != nil {
+			return err
+		}
 
 		b := bytes.NewBufferString(h.Manifest)
 		if err := kubeCli.Create(namespace, b, timeout, false); err != nil {
@@ -356,18 +359,13 @@ func (s *ReleaseServer) execHook(hs []*release.Hook, name, namespace, hook strin
 		// No way to rewind a bytes.Buffer()?
 		b.Reset()
 		b.WriteString(h.Manifest)
+
 		if err := kubeCli.WatchUntilReady(namespace, b, timeout, false); err != nil {
 			s.Log("warning: Release %s %s %s could not complete: %s", name, hook, h.Path, err)
 			// If a hook is failed, checkout the annotation of the hook to determine whether the hook should be deleted
 			// under failed condition. If so, then clear the corresponding resource object in the hook
-			if hookShouldBeDeleted(h, hooks.HookFailed) {
-				b.Reset()
-				b.WriteString(h.Manifest)
-				s.Log("deleting %s hook %s for release %s due to %q policy", hook, h.Name, name, hooks.HookFailed)
-				if errHookDelete := kubeCli.Delete(namespace, b); errHookDelete != nil {
-					s.Log("warning: Release %s %s %S could not be deleted: %s", name, hook, h.Path, errHookDelete)
-					return errHookDelete
-				}
+			if err := s.deleteHookIfShouldBeDeletedByDeletePolicy(h, hooks.HookFailed, name, namespace, hook, kubeCli); err != nil {
+				return err
 			}
 			return err
 		}
@@ -377,13 +375,8 @@ func (s *ReleaseServer) execHook(hs []*release.Hook, name, namespace, hook strin
 	// If all hooks are succeeded, checkout the annotation of each hook to determine whether the hook should be deleted
 	// under succeeded condition. If so, then clear the corresponding resource object in each hook
 	for _, h := range executingHooks {
-		b := bytes.NewBufferString(h.Manifest)
-		if hookShouldBeDeleted(h, hooks.HookSucceeded) {
-			s.Log("deleting %s hook %s for release %s due to %q policy", hook, h.Name, name, hooks.HookSucceeded)
-			if errHookDelete := kubeCli.Delete(namespace, b); errHookDelete != nil {
-				s.Log("warning: Release %s %s %S could not be deleted: %s", name, hook, h.Path, errHookDelete)
-				return errHookDelete
-			}
+		if err := s.deleteHookIfShouldBeDeletedByDeletePolicy(h, hooks.HookSucceeded, name, namespace, hook, kubeCli); err != nil {
+			return err
 		}
 		h.LastRun = timeconv.Now()
 	}
@@ -409,11 +402,23 @@ func validateReleaseName(releaseName string) error {
 	return nil
 }
 
+func (s *ReleaseServer) deleteHookIfShouldBeDeletedByDeletePolicy(h *release.Hook, policy string, name, namespace, hook string, kubeCli environment.KubeClient) error {
+	b := bytes.NewBufferString(h.Manifest)
+	if hookHasDeletePolicy(h, policy) {
+		s.Log("deleting %s hook %s for release %s due to %q policy", hook, h.Name, name, policy)
+		if errHookDelete := kubeCli.Delete(namespace, b); errHookDelete != nil {
+			s.Log("warning: Release %s %s %S could not be deleted: %s", name, hook, h.Path, errHookDelete)
+			return errHookDelete
+		}
+	}
+	return nil
+}
+
 // hookShouldBeDeleted determines whether the defined hook deletion policy matches the hook deletion polices
 // supported by helm. If so, mark the hook as one should be deleted.
-func hookShouldBeDeleted(hook *release.Hook, policy string) bool {
+func hookHasDeletePolicy(h *release.Hook, policy string) bool {
 	if dp, ok := deletePolices[policy]; ok {
-		for _, v := range hook.DeletePolicies {
+		for _, v := range h.DeletePolicies {
 			if dp == v {
 				return true
 			}