Merge pull request #486 from jackgr/deployment

Tweaks to get end to end deployment working

cmd/expandybird/expander/expander.go

@@ -58,11 +58,6 @@ func (e *expander) ExpandChart(request *expansion.ServiceRequest) (*expansion.Se
 	chartFile := request.Chart.Chartfile
 	chartMembers := request.Chart.Members
 
-	if chartFile.Expander.Name != "ExpandyBird" {
-		message := fmt.Sprintf("ExpandyBird cannot do this kind of expansion: ", chartFile.Expander.Name)
-		return nil, fmt.Errorf("%s: %s", chartInv.Name, message)
-	}
-
 	if e.ExpansionBinary == "" {
 		message := fmt.Sprintf("expansion binary cannot be empty")
 		return nil, fmt.Errorf("%s: %s", chartInv.Name, message)

cmd/expandybird/expander/expander_test.go

@@ -18,6 +18,7 @@ package expander
 
 import (
 	"fmt"
+	"path/filepath"
 	"reflect"
 	"runtime"
 	"strings"
@@ -42,10 +43,40 @@ func content(lines []string) []byte {
 	return []byte(strings.Join(lines, "\n") + "\n")
 }
 
-// funcName returns the name of the calling function.
-func funcName() string {
+func getChartNameFromPC(pc uintptr) string {
+	rf := runtime.FuncForPC(pc)
+	fn := rf.Name()
+	bn := filepath.Base(fn)
+	split := strings.Split(bn, ".")
+	if len(split) > 1 {
+		split = split[1:]
+	}
+
+	cn := fmt.Sprintf("%s-1.2.3.tgz", split[0])
+	return cn
+}
+
+func getChartURLFromPC(pc uintptr) string {
+	cn := getChartNameFromPC(pc)
+	cu := fmt.Sprintf("gs://kubernetes-charts-testing/%s", cn)
+	return cu
+}
+
+func getTestChartName(t *testing.T) string {
+	pc, _, _, _ := runtime.Caller(1)
+	cu := getChartURLFromPC(pc)
+	cl, err := chart.Parse(cu)
+	if err != nil {
+		t.Fatalf("cannot parse chart reference %s: %s", cu, err)
+	}
+
+	return cl.Name
+}
+
+func getTestChartURL() string {
 	pc, _, _, _ := runtime.Caller(1)
-	return runtime.FuncForPC(pc).Name()
+	cu := getChartURLFromPC(pc)
+	return cu
 }
 
 func testExpansion(t *testing.T, req *expansion.ServiceRequest,
@@ -85,11 +116,11 @@ func TestEmptyJinja(t *testing.T) {
 		&expansion.ServiceRequest{
 			ChartInvocation: &common.Resource{
 				Name: "test_invocation",
-				Type: funcName(),
+				Type: getTestChartURL(),
 			},
 			Chart: &chart.Content{
 				Chartfile: &chart.Chartfile{
-					Name:     funcName(),
+					Name:     getTestChartName(t),
 					Expander: jinjaExpander,
 				},
 				Members: []*chart.Member{
@@ -113,11 +144,11 @@ func TestEmptyPython(t *testing.T) {
 		&expansion.ServiceRequest{
 			ChartInvocation: &common.Resource{
 				Name: "test_invocation",
-				Type: funcName(),
+				Type: getTestChartURL(),
 			},
 			Chart: &chart.Content{
 				Chartfile: &chart.Chartfile{
-					Name:     funcName(),
+					Name:     getTestChartName(t),
 					Expander: pyExpander,
 				},
 				Members: []*chart.Member{
@@ -144,11 +175,11 @@ func TestSimpleJinja(t *testing.T) {
 		&expansion.ServiceRequest{
 			ChartInvocation: &common.Resource{
 				Name: "test_invocation",
-				Type: funcName(),
+				Type: getTestChartURL(),
 			},
 			Chart: &chart.Content{
 				Chartfile: &chart.Chartfile{
-					Name:     funcName(),
+					Name:     getTestChartName(t),
 					Expander: jinjaExpander,
 				},
 				Members: []*chart.Member{
@@ -181,11 +212,11 @@ func TestSimplePython(t *testing.T) {
 		&expansion.ServiceRequest{
 			ChartInvocation: &common.Resource{
 				Name: "test_invocation",
-				Type: funcName(),
+				Type: getTestChartURL(),
 			},
 			Chart: &chart.Content{
 				Chartfile: &chart.Chartfile{
-					Name:     funcName(),
+					Name:     getTestChartName(t),
 					Expander: pyExpander,
 				},
 				Members: []*chart.Member{
@@ -220,7 +251,7 @@ func TestPropertiesJinja(t *testing.T) {
 		&expansion.ServiceRequest{
 			ChartInvocation: &common.Resource{
 				Name: "test_invocation",
-				Type: funcName(),
+				Type: getTestChartURL(),
 				Properties: map[string]interface{}{
 					"prop1": 3.0,
 					"prop2": "foo",
@@ -228,7 +259,7 @@ func TestPropertiesJinja(t *testing.T) {
 			},
 			Chart: &chart.Content{
 				Chartfile: &chart.Chartfile{
-					Name:     funcName(),
+					Name:     getTestChartName(t),
 					Expander: jinjaExpander,
 				},
 				Members: []*chart.Member{
@@ -266,7 +297,7 @@ func TestPropertiesPython(t *testing.T) {
 		&expansion.ServiceRequest{
 			ChartInvocation: &common.Resource{
 				Name: "test_invocation",
-				Type: funcName(),
+				Type: getTestChartURL(),
 				Properties: map[string]interface{}{
 					"prop1": 3.0,
 					"prop2": "foo",
@@ -274,7 +305,7 @@ func TestPropertiesPython(t *testing.T) {
 			},
 			Chart: &chart.Content{
 				Chartfile: &chart.Chartfile{
-					Name:     funcName(),
+					Name:     getTestChartName(t),
 					Expander: pyExpander,
 				},
 				Members: []*chart.Member{
@@ -314,11 +345,11 @@ func TestMultiFileJinja(t *testing.T) {
 		&expansion.ServiceRequest{
 			ChartInvocation: &common.Resource{
 				Name: "test_invocation",
-				Type: funcName(),
+				Type: getTestChartURL(),
 			},
 			Chart: &chart.Content{
 				Chartfile: &chart.Chartfile{
-					Name:     funcName(),
+					Name:     getTestChartName(t),
 					Expander: jinjaExpander,
 				},
 				Members: []*chart.Member{
@@ -372,7 +403,7 @@ func TestSchema(t *testing.T) {
 		&expansion.ServiceRequest{
 			ChartInvocation: &common.Resource{
 				Name: "test_invocation",
-				Type: funcName(),
+				Type: getTestChartURL(),
 				Properties: map[string]interface{}{
 					"prop1": 3.0,
 					"prop2": "foo",
@@ -380,7 +411,7 @@ func TestSchema(t *testing.T) {
 			},
 			Chart: &chart.Content{
 				Chartfile: &chart.Chartfile{
-					Name:     funcName(),
+					Name:     getTestChartName(t),
 					Expander: jinjaExpander,
 					Schema:   "Schema.yaml",
 				},
@@ -423,7 +454,7 @@ func TestSchemaFail(t *testing.T) {
 		&expansion.ServiceRequest{
 			ChartInvocation: &common.Resource{
 				Name: "test_invocation",
-				Type: funcName(),
+				Type: getTestChartURL(),
 				Properties: map[string]interface{}{
 					"prop1": 3.0,
 					"prop3": "foo",
@@ -431,7 +462,7 @@ func TestSchemaFail(t *testing.T) {
 			},
 			Chart: &chart.Content{
 				Chartfile: &chart.Chartfile{
-					Name:     funcName(),
+					Name:     getTestChartName(t),
 					Expander: jinjaExpander,
 					Schema:   "Schema.yaml",
 				},
@@ -464,11 +495,11 @@ func TestMultiFileJinjaMissing(t *testing.T) {
 		&expansion.ServiceRequest{
 			ChartInvocation: &common.Resource{
 				Name: "test_invocation",
-				Type: funcName(),
+				Type: getTestChartURL(),
 			},
 			Chart: &chart.Content{
 				Chartfile: &chart.Chartfile{
-					Name:     funcName(),
+					Name:     getTestChartName(t),
 					Expander: jinjaExpander,
 				},
 				Members: []*chart.Member{
@@ -490,11 +521,11 @@ func TestMultiFilePython(t *testing.T) {
 		&expansion.ServiceRequest{
 			ChartInvocation: &common.Resource{
 				Name: "test_invocation",
-				Type: funcName(),
+				Type: getTestChartURL(),
 			},
 			Chart: &chart.Content{
 				Chartfile: &chart.Chartfile{
-					Name:     funcName(),
+					Name:     getTestChartName(t),
 					Expander: pyExpander,
 				},
 				Members: []*chart.Member{
@@ -550,11 +581,11 @@ func TestMultiFilePythonMissing(t *testing.T) {
 		&expansion.ServiceRequest{
 			ChartInvocation: &common.Resource{
 				Name: "test_invocation",
-				Type: funcName(),
+				Type: getTestChartURL(),
 			},
 			Chart: &chart.Content{
 				Chartfile: &chart.Chartfile{
-					Name:     funcName(),
+					Name:     getTestChartName(t),
 					Expander: pyExpander,
 				},
 				Members: []*chart.Member{
@@ -578,7 +609,7 @@ func TestWrongChartName(t *testing.T) {
 		&expansion.ServiceRequest{
 			ChartInvocation: &common.Resource{
 				Name: "test_invocation",
-				Type: funcName(),
+				Type: getTestChartURL(),
 			},
 			Chart: &chart.Content{
 				Chartfile: &chart.Chartfile{
@@ -594,7 +625,7 @@ func TestWrongChartName(t *testing.T) {
 			},
 		},
 		nil, // Response
-		"Request chart invocation does not match provided chart",
+		"does not match provided chart",
 	)
 }
 
@@ -604,11 +635,11 @@ func TestEntrypointNotFound(t *testing.T) {
 		&expansion.ServiceRequest{
 			ChartInvocation: &common.Resource{
 				Name: "test_invocation",
-				Type: funcName(),
+				Type: getTestChartURL(),
 			},
 			Chart: &chart.Content{
 				Chartfile: &chart.Chartfile{
-					Name:     funcName(),
+					Name:     getTestChartName(t),
 					Expander: jinjaExpander,
 				},
 				Members: []*chart.Member{},
@@ -625,11 +656,11 @@ func TestMalformedResource(t *testing.T) {
 		&expansion.ServiceRequest{
 			ChartInvocation: &common.Resource{
 				Name: "test_invocation",
-				Type: funcName(),
+				Type: getTestChartURL(),
 			},
 			Chart: &chart.Content{
 				Chartfile: &chart.Chartfile{
-					Name:     funcName(),
+					Name:     getTestChartName(t),
 					Expander: jinjaExpander,
 				},
 				Members: []*chart.Member{
@@ -654,11 +685,11 @@ func TestResourceNoName(t *testing.T) {
 		&expansion.ServiceRequest{
 			ChartInvocation: &common.Resource{
 				Name: "test_invocation",
-				Type: funcName(),
+				Type: getTestChartURL(),
 			},
 			Chart: &chart.Content{
 				Chartfile: &chart.Chartfile{
-					Name:     funcName(),
+					Name:     getTestChartName(t),
 					Expander: jinjaExpander,
 				},
 				Members: []*chart.Member{
@@ -683,11 +714,11 @@ func TestResourceNoType(t *testing.T) {
 		&expansion.ServiceRequest{
 			ChartInvocation: &common.Resource{
 				Name: "test_invocation",
-				Type: funcName(),
+				Type: getTestChartURL(),
 			},
 			Chart: &chart.Content{
 				Chartfile: &chart.Chartfile{
-					Name:     funcName(),
+					Name:     getTestChartName(t),
 					Expander: jinjaExpander,
 				},
 				Members: []*chart.Member{

cmd/manager/chartrepos.go

@@ -58,6 +58,14 @@ func addChartRepoHandlerFunc(w http.ResponseWriter, r *http.Request, c *router.C
 		return nil
 	}
 
+	if string(cr.Format) == "" {
+		cr.Format = repo.GCSRepoFormat
+	}
+
+	if string(cr.Type) == "" {
+		cr.Type = repo.GCSRepoType
+	}
+
 	if err := c.Manager.AddRepo(cr); err != nil {
 		httputil.BadRequest(w, r, err)
 		return nil

examples/charts/replicatedservice/Chart.yaml

+name: replicatedservice
+description: Port of the replicatedservice template from kubernetes/charts
+version: 3
+expander:
+  name: Expandybird
+  entrypoint: templates/replicatedservice.py
+schema: templates/replicatedservice.py.schema

examples/charts/replicatedservice/templates/replicatedservice.py

+"""Defines a ReplicatedService type by creating both a Service and an RC.
+
+This module creates a typical abstraction for running a service in a
+Kubernetes cluster, namely a replication controller and a service packaged
+together into a single unit.
+"""
+
+import yaml
+
+SERVICE_TYPE_COLLECTION = 'Service'
+RC_TYPE_COLLECTION = 'ReplicationController'
+
+
+def GenerateConfig(context):
+  """Generates a Replication Controller and a matching Service.
+
+  Args:
+    context: Template context. See schema for context properties
+
+  Returns:
+    A Container Manifest as a YAML string.
+  """
+  # YAML config that we're going to create for both RC & Service
+  config = {'resources': []}
+
+  name = context.env['name']
+  container_name = context.properties.get('container_name', name)
+  namespace = context.properties.get('namespace', 'default')
+
+  # Define things that the Service cares about
+  service_name = context.properties.get('service_name', name + '-service')
+  service_type = SERVICE_TYPE_COLLECTION
+
+  # Define things that the Replication Controller (rc) cares about
+  rc_name = context.properties.get('rc_name', name + '-rc')
+  rc_type = RC_TYPE_COLLECTION
+
+  service = {
+      'name': service_name,
+      'type': service_type,
+      'properties': {
+          'apiVersion': 'v1',
+          'kind': 'Service',
+          'namespace': namespace,
+          'metadata': {
+              'name': service_name,
+              'labels': GenerateLabels(context, service_name),
+          },
+          'spec': {
+              'ports': [GenerateServicePorts(context, container_name)],
+              'selector': GenerateLabels(context, name)
+          }
+      }
+  }
+  set_up_external_lb = context.properties.get('external_service', None)
+  if set_up_external_lb:
+    service['properties']['spec']['type'] = 'LoadBalancer'
+  cluster_ip = context.properties.get('cluster_ip', None)
+  if cluster_ip:
+    service['properties']['spec']['clusterIP'] = cluster_ip
+  config['resources'].append(service)
+
+  rc = {
+      'name': rc_name,
+      'type': rc_type,
+      'properties': {
+          'apiVersion': 'v1',
+          'kind': 'ReplicationController',
+          'namespace': namespace,
+          'metadata': {
+              'name': rc_name,
+              'labels': GenerateLabels(context, rc_name),
+          },
+          'spec': {
+              'replicas': context.properties['replicas'],
+              'selector': GenerateLabels(context, name),
+              'template': {
+                  'metadata': {
+                      'labels': GenerateLabels(context, name),
+                  },
+                  'spec': {
+                      'containers': [
+                          {
+                              'env': GenerateEnv(context),
+                              'name': container_name,
+                              'image': context.properties['image'],
+                              'ports': [
+                                  {
+                                      'name': container_name,
+                                      'containerPort': context.properties['container_port'],
+                                  }
+                              ],
+                          }
+                      ],
+                  }
+              }
+          }
+      }
+  }
+
+  # Set up volume mounts
+  if context.properties.get('volumes', None):
+    rc['properties']['spec']['template']['spec']['containers'][0]['volumeMounts'] = []
+    rc['properties']['spec']['template']['spec']['volumes'] = []
+    for volume in context.properties['volumes']:
+      # mountPath should be unique
+      volume_name = volume['mount_path'].replace('/', '-').lstrip('-') + '-storage'
+      rc['properties']['spec']['template']['spec']['containers'][0]['volumeMounts'].append(
+          {
+              'name': volume_name,
+              'mountPath': volume['mount_path']
+          }
+      )
+      del volume['mount_path']
+      volume['name'] = volume_name
+      rc['properties']['spec']['template']['spec']['volumes'].append(volume)
+
+  if context.properties.get('privileged', False):
+    rc['properties']['spec']['template']['spec']['containers'][0]['securityContext'] = {
+        'privileged': True
+    }
+
+  config['resources'].append(rc)
+  return yaml.dump(config)
+
+
+# Generates labels either from the context.properties['labels'] or generates
+# a default label 'name':name
+def GenerateLabels(context, name):
+  """Generates labels from context.properties['labels'] or creates default.
+
+  We make a deep copy of the context.properties['labels'] section to avoid
+  linking in the yaml document, which I believe reduces readability of the
+  expanded template. If no labels are given, generate a default 'name':name.
+
+  Args:
+    context: Template context, which can contain the following properties:
+             labels - Labels to generate
+
+  Returns:
+    A dict containing labels in a name:value format
+  """
+  tmp_labels = context.properties.get('labels', None)
+  ret_labels = {'name': name}
+  if isinstance(tmp_labels, dict):
+    for key, value in tmp_labels.iteritems():
+      ret_labels[key] = value
+  return ret_labels
+
+
+def GenerateServicePorts(context, name):
+  """Generates a ports section for a service.
+
+  Args:
+    context: Template context, which can contain the following properties:
+             service_port - Port to use for the service
+             target_port - Target port for the service
+             protocol - Protocol to use.
+
+  Returns:
+    A dict containing a port definition
+  """
+  container_port = context.properties['container_port']
+  target_port = context.properties.get('target_port', container_port)
+  service_port = context.properties.get('service_port', target_port)
+  protocol = context.properties.get('protocol')
+
+  ports = {}
+  if name:
+    ports['name'] = name
+  if service_port:
+    ports['port'] = service_port
+  if target_port:
+    ports['targetPort'] = target_port
+  if protocol:
+    ports['protocol'] = protocol
+
+  return ports
+
+def GenerateEnv(context):
+  """Generates environmental variables for a pod.
+
+  Args:
+    context: Template context, which can contain the following properties:
+             env - Environment variables to set.
+
+  Returns:
+    A list containing env variables in dict format {name: 'name', value: 'value'}
+  """
+  env = []
+  tmp_env = context.properties.get('env', [])
+  for entry in tmp_env:
+    if isinstance(entry, dict):
+      env.append({'name': entry.get('name'), 'value': entry.get('value')})
+  return env

examples/charts/replicatedservice/templates/replicatedservice.py.schema

+info:
+  title: Replicated Service
+  description: |
+    Defines a ReplicatedService type by creating both a Service and an RC.
+
+    This module creates a typical abstraction for running a service in a
+    Kubernetes cluster, namely a replication controller and a service packaged
+    together into a single unit.
+
+required:
+- image
+
+properties:
+  container_name:
+    type: string
+    description: Name to use for container. If omitted, name is used.
+  service_name:
+    type: string
+    description: Name to use for service. If omitted, name-service is used.
+  namespace:
+    type: string
+    description: Namespace to create resources in. If omitted, 'default' is
+      used.
+    default: default
+  protocol:
+    type: string
+    description: Protocol to use for the service.
+  service_port:
+    type: int
+    description: Port to use for the service.
+  target_port:
+    type: int
+    description: Target port to use for the service.
+  container_port:
+    type: int
+    description: Port to use for the container.
+  replicas:
+    type: int
+    description: Number of replicas to create in RC.
+  image:
+    type: string
+    description: Docker image to use for replicas.
+  labels:
+    type: object
+    description: Labels to apply.
+  env:
+    type: array
+    description: Environment variables to apply.
+    properties:
+      name:
+        type: string
+      value:
+        type: string
+  external_service:
+    type: boolean
+    description: If set to true, enable external load balancer.
+  cluster_ip:
+    type: string
+    description: IP to use for the service
+  privileged:
+    type: boolean
+    description: If set to true, enable privileged container
+  volumes:
+    type: array
+    description: Volumes to mount.
+    items:
+      type: object
+      properties:
+        mounth_path:
+          type: string
+          description: Path to mount volume
+        # See https://cloud.google.com/container-engine/docs/spec-schema?hl=en for possible volumes. Since we only use gcePersistentDisk and NFS in our examples we have only added these ones.
+        oneOf:
+          gcePersistentDisk:
+            pdName:
+              type: string
+              description: Persistent's disk name
+            fsType:
+              type: string
+              description: Filesystem type of the persistent disk
+          nfs:
+            server:
+              type: string
+              description: The hostname or IP address of the NFS server
+            path:
+              type: string
+              description: The path that is exported by the NFS server
+            readOnly:
+              type: boolean
+              description: Forces the NFS export to be mounted with read-only permissions
+

pkg/expansion/validate.go

@@ -17,6 +17,8 @@ limitations under the License.
 package expansion
 
 import (
+	"github.com/kubernetes/helm/pkg/chart"
+
 	"fmt"
 )
 
@@ -32,8 +34,13 @@ func ValidateRequest(request *ServiceRequest) error {
 	chartInv := request.ChartInvocation
 	chartFile := request.Chart.Chartfile
 
-	if chartInv.Type != chartFile.Name {
-		return fmt.Errorf("Request chart invocation does not match provided chart")
+	l, err := chart.Parse(chartInv.Type)
+	if err != nil {
+		return fmt.Errorf("cannot parse chart reference %s: %s", chartInv.Type, err)
+	}
+
+	if l.Name != chartFile.Name {
+		return fmt.Errorf("Chart invocation type (%s) does not match provided chart (%s)", chartInv.Type, chartFile.Name)
 	}
 
 	if chartFile.Expander == nil {

pkg/repo/gcs_repo_test.go

@@ -65,15 +65,17 @@ func TestListCharts(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if len(charts) != 1 {
-		t.Fatalf("expected one chart in list, got %d", len(charts))
+	if len(charts) < 1 {
+		t.Fatalf("expected at least one chart in test repository %s", TestRepoURL)
 	}
 
-	haveName := charts[0]
-	wantName := TestArchiveName
-	if haveName != wantName {
-		t.Fatalf("expected chart named %s, got %s", wantName, haveName)
+	for _, ch := range charts {
+		if ch == TestArchiveName {
+			return
+		}
 	}
+
+	t.Fatalf("expected chart named %s in test repository %s", TestArchiveName, TestRepoURL)
 }
 
 func TestListChartsWithShouldFindRegex(t *testing.T) {