Merge pull request #26 from vaikas-google/master

First cut of adding support for types registry. 

client/client.go

@@ -15,6 +15,7 @@ package main
 
 import (
 	"github.com/kubernetes/deployment-manager/expandybird/expander"
+	"github.com/kubernetes/deployment-manager/client/registry"
 
 	"bytes"
 	"encoding/json"
@@ -31,9 +32,10 @@ import (
 )
 
 var (
-	action  = flag.String("action", "deploy", "expand | deploy | list | get | delete | update | listtypes | listtypeinstances")
+	action  = flag.String("action", "deploy", "expand | deploy | list | get | delete | update | listtypes | listtypeinstances | types")
 	name    = flag.String("name", "", "Name of template or deployment")
 	service = flag.String("service", "http://localhost:8080", "URL for deployment manager")
+	type_registry = flag.String("type_registry", "kubernetes/deployment-manager", "Type registry [owner/repo], defaults to kubernetes/deployment-manager/")
 	binary  = flag.String("binary", "../expandybird/expansion/expansion.py",
 		"Path to template expansion binary")
 )
@@ -48,6 +50,23 @@ func main() {
 	flag.Parse()
 	name := getNameArgument()
 	switch *action {
+	case "types":
+		s := strings.Split(*type_registry, "/")
+ 		git := registry.NewGithubRegistry(s[0], s[1])
+		types, err := git.List()
+		if err != nil {
+			log.Fatalf("Cannot list %v err")
+		}
+		log.Printf("Types:")
+		for _, t := range types {
+			log.Printf("%s:%s", t.Name, t.Version)
+			downloadURL, err := git.GetURL(t)
+			if err != nil {
+				log.Printf("Failed to get download URL for %s:%s", t.Name, t.Version)
+			} 
+			log.Printf("\tdownload URL: %s", downloadURL)
+		}
+
 	case "expand":
 		backend := expander.NewExpander(*binary)
 		template := loadTemplate(name)

client/registry/github_registry.go

+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package registry
+
+import (
+	"github.com/google/go-github/github"
+
+	"fmt"
+	"log"
+)
+
+// GithubRegistry implements the Registry interface that talks to github.
+type GithubRegistry struct {
+	owner      string
+	repository string
+	client     *github.Client
+}
+
+// NewGithubRegistry creates a Registry that can be used to talk to github.
+func NewGithubRegistry(owner string, repository string) *GithubRegistry {
+	return &GithubRegistry{
+		owner:      owner,
+		repository: repository,
+		client:     github.NewClient(nil),
+	}
+}
+
+// List the types from the Registry.
+func (g *GithubRegistry) List() ([]Type, error) {
+	// First list all the types at the top level.
+	types, err := g.getDirs(TypesDir)
+	if err != nil {
+		log.Printf("Failed to list types : %v", err)
+		return nil, err
+	}
+	var retTypes []Type
+	for _, t := range types {
+		// Then we need to fetch the versions (directories for this type)
+		versions, err := g.getDirs(TypesDir + "/" + t)
+		if err != nil {
+			log.Printf("Failed to fetch versions for type: %s", t)
+			return nil, err
+		}
+		for _, v := range versions {
+			retTypes = append(retTypes, Type{Name: t, Version: v})
+		}
+	}
+
+	return retTypes, nil
+}
+
+// GetURL fetches the download URL for a given Type.
+func (g *GithubRegistry) GetURL(t Type) (string, error) {
+	_, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, TypesDir+"/"+t.Name+"/"+t.Version, nil)
+	if err != nil {
+		log.Printf("Failed to list types : %v", err)
+		return "", err
+	}
+	for _, f := range dc {
+		if *f.Type == "file" {
+			if *f.Name == t.Name+".jinja" || *f.Name == t.Name+".py" {
+				return *f.DownloadURL, nil
+			}
+		}
+	}
+	return "", fmt.Errorf("Can not find type %s:%s", t.Name, t.Version)
+}
+
+func (g *GithubRegistry) getDirs(dir string) ([]string, error) {
+	_, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, dir, nil)
+	if err != nil {
+		log.Printf("Failed to call ListRefs : %v", err)
+		return nil, err
+	}
+	var dirs []string
+	for _, entry := range dc {
+		if *entry.Type == "dir" {
+			dirs = append(dirs, *entry.Name)
+		}
+	}
+	return dirs, nil
+}

client/registry/registry.go

+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package registry
+
+// Registry abstracts a types registry which holds types that can be
+// used in a Deployment Manager configurations. A registry root must have
+// a 'types' directory which contains all the available types. Each type
+// then contains version directories which in turn contains all the files
+// necessary for that version of the type.
+// For example a type registry holding two types:
+//   redis v1               (implemented in jinja)
+//   replicatedservice v2   (implemented in python)
+// would have a directory structure like so:
+// /types/redis/v1
+//   redis.jinja
+//   redis.jinja.schema
+// /types/replicatedservice/v2
+//   replicatedservice.python
+//   replicatedservice.python.schema
+
+const TypesDir string = "types"
+
+type Type struct {
+	Name string
+	Version string
+}
+
+// Registry abstracts type interactions.
+type Registry interface {
+	// List all the types in the given registry
+	List() ([]Type, error)
+	// Get the download URL for a given type and version
+	GetURL(t Type) (string, error)
+}

types/redis/v1/redis.jinja

+{% set REDIS_PORT = 6379 %}
+{% set WORKERS = properties['workers'] or 2 %}
+
+resources:
+- name: redis-master
+  type: https://raw.githubusercontent.com/kubernetes/deployment-manager/master/examples/replicatedservice/replicatedservice.py
+  properties:
+    # This has to be overwritten since service names are hard coded in the code
+    service_name: redis-master
+    service_port: {{ REDIS_PORT }}
+    target_port: {{ REDIS_PORT }}
+    container_port: {{ REDIS_PORT }}
+    replicas: 1
+    container_name: master
+    image: redis
+
+- name: redis-slave
+  type: https://raw.githubusercontent.com/kubernetes/deployment-manager/master/examples/replicatedservice/replicatedservice.py
+  properties:
+    # This has to be overwritten since service names are hard coded in the code
+    service_name: redis-slave
+    service_port: {{ REDIS_PORT }}
+    container_port: {{ REDIS_PORT }}
+    replicas: {{ WORKERS }}
+    container_name: worker
+    image: kubernetes/redis-slave:v2
+    # An example of how to specify env variables.
+    env:
+    - name: GET_HOSTS_FROM
+      value: env
+    - name: REDIS_MASTER_SERVICE_HOST
+      value: redis-master

types/redis/v1/redis.jinja.schema

+info:
+  title: Redis cluster
+  description: Defines a redis cluster, using a single replica
+    replicatedservice for master and replicatedservice for workers.
+
+properties:
+  workers:
+    type: int
+    default: 2
+    description: Number of worker replicas.

types/replicatedservice/v1/replicatedservice.py

+"""Defines a ReplicatedService type by creating both a Service and an RC.
+
+This module creates a typical abstraction for running a service in a
+Kubernetes cluster, namely a replication controller and a service packaged
+together into a single unit.
+"""
+
+import yaml
+
+SERVICE_TYPE_COLLECTION = 'Service'
+RC_TYPE_COLLECTION = 'ReplicationController'
+
+
+def GenerateConfig(context):
+  """Generates a Replication Controller and a matching Service.
+
+  Args:
+    context: Template context. See schema for context properties.
+
+  Returns:
+    A Container Manifest as a YAML string.
+  """
+  # YAML config that we're going to create for both RC & Service
+  config = {'resources': []}
+
+  name = context.env['name']
+  container_name = context.properties.get('container_name', name)
+  namespace = context.properties.get('namespace', 'default')
+
+  # Define things that the Service cares about
+  service_name = context.properties.get('service_name', name + '-service')
+  service_type = SERVICE_TYPE_COLLECTION
+
+  # Define things that the Replication Controller (rc) cares about
+  rc_name = name + '-rc'
+  rc_type = RC_TYPE_COLLECTION
+
+  service = {
+      'name': service_name,
+      'type': service_type,
+      'properties': {
+          'apiVersion': 'v1',
+          'kind': 'Service',
+          'namespace': namespace,
+          'metadata': {
+              'name': service_name,
+              'labels': GenerateLabels(context, service_name),
+          },
+          'spec': {
+              'ports': [GenerateServicePorts(context, container_name)],
+              'selector': GenerateLabels(context, name)
+          }
+      }
+  }
+  set_up_external_lb = context.properties.get('external_service', None)
+  if set_up_external_lb:
+    service['properties']['spec']['type'] = 'LoadBalancer'
+  config['resources'].append(service)
+
+  rc = {
+      'name': rc_name,
+      'type': rc_type,
+      'properties': {
+          'apiVersion': 'v1',
+          'kind': 'ReplicationController',
+          'namespace': namespace,
+          'metadata': {
+              'name': rc_name,
+              'labels': GenerateLabels(context, rc_name),
+          },
+          'spec': {
+              'replicas': context.properties['replicas'],
+              'selector': GenerateLabels(context, name),
+              'template': {
+                  'metadata': {
+                      'labels': GenerateLabels(context, name),
+                  },
+                  'spec': {
+                      'containers': [
+                          {
+                              'env': GenerateEnv(context),
+                              'name': container_name,
+                              'image': context.properties['image'],
+                              'ports': [
+                                  {
+                                      'name': container_name,
+                                      'containerPort': context.properties['container_port'],
+                                  }
+                              ]
+                          }
+                      ]
+                  }
+              }
+          }
+      }
+  }
+
+  config['resources'].append(rc)
+  return yaml.dump(config)
+
+
+# Generates labels either from the context.properties['labels'] or generates
+# a default label 'name':name
+def GenerateLabels(context, name):
+  """Generates labels from context.properties['labels'] or creates default.
+
+  We make a deep copy of the context.properties['labels'] section to avoid
+  linking in the yaml document, which I believe reduces readability of the
+  expanded template. If no labels are given, generate a default 'name':name.
+
+  Args:
+    context: Template context, which can contain the following properties:
+             labels - Labels to generate
+
+  Returns:
+    A dict containing labels in a name:value format
+  """
+  tmp_labels = context.properties.get('labels', None)
+  ret_labels = {'name': name}
+  if isinstance(tmp_labels, dict):
+    for key, value in tmp_labels.iteritems():
+      ret_labels[key] = value
+  return ret_labels
+
+
+def GenerateServicePorts(context, name):
+  """Generates a ports section for a service.
+
+  Args:
+    context: Template context, which can contain the following properties:
+             service_port - Port to use for the service
+             target_port - Target port for the service
+             protocol - Protocol to use.
+
+  Returns:
+    A dict containing a port definition
+  """
+  service_port = context.properties.get('service_port', None)
+  target_port = context.properties.get('target_port', None)
+  protocol = context.properties.get('protocol')
+
+  ports = {}
+  if name:
+    ports['name'] = name
+  if service_port:
+    ports['port'] = service_port
+  if target_port:
+    ports['targetPort'] = target_port
+  if protocol:
+    ports['protocol'] = protocol
+
+  return ports
+
+def GenerateEnv(context):
+  """Generates environmental variables for a pod.
+
+  Args:
+    context: Template context, which can contain the following properties:
+             env - Environment variables to set.
+
+  Returns:
+    A list containing env variables in dict format {name: 'name', value: 'value'}
+  """
+  env = []
+  tmp_env = context.properties.get('env', [])
+  for entry in tmp_env:
+    if isinstance(entry, dict):
+      env.append({'name': entry.get('name'), 'value': entry.get('value')})
+  return env

types/replicatedservice/v1/replicatedservice.py.schema

+info:
+  title: Replicated Service
+  description: |
+    Defines a ReplicatedService type by creating both a Service and an RC.
+
+    This module creates a typical abstraction for running a service in a
+    Kubernetes cluster, namely a replication controller and a service packaged
+    together into a single unit.
+
+required:
+- image
+
+properties:
+  container_name:
+    type: string
+    description: Name to use for container. If omitted, name is used.
+  service_name:
+    type: string
+    description: Name to use for service. If omitted, name-service is used.
+  namespace:
+    type: string
+    description: Namespace to create resources in. If omitted, 'default' is
+      used.
+    default: default
+  protocol:
+    type: string
+    description: Protocol to use for the service.
+  service_port:
+    type: int
+    description: Port to use for the service.
+  target_port:
+    type: int
+    description: Target port to use for the service.
+  container_port:
+    type: int
+    description: Port to use for the container.
+  replicas:
+    type: int
+    description: Number of replicas to create in RC.
+  image:
+    type: string
+    description: Docker image to use for replicas.
+  labels:
+    type: object
+    description: Labels to apply.
+  env:
+    type: object
+    description: Environment variables to apply.
+    properties:
+      name:
+        type: string
+      value:
+        type: string
+  external_service:
+    type: boolean
+    description: If set to true, enable external load balancer.
+