Add support for short github.com types. Refactor manager/manager/types to…

Add support for short github.com types. Refactor manager/manager/types to common/types and reuse them rather than redefine them.

manager/manager/types.go → common/types.go

@@ -11,7 +11,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package manager
+package common
 
 import (
 	"time"
@@ -120,6 +120,7 @@ type Template struct {
 // ImportFile describes a base64 encoded file imported by a Template.
 type ImportFile struct {
 	Name    string `json:"name,omitempty"`
+	Path    string `json:"path",omitempty` // Actual URL for the file
 	Content string `json:"content"`
 }
 

dm/dm.go

@@ -16,8 +16,8 @@ package main
 import (
 	"github.com/ghodss/yaml"
 
+	"github.com/kubernetes/deployment-manager/common"
 	"github.com/kubernetes/deployment-manager/expandybird/expander"
-	"github.com/kubernetes/deployment-manager/manager/manager"
 	"github.com/kubernetes/deployment-manager/registry"
 	"github.com/kubernetes/deployment-manager/util"
 
@@ -259,8 +259,8 @@ func isHttp(t string) bool {
 	return strings.HasPrefix(t, "http://") || strings.HasPrefix(t, "https://")
 }
 
-func loadTemplate(args []string) *expander.Template {
-	var template *expander.Template
+func loadTemplate(args []string) *common.Template {
+	var template *common.Template
 	var err error
 	if len(args) < 2 {
 		fmt.Fprintln(os.Stderr, "No template name or configuration(s) supplied")
@@ -302,7 +302,7 @@ func getRegistryType(fullType string) *registry.Type {
 	}
 }
 
-func buildTemplateFromType(t registry.Type) *expander.Template {
+func buildTemplateFromType(t registry.Type) *common.Template {
 	downloadURL := getDownloadUrl(t)
 
 	props := make(map[string]interface{})
@@ -328,7 +328,7 @@ func buildTemplateFromType(t registry.Type) *expander.Template {
 	// Name the deployment after the type name.
 	name := fmt.Sprintf("%s:%s", t.Name, t.Version)
 
-	config := manager.Configuration{Resources: []*manager.Resource{&manager.Resource{
+	config := common.Configuration{Resources: []*common.Resource{&common.Resource{
 		Name:       name,
 		Type:       downloadURL,
 		Properties: props,
@@ -339,14 +339,14 @@ func buildTemplateFromType(t registry.Type) *expander.Template {
 		log.Fatalf("error: %s\ncannot create configuration for deployment: %v\n", err, config)
 	}
 
-	return &expander.Template{
+	return &common.Template{
 		Name:    name,
 		Content: string(y),
 		// No imports, as this is a single type from repository.
 	}
 }
 
-func marshalTemplate(template *expander.Template) io.ReadCloser {
+func marshalTemplate(template *common.Template) io.ReadCloser {
 	j, err := json.Marshal(template)
 	if err != nil {
 		log.Fatalf("cannot deploy configuration %s: %s\n", template.Name, err)

examples/guestbook/guestbook.yaml

 resources:
 - name: frontend
-  type: https://raw.githubusercontent.com/kubernetes/deployment-manager/master/templates/replicatedservice/v1/replicatedservice.py
+  type: github.com/kubernetes/application-dm-templates/common/replicatedservice:v1
   properties:
     service_port: 80
     container_port: 80
@@ -8,5 +8,5 @@ resources:
     replicas: 3
     image: gcr.io/google_containers/example-guestbook-php-redis:v3
 - name: redis
-  type: https://raw.githubusercontent.com/kubernetes/deployment-manager/master/templates/redis/v1/redis.jinja
+  type: github.com/kubernetes/application-dm-templates/storage/redis:v1
   properties: null

expandybird/expander/expander.go

@@ -23,11 +23,12 @@ import (
 	"path/filepath"
 
 	"github.com/ghodss/yaml"
+	"github.com/kubernetes/deployment-manager/common"
 )
 
 // Expander abstracts interactions with the expander and deployer services.
 type Expander interface {
-	ExpandTemplate(template *Template) (string, error)
+	ExpandTemplate(template *common.Template) (string, error)
 }
 
 type expander struct {
@@ -39,24 +40,10 @@ func NewExpander(binary string) Expander {
 	return &expander{binary}
 }
 
-// ImportFile describes a file that we import into our templates
-// TODO: Encode the Content so that it doesn't get mangled.
-type ImportFile struct {
-	Name    string `json:"name,omitempty"`
-	Content string `json:"content"`
-}
-
-// A Template defines a single deployment.
-type Template struct {
-	Name    string        `json:"name"`
-	Content string        `json:"content"`
-	Imports []*ImportFile `json:"imports"`
-}
-
 // NewTemplateFromRootTemplate creates and returns a new template whose content
 // and imported files are constructed from reading the root template, parsing out
 // the imports section and reading the imports from there
-func NewTemplateFromRootTemplate(templateFileName string) (*Template, error) {
+func NewTemplateFromRootTemplate(templateFileName string) (*common.Template, error) {
 	templateDir := filepath.Dir(templateFileName)
 	content, err := ioutil.ReadFile(templateFileName)
 	if err != nil {
@@ -85,14 +72,14 @@ func NewTemplateFromRootTemplate(templateFileName string) (*Template, error) {
 func NewTemplateFromFileNames(
 	templateFileName string,
 	importFileNames []string,
-) (*Template, error) {
+) (*common.Template, error) {
 	name := path.Base(templateFileName)
 	content, err := ioutil.ReadFile(templateFileName)
 	if err != nil {
 		return nil, fmt.Errorf("cannot read template file (%s): %s", err, templateFileName)
 	}
 
-	imports := []*ImportFile{}
+	imports := []*common.ImportFile{}
 	for _, importFileName := range importFileNames {
 		importFileData, err := ioutil.ReadFile(importFileName)
 		if err != nil {
@@ -100,13 +87,13 @@ func NewTemplateFromFileNames(
 		}
 
 		imports = append(imports,
-			&ImportFile{
+			&common.ImportFile{
 				Name:    path.Base(importFileName),
 				Content: string(importFileData),
 			})
 	}
 
-	return &Template{
+	return &common.Template{
 		Name:    name,
 		Content: string(content),
 		Imports: imports,
@@ -190,7 +177,7 @@ func (eResponse *ExpansionResponse) Unmarshal() (*ExpansionResult, error) {
 
 // ExpandTemplate passes the given configuration to the expander and returns the
 // expanded configuration as a string on success.
-func (e *expander) ExpandTemplate(template *Template) (string, error) {
+func (e *expander) ExpandTemplate(template *common.Template) (string, error) {
 	if e.ExpansionBinary == "" {
 		message := fmt.Sprintf("expansion binary cannot be empty")
 		return "", fmt.Errorf("error expanding template %s: %s", template.Name, message)
@@ -216,7 +203,7 @@ func (e *expander) ExpandTemplate(template *Template) (string, error) {
 	}
 
 	for _, imp := range template.Imports {
-		cmd.Args = append(cmd.Args, imp.Name, imp.Content)
+		cmd.Args = append(cmd.Args, imp.Name, imp.Path, imp.Content)
 	}
 
 	if err := cmd.Start(); err != nil {

expandybird/expander/expander_test.go

@@ -19,6 +19,8 @@ import (
 	"reflect"
 	"strings"
 	"testing"
+
+	"github.com/kubernetes/deployment-manager/common"
 )
 
 const invalidFileName = "afilethatdoesnotexist"
@@ -36,7 +38,7 @@ type ExpanderTestCase struct {
 	ExpectedError    string
 }
 
-func (etc *ExpanderTestCase) GetTemplate(t *testing.T) *Template {
+func (etc *ExpanderTestCase) GetTemplate(t *testing.T) *common.Template {
 	template, err := NewTemplateFromFileNames(etc.TemplateFileName, etc.ImportFileNames)
 	if err != nil {
 		t.Errorf("cannot create template for test case '%s': %s\n", etc.Description, err)

expandybird/expansion/expansion.py

@@ -136,7 +136,7 @@ def _ProcessResource(resource, imports, env, validate_schema=False):
   layout = {'name': resource['name'],
             'type': resource['type']}
 
-  if IsTemplate(resource['type']) and resource['type'] in imports:
+  if resource['type'] in imports:
     # A template resource, which contains sub-resources.
     expanded_template = ExpandTemplate(resource, imports, env, validate_schema)
 
@@ -182,11 +182,6 @@ def _ValidateUniqueNames(template_resources, template_name='config'):
     # If this resource doesn't have a name, we will report that error later
 
 
-def IsTemplate(resource_type):
-  """Returns whether a given resource type is a Template."""
-  return resource_type.endswith('.py') or resource_type.endswith('.jinja')
-
-
 def ExpandTemplate(resource, imports, env, validate_schema=False):
   """Expands a template, calling expansion mechanism based on type.
 
@@ -205,6 +200,7 @@ def ExpandTemplate(resource, imports, env, validate_schema=False):
     ExpansionError: if there is any error occurred during expansion
   """
   source_file = resource['type']
+  path = resource['type']
 
   # Look for Template in imports.
   if source_file not in imports:
@@ -212,6 +208,12 @@ def ExpandTemplate(resource, imports, env, validate_schema=False):
         source_file,
         'Unable to find source file %s in imports.' % (source_file))
 
+  # source_file could be a short version of the template (say github short name)
+  # so we need to potentially map this into the fully resolvable name.
+  if 'path' in imports[source_file]:
+    if imports[source_file]['path']:
+      path = imports[source_file]['path']
+
   resource['imports'] = imports
 
   # Populate the additional environment variables.
@@ -230,13 +232,13 @@ def ExpandTemplate(resource, imports, env, validate_schema=False):
     except schema_validation.ValidationErrors as e:
       raise ExpansionError(resource, e.message)
 
-  if source_file.endswith('jinja'):
+  if path.endswith('jinja'):
     expanded_template = ExpandJinja(
-        source_file, imports[source_file], resource, imports)
-  elif source_file.endswith('py'):
+        source_file, imports[source_file]['content'], resource, imports)
+  elif path.endswith('py'):
     # This is a Python template.
     expanded_template = ExpandPython(
-        imports[source_file], source_file, resource)
+        imports[source_file]['content'], source_file, resource)
   else:
     # The source file is not a jinja file or a python file.
     # This in fact should never happen due to the IsTemplate check above.
@@ -261,8 +263,8 @@ def ExpandJinja(file_name, source_template, resource, imports):
     source_template: string, the content of jinja file to be render
     resource: resource object, the resource that contains parameters to the
         jinja file
-    imports: map from string to string, the map of imported files names
-        and contents
+    imports: map from string to map {name, path}, the map of imported files names
+        fully resolved path and contents
   Returns:
     The final expanded template
   Raises:
@@ -361,9 +363,10 @@ def main():
       print >>sys.stderr, 'Invalid import definition at argv pos %d' % idx
       sys.exit(1)
     name = sys.argv[idx]
-    value = sys.argv[idx + 1]
-    imports[name] = value
-    idx += 2
+    path = sys.argv[idx + 1]
+    value = sys.argv[idx + 2]
+    imports[name] = {'content': value, 'path': path}
+    idx += 3
 
   env = {}
   env['deployment'] = os.environ['DEPLOYMENT_NAME']

expandybird/expansion/sandbox_loader.py

@@ -95,7 +95,7 @@ def process_imports(imports):
 
   # Now build the hierarchical modules.
   for k in imports.keys():
-    if imports[k].endswith('.jinja'):
+    if imports[k]['path'].endswith('.jinja'):
       continue
     # Normalize paths and trim .py extension, if any.
     normalized = os.path.splitext(os.path.normpath(k))[0]

expandybird/service/service.go

@@ -15,6 +15,7 @@ package service
 
 import (
 	"github.com/kubernetes/deployment-manager/expandybird/expander"
+	"github.com/kubernetes/deployment-manager/common"
 	"github.com/kubernetes/deployment-manager/util"
 
 	"errors"
@@ -40,7 +41,7 @@ func NewService(handler restful.RouteFunction) *Service {
 	webService.Produces(restful.MIME_JSON, restful.MIME_XML)
 	webService.Route(webService.POST("/expand").To(handler).
 		Doc("Expand a template.").
-		Reads(&expander.Template{}))
+		Reads(&common.Template{}))
 	return &Service{webService}
 }
 
@@ -60,7 +61,7 @@ func (s *Service) Register(container *restful.Container) {
 func NewExpansionHandler(backend expander.Expander) restful.RouteFunction {
 	return func(req *restful.Request, resp *restful.Response) {
 		util.LogHandlerEntry("expandybird: expand", req.Request)
-		template := &expander.Template{}
+		template := &common.Template{}
 		if err := req.ReadEntity(&template); err != nil {
 			logAndReturnErrorFromHandler(http.StatusBadRequest, err.Error(), resp)
 			return

expandybird/service/service_test.go

@@ -24,6 +24,7 @@ import (
 	"testing"
 
 	"github.com/kubernetes/deployment-manager/expandybird/expander"
+	"github.com/kubernetes/deployment-manager/common"
 	"github.com/kubernetes/deployment-manager/util"
 
 	restful "github.com/emicklei/go-restful"
@@ -180,7 +181,7 @@ type mockExpander struct {
 
 // ExpandTemplate passes the given configuration to the expander and returns the
 // expanded configuration as a string on success.
-func (e *mockExpander) ExpandTemplate(template *expander.Template) (string, error) {
+func (e *mockExpander) ExpandTemplate(template *common.Template) (string, error) {
 	switch template.Name {
 	case "InvalidFileName.yaml":
 		return "", fmt.Errorf("expansion error")

manager/deployments.go

@@ -22,6 +22,7 @@ import (
 	"log"
 	"net"
 	"net/http"
+	"net/url"
 	"os"
 	"strings"
 
@@ -29,6 +30,7 @@ import (
 	"github.com/gorilla/mux"
 
 	"github.com/kubernetes/deployment-manager/manager/manager"
+	"github.com/kubernetes/deployment-manager/common"
 	"github.com/kubernetes/deployment-manager/manager/repository"
 	"github.com/kubernetes/deployment-manager/util"
 )
@@ -198,17 +200,16 @@ func putDeploymentHandlerFunc(w http.ResponseWriter, r *http.Request) {
 
 func getPathVariable(w http.ResponseWriter, r *http.Request, variable, handler string) (string, error) {
 	vars := mux.Vars(r)
-	variable, ok := vars[variable]
+	retVariable, ok := vars[variable]
 	if !ok {
 		e := fmt.Errorf("%s parameter not found in URL", variable)
 		util.LogAndReturnError(handler, http.StatusBadRequest, e, w)
 		return "", e
 	}
-
-	return variable, nil
+	return retVariable, nil
 }
 
-func getTemplate(w http.ResponseWriter, r *http.Request, handler string) *manager.Template {
+func getTemplate(w http.ResponseWriter, r *http.Request, handler string) *common.Template {
 	util.LogHandlerEntry(handler, r)
 	b := io.LimitReader(r.Body, *maxLength*1024)
 	y, err := ioutil.ReadAll(b)
@@ -237,7 +238,7 @@ func getTemplate(w http.ResponseWriter, r *http.Request, handler string) *manage
 		return nil
 	}
 
-	t := &manager.Template{}
+	t := &common.Template{}
 	if err := json.Unmarshal(j, t); err != nil {
 		e := fmt.Errorf("%v\n%v", err, string(j))
 		util.LogAndReturnError(handler, http.StatusBadRequest, e, w)

manager/manager/deployer.go

@@ -25,14 +25,15 @@ import (
 	"strings"
 
 	"github.com/ghodss/yaml"
+	"github.com/kubernetes/deployment-manager/common"
 )
 
 // Deployer abstracts interactions with the expander and deployer services.
 type Deployer interface {
-	GetConfiguration(cached *Configuration) (*Configuration, error)
-	CreateConfiguration(configuration *Configuration) (*Configuration, error)
-	DeleteConfiguration(configuration *Configuration) (*Configuration, error)
-	PutConfiguration(configuration *Configuration) (*Configuration, error)
+	GetConfiguration(cached *common.Configuration) (*common.Configuration, error)
+	CreateConfiguration(configuration *common.Configuration) (*common.Configuration, error)
+	DeleteConfiguration(configuration *common.Configuration) (*common.Configuration, error)
+	PutConfiguration(configuration *common.Configuration) (*common.Configuration, error)
 }
 
 // NewDeployer returns a new initialized Deployer.
@@ -54,9 +55,9 @@ type formatter func(err error) error
 
 // GetConfiguration reads and returns the actual configuration
 // of the resources described by a cached configuration.
-func (d *deployer) GetConfiguration(cached *Configuration) (*Configuration, error) {
+func (d *deployer) GetConfiguration(cached *common.Configuration) (*common.Configuration, error) {
 	errors := &Error{}
-	actual := &Configuration{}
+	actual := &common.Configuration{}
 	for _, resource := range cached.Resources {
 		rtype := url.QueryEscape(resource.Type)
 		rname := url.QueryEscape(resource.Name)
@@ -70,7 +71,7 @@ func (d *deployer) GetConfiguration(cached *Configuration) (*Configuration, erro
 		}
 
 		if len(body) != 0 {
-			result := &Resource{Name: resource.Name, Type: resource.Type}
+			result := &common.Resource{Name: resource.Name, Type: resource.Type}
 			if err := yaml.Unmarshal(body, &result.Properties); err != nil {
 				return nil, fmt.Errorf("cannot get configuration for resource (%v)", err)
 			}
@@ -88,22 +89,22 @@ func (d *deployer) GetConfiguration(cached *Configuration) (*Configuration, erro
 
 // CreateConfiguration deploys the set of resources described by a configuration and returns
 // the Configuration with status for each resource filled in.
-func (d *deployer) CreateConfiguration(configuration *Configuration) (*Configuration, error) {
+func (d *deployer) CreateConfiguration(configuration *common.Configuration) (*common.Configuration, error) {
 	return d.callServiceWithConfiguration("POST", "create", configuration)
 }
 
 // DeleteConfiguration deletes the set of resources described by a configuration.
-func (d *deployer) DeleteConfiguration(configuration *Configuration) (*Configuration, error) {
+func (d *deployer) DeleteConfiguration(configuration *common.Configuration) (*common.Configuration, error) {
 	return d.callServiceWithConfiguration("DELETE", "delete", configuration)
 }
 
 // PutConfiguration replaces the set of resources described by a configuration and returns
 // the Configuration with status for each resource filled in.
-func (d *deployer) PutConfiguration(configuration *Configuration) (*Configuration, error) {
+func (d *deployer) PutConfiguration(configuration *common.Configuration) (*common.Configuration, error) {
 	return d.callServiceWithConfiguration("PUT", "replace", configuration)
 }
 
-func (d *deployer) callServiceWithConfiguration(method, operation string, configuration *Configuration) (*Configuration, error) {
+func (d *deployer) callServiceWithConfiguration(method, operation string, configuration *common.Configuration) (*common.Configuration, error) {
 	callback := func(e error) error {
 		return fmt.Errorf("cannot %s configuration: %s", operation, e)
 	}
@@ -120,7 +121,7 @@ func (d *deployer) callServiceWithConfiguration(method, operation string, config
 		return nil, err
 	}
 
-	result := &Configuration{}
+	result := &common.Configuration{}
 	if len(resp) != 0 {
 		if err := yaml.Unmarshal(resp, &result); err != nil {
 			return nil, fmt.Errorf("cannot unmarshal response: (%v)", err)

manager/manager/deployer_test.go

@@ -24,6 +24,7 @@ import (
 	"testing"
 
 	"github.com/kubernetes/deployment-manager/util"
+	"github.com/kubernetes/deployment-manager/common"
 
 	"github.com/ghodss/yaml"
 )
@@ -250,8 +251,8 @@ func TestPutConfiguration(t *testing.T) {
 	}
 }
 
-func getValidConfiguration(t *testing.T) *Configuration {
-	valid := &Configuration{}
+func getValidConfiguration(t *testing.T) *common.Configuration {
+	valid := &common.Configuration{}
 	err := yaml.Unmarshal(validConfigurationTestCaseData, valid)
 	if err != nil {
 		t.Errorf("cannot unmarshal test case data:%s\n", err)
@@ -266,7 +267,7 @@ func deployerErrorHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func deployerSuccessHandler(w http.ResponseWriter, r *http.Request) {
-	valid := &Configuration{}
+	valid := &common.Configuration{}
 	err := yaml.Unmarshal(validConfigurationTestCaseData, valid)
 	if err != nil {
 		status := fmt.Sprintf("cannot unmarshal test case data:%s", err)
@@ -282,7 +283,7 @@ func deployerSuccessHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	result := &Configuration{}
+	result := &common.Configuration{}
 	if err := yaml.Unmarshal(body, result); err != nil {
 		status := fmt.Sprintf("cannot unmarshal request body:%s", err)
 		http.Error(w, status, http.StatusInternalServerError)

manager/manager/expander.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/ghodss/yaml"
 	"github.com/kubernetes/deployment-manager/util"
+	"github.com/kubernetes/deployment-manager/common"
 )
 
 const (
@@ -31,13 +32,13 @@ const (
 
 // ExpandedTemplate is the structure returned by the expansion service.
 type ExpandedTemplate struct {
-	Config *Configuration `json:"config"`
-	Layout *Layout        `json:"layout"`
+	Config *common.Configuration `json:"config"`
+	Layout *common.Layout        `json:"layout"`
 }
 
 // Expander abstracts interactions with the expander and deployer services.
 type Expander interface {
-	ExpandTemplate(t Template) (*ExpandedTemplate, error)
+	ExpandTemplate(t common.Template) (*ExpandedTemplate, error)
 }
 
 // NewExpander returns a new initialized Expander.
@@ -54,7 +55,7 @@ func (e *expander) getBaseURL() string {
 	return fmt.Sprintf("%s/expand", e.expanderURL)
 }
 
-func expanderError(t *Template, err error) error {
+func expanderError(t *common.Template, err error) error {
 	return fmt.Errorf("cannot expand template named %s (%s):\n%s", t.Name, err, t.Content)
 }
 
@@ -90,14 +91,14 @@ func expanderError(t *Template, err error) error {
 // between the name#template key to exist in the layout given a particular choice of naming.
 // In practice, it would be nearly impossible to hit, but consider including properties/name/type
 // into a hash of sorts to make this robust...
-func walkLayout(l *Layout, toReplace map[string]*LayoutResource) map[string]*LayoutResource {
-	ret := map[string]*LayoutResource{}
+func walkLayout(l *common.Layout, imports []*common.ImportFile, toReplace map[string]*common.LayoutResource) map[string]*common.LayoutResource {
+	ret := map[string]*common.LayoutResource{}
 	toVisit := l.Resources
 
 	for len(toVisit) > 0 {
 		lr := toVisit[0]
 		nodeKey := lr.Resource.Name + layoutNodeKeySeparator + lr.Resource.Type
-		if len(lr.Layout.Resources) == 0 && util.IsTemplate(lr.Resource.Type) {
+		if len(lr.Layout.Resources) == 0 && util.IsTemplate(lr.Resource.Type, imports) {
 			ret[nodeKey] = lr
 		} else if toReplace[nodeKey] != nil {
 			toReplace[nodeKey].Resources = lr.Resources
@@ -110,20 +111,20 @@ func walkLayout(l *Layout, toReplace map[string]*LayoutResource) map[string]*Lay
 }
 
 // ExpandTemplate expands the supplied template, and returns a configuration.
-func (e *expander) ExpandTemplate(t Template) (*ExpandedTemplate, error) {
+func (e *expander) ExpandTemplate(t common.Template) (*ExpandedTemplate, error) {
 	// We have a fencepost problem here.
 	// 1. Start by trying to resolve any missing templates
 	// 2. Expand the configuration using all the of the imports available to us at this point
 	// 3. Expansion may yield additional templates, so we run the type resolution again
 	// 4. If type resolution resulted in new imports being available, return to 2.
-	config := &Configuration{}
+	config := &common.Configuration{}
 	if err := yaml.Unmarshal([]byte(t.Content), config); err != nil {
 		e := fmt.Errorf("Unable to unmarshal configuration (%s): %s", err, t.Content)
 		return nil, e
 	}
 
-	var finalLayout *Layout
-	needResolve := map[string]*LayoutResource{}
+	var finalLayout *common.Layout
+	needResolve := map[string]*common.LayoutResource{}
 
 	// Start things off by attempting to resolve the templates in a first pass.
 	newImp, err := e.typeResolver.ResolveTypes(config, t.Imports)
@@ -149,7 +150,7 @@ func (e *expander) ExpandTemplate(t Template) (*ExpandedTemplate, error) {
 		if finalLayout == nil {
 			finalLayout = result.Layout
 		}
-		needResolve = walkLayout(result.Layout, needResolve)
+		needResolve = walkLayout(result.Layout, t.Imports, needResolve)
 
 		newImp, err = e.typeResolver.ResolveTypes(result.Config, nil)
 		if err != nil {
@@ -176,7 +177,7 @@ func (e *expander) ExpandTemplate(t Template) (*ExpandedTemplate, error) {
 	}
 }
 
-func (e *expander) expandTemplate(t *Template) (*ExpandedTemplate, error) {
+func (e *expander) expandTemplate(t *common.Template) (*ExpandedTemplate, error) {
 	j, err := json.Marshal(t)
 	if err != nil {
 		return nil, err

manager/manager/expander_test.go

@@ -24,16 +24,17 @@ import (
 	"testing"
 
 	"github.com/kubernetes/deployment-manager/util"
+	"github.com/kubernetes/deployment-manager/common"
 
 	"github.com/ghodss/yaml"
 )
 
 type mockResolver struct {
-	responses [][]*ImportFile
+	responses [][]*common.ImportFile
 	t         *testing.T
 }
 
-func (r *mockResolver) ResolveTypes(c *Configuration, i []*ImportFile) ([]*ImportFile, error) {
+func (r *mockResolver) ResolveTypes(c *common.Configuration, i []*common.ImportFile) ([]*common.ImportFile, error) {
 	if len(r.responses) < 1 {
 		return nil, nil
 	}
@@ -43,7 +44,7 @@ func (r *mockResolver) ResolveTypes(c *Configuration, i []*ImportFile) ([]*Impor
 	return ret, nil
 }
 
-var validTemplateTestCaseData = Template{
+var validTemplateTestCaseData = common.Template{
 	Name:    "TestTemplate",
 	Content: string(validContentTestCaseData),
 	Imports: validImportFilesTestCaseData,
@@ -59,11 +60,19 @@ resources:
     test-property: test-value
 `)
 
-var validImportFilesTestCaseData = []*ImportFile{
-	&ImportFile{
+var validImportFilesTestCaseData = []*common.ImportFile{
+	&common.ImportFile{
 		Name:    "test-type.py",
 		Content: "test-type.py validTemplateTestCaseData content",
 	},
+	&common.ImportFile{
+		Name:    "test.py",
+		Content: "test.py validTemplateTestCaseData content",
+	},
+	&common.ImportFile{
+		Name:    "test2.py",
+		Content: "test2.py validTemplateTestCaseData content",
+	},
 }
 
 var validConfigTestCaseData = []byte(`
@@ -210,7 +219,7 @@ layout:
           test: test
 `
 
-var roundTripTemplate = Template{
+var roundTripTemplate = common.Template{
 	Name:    "TestTemplate",
 	Content: roundTripContent,
 	Imports: nil,
@@ -249,9 +258,9 @@ func TestExpandTemplate(t *testing.T) {
 			"expect success for ExpandTemplate with two expansions",
 			"",
 			roundTripHandler,
-			&mockResolver{[][]*ImportFile{
+			&mockResolver{[][]*common.ImportFile{
 				{},
-				{&ImportFile{Name: "test.py"}},
+				{&common.ImportFile{Name: "test.py"}},
 			}, t},
 			roundTripResponse,
 		},
@@ -334,7 +343,7 @@ func expanderSuccessHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	template := &Template{}
+	template := &common.Template{}
 	if err := json.Unmarshal(body, template); err != nil {
 		status := fmt.Sprintf("cannot unmarshal request body:%s\n%s\n", err, body)
 		http.Error(w, status, http.StatusInternalServerError)

manager/manager/typeresolver.go

@@ -16,8 +16,11 @@ package manager
 import (
 	"fmt"
 	"net/http"
+	"regexp"
 	"time"
 
+	"github.com/kubernetes/deployment-manager/common"
+	"github.com/kubernetes/deployment-manager/registry"
 	"github.com/kubernetes/deployment-manager/util"
 
 	"github.com/ghodss/yaml"
@@ -31,12 +34,14 @@ const (
 // TypeResolver finds Types in a Configuration which aren't yet reduceable to an import file
 // or primitive, and attempts to replace them with a template from a URL.
 type TypeResolver interface {
-	ResolveTypes(config *Configuration, imports []*ImportFile) ([]*ImportFile, error)
+	ResolveTypes(config *common.Configuration, imports []*common.ImportFile) ([]*common.ImportFile, error)
 }
 
 type typeResolver struct {
 	getter  util.HTTPClient
 	maxUrls int
+	re      *regexp.Regexp
+	rp      registry.RegistryProvider
 }
 
 // NewTypeResolver returns a new initialized TypeResolver.
@@ -48,10 +53,12 @@ func NewTypeResolver() TypeResolver {
 	client.Timeout = timeout
 	ret.getter = util.NewHTTPClient(3, client, util.NewSleeper())
 	ret.maxUrls = maxURLImports
+	ret.re = regexp.MustCompile("github.com/(.*)/(.*)/(.*)/(.*):(.*)")
+	ret.rp = &registry.DefaultRegistryProvider{}
 	return ret
 }
 
-func resolverError(c *Configuration, err error) error {
+func resolverError(c *common.Configuration, err error) error {
 	return fmt.Errorf("cannot resolve types in configuration %s due to: \n%s\n",
 		c, err)
 }
@@ -78,31 +85,36 @@ func performHTTPGet(g util.HTTPClient, u string, allowMissing bool) (content str
 // resolved type definitions in t.ImportFiles. Types can be either
 // primitive (i.e., built in), resolved (i.e., already t.ImportFiles), or remote
 // (i.e., described by a URL that must be fetched to resolve the type).
-func (tr *typeResolver) ResolveTypes(config *Configuration, imports []*ImportFile) ([]*ImportFile, error) {
+func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*common.ImportFile) ([]*common.ImportFile, error) {
 	existing := map[string]bool{}
 	for _, v := range imports {
 		existing[v.Name] = true
 	}
 
-	fetched := map[string][]*ImportFile{}
+	fetched := map[string][]*common.ImportFile{}
 	toFetch := make([]string, 0, tr.maxUrls)
 	for _, r := range config.Resources {
-		// Only fetch HTTP URLs that we haven't already imported.
-		if util.IsHttpUrl(r.Type) && !existing[r.Type] {
-			toFetch = append(toFetch, r.Type)
-			fetched[r.Type] = append(fetched[r.Type], &ImportFile{Name: r.Type})
+		// Map the type to a fetchable URL (if applicable) or skip it if it's a non-fetchable type (primitive for example).
+		u, err := tr.MapFetchableURL(r.Type)
+		if err != nil {
+			return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", r.Type, err))
+		}
+		if len(u) > 0 && !existing[r.Type] {
+			toFetch = append(toFetch, u)
+			fetched[u] = append(fetched[u], &common.ImportFile{Name: r.Type, Path: u})
 		}
 	}
 
 	count := 0
 	for len(toFetch) > 0 {
-		//1. Fetch import URL. Exit if no URLs left
-		//2. Check/handle HTTP status
-		//3. Store results in all ImportFiles from that URL
-		//4. Check for the optional schema file at import URL + .schema
-		//5. Repeat 2,3 for schema file
-		//6. Add each schema import to fetch if not already done
-		//7. Mark URL done. Return to 1.
+		//1. If short github URL, resolve to a download URL
+		//2. Fetch import URL. Exit if no URLs left
+		//3. Check/handle HTTP status
+		//4. Store results in all ImportFiles from that URL
+		//5. Check for the optional schema file at import URL + .schema
+		//6. Repeat 2,3 for schema file
+		//7. Add each schema import to fetch if not already done
+		//8. Mark URL done. Return to 1.
 		if count >= tr.maxUrls {
 			return nil, resolverError(config,
 				fmt.Errorf("Number of imports exceeds maximum of %d", tr.maxUrls))
@@ -125,32 +137,41 @@ func (tr *typeResolver) ResolveTypes(config *Configuration, imports []*ImportFil
 		}
 
 		if sch != "" {
-			var s Schema
+			var s common.Schema
 			if err := yaml.Unmarshal([]byte(sch), &s); err != nil {
 				return nil, resolverError(config, err)
 			}
 			// Here we handle any nested imports in the schema we've just fetched.
 			for _, v := range s.Imports {
-				i := &ImportFile{Name: v.Name}
+				i := &common.ImportFile{Name: v.Name}
 				var existingSchema string
-				if len(fetched[v.Path]) == 0 {
+				u, conversionErr := tr.MapFetchableURL(v.Path)
+				if conversionErr != nil {
+					return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", v.Path, conversionErr))
+				}
+				// If it's not a fetchable URL, we need to use the type name as is, since it is a short name
+				// for a schema.
+				if len(u) == 0 {
+					u = v.Path
+				}
+				if len(fetched[u]) == 0 {
 					// If this import URL is new to us, add it to the URLs to fetch.
-					toFetch = append(toFetch, v.Path)
+					toFetch = append(toFetch, u)
 				} else {
 					// If this is not a new import URL and we've already fetched its contents,
 					// reuse them. Also, check if we also found a schema for that import URL and
 					// record those contents for re-use as well.
-					if fetched[v.Path][0].Content != "" {
-						i.Content = fetched[v.Path][0].Content
-						if len(fetched[v.Path+schemaSuffix]) > 0 {
-							existingSchema = fetched[v.Path+schemaSuffix][0].Content
+					if fetched[u][0].Content != "" {
+						i.Content = fetched[u][0].Content
+						if len(fetched[u+schemaSuffix]) > 0 {
+							existingSchema = fetched[u+schemaSuffix][0].Content
 						}
 					}
 				}
-				fetched[v.Path] = append(fetched[v.Path], i)
+				fetched[u] = append(fetched[u], i)
 				if existingSchema != "" {
-					fetched[v.Path+schemaSuffix] = append(fetched[v.Path+schemaSuffix],
-						&ImportFile{Name: v.Name + schemaSuffix, Content: existingSchema})
+					fetched[u+schemaSuffix] = append(fetched[u+schemaSuffix],
+						&common.ImportFile{Name: v.Name + schemaSuffix, Content: existingSchema})
 				}
 			}
 
@@ -158,7 +179,7 @@ func (tr *typeResolver) ResolveTypes(config *Configuration, imports []*ImportFil
 			for _, i := range fetched[url] {
 				schemaImportName := i.Name + schemaSuffix
 				fetched[schemaURL] = append(fetched[schemaURL],
-					&ImportFile{Name: schemaImportName, Content: sch})
+					&common.ImportFile{Name: schemaImportName, Content: sch})
 			}
 		}
 
@@ -166,10 +187,37 @@ func (tr *typeResolver) ResolveTypes(config *Configuration, imports []*ImportFil
 		toFetch = toFetch[1:]
 	}
 
-	ret := []*ImportFile{}
+	ret := []*common.ImportFile{}
 	for _, v := range fetched {
 		ret = append(ret, v...)
 	}
 
 	return ret, nil
 }
+
+// MapFetchableUrl checks a type to see if it is either a short git hub url or a fully specified URL
+// and returns the URL that should be used to fetch it. If the url is not fetchable (primitive type for
+// example) will return empty string.
+func (tr *typeResolver) MapFetchableURL(t string) (string, error) {
+	if util.IsGithubShortType(t) {
+		return tr.ShortTypeToDownloadURL(t)
+	} else if util.IsHttpUrl(t) {
+		return t, nil
+	}
+	return "", nil
+}
+
+// ShortTypeToDownloadURL converts a github URL into downloadable URL from github.
+// Input must be of the type and is assumed to have been validated before this call:
+// github.com/owner/repo/qualifier/type:version
+// for example:
+// github.com/kubernetes/application-dm-templates/storage/redis:v1
+func (tr *typeResolver) ShortTypeToDownloadURL(template string) (string, error) {
+	m := tr.re.FindStringSubmatch(template)
+	if len(m) != 6 {
+		return "", fmt.Errorf("Failed to parse short github url: %s", template)
+	}
+	r := tr.rp.GetGithubRegistry(m[1], m[2])
+	t := registry.Type{m[3], m[4], m[5]}
+	return r.GetURL(t)
+}

manager/repository/repository.go

@@ -22,37 +22,37 @@ import (
 	"sync"
 	"time"
 
-	"github.com/kubernetes/deployment-manager/manager/manager"
+	"github.com/kubernetes/deployment-manager/common"
 )
 
 // deploymentTypeInstanceMap stores type instances mapped by deployment name.
 // This allows for simple updating and deleting of per-deployment instances
 // when deployments are created/updated/deleted.
-type deploymentTypeInstanceMap map[string][]*manager.TypeInstance
+type deploymentTypeInstanceMap map[string][]*common.TypeInstance
 type typeInstanceMap map[string]deploymentTypeInstanceMap
 
 type mapBasedRepository struct {
 	sync.RWMutex
-	deployments map[string]manager.Deployment
-	manifests   map[string]map[string]*manager.Manifest
+	deployments map[string]common.Deployment
+	manifests   map[string]map[string]*common.Manifest
 	instances   typeInstanceMap
 }
 
 // NewMapBasedRepository returns a new map based repository.
-func NewMapBasedRepository() manager.Repository {
+func NewMapBasedRepository() common.Repository {
 	return &mapBasedRepository{
-		deployments: make(map[string]manager.Deployment, 0),
-		manifests:   make(map[string]map[string]*manager.Manifest, 0),
+		deployments: make(map[string]common.Deployment, 0),
+		manifests:   make(map[string]map[string]*common.Manifest, 0),
 		instances:   typeInstanceMap{},
 	}
 }
 
 // ListDeployments returns of all of the deployments in the repository.
-func (r *mapBasedRepository) ListDeployments() ([]manager.Deployment, error) {
+func (r *mapBasedRepository) ListDeployments() ([]common.Deployment, error) {
 	r.RLock()
 	defer r.RUnlock()
 
-	l := []manager.Deployment{}
+	l := []common.Deployment{}
 	for _, deployment := range r.deployments {
 		l = append(l, deployment)
 	}
@@ -62,7 +62,7 @@ func (r *mapBasedRepository) ListDeployments() ([]manager.Deployment, error) {
 
 // GetDeployment returns the deployment with the supplied name.
 // If the deployment is not found, it returns an error.
-func (r *mapBasedRepository) GetDeployment(name string) (*manager.Deployment, error) {
+func (r *mapBasedRepository) GetDeployment(name string) (*common.Deployment, error) {
 	d, ok := r.deployments[name]
 	if !ok {
 		return nil, fmt.Errorf("deployment %s not found", name)
@@ -72,13 +72,13 @@ func (r *mapBasedRepository) GetDeployment(name string) (*manager.Deployment, er
 
 // GetValidDeployment returns the deployment with the supplied name.
 // If the deployment is not found or marked as deleted, it returns an error.
-func (r *mapBasedRepository) GetValidDeployment(name string) (*manager.Deployment, error) {
+func (r *mapBasedRepository) GetValidDeployment(name string) (*common.Deployment, error) {
 	d, err := r.GetDeployment(name)
 	if err != nil {
 		return nil, err
 	}
 
-	if d.Status == manager.DeletedStatus {
+	if d.Status == common.DeletedStatus {
 		return nil, fmt.Errorf("deployment %s is deleted", name)
 	}
 
@@ -86,7 +86,7 @@ func (r *mapBasedRepository) GetValidDeployment(name string) (*manager.Deploymen
 }
 
 // SetDeploymentStatus sets the DeploymentStatus of the deployment and updates ModifiedAt
-func (r *mapBasedRepository) SetDeploymentStatus(name string, status manager.DeploymentStatus) error {
+func (r *mapBasedRepository) SetDeploymentStatus(name string, status common.DeploymentStatus) error {
 	return func() error {
 		r.Lock()
 		defer r.Unlock()
@@ -104,8 +104,8 @@ func (r *mapBasedRepository) SetDeploymentStatus(name string, status manager.Dep
 }
 
 // CreateDeployment creates a new deployment and stores it in the repository.
-func (r *mapBasedRepository) CreateDeployment(name string) (*manager.Deployment, error) {
-	d, err := func() (*manager.Deployment, error) {
+func (r *mapBasedRepository) CreateDeployment(name string) (*common.Deployment, error) {
+	d, err := func() (*common.Deployment, error) {
 		r.Lock()
 		defer r.Unlock()
 
@@ -114,8 +114,8 @@ func (r *mapBasedRepository) CreateDeployment(name string) (*manager.Deployment,
 			return nil, fmt.Errorf("Deployment %s already exists", name)
 		}
 
-		d := manager.NewDeployment(name)
-		d.Status = manager.CreatedStatus
+		d := common.NewDeployment(name)
+		d.Status = common.CreatedStatus
 		d.DeployedAt = time.Now()
 		r.deployments[name] = *d
 		return d, nil
@@ -129,7 +129,7 @@ func (r *mapBasedRepository) CreateDeployment(name string) (*manager.Deployment,
 	return d, nil
 }
 
-func (r *mapBasedRepository) AddManifest(deploymentName string, manifest *manager.Manifest) error {
+func (r *mapBasedRepository) AddManifest(deploymentName string, manifest *common.Manifest) error {
 	err := func() error {
 		r.Lock()
 		defer r.Unlock()
@@ -166,8 +166,8 @@ func (r *mapBasedRepository) AddManifest(deploymentName string, manifest *manage
 // DeleteDeployment deletes the deployment with the supplied name.
 // If forget is true, then the deployment is removed from the repository.
 // Otherwise, it is marked as deleted and retained.
-func (r *mapBasedRepository) DeleteDeployment(name string, forget bool) (*manager.Deployment, error) {
-	d, err := func() (*manager.Deployment, error) {
+func (r *mapBasedRepository) DeleteDeployment(name string, forget bool) (*common.Deployment, error) {
+	d, err := func() (*common.Deployment, error) {
 		r.Lock()
 		defer r.Unlock()
 
@@ -178,7 +178,7 @@ func (r *mapBasedRepository) DeleteDeployment(name string, forget bool) (*manage
 
 		if !forget {
 			d.DeletedAt = time.Now()
-			d.Status = manager.DeletedStatus
+			d.Status = common.DeletedStatus
 			r.deployments[name] = *d
 		} else {
 			delete(r.deployments, name)
@@ -197,7 +197,7 @@ func (r *mapBasedRepository) DeleteDeployment(name string, forget bool) (*manage
 	return d, nil
 }
 
-func (r *mapBasedRepository) ListManifests(deploymentName string) (map[string]*manager.Manifest, error) {
+func (r *mapBasedRepository) ListManifests(deploymentName string) (map[string]*common.Manifest, error) {
 	r.Lock()
 	defer r.Unlock()
 
@@ -209,17 +209,17 @@ func (r *mapBasedRepository) ListManifests(deploymentName string) (map[string]*m
 	return r.listManifestsForDeployment(deploymentName)
 }
 
-func (r *mapBasedRepository) listManifestsForDeployment(deploymentName string) (map[string]*manager.Manifest, error) {
+func (r *mapBasedRepository) listManifestsForDeployment(deploymentName string) (map[string]*common.Manifest, error) {
 	l, ok := r.manifests[deploymentName]
 	if !ok {
-		l = make(map[string]*manager.Manifest, 0)
+		l = make(map[string]*common.Manifest, 0)
 		r.manifests[deploymentName] = l
 	}
 
 	return l, nil
 }
 
-func (r *mapBasedRepository) GetManifest(deploymentName string, manifestName string) (*manager.Manifest, error) {
+func (r *mapBasedRepository) GetManifest(deploymentName string, manifestName string) (*common.Manifest, error) {
 	r.Lock()
 	defer r.Unlock()
 
@@ -231,7 +231,7 @@ func (r *mapBasedRepository) GetManifest(deploymentName string, manifestName str
 	return r.getManifestForDeployment(deploymentName, manifestName)
 }
 
-func (r *mapBasedRepository) getManifestForDeployment(deploymentName string, manifestName string) (*manager.Manifest, error) {
+func (r *mapBasedRepository) getManifestForDeployment(deploymentName string, manifestName string) (*common.Manifest, error) {
 	l, err := r.listManifestsForDeployment(deploymentName)
 	if err != nil {
 		return nil, err
@@ -247,7 +247,7 @@ func (r *mapBasedRepository) getManifestForDeployment(deploymentName string, man
 
 // GetLatestManifest returns the latest manifest for a given deployment,
 // which by definition is the manifest with the largest time stamp.
-func (r *mapBasedRepository) GetLatestManifest(deploymentName string) (*manager.Manifest, error) {
+func (r *mapBasedRepository) GetLatestManifest(deploymentName string) (*common.Manifest, error) {
 	r.Lock()
 	defer r.Unlock()
 
@@ -271,11 +271,11 @@ func (r *mapBasedRepository) ListTypes() []string {
 
 // GetTypeInstances returns all instances of a given type. If type is empty,
 // returns all instances for all types.
-func (r *mapBasedRepository) GetTypeInstances(typeName string) []*manager.TypeInstance {
+func (r *mapBasedRepository) GetTypeInstances(typeName string) []*common.TypeInstance {
 	r.Lock()
 	defer r.Unlock()
 
-	var instances []*manager.TypeInstance
+	var instances []*common.TypeInstance
 	for t, dInstMap := range r.instances {
 		if t == typeName || typeName == "all" {
 			for _, i := range dInstMap {
@@ -305,7 +305,7 @@ func (r *mapBasedRepository) ClearTypeInstances(deploymentName string) {
 //
 // To clear the current set of instances first, caller should first use
 // ClearTypeInstances().
-func (r *mapBasedRepository) SetTypeInstances(deploymentName string, instances map[string][]*manager.TypeInstance) {
+func (r *mapBasedRepository) SetTypeInstances(deploymentName string, instances map[string][]*common.TypeInstance) {
 	r.Lock()
 	defer r.Unlock()
 

manager/repository/repository_test.go

@@ -14,7 +14,7 @@ limitations under the License.
 package repository
 
 import (
-	"github.com/kubernetes/deployment-manager/manager/manager"
+	"github.com/kubernetes/deployment-manager/common"
 
 	"fmt"
 	"testing"
@@ -77,7 +77,7 @@ func testCreateDeploymentWithManifests(t *testing.T, count int) {
 
 	for i := 0; i < count; i++ {
 		var manifestName = fmt.Sprintf("manifest-%d", i)
-		manifest := manager.Manifest{Deployment: deploymentName, Name: manifestName}
+		manifest := common.Manifest{Deployment: deploymentName, Name: manifestName}
 		err := r.AddManifest(deploymentName, &manifest)
 		if err != nil {
 			t.Fatalf("AddManifest failed: %v", err)
@@ -136,7 +136,7 @@ func TestRepositoryDeleteWorksWithNoLatestManifest(t *testing.T) {
 	if err != nil {
 		t.Fatalf("DeleteDeployment failed: %v", err)
 	}
-	if dDeleted.Status != manager.DeletedStatus {
+	if dDeleted.Status != common.DeletedStatus {
 		t.Fatalf("Deployment Status is not deleted")
 	}
 	if _, err := r.ListManifests(deploymentName); err == nil {
@@ -148,7 +148,7 @@ func TestRepositoryDeleteDeploymentWorksNoForget(t *testing.T) {
 	var deploymentName = "mydeployment"
 	var manifestName = "manifest-0"
 	r := NewMapBasedRepository()
-	manifest := manager.Manifest{Deployment: deploymentName, Name: manifestName}
+	manifest := common.Manifest{Deployment: deploymentName, Name: manifestName}
 	_, err := r.CreateDeployment(deploymentName)
 	if err != nil {
 		t.Fatalf("CreateDeployment failed: %v", err)
@@ -161,7 +161,7 @@ func TestRepositoryDeleteDeploymentWorksNoForget(t *testing.T) {
 	if err != nil {
 		t.Fatalf("DeleteDeployment failed: %v", err)
 	}
-	if dDeleted.Status != manager.DeletedStatus {
+	if dDeleted.Status != common.DeletedStatus {
 		t.Fatalf("Deployment Status is not deleted")
 	}
 }
@@ -170,7 +170,7 @@ func TestRepositoryDeleteDeploymentWorksForget(t *testing.T) {
 	var deploymentName = "mydeployment"
 	var manifestName = "manifest-0"
 	r := NewMapBasedRepository()
-	manifest := manager.Manifest{Deployment: deploymentName, Name: manifestName}
+	manifest := common.Manifest{Deployment: deploymentName, Name: manifestName}
 	_, err := r.CreateDeployment(deploymentName)
 	if err != nil {
 		t.Fatalf("CreateDeployment failed: %v", err)
@@ -183,7 +183,7 @@ func TestRepositoryDeleteDeploymentWorksForget(t *testing.T) {
 	if err != nil {
 		t.Fatalf("DeleteDeployment failed: %v", err)
 	}
-	if dDeleted.Status != manager.CreatedStatus {
+	if dDeleted.Status != common.CreatedStatus {
 		t.Fatalf("Deployment Status is not created")
 	}
 }
@@ -191,9 +191,9 @@ func TestRepositoryDeleteDeploymentWorksForget(t *testing.T) {
 func TestRepositoryTypeInstances(t *testing.T) {
 	r := NewMapBasedRepository()
 
-	d1Map := map[string][]*manager.TypeInstance{
-		"t1": []*manager.TypeInstance{
-			&manager.TypeInstance{
+	d1Map := map[string][]*common.TypeInstance{
+		"t1": []*common.TypeInstance{
+			&common.TypeInstance{
 				Name:       "i1",
 				Type:       "t1",
 				Deployment: "d1",
@@ -203,9 +203,9 @@ func TestRepositoryTypeInstances(t *testing.T) {
 		},
 	}
 
-	d2Map := map[string][]*manager.TypeInstance{
-		"t2": []*manager.TypeInstance{
-			&manager.TypeInstance{
+	d2Map := map[string][]*common.TypeInstance{
+		"t2": []*common.TypeInstance{
+			&common.TypeInstance{
 				Name:       "i2",
 				Type:       "t2",
 				Deployment: "d2",
@@ -215,9 +215,9 @@ func TestRepositoryTypeInstances(t *testing.T) {
 		},
 	}
 
-	d3Map := map[string][]*manager.TypeInstance{
-		"t2": []*manager.TypeInstance{
-			&manager.TypeInstance{
+	d3Map := map[string][]*common.TypeInstance{
+		"t2": []*common.TypeInstance{
+			&common.TypeInstance{
 				Name:       "i3",
 				Type:       "t2",
 				Deployment: "d3",

registry/github_registry.go

@@ -18,6 +18,7 @@ import (
 
 	"fmt"
 	"log"
+	"strings"
 )
 
 // GithubRegistry implements the Registry interface that talks to github.
@@ -66,7 +67,11 @@ func (g *GithubRegistry) List() ([]Type, error) {
 
 // GetURL fetches the download URL for a given Type and checks for existence of a schema file.
 func (g *GithubRegistry) GetURL(t Type) (string, error) {
-	path := g.path + "/" + t.Name + "/" + t.Version
+	path,err := g.MakeRepositoryPath(t)
+	if err != nil {
+		return "", err
+	}
+	log.Printf("Got repository path: %s", path)
 	_, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, path, nil)
 	if err != nil {
 		log.Printf("Failed to list versions at path: %s: %v", path, err)
@@ -114,3 +119,34 @@ func (g *GithubRegistry) getDirs(dir string) ([]string, error) {
 
 	return dirs, nil
 }
+
+func (g *GithubRegistry) mapCollection(collection string) (string, error) {
+	if strings.ContainsAny(collection, "/") {
+		return "", fmt.Errorf("collection must not contain slashes, got %s", collection)
+	}
+	// TODO(vaikas): Implement lookup from the root metadata file to map collection to a path
+	return collection, nil
+}
+
+// MakeRepositoryPath constructs a github path to a given type based on a repository, and type name and version.
+// The returned repository path will be of the form:
+// [GithubRegistry.path/][Type.Collection]/Type.Name/Type.Version
+// Type.Collection will be mapped using mapCollection in the future, for now it's a straight
+// 1:1 mapping (if given)
+func (g *GithubRegistry) MakeRepositoryPath(t Type) (string, error) {
+	log.Printf("Making repository path: %v", t)
+	// First map the collection
+	collection, err := g.mapCollection(t.Collection)
+	if err != nil {
+		return "", err
+	}
+	// Construct the return path
+	p := ""
+	if len(g.path) > 0 {
+		p += g.path + "/"
+	}
+	if len(collection) > 0 {
+		p += collection + "/"
+	}
+	return p + t.Name + "/" + t.Version, nil
+}

registry/registry.go

@@ -22,6 +22,7 @@ package registry
 // For example, a template registry containing two versions of redis
 // (implemented in jinja), and one version of replicatedservice (implemented
 // in python) would have a directory structure that looks something like this:
+// qualifier [optional] prefix to a virtual root within the repository.
 // /redis
 //   /v1
 //     redis.jinja
@@ -35,6 +36,7 @@ package registry
 //     replicatedservice.python.schema
 
 type Type struct {
+	Collection string
 	Name    string
 	Version string
 }
@@ -46,3 +48,6 @@ type Registry interface {
 	// Get the download URL for a given template and version
 	GetURL(t Type) (string, error)
 }
+
+
+

registry/registryprovider.go

+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package registry
+
+// RegistryProvider returns factories for creating registries for a given RegistryType.
+type RegistryProvider interface {
+	GetGithubRegistry(owner string, repository string) Registry
+}
+
+type DefaultRegistryProvider struct {
+}
+
+func (drp *DefaultRegistryProvider) GetGithubRegistry(owner string, repository string) Registry {
+	return NewGithubRegistry(owner, repository, "")
+}

resourcifier/configurations.go

@@ -14,7 +14,7 @@ limitations under the License.
 package main
 
 import (
-	"github.com/kubernetes/deployment-manager/manager/manager"
+	"github.com/kubernetes/deployment-manager/common"
 	"github.com/kubernetes/deployment-manager/resourcifier/configurator"
 	"github.com/kubernetes/deployment-manager/util"
 
@@ -76,8 +76,8 @@ func listConfigurationsHandlerFunc(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	c := &manager.Configuration{
-		[]*manager.Resource{
+	c := &common.Configuration{
+		[]*common.Resource{
 			{Type: rtype},
 		},
 	}
@@ -105,8 +105,8 @@ func getConfigurationHandlerFunc(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	c := &manager.Configuration{
-		[]*manager.Resource{
+	c := &common.Configuration{
+		[]*common.Resource{
 			{Name: rname, Type: rtype},
 		},
 	}
@@ -253,7 +253,7 @@ func getPathVariable(w http.ResponseWriter, r *http.Request, variable, handler s
 	return unescaped, nil
 }
 
-func getConfiguration(w http.ResponseWriter, r *http.Request, handler string) *manager.Configuration {
+func getConfiguration(w http.ResponseWriter, r *http.Request, handler string) *common.Configuration {
 	b := io.LimitReader(r.Body, *maxLength*1024)
 	y, err := ioutil.ReadAll(b)
 	if err != nil {
@@ -276,7 +276,7 @@ func getConfiguration(w http.ResponseWriter, r *http.Request, handler string) *m
 		return nil
 	}
 
-	c := &manager.Configuration{}
+	c := &common.Configuration{}
 	if err := json.Unmarshal(j, c); err != nil {
 		e := errors.New(err.Error() + "\n" + string(j))
 		util.LogAndReturnError(handler, http.StatusBadRequest, e, w)

resourcifier/configurator/configurator.go

@@ -20,7 +20,7 @@ import (
 	"os/exec"
 	"strings"
 
-	"github.com/kubernetes/deployment-manager/manager/manager"
+	"github.com/kubernetes/deployment-manager/common"
 	"github.com/ghodss/yaml"
 )
 
@@ -77,7 +77,7 @@ func (e *Error) appendError(err error) error {
 // action on it (create/delete/replace) and updates the State of the resource with the resulting
 // status. In case of errors with a resource, Resource.State.Errors is set. 
 // and then updates the deployment with the completion status and completion time.
-func (a *Configurator) Configure(c *manager.Configuration, o operation) (string, error) {
+func (a *Configurator) Configure(c *common.Configuration, o operation) (string, error) {
 	errors := &Error{}
 	var output []string
 	for i, resource := range c.Resources {
@@ -99,8 +99,8 @@ func (a *Configurator) Configure(c *manager.Configuration, o operation) (string,
 			if err != nil {
 				e := fmt.Errorf("yaml marshal failed for resource: %v: %v", resource.Name, err)
 				log.Println(errors.appendError(e))
-				c.Resources[i].State = &manager.ResourceState{
-					Status: manager.Aborted,
+				c.Resources[i].State = &common.ResourceState{
+					Status: common.Aborted,
 					Errors: []string{e.Error()},
 				}
 				continue
@@ -122,8 +122,8 @@ func (a *Configurator) Configure(c *manager.Configuration, o operation) (string,
 
 		if err := cmd.Start(); err != nil {
 			e := fmt.Errorf("cannot start kubetcl for resource: %v: %v", resource.Name, err)
-			c.Resources[i].State = &manager.ResourceState{
-				Status: manager.Failed,
+			c.Resources[i].State = &common.ResourceState{
+				Status: common.Failed,
 				Errors: []string{e.Error()},
 			}
 			log.Println(errors.appendError(e))
@@ -137,8 +137,8 @@ func (a *Configurator) Configure(c *manager.Configuration, o operation) (string,
 				log.Println(resource.Name + " not found, treating as success for delete")
 			} else {
 				e := fmt.Errorf("kubetcl failed for resource: %v: %v: %v", resource.Name, err, combined.String())
-				c.Resources[i].State = &manager.ResourceState{
-					Status: manager.Failed,
+				c.Resources[i].State = &common.ResourceState{
+					Status: common.Failed,
 					Errors: []string{e.Error()},
 				}
 				log.Println(errors.appendError(e))
@@ -147,7 +147,7 @@ func (a *Configurator) Configure(c *manager.Configuration, o operation) (string,
 		}
 
 		output = append(output, combined.String())
-		c.Resources[i].State = &manager.ResourceState{Status: manager.Created}
+		c.Resources[i].State = &common.ResourceState{Status: common.Created}
 		log.Printf("kubectl succeeded for resource: %v: SysTime: %v UserTime: %v\n%v",
 			resource.Name, cmd.ProcessState.SystemTime(), cmd.ProcessState.UserTime(), combined.String())
 	}

util/templateutil.go

@@ -15,9 +15,39 @@ package util
 
 import (
 	"strings"
+	"log"
+
+	"github.com/kubernetes/deployment-manager/common"
 )
 
 // IsTemplate returns whether a given type is a template.
-func IsTemplate(t string) bool {
-	return strings.HasSuffix(t, ".py") || strings.HasSuffix(t, ".jinja")
+func IsTemplate(t string, imports []*common.ImportFile) bool {
+	log.Printf("IsTemplate: %s : %+v", t, imports)
+	for _, imp := range imports {
+		log.Printf("Checking: %s", imp.Name)
+		if imp.Name == t {
+			return true
+		}
+	}
+	return false
+}
+
+// IsGithubShortType returns whether a given type is a type description in a short format to a github repository type.
+// For now, this means using github types:
+// github.com/owner/repo/qualifier/type:version
+// for example:
+// github.com/kubernetes/application-dm-templates/storage/redis:v1
+func IsGithubShortType(t string) bool {
+	if !strings.HasPrefix(t, "github.com/") {
+		return false
+	}
+	s := strings.Split(t, "/")
+	if len(s) != 5 {
+		return false
+	}
+	v := strings.Split(s[4], ":")
+	if len(v) != 2 {
+		return false
+	}
+	return true
 }