Merge pull request #178 from jackgr/helm-integration

helm integration

common/types.go

@@ -173,30 +173,76 @@ type BasicAuthCredential struct {
 	Password string `json:"password"`
 }
 
-// Credentials used to access the repository
+type APITokenCredential string
+
+// Credential used to access the repository
 type RegistryCredential struct {
+	APIToken  APITokenCredential  `json:"apitoken,omitempty"`
 	BasicAuth BasicAuthCredential `json:"basicauth,omitempty"`
 }
 
+// Registry describes a template registry
+// TODO(jackr): Fix ambiguity re: whether or not URL has a scheme.
 type Registry struct {
-	Name       string             `json:"name,omitempty"` // Friendly name for the repo
+	Name   string         `json:"name,omitempty"`   // Friendly name for the registry
 	Type   RegistryType   `json:"type,omitempty"`   // Technology implementing the registry
-	URL        string             `json:"name,omitempty"` // URL to the root of the repo, for example: github.com/helm/charts
+	URL    string         `json:"name,omitempty"`   // URL to the root of the registry
+	Format RegistryFormat `json:"format,omitempty"` // Format of the registry
+}
+
+// AuthenticatedRegistry describes a type registry with credential.
+// Broke this out of Registry, so that we can pass around instances of Registry
+// without worrying about secrets.
+type AuthenticatedRegistry struct {
+	Registry
 	Credential RegistryCredential `json:"credential,omitempty"`
-	Format     RegistryFormat     `json:"format,omitempty"`
 }
 
 // RegistryType defines the technology that implements the registry
 type RegistryType string
 
 const (
-	Github RegistryType = "github"
+	GithubRegistryType RegistryType = "github"
 )
 
-// RegistryFormat defines the format of the registry
+// RegistryFormat is a semi-colon delimited string that describes the format
+// of a registry.
 type RegistryFormat string
 
 const (
+	// Versioning.
+	// If a registry if versioned, then types appear under versions.
 	VersionedRegistry RegistryFormat = "versioned"
+	// If a registry is unversioned, then types appear under their names.
 	UnversionedRegistry RegistryFormat = "unversioned"
+
+	// Organization.
+	// In a collection registry, types are grouped into collections.
+	CollectionRegistry RegistryFormat = "collection"
+	// In a one level registry, all types appear at the top level.
+	OneLevelRegistry RegistryFormat = "onelevel"
 )
+
+// RegistryService maintains a set of registries that defines the scope of all
+// registry based operations, such as search and type resolution.
+type RegistryService interface {
+	// List all the registries
+	List() ([]*Registry, error)
+	// Create a new registry
+	Create(registry *Registry) error
+	// Get a registry
+	Get(name string) (*Registry, error)
+	// Get a registry with credential.
+	GetAuthenticatedRegistry(name string) (*AuthenticatedRegistry, error)
+	// Delete a registry
+	Delete(name string) error
+	// Find a registry that backs the given URL
+	GetByURL(URL string) (*Registry, error)
+	// GetAuthenticatedRegistryByURL returns an authenticated registry that handles the types for a given URL.
+	GetAuthenticatedRegistryByURL(URL string) (*AuthenticatedRegistry, error)
+	// Set the credential for a registry.
+	// May not be supported by some registry services.
+	SetCredential(name string, credential RegistryCredential) error
+	// Get the credential for a registry.
+	GetCredential(name string) (RegistryCredential, error)
+}

dm/dm.go

@@ -35,6 +35,8 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"path"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -48,6 +50,8 @@ var (
 	service           = flag.String("service", "http://localhost:8001/api/v1/proxy/namespaces/dm/services/manager-service:manager", "URL for deployment manager")
 	binary            = flag.String("binary", "../expandybird/expansion/expansion.py", "Path to template expansion binary")
 	timeout           = flag.Int("timeout", 10, "Time in seconds to wait for response")
+	regex_string      = flag.String("regex", "", "Regular expression to filter the templates listed in a template registry")
+	apitoken          = flag.String("apitoken", "", "Github api token that overrides GITHUB_API_TOKEN environment variable")
 )
 
 var commands = []string{
@@ -81,8 +85,58 @@ var usage = func() {
 	panic("\n")
 }
 
-func getGitRegistry() (registry.Registry, error) {
-	return registry.NewDefaultRegistryProvider().GetRegistry(*template_registry)
+// TODO(jackgr): Move all registry related operations to the server side.
+var registryProvider registry.RegistryProvider
+
+func getRegistryProvider() registry.RegistryProvider {
+	if registryProvider == nil {
+		rs := registry.NewInmemRegistryService()
+		r, err := rs.GetByURL(*template_registry)
+		if err != nil {
+			r := newRegistry(*template_registry)
+			if err := rs.Create(r); err != nil {
+				panic(fmt.Errorf("cannot configure registry at %s: %s", r.URL, err))
+			}
+		}
+
+		if *apitoken == "" {
+			*apitoken = os.Getenv("DM_GITHUB_API_TOKEN")
+		}
+
+		if *apitoken != "" {
+			credential := common.RegistryCredential{
+				APIToken: common.APITokenCredential(*apitoken),
+			}
+
+			if err := rs.SetCredential(r.Name, credential); err != nil {
+				panic(fmt.Errorf("cannot configure registry at %s: %s", r.Name, err))
+			}
+		}
+
+		registryProvider = registry.NewRegistryProvider(rs, nil)
+	}
+
+	return registryProvider
+}
+
+func newRegistry(URL string) *common.Registry {
+	tFormat := fmt.Sprintf("%s;%s", common.VersionedRegistry, common.CollectionRegistry)
+	return &common.Registry{
+		Name:   util.TrimURLScheme(URL),
+		Type:   common.GithubRegistryType,
+		URL:    URL,
+		Format: common.RegistryFormat(tFormat),
+	}
+}
+
+func getGithubRegistry() registry.Registry {
+	provider := getRegistryProvider()
+	git, err := provider.GetRegistryByShortURL(*template_registry)
+	if err != nil {
+		panic(fmt.Errorf("cannot open registry %s: %s", *template_registry, err))
+	}
+
+	return git
 }
 
 func main() {
@@ -107,30 +161,29 @@ func execute() {
 
 	switch args[0] {
 	case "templates":
-		git, err := getGitRegistry()
+		var regex *regexp.Regexp
+		if *regex_string != "" {
+			var err error
+			regex, err = regexp.Compile(*regex_string)
 			if err != nil {
-			panic(fmt.Errorf("Cannot get registry %v", err))
+				panic(fmt.Errorf("cannot compile regular expression %s: %s", *regex_string, err))
 			}
-		templates, err := git.List()
-		if err != nil {
-			panic(fmt.Errorf("Cannot list %v", err))
 		}
 
-		fmt.Printf("Templates:\n")
-		for _, t := range templates {
-			var typeSpec = ""
-			if len(t.Collection) > 0 {
-				typeSpec = t.Collection + "/"
+		git := getGithubRegistry()
+		types, err := git.ListTypes(regex)
+		if err != nil {
+			panic(fmt.Errorf("cannot list templates in registry %s: %s", *template_registry, err))
 		}
-			typeSpec = typeSpec + t.Name
-			if len(t.Version) > 0 {
-				typeSpec = typeSpec + ":" + t.Version
+
+		for _, t := range types {
+			fmt.Printf("%s\n", t.String())
+			urls, err := git.GetDownloadURLs(t)
+			if err != nil {
+				panic(fmt.Errorf("cannot get download urls for %s: %s", t, err))
 			}
 
-			fmt.Printf("%s\n", typeSpec)
-			fmt.Printf("\tshort URL: github.com/%s/%s\n", *template_registry, typeSpec)
-			fmt.Printf("\tdownload URL(s):\n")
-			for _, downloadURL := range getDownloadURLs(t) {
+			for _, downloadURL := range urls {
 				fmt.Printf("\t%s\n", downloadURL)
 			}
 		}
@@ -197,7 +250,7 @@ func execute() {
 			usage()
 		}
 
-		tUrls := getTypeURLs(args[1])
+		tUrls := getDownloadURLs(args[1])
 		var tUrl = ""
 		if len(tUrls) == 0 {
 			// Type is most likely a primitive.
@@ -268,46 +321,25 @@ func describeType(args []string) {
 		usage()
 	}
 
-	tUrls := getTypeURLs(args[1])
+	tUrls := getDownloadURLs(args[1])
 	if len(tUrls) == 0 {
 		panic(fmt.Errorf("Invalid type name, must be a template URL or in the form \"<type-name>:<version>\": %s", args[1]))
 	}
+
 	schemaUrl := tUrls[0] + ".schema"
 	fmt.Println(callHttp(schemaUrl, "GET", "get schema for type ("+tUrls[0]+")", nil))
 }
 
-// getTypeURLs returns URLs or empty list if a primitive type.
-func getTypeURLs(tName string) []string {
-	if util.IsHttpUrl(tName) {
-		// User can pass raw URL to template.
-		return []string{tName}
-	}
-
-	// User can pass registry type.
-	t := getRegistryType(tName)
-	if t == nil {
-		// Primitive types have no associated URL.
-		return []string{}
-	}
-
-	return getDownloadURLs(*t)
-}
-
-func getDownloadURLs(t registry.Type) []string {
-	git, err := getGitRegistry()
+// getDownloadURLs returns URLs or empty list if a primitive type.
+func getDownloadURLs(tName string) []string {
+	qName := path.Join(*template_registry, tName)
+	provider := getRegistryProvider()
+	result, err := registry.GetDownloadURLs(provider, qName)
 	if err != nil {
-		panic(fmt.Errorf("Failed to get registry"))
+		panic(fmt.Errorf("cannot get URLs for %s: %s\n", tName, err))
 	}
-	urls, err := git.GetURLs(t)
-	if err != nil {
-		panic(fmt.Errorf("Failed to fetch type information for \"%s:%s\": %s", t.Name, t.Version, err))
-	}
-
-	return urls
-}
 
-func isHttp(t string) bool {
-	return strings.HasPrefix(t, "http://") || strings.HasPrefix(t, "https://")
+	return result
 }
 
 func loadTemplate(args []string) *common.Template {
@@ -343,8 +375,8 @@ func loadTemplate(args []string) *common.Template {
 		}
 	} else {
 		if len(args) < 3 {
-			if t := getRegistryType(args[1]); t != nil {
-				template = buildTemplateFromType(*t)
+			if t, err := registry.ParseType(args[1]); err == nil {
+				template = buildTemplateFromType(t)
 			} else {
 				template, err = expander.NewTemplateFromRootTemplate(args[1])
 			}
@@ -365,28 +397,6 @@ func loadTemplate(args []string) *common.Template {
 	return template
 }
 
-// TODO: needs better validation that this is actually a registry type.
-func getRegistryType(fullType string) *registry.Type {
-	tList := strings.Split(fullType, ":")
-	if len(tList) != 2 {
-		return nil
-	}
-
-	cList := strings.Split(tList[0], "/")
-	if len(cList) == 1 {
-		return &registry.Type{
-			Name:    tList[0],
-			Version: tList[1],
-		}
-	} else {
-		return &registry.Type{
-			Collection: cList[0],
-			Name:       cList[1],
-			Version:    tList[1],
-		}
-	}
-}
-
 func buildTemplateFromType(t registry.Type) *common.Template {
 	props := make(map[string]interface{})
 	if *properties != "" {
@@ -409,11 +419,11 @@ func buildTemplateFromType(t registry.Type) *common.Template {
 	}
 
 	// Name the deployment after the type name.
-	name := fmt.Sprintf("%s:%s", t.Name, t.Version)
+	name := fmt.Sprintf("%s:%s", t.Name, t.GetVersion())
 
 	config := common.Configuration{Resources: []*common.Resource{&common.Resource{
 		Name:       name,
-		Type:       getDownloadURLs(t)[0],
+		Type:       getDownloadURLs(t.String())[0],
 		Properties: props,
 	}}}
 

manager/deployments.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"encoding/json"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -25,7 +26,9 @@ import (
 	"log"
 	"net"
 	"net/http"
+	"net/url"
 	"os"
+	"regexp"
 	"strings"
 
 	"github.com/ghodss/yaml"
@@ -50,6 +53,9 @@ var deployments = []Route{
 	{"ListTypes", "/types", "GET", listTypesHandlerFunc, ""},
 	{"ListTypeInstances", "/types/{type}/instances", "GET", listTypeInstancesHandlerFunc, ""},
 	{"ListRegistries", "/registries", "GET", listRegistriesHandlerFunc, ""},
+	{"GetRegistry", "/registries/{registry}", "GET", getRegistryHandlerFunc, ""},
+	{"ListRegistryTypes", "/registries/{registry}/types", "GET", listRegistryTypesHandlerFunc, ""},
+	{"GetDownloadURLs", "/registries/{registry}/types/{type}", "GET", getDownloadURLsHandlerFunc, ""},
 }
 
 var (
@@ -72,11 +78,13 @@ func init() {
 }
 
 func newManager() manager.Manager {
-	expander := manager.NewExpander(getServiceURL(*expanderURL, *expanderName), manager.NewTypeResolver(registry.NewDefaultRegistryProvider()))
+	provider := registry.NewDefaultRegistryProvider()
+	resolver := manager.NewTypeResolver(provider)
+	expander := manager.NewExpander(getServiceURL(*expanderURL, *expanderName), resolver)
 	deployer := manager.NewDeployer(getServiceURL(*deployerURL, *deployerName))
 	r := repository.NewMapBasedRepository()
-	registryService := registry.NewInmemRepositoryService()
-	return manager.NewManager(expander, deployer, r, registryService)
+	service := registry.NewInmemRegistryService()
+	return manager.NewManager(expander, deployer, r, provider, service)
 }
 
 func getServiceURL(serviceURL, serviceName string) string {
@@ -205,13 +213,21 @@ func putDeploymentHandlerFunc(w http.ResponseWriter, r *http.Request) {
 
 func getPathVariable(w http.ResponseWriter, r *http.Request, variable, handler string) (string, error) {
 	vars := mux.Vars(r)
-	ret, ok := vars[variable]
+	escaped, ok := vars[variable]
 	if !ok {
-		e := fmt.Errorf("%s parameter not found in URL", variable)
+		e := errors.New(fmt.Sprintf("%s name not found in URL", variable))
 		util.LogAndReturnError(handler, http.StatusBadRequest, e, w)
 		return "", e
 	}
-	return ret, nil
+
+	unescaped, err := url.QueryUnescape(escaped)
+	if err != nil {
+		e := fmt.Errorf("cannot decode name (%v)", variable)
+		util.LogAndReturnError(handler, http.StatusBadRequest, e, w)
+		return "", e
+	}
+
+	return unescaped, nil
 }
 
 func getTemplate(w http.ResponseWriter, r *http.Request, handler string) *common.Template {
@@ -342,5 +358,78 @@ func listRegistriesHandlerFunc(w http.ResponseWriter, r *http.Request) {
 	if err != nil {
 		return
 	}
+
 	util.LogHandlerExitWithJSON(handler, w, registries, http.StatusOK)
 }
+
+func getRegistryHandlerFunc(w http.ResponseWriter, r *http.Request) {
+	handler := "manager: get registry"
+	util.LogHandlerEntry(handler, r)
+	registryName, err := getPathVariable(w, r, "registry", handler)
+	if err != nil {
+		return
+	}
+
+	cr, err := backend.GetRegistry(registryName)
+	if err != nil {
+		util.LogAndReturnError(handler, http.StatusBadRequest, err, w)
+		return
+	}
+
+	util.LogHandlerExitWithJSON(handler, w, cr, http.StatusOK)
+}
+
+func listRegistryTypesHandlerFunc(w http.ResponseWriter, r *http.Request) {
+	handler := "manager: list registry types"
+	util.LogHandlerEntry(handler, r)
+	registryName, err := getPathVariable(w, r, "registry", handler)
+	if err != nil {
+		return
+	}
+
+	var regex *regexp.Regexp
+	regexString, err := getPathVariable(w, r, "regex", handler)
+	if err == nil {
+		regex, err = regexp.Compile(regexString)
+		if err != nil {
+			util.LogAndReturnError(handler, http.StatusInternalServerError, err, w)
+			return
+		}
+	}
+
+	registryTypes, err := backend.ListRegistryTypes(registryName, regex)
+	if err != nil {
+		util.LogAndReturnError(handler, http.StatusInternalServerError, err, w)
+		return
+	}
+
+	util.LogHandlerExitWithJSON(handler, w, registryTypes, http.StatusOK)
+}
+
+func getDownloadURLsHandlerFunc(w http.ResponseWriter, r *http.Request) {
+	handler := "manager: get download URLs"
+	util.LogHandlerEntry(handler, r)
+	registryName, err := getPathVariable(w, r, "registry", handler)
+	if err != nil {
+		return
+	}
+
+	typeName, err := getPathVariable(w, r, "type", handler)
+	if err != nil {
+		return
+	}
+
+	tt, err := registry.ParseType(typeName)
+	if err != nil {
+		util.LogAndReturnError(handler, http.StatusInternalServerError, err, w)
+		return
+	}
+
+	c, err := backend.GetDownloadURLs(registryName, tt)
+	if err != nil {
+		util.LogAndReturnError(handler, http.StatusBadRequest, err, w)
+		return
+	}
+
+	util.LogHandlerExitWithJSON(handler, w, c, http.StatusOK)
+}

manager/manager/expander.go

@@ -25,7 +25,6 @@ import (
 
 	"github.com/ghodss/yaml"
 	"github.com/kubernetes/deployment-manager/common"
-	"github.com/kubernetes/deployment-manager/util"
 )
 
 const (
@@ -101,7 +100,7 @@ func walkLayout(l *common.Layout, imports []*common.ImportFile, toReplace map[st
 	for len(toVisit) > 0 {
 		lr := toVisit[0]
 		nodeKey := lr.Resource.Name + layoutNodeKeySeparator + lr.Resource.Type
-		if len(lr.Layout.Resources) == 0 && util.IsTemplate(lr.Resource.Type, imports) {
+		if len(lr.Layout.Resources) == 0 && isTemplate(lr.Resource.Type, imports) {
 			ret[nodeKey] = lr
 		} else if toReplace[nodeKey] != nil {
 			toReplace[nodeKey].Resources = lr.Resources
@@ -113,6 +112,17 @@ func walkLayout(l *common.Layout, imports []*common.ImportFile, toReplace map[st
 	return ret
 }
 
+// isTemplate returns whether a given type is a template.
+func isTemplate(t string, imports []*common.ImportFile) bool {
+	for _, imp := range imports {
+		if imp.Name == t {
+			return true
+		}
+	}
+
+	return false
+}
+
 // ExpandTemplate expands the supplied template, and returns a configuration.
 // It will also update the imports in the provided template if any were added
 // during type resolution.

manager/manager/manager.go

@@ -19,6 +19,8 @@ package manager
 import (
 	"fmt"
 	"log"
+	"net/url"
+	"regexp"
 	"time"
 
 	"github.com/kubernetes/deployment-manager/common"
@@ -28,33 +30,48 @@ import (
 
 // Manager manages a persistent set of Deployments.
 type Manager interface {
+	// Deployments
 	ListDeployments() ([]common.Deployment, error)
 	GetDeployment(name string) (*common.Deployment, error)
 	CreateDeployment(t *common.Template) (*common.Deployment, error)
 	DeleteDeployment(name string, forget bool) (*common.Deployment, error)
 	PutDeployment(name string, t *common.Template) (*common.Deployment, error)
+
+	// Manifests
 	ListManifests(deploymentName string) (map[string]*common.Manifest, error)
 	GetManifest(deploymentName string, manifest string) (*common.Manifest, error)
 	Expand(t *common.Template) (*common.Manifest, error)
+
+	// Types
 	ListTypes() []string
 	ListInstances(typeName string) []*common.TypeInstance
-	// Registry related functions
+
+	// Registries
 	ListRegistries() ([]*common.Registry, error)
 	CreateRegistry(pr *common.Registry) error
 	GetRegistry(name string) (*common.Registry, error)
 	DeleteRegistry(name string) error
+
+	// Registry Types
+	ListRegistryTypes(registryName string, regex *regexp.Regexp) ([]registry.Type, error)
+	GetDownloadURLs(registryName string, t registry.Type) ([]*url.URL, error)
 }
 
 type manager struct {
 	expander   Expander
 	deployer   Deployer
 	repository repository.Repository
-	registryService registry.RegistryService
+	provider   registry.RegistryProvider
+	service    common.RegistryService
 }
 
 // NewManager returns a new initialized Manager.
-func NewManager(expander Expander, deployer Deployer, repository repository.Repository, registryService registry.RegistryService) Manager {
-	return &manager{expander, deployer, repository, registryService}
+func NewManager(expander Expander,
+	deployer Deployer,
+	repository repository.Repository,
+	provider registry.RegistryProvider,
+	service common.RegistryService) Manager {
+	return &manager{expander, deployer, repository, provider, service}
 }
 
 // ListDeployments returns the list of deployments
@@ -309,20 +326,32 @@ func (m *manager) ListInstances(typeName string) []*common.TypeInstance {
 	return m.repository.GetTypeInstances(typeName)
 }
 
+// ListRegistries returns the list of registries
 func (m *manager) ListRegistries() ([]*common.Registry, error) {
-	return m.registryService.List()
+	return m.service.List()
 }
 
 func (m *manager) CreateRegistry(pr *common.Registry) error {
-	return m.registryService.Create(pr)
+	return m.service.Create(pr)
 }
 
 func (m *manager) GetRegistry(name string) (*common.Registry, error) {
-	return m.registryService.Get(name)
+	return m.service.Get(name)
 }
 
 func (m *manager) DeleteRegistry(name string) error {
-	return m.registryService.Delete(name)
+	return m.service.Delete(name)
+}
+
+// Set the credential for a registry.
+// May not be supported by some registry services.
+func (m *manager) SetCredential(name string, credential common.RegistryCredential) error {
+	return m.service.SetCredential(name, credential)
+}
+
+// Get the credential for a registry.
+func (m *manager) GetCredential(name string) (common.RegistryCredential, error) {
+	return m.service.GetCredential(name)
 }
 
 func generateManifestName() string {
@@ -346,3 +375,26 @@ func getResourceErrors(c *common.Configuration) []string {
 
 	return errs
 }
+
+// ListRegistryTypes lists types in a given registry whose string values
+// conform to the supplied regular expression, or all types, if the regular
+// expression is nil.
+func (m *manager) ListRegistryTypes(registryName string, regex *regexp.Regexp) ([]registry.Type, error) {
+	r, err := m.provider.GetRegistryByName(registryName)
+	if err != nil {
+		return nil, err
+	}
+
+	return r.ListTypes(regex)
+}
+
+// GetDownloadURLs returns the URLs required to download the contents
+// of a given type in a given registry.
+func (m *manager) GetDownloadURLs(registryName string, t registry.Type) ([]*url.URL, error) {
+	r, err := m.provider.GetRegistryByName(registryName)
+	if err != nil {
+		return nil, err
+	}
+
+	return r.GetDownloadURLs(t)
+}

manager/manager/manager_test.go

@@ -66,7 +66,7 @@ var deploymentList = []common.Deployment{deployment, {Name: "test2"}}
 
 var typeInstMap = map[string][]string{"test": []string{"test"}}
 
-var errTest = errors.New("test")
+var errTest = errors.New("test error")
 
 type expanderStub struct{}
 
@@ -164,6 +164,7 @@ func (repository *repositoryStub) ListDeployments() ([]common.Deployment, error)
 	if repository.FailListDeployments {
 		return deploymentList, errTest
 	}
+
 	return deploymentList, nil
 }
 
@@ -229,31 +230,32 @@ func (repository *repositoryStub) GetManifest(d string, m string) (*common.Manif
 	return nil, errTest
 }
 
-func (r *repositoryStub) ListTypes() []string {
-	r.ListTypesCalled = true
+func (tgr *repositoryStub) ListTypes() []string {
+	tgr.ListTypesCalled = true
 	return []string{}
 }
 
-func (r *repositoryStub) GetTypeInstances(t string) []*common.TypeInstance {
-	r.GetTypeInstancesCalled = true
+func (tgr *repositoryStub) GetTypeInstances(t string) []*common.TypeInstance {
+	tgr.GetTypeInstancesCalled = true
 	return []*common.TypeInstance{}
 }
 
-func (r *repositoryStub) ClearTypeInstances(d string) {
-	r.TypeInstancesCleared = true
+func (tgr *repositoryStub) ClearTypeInstances(d string) {
+	tgr.TypeInstancesCleared = true
 }
 
-func (r *repositoryStub) SetTypeInstances(d string, is map[string][]*common.TypeInstance) {
+func (tgr *repositoryStub) SetTypeInstances(d string, is map[string][]*common.TypeInstance) {
 	for k, _ := range is {
-		r.TypeInstances[d] = append(r.TypeInstances[d], k)
+		tgr.TypeInstances[d] = append(tgr.TypeInstances[d], k)
 	}
 }
 
 var testExpander = &expanderStub{}
 var testRepository = newRepositoryStub()
 var testDeployer = newDeployerStub()
-var testRegistryService = registry.NewInmemRepositoryService()
-var testManager = NewManager(testExpander, testDeployer, testRepository, testRegistryService)
+var testRegistryService = registry.NewInmemRegistryService()
+var testProvider = registry.NewRegistryProvider(nil, registry.NewTestGithubRegistryProvider("", nil))
+var testManager = NewManager(testExpander, testDeployer, testRepository, testProvider, testRegistryService)
 
 func TestListDeployments(t *testing.T) {
 	testRepository.reset()
@@ -523,3 +525,23 @@ func TestListInstances(t *testing.T) {
 		t.Fatal("expected repository GetTypeInstances() call.")
 	}
 }
+
+// TODO(jackgr): Implement TestListRegistryTypes
+func TestListRegistryTypes(t *testing.T) {
+	/*
+		types, err := testManager.ListRegistryTypes("", nil)
+		if err != nil {
+		    t.Fatalf("cannot list registry types: %s", err)
+		}
+	*/
+}
+
+// TODO(jackgr): Implement TestGetDownloadURLs
+func TestGetDownloadURLs(t *testing.T) {
+	/*
+		    urls, err := testManager.GetDownloadURLs("", registry.Type{})
+			if err != nil {
+			    t.Fatalf("cannot list get download urls: %s", err)
+			}
+	*/
+}

manager/manager/typeresolver.go

@@ -100,7 +100,7 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
 	toFetch := make([]*fetchUnit, 0, tr.maxUrls)
 	for _, r := range config.Resources {
 		// Map the type to a fetchable URL (if applicable) or skip it if it's a non-fetchable type (primitive for example).
-		urls, err := tr.MapFetchableURLs(r.Type)
+		urls, err := registry.GetDownloadURLs(tr.rp, r.Type)
 		if err != nil {
 			return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", r.Type, err))
 		}
@@ -168,7 +168,7 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
 			for _, v := range s.Imports {
 				i := &common.ImportFile{Name: v.Name}
 				var existingSchema string
-				urls, conversionErr := tr.MapFetchableURLs(v.Path)
+				urls, conversionErr := registry.GetDownloadURLs(tr.rp, v.Path)
 				if conversionErr != nil {
 					return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", v.Path, conversionErr))
 				}
@@ -220,56 +220,6 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
 	return ret, nil
 }
 
-// MapFetchableUrls checks a type to see if it is either a short git hub url or a fully specified URL
-// and returns the URL that should be used to fetch it. If the url is not fetchable (primitive type for
-// example) will return empty string.
-func (tr *typeResolver) MapFetchableURLs(t string) ([]string, error) {
-	if util.IsGithubShortType(t) {
-		return tr.ShortTypeToDownloadURLs(t)
-	} else if util.IsGithubShortPackageType(t) {
-		return tr.ShortTypeToPackageDownloadURLs(t)
-	} else if util.IsHttpUrl(t) {
-		return []string{t}, nil
-	}
-	return []string{}, nil
-}
-
-// ShortTypeToDownloadURLs converts a github URL into downloadable URL from github.
-// Input must be of the type and is assumed to have been validated before this call:
-// github.com/owner/repo/qualifier/type:version
-// for example:
-// github.com/kubernetes/application-dm-templates/storage/redis:v1
-func (tr *typeResolver) ShortTypeToDownloadURLs(template string) ([]string, error) {
-	m := util.TemplateRegistryMatcher.FindStringSubmatch(template)
-	if len(m) != 6 {
-		return []string{}, fmt.Errorf("Failed to parse short github url: %s", template)
-	}
-	r, err := tr.rp.GetRegistry(template)
-	if err != nil {
-		return []string{}, err
-	}
-	t := registry.Type{m[3], m[4], m[5]}
-	return r.GetURLs(t)
-}
-
-// ShortTypeToPackageDownloadURLs converts a github URL into downloadable URLs from github.
-// Input must be of the type and is assumed to have been validated before this call:
-// github.com/owner/repo/type
-// for example:
-// github.com/helm/charts/cassandra
-func (tr *typeResolver) ShortTypeToPackageDownloadURLs(template string) ([]string, error) {
-	m := util.PackageRegistryMatcher.FindStringSubmatch(template)
-	if len(m) != 4 {
-		return []string{}, fmt.Errorf("Failed to parse short github url: %s", template)
-	}
-	r, err := tr.rp.GetRegistry(template)
-	if err != nil {
-		return []string{}, err
-	}
-	t := registry.Type{Name: m[3]}
-	return r.GetURLs(t)
-}
-
 func parseContent(templates []string) (string, error) {
 	if len(templates) == 1 {
 		return templates[0], nil

manager/manager/typeresolver_test.go

@@ -18,7 +18,6 @@ package manager
 
 import (
 	"errors"
-	"fmt"
 	"net/http"
 	"reflect"
 	"strings"
@@ -58,76 +57,6 @@ func (tg *testGetter) Get(url string) (body string, code int, err error) {
 	return ret.resp, ret.code, ret.err
 }
 
-type urlAndError struct {
-	u string
-	e error
-}
-
-type testRegistryProvider struct {
-	URLPrefix string
-	r         map[string]registry.Registry
-}
-
-func newTestRegistryProvider(URLPrefix string, tests map[registry.Type]urlAndError, count int) registry.RegistryProvider {
-	r := make(map[string]registry.Registry)
-	r[URLPrefix] = &testGithubRegistry{tests, count}
-	return &testRegistryProvider{URLPrefix, r}
-}
-
-func (trp *testRegistryProvider) GetRegistry(URL string) (registry.Registry, error) {
-	for key, r := range trp.r {
-		if strings.HasPrefix(URL, key) {
-			return r, nil
-		}
-	}
-	return nil, fmt.Errorf("No registry found for %s", URL)
-}
-
-type testGithubRegistry struct {
-	responses map[registry.Type]urlAndError
-	count     int
-}
-
-func (tgr *testGithubRegistry) GetURLs(t registry.Type) ([]string, error) {
-	tgr.count = tgr.count + 1
-	ret := tgr.responses[t]
-	return []string{ret.u}, ret.e
-}
-
-func (tgr *testGithubRegistry) List() ([]registry.Type, error) {
-	return []registry.Type{}, fmt.Errorf("List should not be called in the test")
-}
-
-type testGithubPackageRegistry struct {
-	responses map[registry.Type]urlAndError
-	count     int
-}
-
-func (tgr *testGithubPackageRegistry) GetURLs(t registry.Type) ([]string, error) {
-	tgr.count = tgr.count + 1
-	ret := tgr.responses[t]
-	return []string{ret.u}, ret.e
-}
-
-func (tgr *testGithubPackageRegistry) List() ([]registry.Type, error) {
-	return []registry.Type{}, fmt.Errorf("List should not be called in the test")
-}
-
-func testUrlConversionDriver(c resolverTestCase, tests map[string]urlAndError, t *testing.T) {
-	r := &typeResolver{
-		rp: c.registryProvider,
-	}
-	for in, expected := range tests {
-		actual, err := r.ShortTypeToDownloadURLs(in)
-		if err != expected.e {
-			t.Errorf("failed on: %s : expected error %v but got %v", in, expected.e, err)
-		}
-		if actual[0] != expected.u {
-			t.Errorf("failed on: %s : expected %s but got %v", in, expected.u, actual)
-		}
-	}
-}
-
 func testDriver(c resolverTestCase, t *testing.T) {
 	g := &testGetter{test: t, responses: c.responses}
 	r := &typeResolver{
@@ -299,6 +228,7 @@ func TestTooManyImports(t *testing.T) {
 		responses:   responses,
 		expectedErr: errors.New("Number of imports exceeds maximum of 5"),
 	}
+
 	testDriver(test, t)
 }
 
@@ -341,41 +271,8 @@ func TestSharedImport(t *testing.T) {
 		responses: responses,
 		importOut: finalImports,
 	}
-	testDriver(test, t)
-}
-
-func TestShortGithubUrlMapping(t *testing.T) {
-	githubUrlMaps := map[registry.Type]urlAndError{
-		registry.Type{"common", "replicatedservice", "v1"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil},
-		registry.Type{"storage", "redis", "v1"}:            urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil},
-	}
-
-	tests := map[string]urlAndError{
-		"github.com/kubernetes/application-dm-templates/common/replicatedservice:v1": urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil},
-		"github.com/kubernetes/application-dm-templates/storage/redis:v1":            urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil},
-	}
 
-	test := resolverTestCase{
-		registryProvider: newTestRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps, 2),
-	}
-	testUrlConversionDriver(test, tests, t)
-}
-
-func TestShortGithubUrlMappingDifferentOwnerAndRepo(t *testing.T) {
-	githubUrlMaps := map[registry.Type]urlAndError{
-		registry.Type{"common", "replicatedservice", "v1"}: urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/common/replicatedservice/v1/replicatedservice.py", nil},
-		registry.Type{"storage", "redis", "v1"}:            urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/storage/redis/v1/redis.jinja", nil},
-	}
-
-	tests := map[string]urlAndError{
-		"github.com/example/mytemplates/common/replicatedservice:v1": urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/common/replicatedservice/v1/replicatedservice.py", nil},
-		"github.com/example/mytemplates/storage/redis:v1":            urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/storage/redis/v1/redis.jinja", nil},
-	}
-
-	test := resolverTestCase{
-		registryProvider: newTestRegistryProvider("github.com/example/mytemplates", githubUrlMaps, 2),
-	}
-	testUrlConversionDriver(test, tests, t)
+	testDriver(test, t)
 }
 
 var templateShortGithubTemplate = `
@@ -405,17 +302,19 @@ func TestShortGithubUrl(t *testing.T) {
 		"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py.schema": responseAndError{nil, http.StatusNotFound, ""},
 	}
 
-	githubUrlMaps := map[registry.Type]urlAndError{
-		registry.Type{"common", "replicatedservice", "v1"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil},
-		registry.Type{"common", "replicatedservice", "v2"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py", nil},
+	githubUrlMaps := map[registry.Type]registry.TestURLAndError{
+		registry.NewTypeOrDie("common", "replicatedservice", "v1"): registry.TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil},
+		registry.NewTypeOrDie("common", "replicatedservice", "v2"): registry.TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py", nil},
 	}
 
+	grp := registry.NewTestGithubRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps)
 	test := resolverTestCase{
 		config:           templateShortGithubTemplate,
 		importOut:        finalImports,
 		urlcount:         4,
 		responses:        responses,
-		registryProvider: newTestRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps, 2),
+		registryProvider: registry.NewRegistryProvider(nil, grp),
 	}
+
 	testDriver(test, t)
 }

registry/github_package_registry.go

@@ -17,14 +17,18 @@ limitations under the License.
 package registry
 
 import (
+	"github.com/google/go-github/github"
+	"github.com/kubernetes/deployment-manager/common"
+
+	"fmt"
 	"log"
+	"net/url"
+	"regexp"
 	"strings"
-
-	"github.com/google/go-github/github"
 )
 
 // GithubPackageRegistry implements the Registry interface that talks to github and
-// expects packages in helm format without versioning and no qualifier in the path.
+// expects packages in helm format without versioning and no collection in the path.
 // Format of the directory for a type is like so:
 // package/
 //   Chart.yaml
@@ -33,22 +37,28 @@ import (
 //     bar.yaml
 //     ...
 type GithubPackageRegistry struct {
-	owner      string
-	repository string
-	client     *github.Client
+	githubRegistry
 }
 
-// NewGithubRegistry creates a Registry that can be used to talk to github.
-func NewGithubPackageRegistry(owner, repository string, client *github.Client) *GithubPackageRegistry {
-	return &GithubPackageRegistry{
-		owner:      owner,
-		repository: repository,
-		client:     client,
+// NewGithubPackageRegistry creates a GithubPackageRegistry.
+func NewGithubPackageRegistry(name, shortURL string, service RepositoryService) (*GithubPackageRegistry, error) {
+	format := fmt.Sprintf("%s;%s", common.UnversionedRegistry, common.OneLevelRegistry)
+	if service == nil {
+		client := github.NewClient(nil)
+		service = client.Repositories
 	}
+
+	gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), service)
+	if err != nil {
+		return nil, err
+	}
+
+	return &GithubPackageRegistry{githubRegistry: *gr}, nil
 }
 
-// List the types from the Registry.
-func (g *GithubPackageRegistry) List() ([]Type, error) {
+// ListTypes lists types in this registry whose string values conform to the
+// supplied regular expression, or all types, if the regular expression is nil.
+func (g GithubPackageRegistry) ListTypes(regex *regexp.Regexp) ([]Type, error) {
 	// Just list all the types at the top level.
 	types, err := g.getDirs("")
 	if err != nil {
@@ -59,7 +69,7 @@ func (g *GithubPackageRegistry) List() ([]Type, error) {
 	var retTypes []Type
 	for _, t := range types {
 		// Check to see if there's a Chart.yaml file in the directory
-		_, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, t, nil)
+		_, dc, _, err := g.service.GetContents(g.owner, g.repository, t, nil)
 		if err != nil {
 			log.Printf("Failed to list package files at path: %s: %v", t, err)
 			return nil, err
@@ -71,33 +81,52 @@ func (g *GithubPackageRegistry) List() ([]Type, error) {
 		}
 	}
 
+	if regex != nil {
+		var matchTypes []Type
+		for _, retType := range retTypes {
+			if regex.MatchString(retType.String()) {
+				matchTypes = append(matchTypes, retType)
+			}
+		}
+
+		return matchTypes, nil
+	}
+
 	return retTypes, nil
 }
 
-// GetURLs fetches the download URLs for a given Type.
-func (g *GithubPackageRegistry) GetURLs(t Type) ([]string, error) {
+// GetDownloadURLs fetches the download URLs for a given Type.
+func (g GithubPackageRegistry) GetDownloadURLs(t Type) ([]*url.URL, error) {
 	path, err := g.MakeRepositoryPath(t)
 	if err != nil {
-		return []string{}, err
+		return nil, err
 	}
-	_, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, path, nil)
+
+	_, dc, _, err := g.service.GetContents(g.owner, g.repository, path, nil)
 	if err != nil {
 		log.Printf("Failed to list package files at path: %s: %v", path, err)
-		return []string{}, err
+		return nil, err
 	}
-	downloadURLs := []string{}
+
+	downloadURLs := []*url.URL{}
 	for _, f := range dc {
 		if *f.Type == "file" {
 			if strings.HasSuffix(*f.Name, ".yaml") {
-				downloadURLs = append(downloadURLs, *f.DownloadURL)
+				u, err := url.Parse(*f.DownloadURL)
+				if err != nil {
+					return nil, fmt.Errorf("cannot parse URL from %s: %s", *f.DownloadURL, err)
 				}
+
+				downloadURLs = append(downloadURLs, u)
 			}
 		}
+	}
+
 	return downloadURLs, nil
 }
 
-func (g *GithubPackageRegistry) getDirs(dir string) ([]string, error) {
-	_, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, dir, nil)
+func (g GithubPackageRegistry) getDirs(dir string) ([]string, error) {
+	_, dc, _, err := g.service.GetContents(g.owner, g.repository, dir, nil)
 	if err != nil {
 		log.Printf("Failed to get contents at path: %s: %v", dir, err)
 		return nil, err
@@ -116,7 +145,7 @@ func (g *GithubPackageRegistry) getDirs(dir string) ([]string, error) {
 // MakeRepositoryPath constructs a github path to a given type based on a repository, and type name.
 // The returned repository path will be of the form:
 // Type.Name/manifests
-func (g *GithubPackageRegistry) MakeRepositoryPath(t Type) (string, error) {
+func (g GithubPackageRegistry) MakeRepositoryPath(t Type) (string, error) {
 	// Construct the return path
 	return t.Name + "/manifests", nil
 }

registry/github_registry.go

@@ -18,166 +18,109 @@ package registry
 
 import (
 	"github.com/google/go-github/github"
+	"github.com/kubernetes/deployment-manager/common"
+	"github.com/kubernetes/deployment-manager/util"
 
 	"fmt"
-	"log"
 	"strings"
 )
 
-// GithubRegistry implements the Registry interface that talks to github and
-// implements Deployment Manager templates registry.
-// A registry root must be a directory that contains all the available templates,
-// one directory per template. Each template directory then contains version
-// directories, each of which in turn contains all the files necessary for that
-// version of the template.
-//
-// For example, a template registry containing two versions of redis
-// (implemented in jinja), and one version of replicatedservice (implemented
-// in python) would have a directory structure that looks something like this:
-// qualifier [optional] prefix to a virtual root within the repository.
-// /redis
-//   /v1
-//     redis.jinja
-//     redis.jinja.schema
-//   /v2
-//     redis.jinja
-//     redis.jinja.schema
-// /replicatedservice
-//   /v1
-//     replicatedservice.python
-//     replicatedservice.python.schema
-
-type GithubRegistry struct {
+// githubRegistry implements the Registry interface and talks to github.
+// The registry short URL and format determine how types are laid out in the
+// registry.
+type githubRegistry struct {
+	name       string
+	shortURL   string
 	owner      string
 	repository string
 	path       string
-	client     *github.Client
+	format     common.RegistryFormat
+	service    RepositoryService
 }
 
-// NewGithubRegistry creates a Registry that can be used to talk to github.
-func NewGithubRegistry(owner, repository, path string, client *github.Client) *GithubRegistry {
-	return &GithubRegistry{
+type RepositoryService interface {
+	GetContents(
+		owner, repo, path string,
+		opt *github.RepositoryContentGetOptions,
+	) (
+		fileContent *github.RepositoryContent,
+		directoryContent []*github.RepositoryContent,
+		resp *github.Response,
+		err error,
+	)
+}
+
+// newGithubRegistry creates a githubRegistry.
+func newGithubRegistry(name, shortURL string, format common.RegistryFormat, service RepositoryService) (*githubRegistry, error) {
+	trimmed := util.TrimURLScheme(shortURL)
+	owner, repository, path, err := parseGithubShortURL(trimmed)
+	if err != nil {
+		return nil, fmt.Errorf("cannot create Github template registry %s: %s", name, err)
+	}
+
+	return &githubRegistry{
+		name:       name,
+		shortURL:   trimmed,
 		owner:      owner,
 		repository: repository,
 		path:       path,
-		client:     client,
-	}
+		format:     format,
+		service:    service,
+	}, nil
 }
 
-// List the types from the Registry.
-func (g *GithubRegistry) List() ([]Type, error) {
-	// First list all the collections at the top level.
-	collections, err := g.getDirs("")
-	if err != nil {
-		log.Printf("Failed to list qualifiers: %v", err)
-		return nil, err
+func parseGithubShortURL(shortURL string) (string, string, string, error) {
+	if !strings.HasPrefix(shortURL, "github.com/") {
+		return "", "", "", fmt.Errorf("invalid Github short URL: %s", shortURL)
 	}
 
-	var retTypes []Type
-	for _, c := range collections {
-		// Then we need to fetch the versions (directories for this type)
-		types, err := g.getDirs(c)
-		if err != nil {
-			log.Printf("Failed to fetch types for collection: %s", c)
-			return nil, err
-		}
+	tPath := strings.TrimPrefix(shortURL, "github.com/")
+	parts := strings.Split(tPath, "/")
 
-		for _, t := range types {
-			// Then we need to fetch the versions (directories for this type)
-			versions, err := g.getDirs(c + "/" + t)
-			if err != nil {
-				log.Printf("Failed to fetch versions for template: %s", t)
-				return nil, err
-			}
-			for _, v := range versions {
-				retTypes = append(retTypes, Type{Name: t, Version: v, Collection: c})
-			}
+	// Handle the case where there's no path after owner and repository.
+	if len(parts) == 2 {
+		return parts[0], parts[1], "", nil
 	}
+
+	// Handle the case where there's a path after owner and repository.
+	if len(parts) == 3 {
+		return parts[0], parts[1], parts[2], nil
 	}
 
-	return retTypes, nil
+	return "", "", "", fmt.Errorf("invalid Github short URL: %s", shortURL)
 }
 
-// GetURL fetches the download URL for a given Type and checks for existence of a schema file.
-func (g *GithubRegistry) GetURLs(t Type) ([]string, error) {
-	path, err := g.MakeRepositoryPath(t)
-	if err != nil {
-		return []string{}, err
-	}
-	_, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, path, nil)
-	if err != nil {
-		log.Printf("Failed to list versions at path: %s: %v", path, err)
-		return []string{}, err
-	}
-	var downloadURL, typeName, schemaName string
-	for _, f := range dc {
-		if *f.Type == "file" {
-			if *f.Name == t.Name+".jinja" || *f.Name == t.Name+".py" {
-				typeName = *f.Name
-				downloadURL = *f.DownloadURL
-			}
-			if *f.Name == t.Name+".jinja.schema" || *f.Name == t.Name+".py.schema" {
-				schemaName = *f.Name
-			}
-		}
-	}
-	if downloadURL == "" {
-		return []string{}, fmt.Errorf("Can not find template %s:%s", t.Name, t.Version)
-	}
-	if schemaName == typeName+".schema" {
-		return []string{downloadURL}, nil
-	}
-	return []string{}, fmt.Errorf("Can not find schema for %s:%s, expected to find %s", t.Name, t.Version, typeName+".schema")
+// GetRegistryName returns the name of this registry
+func (g githubRegistry) GetRegistryName() string {
+	return g.name
 }
 
-func (g *GithubRegistry) getDirs(dir string) ([]string, error) {
-	var path = g.path
-	if dir != "" {
-		path = g.path + "/" + dir
-	}
+// GetRegistryType returns the type of this registry.
+func (g githubRegistry) GetRegistryType() common.RegistryType {
+	return common.GithubRegistryType
+}
 
-	_, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, path, nil)
-	if err != nil {
-		log.Printf("Failed to get contents at path: %s: %v", path, err)
-		return nil, err
-	}
+// GetRegistryShortURL returns the short URL for this registry.
+func (g githubRegistry) GetRegistryShortURL() string {
+	return g.shortURL
+}
 
-	var dirs []string
-	for _, entry := range dc {
-		if *entry.Type == "dir" {
-			dirs = append(dirs, *entry.Name)
-		}
-	}
+// GetRegistryFormat returns the format of this registry.
+func (g githubRegistry) GetRegistryFormat() common.RegistryFormat {
+	return g.format
+}
 
-	return dirs, nil
+// GetRegistryOwner returns the owner name for this registry
+func (g githubRegistry) GetRegistryOwner() string {
+	return g.owner
 }
 
-func (g *GithubRegistry) mapCollection(collection string) (string, error) {
-	if strings.ContainsAny(collection, "/") {
-		return "", fmt.Errorf("collection must not contain slashes, got %s", collection)
-	}
-	// TODO(vaikas): Implement lookup from the root metadata file to map collection to a path
-	return collection, nil
+// GetRegistryRepository returns the repository name for this registry.
+func (g githubRegistry) GetRegistryRepository() string {
+	return g.repository
 }
 
-// MakeRepositoryPath constructs a github path to a given type based on a repository, and type name and version.
-// The returned repository path will be of the form:
-// [GithubRegistry.path/][Type.Collection]/Type.Name/Type.Version
-// Type.Collection will be mapped using mapCollection in the future, for now it's a straight
-// 1:1 mapping (if given)
-func (g *GithubRegistry) MakeRepositoryPath(t Type) (string, error) {
-	// First map the collection
-	collection, err := g.mapCollection(t.Collection)
-	if err != nil {
-		return "", err
-	}
-	// Construct the return path
-	p := ""
-	if len(g.path) > 0 {
-		p += g.path + "/"
-	}
-	if len(collection) > 0 {
-		p += collection + "/"
-	}
-	return p + t.Name + "/" + t.Version, nil
+// GetRegistryName returns the name of this registry
+func (g githubRegistry) GetRegistryPath() string {
+	return g.path
 }

registry/github_template_registry.go

+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package registry
+
+import (
+	"github.com/google/go-github/github"
+	"github.com/kubernetes/deployment-manager/common"
+
+	"fmt"
+	"log"
+	"net/url"
+	"regexp"
+	"strings"
+)
+
+// GithubTemplateRegistry implements the Registry interface and implements a
+// Deployment Manager templates registry.
+// A registry root must be a directory that contains all the available templates,
+// one directory per template. Each template directory then contains version
+// directories, each of which in turn contains all the files necessary for that
+// version of the template.
+//
+// For example, a template registry containing two versions of redis
+// (implemented in jinja), and one version of replicatedservice (implemented
+// in python) would have a directory structure that looks something like this:
+// qualifier [optional] prefix to a virtual root within the repository.
+// /redis
+//   /v1
+//     redis.jinja
+//     redis.jinja.schema
+//   /v2
+//     redis.jinja
+//     redis.jinja.schema
+// /replicatedservice
+//   /v1
+//     replicatedservice.python
+//     replicatedservice.python.schema
+type GithubTemplateRegistry struct {
+	githubRegistry
+}
+
+// NewGithubTemplateRegistry creates a GithubTemplateRegistry.
+func NewGithubTemplateRegistry(name, shortURL string, service RepositoryService) (*GithubTemplateRegistry, error) {
+	format := fmt.Sprintf("%s;%s", common.VersionedRegistry, common.CollectionRegistry)
+	if service == nil {
+		client := github.NewClient(nil)
+		service = client.Repositories
+	}
+
+	gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), service)
+	if err != nil {
+		return nil, err
+	}
+
+	return &GithubTemplateRegistry{githubRegistry: *gr}, nil
+}
+
+// ListTypes lists types in this registry whose string values conform to the
+// supplied regular expression, or all types, if the regular expression is nil.
+func (g GithubTemplateRegistry) ListTypes(regex *regexp.Regexp) ([]Type, error) {
+	// First list all the collections at the top level.
+	collections, err := g.getDirs("")
+	if err != nil {
+		log.Printf("cannot list qualifiers: %v", err)
+		return nil, err
+	}
+
+	var retTypes []Type
+	for _, c := range collections {
+		// Then we need to fetch the versions (directories for this type)
+		types, err := g.getDirs(c)
+		if err != nil {
+			log.Printf("cannot fetch types for collection: %s", c)
+			return nil, err
+		}
+
+		for _, t := range types {
+			path := c + "/" + t
+			// Then we need to fetch the versions (directories for this type)
+			versions, err := g.getDirs(path)
+			if err != nil {
+				log.Printf("cannot fetch versions at path %s", path)
+				return nil, err
+			}
+
+			for _, v := range versions {
+				tt, err := NewType(c, t, v)
+				if err != nil {
+					return nil, fmt.Errorf("malformed type at path %s", path)
+				}
+
+				retTypes = append(retTypes, tt)
+			}
+		}
+	}
+
+	if regex != nil {
+		var matchTypes []Type
+		for _, retType := range retTypes {
+			if regex.MatchString(retType.String()) {
+				matchTypes = append(matchTypes, retType)
+			}
+		}
+
+		return matchTypes, nil
+	}
+
+	return retTypes, nil
+}
+
+// GetDownloadURLs fetches the download URLs for a given Type and checks for existence of a schema file.
+func (g GithubTemplateRegistry) GetDownloadURLs(t Type) ([]*url.URL, error) {
+	path, err := g.MakeRepositoryPath(t)
+	if err != nil {
+		return nil, err
+	}
+	_, dc, _, err := g.service.GetContents(g.owner, g.repository, path, nil)
+	if err != nil {
+		return nil, fmt.Errorf("cannot list versions at path %s: %v", path, err)
+	}
+
+	var downloadURL, typeName, schemaName string
+	for _, f := range dc {
+		if *f.Type == "file" {
+			if *f.Name == t.Name+".jinja" || *f.Name == t.Name+".py" {
+				typeName = *f.Name
+				downloadURL = *f.DownloadURL
+			}
+			if *f.Name == t.Name+".jinja.schema" || *f.Name == t.Name+".py.schema" {
+				schemaName = *f.Name
+			}
+		}
+	}
+
+	if downloadURL == "" {
+		return nil, fmt.Errorf("cannot find type %s", t.String())
+	}
+
+	if schemaName != typeName+".schema" {
+		return nil, fmt.Errorf("cannot find schema for %s, expected %s", t.String(), typeName+".schema")
+	}
+
+	result, err := url.Parse(downloadURL)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse URL from %s: %s", downloadURL, err)
+	}
+
+	return []*url.URL{result}, nil
+}
+
+func (g GithubTemplateRegistry) getDirs(dir string) ([]string, error) {
+	var path = g.path
+	if dir != "" {
+		path = g.path + "/" + dir
+	}
+
+	_, dc, _, err := g.service.GetContents(g.owner, g.repository, path, nil)
+	if err != nil {
+		log.Printf("Failed to get contents at path: %s: %v", path, err)
+		return nil, err
+	}
+
+	var dirs []string
+	for _, entry := range dc {
+		if *entry.Type == "dir" {
+			dirs = append(dirs, *entry.Name)
+		}
+	}
+
+	return dirs, nil
+}
+
+func (g GithubTemplateRegistry) mapCollection(collection string) (string, error) {
+	if strings.ContainsAny(collection, "/") {
+		return "", fmt.Errorf("collection must not contain slashes, got %s", collection)
+	}
+	// TODO(vaikas): Implement lookup from the root metadata file to map collection to a path
+	return collection, nil
+}
+
+// MakeRepositoryPath constructs a github path to a given type based on a repository, and type name and version.
+// The returned repository path will be of the form:
+// [GithubTemplateRegistry.path/][Type.Collection]/Type.Name/Type.Version
+// Type.Collection will be mapped using mapCollection in the future, for now it's a straight
+// 1:1 mapping (if given)
+func (g GithubTemplateRegistry) MakeRepositoryPath(t Type) (string, error) {
+	// First map the collection
+	collection, err := g.mapCollection(t.Collection)
+	if err != nil {
+		return "", err
+	}
+	// Construct the return path
+	p := ""
+	if len(g.path) > 0 {
+		p += g.path + "/"
+	}
+	if len(collection) > 0 {
+		p += collection + "/"
+	}
+	return p + t.Name + "/" + t.GetVersion(), nil
+}

registry/inmem_registry_service.go

+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package registry
+
+import (
+	"github.com/kubernetes/deployment-manager/common"
+	"github.com/kubernetes/deployment-manager/util"
+
+	"fmt"
+	"strings"
+)
+
+type inmemRegistryService struct {
+	registries map[string]*common.AuthenticatedRegistry
+}
+
+func NewInmemRegistryService() common.RegistryService {
+	rs := &inmemRegistryService{
+		registries: make(map[string]*common.AuthenticatedRegistry),
+	}
+
+	pFormat := fmt.Sprintf("%s;%s", common.UnversionedRegistry, common.OneLevelRegistry)
+	rs.Create(&common.Registry{
+		Name:   "charts",
+		Type:   common.GithubRegistryType,
+		URL:    "github.com/helm/charts",
+		Format: common.RegistryFormat(pFormat),
+	})
+
+	tFormat := fmt.Sprintf("%s;%s", common.VersionedRegistry, common.CollectionRegistry)
+	rs.Create(&common.Registry{
+		Name:   "application-dm-templates",
+		Type:   common.GithubRegistryType,
+		URL:    "github.com/kubernetes/application-dm-templates",
+		Format: common.RegistryFormat(tFormat),
+	})
+
+	return rs
+}
+
+// List returns the list of known registries.
+func (rs *inmemRegistryService) List() ([]*common.Registry, error) {
+	ret := []*common.Registry{}
+	for _, r := range rs.registries {
+		ret = append(ret, &r.Registry)
+	}
+
+	return ret, nil
+}
+
+// Create creates an authenticated registry.
+func (rs *inmemRegistryService) Create(registry *common.Registry) error {
+	rs.registries[registry.Name] = &common.AuthenticatedRegistry{Registry: *registry}
+	return nil
+}
+
+// Get returns a registry with a given name.
+func (rs *inmemRegistryService) Get(name string) (*common.Registry, error) {
+	r, ok := rs.registries[name]
+	if !ok {
+		return nil, fmt.Errorf("Failed to find registry named %s", name)
+	}
+
+	return &r.Registry, nil
+}
+
+// GetAuthenticatedRegistry returns an authenticated registry with a given name.
+func (rs *inmemRegistryService) GetAuthenticatedRegistry(name string) (*common.AuthenticatedRegistry, error) {
+	r, ok := rs.registries[name]
+	if !ok {
+		return nil, fmt.Errorf("Failed to find registry named %s", name)
+	}
+
+	return r, nil
+}
+
+// Create deletes the authenticated registry with a given name.
+func (rs *inmemRegistryService) Delete(name string) error {
+	_, ok := rs.registries[name]
+	if !ok {
+		return fmt.Errorf("Failed to find registry named %s", name)
+	}
+
+	delete(rs.registries, name)
+	return nil
+}
+
+// GetByURL returns a registry that handles the types for a given URL.
+func (rs *inmemRegistryService) GetByURL(URL string) (*common.Registry, error) {
+	trimmed := util.TrimURLScheme(URL)
+	for _, r := range rs.registries {
+		if strings.HasPrefix(trimmed, util.TrimURLScheme(r.URL)) {
+			return &r.Registry, nil
+		}
+	}
+
+	return nil, fmt.Errorf("Failed to find registry for url: %s", URL)
+}
+
+// GetAuthenticatedRegistryByURL returns an authenticated registry that handles the types for a given URL.
+func (rs *inmemRegistryService) GetAuthenticatedRegistryByURL(URL string) (*common.AuthenticatedRegistry, error) {
+	trimmed := util.TrimURLScheme(URL)
+	for _, r := range rs.registries {
+		if strings.HasPrefix(trimmed, util.TrimURLScheme(r.URL)) {
+			return r, nil
+		}
+	}
+
+	return nil, fmt.Errorf("Failed to find registry for url: %s", URL)
+}
+
+// Set the credential for a registry.
+func (rs *inmemRegistryService) SetCredential(name string, credential common.RegistryCredential) error {
+	r, ok := rs.registries[name]
+	if !ok {
+		return fmt.Errorf("Failed to find registry named %s", name)
+	}
+
+	r.Credential = credential
+	return nil
+}
+
+// Get the credential for a registry.
+func (rs *inmemRegistryService) GetCredential(name string) (common.RegistryCredential, error) {
+	r, ok := rs.registries[name]
+	if !ok {
+		return common.RegistryCredential{}, fmt.Errorf("Failed to find registry named %s", name)
+	}
+
+	return r.Credential, nil
+}

registry/registry.go

 /*
 Copyright 2015 The Kubernetes Authors All rights reserved.
 
-Licensed under the Apache License, Version 2.0 (the "License");
+Licensed under the Apache License, version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
@@ -17,61 +17,126 @@ limitations under the License.
 package registry
 
 import (
-	"strings"
-
 	"github.com/kubernetes/deployment-manager/common"
+
+	"fmt"
+	"net/url"
+	"regexp"
+	"strings"
 )
 
-type RegistryService interface {
-	// List all the registries
-	List() ([]*common.Registry, error)
-	// Create a new registry
-	Create(repository *common.Registry) error
-	// Get a registry
-	Get(name string) (*common.Registry, error)
-	// Delete a registry
-	Delete(name string) error
-	// Find a registry that backs the given URL
-	GetByURL(URL string) (*common.Registry, error)
+// Registry abstracts a registry that holds charts, which can be
+// used in a Deployment Manager configuration. There can be multiple
+// registry implementations.
+type Registry interface {
+	// GetRegistryName returns the name of this registry
+	GetRegistryName() string
+	// GetRegistryType returns the type of this registry.
+	GetRegistryType() common.RegistryType
+	// GetRegistryShortURL returns the short URL for this registry.
+	GetRegistryShortURL() string
+	// GetRegistryFormat returns the format of this registry.
+	GetRegistryFormat() common.RegistryFormat
+
+	// ListTypes lists types in this registry whose string values conform to the
+	// supplied regular expression, or all types, if the regular expression is nil.
+	ListTypes(regex *regexp.Regexp) ([]Type, error)
+	// GetDownloadURLs returns the URLs required to download the type contents.
+	GetDownloadURLs(t Type) ([]*url.URL, error)
+}
+
+// GithubRegistry abstracts a registry that resides in a Github repository.
+type GithubRegistry interface {
+	Registry // A GithubRegistry is a Registry.
+	// GetRegistryOwner returns the owner name for this registry
+	GetRegistryOwner() string
+	// GetRegistryRepository returns the repository name for this registry.
+	GetRegistryRepository() string
+	// GetRegistryPath returns the path to the registry in the repository.
+	GetRegistryPath() string
 }
 
-// Registry abstracts a registry that holds templates, which can be
-// used in a Deployment Manager configurations. There can be multiple
-// implementations of a registry. Currently we support Deployment Manager
-// github.com/kubernetes/application-dm-templates
-// and helm packages
-// github.com/helm/charts
-//
 type Type struct {
 	Collection string
 	Name       string
-	Version    string
+	version    SemVer
 }
 
-// ParseType takes a registry name and parses it into a *registry.Type.
-func ParseType(name string) *Type {
-	tt := &Type{}
+// NewType initializes a type
+func NewType(collection, name, version string) (Type, error) {
+	result := Type{Collection: collection, Name: name}
+	err := result.SetVersion(version)
+	return result, err
+}
+
+// NewTypeOrDie initializes a type and panics if initialization fails
+func NewTypeOrDie(collection, name, version string) Type {
+	result, err := NewType(collection, name, version)
+	if err != nil {
+		panic(err)
+	}
+
+	return result
+}
+
+// Type conforms to the Stringer interface.
+func (t Type) String() string {
+	var result string
+	if t.Collection != "" {
+		result = t.Collection + "/"
+	}
 
-	tList := strings.Split(name, ":")
+	result = result + t.Name
+	version := t.GetVersion()
+	if version != "" && version != "v0" {
+		result = result + ":" + version
+	}
+
+	return result
+}
+
+// GetVersion returns the type version with the letter "v" prepended.
+func (t Type) GetVersion() string {
+	var result string
+	version := t.version.String()
+	if version != "0" {
+		result = "v" + version
+	}
+
+	return result
+}
+
+// SetVersion strips the letter "v" from version, if present,
+// and sets the the version of the type to the result.
+func (t *Type) SetVersion(version string) error {
+	vstring := strings.TrimPrefix(version, "v")
+	s, err := ParseSemVer(vstring)
+	if err != nil {
+		return err
+	}
+
+	t.version = s
+	return nil
+}
+
+// ParseType takes a registry type string and parses it into a *registry.Type.
+// TODO: needs better validation that this is actually a registry type.
+func ParseType(ts string) (Type, error) {
+	tt := Type{}
+	tList := strings.Split(ts, ":")
 	if len(tList) == 2 {
-		tt.Version = tList[1]
+		if err := tt.SetVersion(tList[1]); err != nil {
+			return tt, fmt.Errorf("malformed type string: %s", ts)
+		}
 	}
 
 	cList := strings.Split(tList[0], "/")
-
 	if len(cList) == 1 {
 		tt.Name = tList[0]
 	} else {
 		tt.Collection = cList[0]
 		tt.Name = cList[1]
 	}
-	return tt
-}
 
-// Registry abstracts type interactions.
-type Registry interface {
-	// List all the templates at the given path
-	List() ([]Type, error)
-	// Get the download URL(s) for a given type
-	GetURLs(t Type) ([]string, error)
+	return tt, nil
 }

registry/registry_test.go

@@ -4,25 +4,36 @@ import (
 	"testing"
 )
 
-func TestParseType(t *testing.T) {
-	// TODO: Are there some real-world examples we want to valide here?
-	tests := map[string]*Type{
-		"foo":                   &Type{Name: "foo"},
-		"foo:v1":                &Type{Name: "foo", Version: "v1"},
-		"github.com/foo":        &Type{Name: "foo", Collection: "github.com"},
-		"github.com/foo:v1.2.3": &Type{Name: "foo", Collection: "github.com", Version: "v1.2.3"},
+func TestTypeConversion(t *testing.T) {
+	// TODO: Are there some real-world examples we want to validate here?
+	tests := map[string]Type{
+		"foo":                   NewTypeOrDie("", "foo", ""),
+		"foo:v1":                NewTypeOrDie("", "foo", "v1"),
+		"github.com/foo":        NewTypeOrDie("github.com", "foo", ""),
+		"github.com/foo:v1.2.3": NewTypeOrDie("github.com", "foo", "v1.2.3"),
 	}
 
 	for in, expect := range tests {
-		out := ParseType(in)
+		out, err := ParseType(in)
+		if err != nil {
+			t.Errorf("Error parsing type string %s: $s", in, err)
+		}
+
 		if out.Name != expect.Name {
 			t.Errorf("Expected name to be %q, got %q", expect.Name, out.Name)
 		}
-		if out.Version != expect.Version {
-			t.Errorf("Expected version to be %q, got %q", expect.Version, out.Version)
+
+		if out.GetVersion() != expect.GetVersion() {
+			t.Errorf("Expected version to be %q, got %q", expect.GetVersion(), out.GetVersion())
 		}
+
 		if out.Collection != expect.Collection {
 			t.Errorf("Expected collection to be %q, got %q", expect.Collection, out.Collection)
 		}
+
+		svalue := out.String()
+		if svalue != in {
+			t.Errorf("Expected string value to be %q, got %q", in, svalue)
+		}
 	}
 }

registry/registryprovider.go

@@ -17,39 +17,249 @@ limitations under the License.
 package registry
 
 import (
-	"fmt"
-
-	"github.com/google/go-github/github"
 	"github.com/kubernetes/deployment-manager/common"
+	"github.com/kubernetes/deployment-manager/util"
+
+	"fmt"
+	"net/url"
+	"regexp"
+	"strings"
+	"sync"
 )
 
-// RegistryProvider returns factories for creating registries for a given RegistryType.
+// RegistryProvider is a factory for Registry instances.
 type RegistryProvider interface {
-	GetRegistry(prefix string) (Registry, error)
+	GetRegistryByShortURL(URL string) (Registry, error)
+	GetRegistryByName(registryName string) (Registry, error)
+}
+
+type registryProvider struct {
+	sync.RWMutex
+	rs         common.RegistryService
+	grp        GithubRegistryProvider
+	registries map[string]Registry
 }
 
 func NewDefaultRegistryProvider() RegistryProvider {
-	rs := NewInmemRepositoryService()
-	return &DefaultRegistryProvider{rs: rs}
+	return NewRegistryProvider(nil, NewGithubRegistryProvider())
+}
 
+func NewRegistryProvider(rs common.RegistryService, grp GithubRegistryProvider) RegistryProvider {
+	if rs == nil {
+		rs = NewInmemRegistryService()
+	}
+
+	if grp == nil {
+		grp = NewGithubRegistryProvider()
+	}
+
+	registries := make(map[string]Registry)
+	rp := &registryProvider{rs: rs, grp: grp, registries: registries}
+	return rp
 }
 
-type DefaultRegistryProvider struct {
-	rs RegistryService
+func (rp registryProvider) GetRegistryByShortURL(URL string) (Registry, error) {
+	rp.RLock()
+	defer rp.RUnlock()
+
+	result := rp.findRegistryByShortURL(URL)
+	if result == nil {
+		cr, err := rp.rs.GetByURL(URL)
+		if err != nil {
+			return nil, err
+		}
+
+		r, err := rp.grp.GetGithubRegistry(*cr)
+		if err != nil {
+			return nil, err
+		}
+
+		rp.registries[r.GetRegistryName()] = r
+		result = r
+	}
+
+	return result, nil
 }
 
-func (drp *DefaultRegistryProvider) GetRegistry(URL string) (Registry, error) {
-	r, err := drp.rs.GetByURL(URL)
+// findRegistryByShortURL trims the scheme from both the supplied URL
+// and the short URL returned by GetRegistryShortURL.
+func (rp registryProvider) findRegistryByShortURL(URL string) Registry {
+	trimmed := util.TrimURLScheme(URL)
+	for _, r := range rp.registries {
+		if strings.HasPrefix(trimmed, util.TrimURLScheme(r.GetRegistryShortURL())) {
+			return r
+		}
+	}
+
+	return nil
+}
+
+func (rp registryProvider) GetRegistryByName(registryName string) (Registry, error) {
+	rp.RLock()
+	defer rp.RUnlock()
+
+	result, ok := rp.registries[registryName]
+	if !ok {
+		cr, err := rp.rs.Get(registryName)
+		if err != nil {
+			return nil, err
+		}
+
+		r, err := rp.grp.GetGithubRegistry(*cr)
 		if err != nil {
 			return nil, err
 		}
-	if r.Type == common.Github {
-		if r.Format == common.UnversionedRegistry {
-			return NewGithubPackageRegistry("helm", "charts", github.NewClient(nil)), nil
+
+		rp.registries[r.GetRegistryName()] = r
+		result = r
+	}
+
+	return result, nil
+}
+
+func ParseRegistryFormat(rf common.RegistryFormat) map[common.RegistryFormat]bool {
+	split := strings.Split(string(rf), ";")
+	var result = map[common.RegistryFormat]bool{}
+	for _, format := range split {
+		result[common.RegistryFormat(format)] = true
+	}
+
+	return result
+}
+
+// GithubRegistryProvider is a factory for GithubRegistry instances.
+type GithubRegistryProvider interface {
+	GetGithubRegistry(cr common.Registry) (GithubRegistry, error)
+}
+
+type githubRegistryProvider struct {
+}
+
+// NewGithubRegistryProvider creates a GithubRegistryProvider.
+func NewGithubRegistryProvider() GithubRegistryProvider {
+	return &githubRegistryProvider{}
+}
+
+func (grp githubRegistryProvider) GetGithubRegistry(cr common.Registry) (GithubRegistry, error) {
+	if cr.Type == common.GithubRegistryType {
+		fMap := ParseRegistryFormat(cr.Format)
+		if fMap[common.UnversionedRegistry] && fMap[common.OneLevelRegistry] {
+			return NewGithubPackageRegistry(cr.Name, cr.URL, nil)
 		}
-		if r.Format == common.VersionedRegistry {
-			return NewGithubRegistry("kubernetes", "application-dm-templates", "", github.NewClient(nil)), nil
+
+		if fMap[common.VersionedRegistry] && fMap[common.CollectionRegistry] {
+			return NewGithubTemplateRegistry(cr.Name, cr.URL, nil)
 		}
+
+		return nil, fmt.Errorf("unknown registry format: %s", cr.Format)
 	}
-	return nil, fmt.Errorf("cannot find registry backing url %s", URL)
+
+	return nil, fmt.Errorf("unknown registry type: %s", cr.Type)
+}
+
+// RE for a registry type that does support versions and has collections.
+var TemplateRegistryMatcher = regexp.MustCompile("github.com/(.*)/(.*)/(.*)/(.*):(.*)")
+
+// RE for a registry type that does not support versions and does not have collections.
+var PackageRegistryMatcher = regexp.MustCompile("github.com/(.*)/(.*)/(.*)")
+
+// IsGithubShortType returns whether a given type is a type description in a short format to a github repository type.
+// For now, this means using github types:
+// github.com/owner/repo/qualifier/type:version
+// for example:
+// github.com/kubernetes/application-dm-templates/storage/redis:v1
+func IsGithubShortType(t string) bool {
+	return TemplateRegistryMatcher.MatchString(t)
+}
+
+// IsGithubShortPackageType returns whether a given type is a type description in a short format to a github
+// package repository type.
+// For now, this means using github types:
+// github.com/owner/repo/type
+// for example:
+// github.com/helm/charts/cassandra
+func IsGithubShortPackageType(t string) bool {
+	return PackageRegistryMatcher.MatchString(t)
+}
+
+// GetDownloadURLs checks a type to see if it is either a short git hub url or a fully specified URL
+// and returns the URLs that should be used to fetch it. If the url is not fetchable (primitive type
+// for example), it returns an empty slice.
+func GetDownloadURLs(rp RegistryProvider, t string) ([]string, error) {
+	if IsGithubShortType(t) {
+		return ShortTypeToDownloadURLs(rp, t)
+	} else if IsGithubShortPackageType(t) {
+		return ShortTypeToPackageDownloadURLs(rp, t)
+	} else if util.IsHttpUrl(t) {
+		result, err := url.Parse(t)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse download URL %s: %s", t, err)
+		}
+
+		return []string{result.String()}, nil
+	}
+
+	return []string{}, nil
+}
+
+// ShortTypeToDownloadURLs converts a github URL into downloadable URL from github.
+// Input must be of the type and is assumed to have been validated before this call:
+// github.com/owner/repo/qualifier/type:version
+// for example:
+// github.com/kubernetes/application-dm-templates/storage/redis:v1
+func ShortTypeToDownloadURLs(rp RegistryProvider, t string) ([]string, error) {
+	m := TemplateRegistryMatcher.FindStringSubmatch(t)
+	if len(m) != 6 {
+		return nil, fmt.Errorf("cannot parse short github url: %s", t)
+	}
+
+	r, err := rp.GetRegistryByShortURL(t)
+	if err != nil {
+		return nil, err
+	}
+
+	if r == nil {
+		panic(fmt.Errorf("cannot get github registry for %s", t))
+	}
+
+	tt, err := NewType(m[3], m[4], m[5])
+	if err != nil {
+		return nil, err
+	}
+
+	urls, err := r.GetDownloadURLs(tt)
+	if err != nil {
+		return nil, err
+	}
+
+	return util.ConvertURLsToStrings(urls), err
+}
+
+// ShortTypeToPackageDownloadURLs converts a github URL into downloadable URLs from github.
+// Input must be of the type and is assumed to have been validated before this call:
+// github.com/owner/repo/type
+// for example:
+// github.com/helm/charts/cassandra
+func ShortTypeToPackageDownloadURLs(rp RegistryProvider, t string) ([]string, error) {
+	m := PackageRegistryMatcher.FindStringSubmatch(t)
+	if len(m) != 4 {
+		return nil, fmt.Errorf("Failed to parse short github url: %s", t)
+	}
+
+	r, err := rp.GetRegistryByShortURL(t)
+	if err != nil {
+		return nil, err
+	}
+
+	tt, err := NewType("", m[3], "")
+	if err != nil {
+		return nil, err
+	}
+
+	urls, err := r.GetDownloadURLs(tt)
+	if err != nil {
+		return nil, err
+	}
+
+	return util.ConvertURLsToStrings(urls), err
 }

registry/registryprovider_test.go

+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package registry
+
+import (
+	"testing"
+)
+
+func testUrlConversionDriver(rp RegistryProvider, tests map[string]TestURLAndError, t *testing.T) {
+	for in, expected := range tests {
+		actual, err := GetDownloadURLs(rp, in)
+		if err != expected.Err {
+			t.Fatalf("failed on: %s : expected error %v but got %v", in, expected.Err, err)
+		}
+
+		if actual[0] != expected.URL {
+			t.Fatalf("failed on: %s : expected %s but got %v", in, expected.URL, actual)
+		}
+	}
+}
+
+func TestShortGithubUrlTemplateMapping(t *testing.T) {
+	githubUrlMaps := map[Type]TestURLAndError{
+		NewTypeOrDie("common", "replicatedservice", "v1"): TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil},
+		NewTypeOrDie("storage", "redis", "v1"):            TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil},
+	}
+
+	tests := map[string]TestURLAndError{
+		"github.com/kubernetes/application-dm-templates/common/replicatedservice:v1": TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil},
+		"github.com/kubernetes/application-dm-templates/storage/redis:v1":            TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil},
+	}
+
+	grp := NewTestGithubRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps)
+	testUrlConversionDriver(NewRegistryProvider(nil, grp), tests, t)
+}
+
+func TestShortGithubUrlPackageMapping(t *testing.T) {
+	githubUrlMaps := map[Type]TestURLAndError{
+		NewTypeOrDie("", "mongodb", ""): TestURLAndError{"https://raw.githubusercontent.com/helm/charts/master/mongodb/manifests/mongodb.yaml", nil},
+		NewTypeOrDie("", "redis", ""):   TestURLAndError{"https://raw.githubusercontent.com/helm/charts/master/redis/manifests/redis.yaml", nil},
+	}
+
+	tests := map[string]TestURLAndError{
+		"github.com/helm/charts/mongodb": TestURLAndError{"https://raw.githubusercontent.com/helm/charts/master/mongodb/manifests/mongodb.yaml", nil},
+		"github.com/helm/charts/redis":   TestURLAndError{"https://raw.githubusercontent.com/helm/charts/master/redis/manifests/redis.yaml", nil},
+	}
+
+	grp := NewTestGithubRegistryProvider("github.com/helm/charts", githubUrlMaps)
+	testUrlConversionDriver(NewRegistryProvider(nil, grp), tests, t)
+}

registry/semver.go

+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package registry
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+type SemVer struct {
+	Major uint
+	Minor uint
+	Patch uint
+}
+
+func ParseSemVer(version string) (SemVer, error) {
+	var err error
+	major, minor, patch := uint64(0), uint64(0), uint64(0)
+	if len(version) > 0 {
+		parts := strings.SplitN(version, ".", 3)
+		if len(parts) > 3 {
+			return SemVer{}, fmt.Errorf("invalid semantic version: %s", version)
+		}
+
+		if len(parts) < 1 {
+			return SemVer{}, fmt.Errorf("invalid semantic version: %s", version)
+		}
+
+		if parts[0] != "0" {
+			major, err = strconv.ParseUint(parts[0], 10, 0)
+			if err != nil {
+				return SemVer{}, fmt.Errorf("invalid semantic version: %s", version)
+			}
+		}
+
+		if len(parts) > 1 {
+			if parts[1] != "0" {
+				minor, err = strconv.ParseUint(parts[1], 10, 0)
+				if err != nil {
+					return SemVer{}, fmt.Errorf("invalid semantic version: %s", version)
+				}
+			}
+
+			if len(parts) > 2 {
+				if parts[2] != "0" {
+					patch, err = strconv.ParseUint(parts[2], 10, 0)
+					if err != nil {
+						return SemVer{}, fmt.Errorf("invalid semantic version: %s", version)
+					}
+				}
+			}
+		}
+	}
+
+	return SemVer{Major: uint(major), Minor: uint(minor), Patch: uint(patch)}, nil
+}
+
+func (s SemVer) IsZero() bool {
+	return s.Major == 0 && s.Minor == 0 && s.Patch == 0
+}
+
+// SemVer conforms to the Stringer interface.
+func (s SemVer) String() string {
+	result := strconv.Itoa(int(s.Major))
+	if s.Minor != 0 || s.Patch != 0 {
+		result = result + "." + strconv.Itoa(int(s.Minor))
+	}
+
+	if s.Patch != 0 {
+		result = result + "." + strconv.Itoa(int(s.Patch))
+	}
+
+	return result
+}

registry/semver_test.go

+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package registry
+
+import (
+	"testing"
+)
+
+func TestParseInvalidVersionFails(t *testing.T) {
+	for _, test := range []string{
+		".",
+		"..",
+		"...",
+		"1.2.3.4",
+		"notAUnit",
+		"1.notAUint",
+		"1.1.notAUint",
+		"-1",
+		"1.-1",
+		"1.1.-1",
+		"1,1",
+		"1.1,1",
+	} {
+		_, err := ParseSemVer(test)
+		if err == nil {
+			t.Errorf("Invalid version parsed successfully: %s\n", test)
+		}
+	}
+}
+
+func TestParseValidVersionSucceeds(t *testing.T) {
+	for _, test := range []struct {
+		String  string
+		Version SemVer
+	}{
+		{"", SemVer{0, 0, 0}},
+		{"0", SemVer{0, 0, 0}},
+		{"0.0", SemVer{0, 0, 0}},
+		{"0.0.0", SemVer{0, 0, 0}},
+		{"1", SemVer{1, 0, 0}},
+		{"1.0", SemVer{1, 0, 0}},
+		{"1.0.0", SemVer{1, 0, 0}},
+		{"1.1", SemVer{1, 1, 0}},
+		{"1.1.0", SemVer{1, 1, 0}},
+		{"1.1.1", SemVer{1, 1, 1}},
+	} {
+		result, err := ParseSemVer(test.String)
+		if err != nil {
+			t.Errorf("Valid version %s did not parse successfully\n", test.String)
+		}
+
+		if result.Major != test.Version.Major ||
+			result.Minor != test.Version.Minor ||
+			result.Patch != test.Version.Patch {
+			t.Errorf("Valid version %s did not parse correctly: %s\n", test.String, test.Version)
+		}
+	}
+}
+
+func TestConvertSemVerToStringSucceeds(t *testing.T) {
+	for _, test := range []struct {
+		String  string
+		Version SemVer
+	}{
+		{"0", SemVer{0, 0, 0}},
+		{"0.1", SemVer{0, 1, 0}},
+		{"0.0.1", SemVer{0, 0, 1}},
+		{"1", SemVer{1, 0, 0}},
+		{"1.1", SemVer{1, 1, 0}},
+		{"1.1.1", SemVer{1, 1, 1}},
+	} {
+		result := test.Version.String()
+		if result != test.String {
+			t.Errorf("Valid version %s did not format correctly: %s\n", test.Version, test.String)
+		}
+	}
+}

registry/inmem_repository_service.go → registry/testhelper.go

@@ -16,63 +16,67 @@ limitations under the License.
 
 package registry
 
+// TODO(jackgr): Mock github repository service to test package and template registry implementations.
+
 import (
+	"github.com/kubernetes/deployment-manager/common"
+	"github.com/kubernetes/deployment-manager/util"
+
 	"fmt"
+	"net/url"
+	"regexp"
 	"strings"
-
-	"github.com/kubernetes/deployment-manager/common"
 )
 
-type inmemRepositoryService struct {
-	repositories map[string]*common.Registry
+type TestURLAndError struct {
+	URL string
+	Err error
 }
 
-func NewInmemRepositoryService() RegistryService {
-	rs := &inmemRepositoryService{
-		repositories: make(map[string]*common.Registry),
-	}
-	rs.Create(&common.Registry{
-		Name:   "charts",
-		Type:   common.Github,
-		URL:    "github.com/helm/charts",
-		Format: common.UnversionedRegistry,
-	})
-	rs.Create(&common.Registry{
-		Name:   "application-dm-templates",
-		Type:   common.Github,
-		URL:    "github.com/kubernetes/application-dm-templates",
-		Format: common.VersionedRegistry,
-	})
-	return rs
+type testGithubRegistryProvider struct {
+	shortURL  string
+	responses map[Type]TestURLAndError
 }
 
-func (rs *inmemRepositoryService) List() ([]*common.Registry, error) {
-	ret := []*common.Registry{}
-	for _, r := range rs.repositories {
-		ret = append(ret, r)
-	}
-	return ret, nil
+type testGithubRegistry struct {
+	githubRegistry
+	responses map[Type]TestURLAndError
 }
 
-func (rs *inmemRepositoryService) Create(repository *common.Registry) error {
-	rs.repositories[repository.URL] = repository
-	return nil
+func NewTestGithubRegistryProvider(shortURL string, responses map[Type]TestURLAndError) GithubRegistryProvider {
+	return testGithubRegistryProvider{
+		shortURL:  util.TrimURLScheme(shortURL),
+		responses: responses,
+	}
 }
 
-func (rs *inmemRepositoryService) Get(name string) (*common.Registry, error) {
-	return &common.Registry{}, nil
+func (tgrp testGithubRegistryProvider) GetGithubRegistry(cr common.Registry) (GithubRegistry, error) {
+	trimmed := util.TrimURLScheme(cr.URL)
+	if strings.HasPrefix(trimmed, tgrp.shortURL) {
+		ghr, err := newGithubRegistry(cr.Name, trimmed, cr.Format, nil)
+		if err != nil {
+			panic(fmt.Errorf("cannot create a github registry: %s", err))
+		}
+
+		return &testGithubRegistry{
+			githubRegistry: *ghr,
+			responses:      tgrp.responses,
+		}, nil
+	}
+
+	panic(fmt.Errorf("unknown registry: %v", cr))
 }
 
-func (rs *inmemRepositoryService) Delete(name string) error {
-	return nil
+func (tgr testGithubRegistry) ListTypes(regex *regexp.Regexp) ([]Type, error) {
+	panic(fmt.Errorf("ListTypes should not be called in the test"))
 }
 
-// GetByURL returns a registry that handles the types for a given URL.
-func (rs *inmemRepositoryService) GetByURL(URL string) (*common.Registry, error) {
-	for _, r := range rs.repositories {
-		if strings.HasPrefix(URL, r.URL) {
-			return r, nil
-		}
+func (tgr testGithubRegistry) GetDownloadURLs(t Type) ([]*url.URL, error) {
+	result := tgr.responses[t]
+	URL, err := url.Parse(result.URL)
+	if err != nil {
+		panic(err)
 	}
-	return nil, fmt.Errorf("Failed to find registry for github url: %s", URL)
+
+	return []*url.URL{URL}, result.Err
 }

resourcifier/configurations.go

@@ -238,14 +238,14 @@ func getConfigurator() *configurator.Configurator {
 
 func getPathVariable(w http.ResponseWriter, r *http.Request, variable, handler string) (string, error) {
 	vars := mux.Vars(r)
-	variable, ok := vars[variable]
+	escaped, ok := vars[variable]
 	if !ok {
 		e := errors.New(fmt.Sprintf("%s name not found in URL", variable))
 		util.LogAndReturnError(handler, http.StatusBadRequest, e, w)
 		return "", e
 	}
 
-	unescaped, err := url.QueryUnescape(variable)
+	unescaped, err := url.QueryUnescape(escaped)
 	if err != nil {
 		e := fmt.Errorf("cannot decode name (%v)", variable)
 		util.LogAndReturnError(handler, http.StatusBadRequest, e, w)

util/httputil.go

@@ -29,6 +29,26 @@ import (
 	"github.com/ghodss/yaml"
 )
 
+// ConvertURLsToStrings converts a slice of *url.URL to a slice of string.
+func ConvertURLsToStrings(urls []*url.URL) []string {
+	var result []string
+	for _, u := range urls {
+		result = append(result, u.String())
+	}
+
+	return result
+}
+
+// TrimURLScheme removes the scheme, if any, from an URL.
+func TrimURLScheme(URL string) string {
+	parts := strings.SplitAfter(URL, "://")
+	if len(parts) > 1 {
+		return parts[1]
+	}
+
+	return URL
+}
+
 // A HandlerTester is a function that takes an HTTP method, an URL path, and a
 // reader for a request body, creates a request from them, and serves it to the
 // handler to which it was bound and returns a response recorder describing the

util/templateutil.go

-/*
-Copyright 2015 The Kubernetes Authors All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package util
-
-import (
-	"regexp"
-
-	"github.com/kubernetes/deployment-manager/common"
-)
-
-var TemplateRegistryMatcher = regexp.MustCompile("github.com/(.*)/(.*)/(.*)/(.*):(.*)")
-
-// RE for Registry that does not support versions and can have multiple files without imports.
-var PackageRegistryMatcher = regexp.MustCompile("github.com/(.*)/(.*)/(.*)")
-
-// IsTemplate returns whether a given type is a template.
-func IsTemplate(t string, imports []*common.ImportFile) bool {
-	for _, imp := range imports {
-		if imp.Name == t {
-			return true
-		}
-	}
-	return false
-}
-
-// IsGithubShortType returns whether a given type is a type description in a short format to a github repository type.
-// For now, this means using github types:
-// github.com/owner/repo/qualifier/type:version
-// for example:
-// github.com/kubernetes/application-dm-templates/storage/redis:v1
-func IsGithubShortType(t string) bool {
-	return TemplateRegistryMatcher.MatchString(t)
-}
-
-// IsGithubShortPackageType returns whether a given type is a type description in a short format to a github
-// package repository type.
-// For now, this means using github types:
-// github.com/owner/repo/type
-// for example:
-// github.com/helm/charts/cassandra
-func IsGithubShortPackageType(t string) bool {
-	return PackageRegistryMatcher.MatchString(t)
-}