Add GCS as a registry provider. Add a way to get a file using credentials from a…

Add GCS as a registry provider. Add a way to get a file using credentials from a registry to support private repos. Add ability to create / update a registry through manager

common/types.go

@@ -182,11 +182,13 @@ type BasicAuthCredential struct {
 }
 
 type APITokenCredential string
+type JWTTokenCredential string
 
 // Credential used to access the repository
 type RegistryCredential struct {
-	APIToken  APITokenCredential  `json:"apitoken,omitempty"`
-	BasicAuth BasicAuthCredential `json:"basicauth,omitempty"`
+	APIToken       APITokenCredential  `json:"apitoken,omitempty"`
+	BasicAuth      BasicAuthCredential `json:"basicauth,omitempty"`
+	ServiceAccount JWTTokenCredential  `json:"serviceaccount,omitempty"`
 }
 
 // Registry describes a template registry
@@ -204,6 +206,7 @@ type RegistryType string
 
 const (
 	GithubRegistryType RegistryType = "github"
+	GCSRegistryType    RegistryType = "gcs"
 )
 
 // RegistryFormat is a semi-colon delimited string that describes the format

dm/dm.go

@@ -50,6 +50,8 @@ var (
 	username          = flag.String("username", "", "Github user name that overrides GITHUB_USERNAME environment variable")
 	password          = flag.String("password", "", "Github password that overrides GITHUB_PASSWORD environment variable")
 	apitoken          = flag.String("apitoken", "", "Github api token that overrides GITHUB_API_TOKEN environment variable")
+	serviceaccount    = flag.String("serviceaccount", "", "Service account file containing JWT token")
+	registryfile      = flag.String("registryfile", "", "File containing registry specification")
 )
 
 var commands = []string{
@@ -67,6 +69,7 @@ var commands = []string{
 	"describe \t\t Describes the named template in a given template registry",
 	"getcredential \t\t Gets the named credential used by a registry",
 	"setcredential \t\t Sets a credential used by a registry",
+	"createregistry \t\t Creates a registry that holds charts",
 }
 
 var usage = func() {
@@ -86,7 +89,7 @@ var usage = func() {
 	os.Exit(0)
 }
 
-func getGithubCredential() *common.RegistryCredential {
+func getCredential() *common.RegistryCredential {
 	*apitoken = strings.TrimSpace(*apitoken)
 	if *apitoken == "" {
 		*apitoken = strings.TrimSpace(os.Getenv("GITHUB_API_TOKEN"))
@@ -117,6 +120,15 @@ func getGithubCredential() *common.RegistryCredential {
 		}
 	}
 
+	if *serviceaccount != "" {
+		b, err := ioutil.ReadFile(*serviceaccount)
+		if err != nil {
+			log.Fatalf("Unable to read service account file: %v", err)
+		}
+		return &common.RegistryCredential{
+			ServiceAccount: common.JWTTokenCredential(string(b)),
+		}
+	}
 	return nil
 }
 
@@ -124,6 +136,13 @@ func init() {
 	flag.Usage = usage
 }
 
+func getRegistry() ([]byte, error) {
+	if *registryfile == "" {
+		log.Fatalf("No registryfile specified (-registryfile)")
+	}
+	return ioutil.ReadFile(*registryfile)
+}
+
 func main() {
 	defer func() {
 		result := recover()
@@ -145,6 +164,10 @@ func execute() {
 
 	switch args[0] {
 	case "templates":
+		if len(args) < 2 {
+			fmt.Fprintln(os.Stderr, "No registry name supplied")
+			usage()
+		}
 		path := fmt.Sprintf("registries/%s/types", args[1])
 		callService(path, "GET", "list templates", nil)
 	case "describe":
@@ -162,7 +185,7 @@ func execute() {
 		path := fmt.Sprintf("credentials/%s", args[1])
 		callService(path, "GET", "get credential", nil)
 	case "setcredential":
-		c := getGithubCredential()
+		c := getCredential()
 		if c == nil {
 			panic(fmt.Errorf("Failed to create a credential from flags/arguments"))
 		}
@@ -172,7 +195,14 @@ func execute() {
 		}
 
 		path := fmt.Sprintf("credentials/%s", args[1])
-		callService(path, "POST", "get credential", ioutil.NopCloser(bytes.NewReader(y)))
+		callService(path, "POST", "set credential", ioutil.NopCloser(bytes.NewReader(y)))
+	case "createregistry":
+		reg, err := getRegistry()
+		if err != nil {
+			panic(fmt.Errorf("Failed to create a registry from arguments: %#v", err))
+		}
+		path := fmt.Sprintf("registries/%s", args[1])
+		callService(path, "POST", "set registry", ioutil.NopCloser(bytes.NewReader(reg)))
 	case "get":
 		if len(args) < 2 {
 			fmt.Fprintln(os.Stderr, "No deployment name supplied")
@@ -300,13 +330,20 @@ func describeType(args []string) {
 		os.Exit(1)
 	}
 
-	tUrls := getDownloadURLs(args[1])
+	tUrls := getDownloadURLs(url.QueryEscape(args[1]))
 	if len(tUrls) == 0 {
 		panic(fmt.Errorf("Invalid type name, must be a template URL or in the form \"<type-name>:<version>\": %s", args[1]))
 	}
 
-	schemaUrl := tUrls[0] + ".schema"
-	fmt.Println(callHttp(schemaUrl, "GET", "get schema for type ("+tUrls[0]+")", nil))
+	if !strings.Contains(tUrls[0], ".prov") {
+		// It's not a chart, so grab the schema
+		path := fmt.Sprintf("registries/%s/download?file=%s.schema", *template_registry, url.QueryEscape(tUrls[0]))
+		callService(path, "GET", "get schema for type ("+tUrls[0]+")", nil)
+	} else {
+		// It's a chart, so grab the provenance file
+		path := fmt.Sprintf("registries/%s/download?file=%s", *template_registry, url.QueryEscape(tUrls[0]))
+		callService(path, "GET", "get file", nil)
+	}
 }
 
 // getDownloadURLs returns URLs for a type in the given registry

manager/deployments.go

@@ -54,8 +54,10 @@ var deployments = []Route{
 	{"ListTypeInstances", "/types/{type}/instances", "GET", listTypeInstancesHandlerFunc, ""},
 	{"ListRegistries", "/registries", "GET", listRegistriesHandlerFunc, ""},
 	{"GetRegistry", "/registries/{registry}", "GET", getRegistryHandlerFunc, ""},
+	{"CreateRegistry", "/registries/{registry}", "POST", createRegistryHandlerFunc, "JSON"},
 	{"ListRegistryTypes", "/registries/{registry}/types", "GET", listRegistryTypesHandlerFunc, ""},
 	{"GetDownloadURLs", "/registries/{registry}/types/{type}", "GET", getDownloadURLsHandlerFunc, ""},
+	{"GetFile", "/registries/{registry}/download", "GET", getFileHandlerFunc, ""},
 	{"CreateCredential", "/credentials/{credential}", "POST", createCredentialHandlerFunc, "JSON"},
 	{"GetCredential", "/credentials/{credential}", "GET", getCredentialHandlerFunc, ""},
 }
@@ -97,12 +99,12 @@ func init() {
 }
 
 func newManager(cp common.CredentialProvider) manager.Manager {
-	registryProvider := registry.NewDefaultRegistryProvider(cp)
-	resolver := manager.NewTypeResolver(registryProvider)
+	service := registry.NewInmemRegistryService()
+	registryProvider := registry.NewDefaultRegistryProvider(cp, service)
+	resolver := manager.NewTypeResolver(registryProvider, util.DefaultHTTPClient())
 	expander := manager.NewExpander(getServiceURL(*expanderURL, *expanderName), resolver)
 	deployer := manager.NewDeployer(getServiceURL(*deployerURL, *deployerName))
 	r := repository.NewMapBasedRepository()
-	service := registry.NewInmemRegistryService()
 	credentialProvider := cp
 	return manager.NewManager(expander, deployer, r, registryProvider, service, credentialProvider)
 }
@@ -378,6 +380,49 @@ func getRegistryHandlerFunc(w http.ResponseWriter, r *http.Request) {
 	util.LogHandlerExitWithJSON(handler, w, cr, http.StatusOK)
 }
 
+func getRegistry(w http.ResponseWriter, r *http.Request, handler string) *common.Registry {
+	util.LogHandlerEntry(handler, r)
+	j, err := getJsonFromRequest(w, r, handler)
+	if err != nil {
+		return nil
+	}
+
+	t := &common.Registry{}
+	if err := json.Unmarshal(j, t); err != nil {
+		e := fmt.Errorf("%v\n%v", err, string(j))
+		util.LogAndReturnError(handler, http.StatusBadRequest, e, w)
+		return nil
+	}
+
+	return t
+}
+
+func createRegistryHandlerFunc(w http.ResponseWriter, r *http.Request) {
+	handler := "manager: create registry"
+	util.LogHandlerEntry(handler, r)
+	defer r.Body.Close()
+	registryName, err := getPathVariable(w, r, "registry", handler)
+	if err != nil {
+		return
+	}
+
+	reg := getRegistry(w, r, handler)
+	if reg.Name != registryName {
+		e := fmt.Errorf("Registry name does not match %s != %s", reg.Name, registryName)
+		util.LogAndReturnError(handler, http.StatusBadRequest, e, w)
+		return
+	}
+	if reg != nil {
+		err = backend.CreateRegistry(reg)
+		if err != nil {
+			util.LogAndReturnError(handler, http.StatusBadRequest, err, w)
+			return
+		}
+	}
+
+	util.LogHandlerExitWithJSON(handler, w, reg, http.StatusOK)
+}
+
 func listRegistryTypesHandlerFunc(w http.ResponseWriter, r *http.Request) {
 	handler := "manager: list registry types"
 	util.LogHandlerEntry(handler, r)
@@ -437,6 +482,28 @@ func getDownloadURLsHandlerFunc(w http.ResponseWriter, r *http.Request) {
 	util.LogHandlerExitWithJSON(handler, w, urls, http.StatusOK)
 }
 
+func getFileHandlerFunc(w http.ResponseWriter, r *http.Request) {
+	handler := "manager: get file"
+	util.LogHandlerEntry(handler, r)
+	registryName, err := getPathVariable(w, r, "registry", handler)
+	if err != nil {
+		return
+	}
+
+	file := r.FormValue("file")
+	if file == "" {
+		return
+	}
+
+	b, err := backend.GetFile(registryName, file)
+	if err != nil {
+		util.LogAndReturnError(handler, http.StatusBadRequest, err, w)
+		return
+	}
+
+	util.LogHandlerExitWithJSON(handler, w, b, http.StatusOK)
+}
+
 func getCredential(w http.ResponseWriter, r *http.Request, handler string) *common.RegistryCredential {
 	util.LogHandlerEntry(handler, r)
 	j, err := getJsonFromRequest(w, r, handler)

manager/manager/manager.go

@@ -26,6 +26,7 @@ import (
 	"github.com/kubernetes/deployment-manager/common"
 	"github.com/kubernetes/deployment-manager/manager/repository"
 	"github.com/kubernetes/deployment-manager/registry"
+	"github.com/kubernetes/deployment-manager/util"
 )
 
 // Manager manages a persistent set of Deployments.
@@ -55,6 +56,7 @@ type Manager interface {
 	// Registry Types
 	ListRegistryTypes(registryName string, regex *regexp.Regexp) ([]registry.Type, error)
 	GetDownloadURLs(registryName string, t registry.Type) ([]*url.URL, error)
+	GetFile(registryName string, url string) (string, error)
 
 	// Credentials
 	CreateCredential(name string, c *common.RegistryCredential) error
@@ -394,6 +396,21 @@ func (m *manager) GetDownloadURLs(registryName string, t registry.Type) ([]*url.
 	return r.GetDownloadURLs(t)
 }
 
+// GetFile returns a file from the backing registry
+func (m *manager) GetFile(registryName string, url string) (string, error) {
+	r, err := m.registryProvider.GetRegistryByName(registryName)
+	if err != nil {
+		return "", err
+	}
+
+	getter := util.NewHTTPClient(3, r, util.NewSleeper())
+	body, _, err := getter.Get(url)
+	if err != nil {
+		return "", err
+	}
+	return body, nil
+}
+
 // CreateCredential creates a credential that can be used to authenticate to registry
 func (m *manager) CreateCredential(name string, c *common.RegistryCredential) error {
 	return m.credentialProvider.SetCredential(name, c)

manager/manager/manager_test.go

@@ -255,7 +255,7 @@ var testRepository = newRepositoryStub()
 var testDeployer = newDeployerStub()
 var testRegistryService = registry.NewInmemRegistryService()
 var testCredentialProvider = registry.NewInmemCredentialProvider()
-var testProvider = registry.NewRegistryProvider(nil, registry.NewTestGithubRegistryProvider("", nil), testCredentialProvider)
+var testProvider = registry.NewRegistryProvider(nil, registry.NewTestGithubRegistryProvider("", nil), registry.NewTestGCSRegistryProvider("", nil), testCredentialProvider)
 var testManager = NewManager(testExpander, testDeployer, testRepository, testProvider, testRegistryService, testCredentialProvider)
 
 func TestListDeployments(t *testing.T) {

manager/manager/typeresolver.go

@@ -19,7 +19,6 @@ package manager
 import (
 	"fmt"
 	"net/http"
-	"time"
 
 	"github.com/kubernetes/deployment-manager/common"
 	"github.com/kubernetes/deployment-manager/registry"
@@ -40,26 +39,23 @@ type TypeResolver interface {
 }
 
 type typeResolver struct {
-	getter  util.HTTPClient
 	maxUrls int
 	rp      registry.RegistryProvider
+	c       util.HTTPClient
+}
+
+type fetchableURL struct {
+	registry registry.Registry
+	url      string
 }
 
 type fetchUnit struct {
-	urls []string
+	urls []fetchableURL
 }
 
 // NewTypeResolver returns a new initialized TypeResolver.
-func NewTypeResolver(rp registry.RegistryProvider) TypeResolver {
-	ret := &typeResolver{}
-	client := http.DefaultClient
-	//TODO (iantw): Make this a flag
-	timeout, _ := time.ParseDuration("10s")
-	client.Timeout = timeout
-	ret.getter = util.NewHTTPClient(3, client, util.NewSleeper())
-	ret.maxUrls = maxURLImports
-	ret.rp = rp
-	return ret
+func NewTypeResolver(rp registry.RegistryProvider, c util.HTTPClient) TypeResolver {
+	return &typeResolver{maxUrls: maxURLImports, rp: rp, c: c}
 }
 
 func resolverError(c *common.Configuration, err error) error {
@@ -67,7 +63,13 @@ func resolverError(c *common.Configuration, err error) error {
 		c, err)
 }
 
-func performHTTPGet(g util.HTTPClient, u string, allowMissing bool) (content string, err error) {
+func (tr *typeResolver) performHTTPGet(d util.HTTPDoer, u string, allowMissing bool) (content string, err error) {
+	var g util.HTTPClient
+	if d == nil {
+		g = tr.c
+	} else {
+		g = util.NewHTTPClient(3, d, util.NewSleeper())
+	}
 	r, code, err := g.Get(u)
 	if err != nil {
 		return "", err
@@ -100,7 +102,7 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
 	toFetch := make([]*fetchUnit, 0, tr.maxUrls)
 	for _, r := range config.Resources {
 		// Map the type to a fetchable URL (if applicable) or skip it if it's a non-fetchable type (primitive for example).
-		urls, err := registry.GetDownloadURLs(tr.rp, r.Type)
+		urls, registry, err := registry.GetDownloadURLs(tr.rp, r.Type)
 		if err != nil {
 			return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", r.Type, err))
 		}
@@ -108,14 +110,14 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
 			f := &fetchUnit{}
 			for _, u := range urls {
 				if len(u) > 0 {
-					f.urls = append(f.urls, u)
+					f.urls = append(f.urls, fetchableURL{registry, u})
 					// Add to existing map so it is not fetched multiple times.
 					existing[r.Type] = true
 				}
 			}
 			if len(f.urls) > 0 {
 				toFetch = append(toFetch, f)
-				fetched[f.urls[0]] = append(fetched[f.urls[0]], &common.ImportFile{Name: r.Type, Path: f.urls[0]})
+				fetched[f.urls[0].url] = append(fetched[f.urls[0].url], &common.ImportFile{Name: r.Type, Path: f.urls[0].url})
 			}
 		}
 	}
@@ -138,14 +140,14 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
 		templates := []string{}
 		url := toFetch[0].urls[0]
 		for _, u := range toFetch[0].urls {
-			template, err := performHTTPGet(tr.getter, u, false)
+			template, err := tr.performHTTPGet(u.registry, u.url, false)
 			if err != nil {
 				return nil, resolverError(config, err)
 			}
 			templates = append(templates, template)
 		}
 
-		for _, i := range fetched[url] {
+		for _, i := range fetched[url.url] {
 			template, err := parseContent(templates)
 			if err != nil {
 				return nil, resolverError(config, err)
@@ -153,8 +155,8 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
 			i.Content = template
 		}
 
-		schemaURL := url + schemaSuffix
-		sch, err := performHTTPGet(tr.getter, schemaURL, true)
+		schemaURL := url.url + schemaSuffix
+		sch, err := tr.performHTTPGet(url.registry, schemaURL, true)
 		if err != nil {
 			return nil, resolverError(config, err)
 		}
@@ -168,7 +170,7 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
 			for _, v := range s.Imports {
 				i := &common.ImportFile{Name: v.Name}
 				var existingSchema string
-				urls, conversionErr := registry.GetDownloadURLs(tr.rp, v.Path)
+				urls, registry, conversionErr := registry.GetDownloadURLs(tr.rp, v.Path)
 				if conversionErr != nil {
 					return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", v.Path, conversionErr))
 				}
@@ -180,7 +182,7 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
 				for _, u := range urls {
 					if len(fetched[u]) == 0 {
 						// If this import URL is new to us, add it to the URLs to fetch.
-						toFetch = append(toFetch, &fetchUnit{[]string{u}})
+						toFetch = append(toFetch, &fetchUnit{[]fetchableURL{fetchableURL{registry, u}}})
 					} else {
 						// If this is not a new import URL and we've already fetched its contents,
 						// reuse them. Also, check if we also found a schema for that import URL and
@@ -201,7 +203,7 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
 			}
 
 			// Add the schema we've fetched as the schema for any templates which used this URL.
-			for _, i := range fetched[url] {
+			for _, i := range fetched[url.url] {
 				schemaImportName := i.Name + schemaSuffix
 				fetched[schemaURL] = append(fetched[schemaURL],
 					&common.ImportFile{Name: schemaImportName, Content: sch})

manager/manager/typeresolver_test.go

@@ -18,6 +18,7 @@ package manager
 
 import (
 	"errors"
+	"log"
 	"net/http"
 	"reflect"
 	"strings"
@@ -50,19 +51,20 @@ type testGetter struct {
 	test      *testing.T
 }
 
-func (tg *testGetter) Get(url string) (body string, code int, err error) {
+func (tg testGetter) Get(url string) (body string, code int, err error) {
 	tg.count = tg.count + 1
 	ret := tg.responses[url]
-
+	log.Printf("GET RETURNING: '%s' '%d'", ret.resp, tg.count)
 	return ret.resp, ret.code, ret.err
 }
 
 func testDriver(c resolverTestCase, t *testing.T) {
 	g := &testGetter{test: t, responses: c.responses}
+	log.Printf("getter: %#v", g)
 	r := &typeResolver{
-		getter:  g,
 		maxUrls: 5,
 		rp:      c.registryProvider,
+		c:       g,
 	}
 
 	conf := &common.Configuration{}
@@ -74,7 +76,7 @@ func testDriver(c resolverTestCase, t *testing.T) {
 	result, err := r.ResolveTypes(conf, c.imports)
 
 	if g.count != c.urlcount {
-		t.Errorf("Expected %d url GETs but only %d found", c.urlcount, g.count)
+		t.Errorf("Expected %d url GETs but only %d found %#v", c.urlcount, g.count, g)
 	}
 
 	if (err != nil && c.expectedErr == nil) || (err == nil && c.expectedErr != nil) {
@@ -307,13 +309,19 @@ func TestShortGithubUrl(t *testing.T) {
 		registry.NewTypeOrDie("common", "replicatedservice", "v2"): registry.TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py", nil},
 	}
 
+	gcsUrlMaps := map[registry.Type]registry.TestURLAndError{
+		registry.NewTypeOrDie("common", "replicatedservice", "v1"): registry.TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil},
+		registry.NewTypeOrDie("common", "replicatedservice", "v2"): registry.TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py", nil},
+	}
+
 	grp := registry.NewTestGithubRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps)
+	gcsrp := registry.NewTestGCSRegistryProvider("gs://charts", gcsUrlMaps)
 	test := resolverTestCase{
 		config:           templateShortGithubTemplate,
 		importOut:        finalImports,
 		urlcount:         4,
 		responses:        responses,
-		registryProvider: registry.NewRegistryProvider(nil, grp, registry.NewInmemCredentialProvider()),
+		registryProvider: registry.NewRegistryProvider(nil, grp, gcsrp, registry.NewInmemCredentialProvider()),
 	}
 
 	testDriver(test, t)

registry/gcs_registry.go

+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package registry
+
+import (
+	"github.com/kubernetes/deployment-manager/common"
+	"github.com/kubernetes/deployment-manager/util"
+
+	//        "golang.org/x/net/context"
+	//        "golang.org/x/oauth2/google"
+	storage "google.golang.org/api/storage/v1"
+
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+	"regexp"
+)
+
+// GCSRegistry implements the ObbectStorageRegistry interface and implements a
+// Deployment Manager templates registry.
+//
+// A registry root must be a directory that contains all the available charts,
+// one or two files per template.
+// name-version.tgz
+// name-version.prov
+type GCSRegistry struct {
+	name           string
+	shortURL       string
+	bucket         string
+	format         common.RegistryFormat
+	credentialName string
+
+	httpClient *http.Client
+	service    *storage.Service
+}
+
+// RE for GCS storage
+var ChartFormatMatcher = regexp.MustCompile("(.*)-(.*).tgz")
+var URLFormatMatcher = regexp.MustCompile("gs://(.*)")
+
+// NewGithubTemplateRegistry creates a GithubTemplateRegistry.
+func NewGCSRegistry(name, shortURL string, httpClient *http.Client, gcsService *storage.Service) (*GCSRegistry, error) {
+	format := fmt.Sprintf("%s;%s", common.VersionedRegistry, common.OneLevelRegistry)
+	trimmed := util.TrimURLScheme(shortURL)
+	m := URLFormatMatcher.FindStringSubmatch(shortURL)
+	if len(m) != 2 {
+		return nil, fmt.Errorf("URL must be of the form gs://<bucket> was: %s", shortURL)
+	}
+
+	return &GCSRegistry{
+			name:       name,
+			shortURL:   trimmed,
+			format:     common.RegistryFormat(format),
+			httpClient: httpClient,
+			service:    gcsService,
+			bucket:     m[1],
+		},
+		nil
+}
+
+func (g GCSRegistry) GetRegistryName() string {
+	return g.name
+}
+
+func (g GCSRegistry) GetBucket() string {
+	return g.bucket
+}
+
+func (g GCSRegistry) GetRegistryType() common.RegistryType {
+	return common.GCSRegistryType
+}
+
+// ListTypes lists types in this registry whose string values conform to the
+// supplied regular expression, or all types, if the regular expression is nil.
+func (g GCSRegistry) ListTypes(regex *regexp.Regexp) ([]Type, error) {
+	// List all files in the bucket/prefix that contain the
+	types := []Type{}
+
+	// List all objects in a bucket using pagination
+	pageToken := ""
+	for {
+		call := g.service.Objects.List(g.bucket)
+		call.Delimiter("/")
+		if pageToken != "" {
+			call = call.PageToken(pageToken)
+		}
+		res, err := call.Do()
+		if err != nil {
+			return []Type{}, err
+		}
+		for _, object := range res.Items {
+			// Charts should be named bucket/chart-X.Y.Z.tgz, so tease apart the version here
+			m := ChartFormatMatcher.FindStringSubmatch(object.Name)
+			if len(m) != 3 {
+				continue
+			}
+
+			t, err := NewType("", m[1], m[2])
+			if err != nil {
+				return []Type{}, fmt.Errorf("can't create a type type at path %#v", err)
+			}
+			types = append(types, t)
+		}
+		if pageToken = res.NextPageToken; pageToken == "" {
+			break
+		}
+	}
+	return types, nil
+}
+
+func (g GCSRegistry) GetRegistryFormat() common.RegistryFormat {
+	return common.CollectionRegistry
+}
+
+func (g GCSRegistry) GetRegistryShortURL() string {
+	return g.shortURL
+}
+
+// GetDownloadURLs fetches the download URLs for a given Chart
+func (g GCSRegistry) GetDownloadURLs(t Type) ([]*url.URL, error) {
+	call := g.service.Objects.List(g.bucket)
+	call.Delimiter("/")
+	call.Prefix(t.String())
+	res, err := call.Do()
+	ret := []*url.URL{}
+	if err != nil {
+		return ret, err
+	}
+	for _, object := range res.Items {
+		log.Printf("Found: %s", object.Name)
+		u, err := url.Parse(object.MediaLink)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse URL from %s: %s", object.MediaLink, err)
+		}
+		ret = append(ret, u)
+	}
+	return ret, err
+}
+
+func (g GCSRegistry) Do(req *http.Request) (resp *http.Response, err error) {
+	return g.httpClient.Do(req)
+}

registry/github_package_registry.go

@@ -22,6 +22,7 @@ import (
 
 	"fmt"
 	"log"
+	"net/http"
 	"net/url"
 	"regexp"
 	"strings"
@@ -41,7 +42,7 @@ type GithubPackageRegistry struct {
 }
 
 // NewGithubPackageRegistry creates a GithubPackageRegistry.
-func NewGithubPackageRegistry(name, shortURL string, service GithubRepositoryService, client *github.Client) (*GithubPackageRegistry, error) {
+func NewGithubPackageRegistry(name, shortURL string, service GithubRepositoryService, httpClient *http.Client, client *github.Client) (*GithubPackageRegistry, error) {
 	format := fmt.Sprintf("%s;%s", common.UnversionedRegistry, common.OneLevelRegistry)
 	if service == nil {
 		if client == nil {
@@ -51,7 +52,7 @@ func NewGithubPackageRegistry(name, shortURL string, service GithubRepositorySer
 		}
 	}
 
-	gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), service)
+	gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), httpClient, service)
 	if err != nil {
 		return nil, err
 	}
@@ -151,3 +152,7 @@ func (g GithubPackageRegistry) MakeRepositoryPath(t Type) (string, error) {
 	// Construct the return path
 	return t.Name + "/manifests", nil
 }
+
+func (g GithubPackageRegistry) Do(req *http.Request) (resp *http.Response, err error) {
+	return g.httpClient.Do(req)
+}

registry/github_registry.go

@@ -22,6 +22,7 @@ import (
 	"github.com/kubernetes/deployment-manager/util"
 
 	"fmt"
+	"net/http"
 	"strings"
 )
 
@@ -38,6 +39,7 @@ type githubRegistry struct {
 	format         common.RegistryFormat
 	credentialName string
 	service        GithubRepositoryService
+	httpClient     *http.Client
 }
 
 // GithubRepositoryService defines the interface that's defined in github.com/go-github/repos_contents.go GetContents method.
@@ -54,7 +56,7 @@ type GithubRepositoryService interface {
 }
 
 // newGithubRegistry creates a githubRegistry.
-func newGithubRegistry(name, shortURL string, format common.RegistryFormat, service GithubRepositoryService) (*githubRegistry, error) {
+func newGithubRegistry(name, shortURL string, format common.RegistryFormat, httpClient *http.Client, service GithubRepositoryService) (*githubRegistry, error) {
 	trimmed := util.TrimURLScheme(shortURL)
 	owner, repository, path, err := parseGithubShortURL(trimmed)
 	if err != nil {
@@ -69,6 +71,7 @@ func newGithubRegistry(name, shortURL string, format common.RegistryFormat, serv
 		path:       path,
 		format:     format,
 		service:    service,
+		httpClient: httpClient,
 	}, nil
 }
 

registry/github_template_registry.go

@@ -22,6 +22,7 @@ import (
 
 	"fmt"
 	"log"
+	"net/http"
 	"net/url"
 	"regexp"
 	"strings"
@@ -54,13 +55,13 @@ type GithubTemplateRegistry struct {
 }
 
 // NewGithubTemplateRegistry creates a GithubTemplateRegistry.
-func NewGithubTemplateRegistry(name, shortURL string, service GithubRepositoryService, client *github.Client) (*GithubTemplateRegistry, error) {
+func NewGithubTemplateRegistry(name, shortURL string, service GithubRepositoryService, httpClient *http.Client, client *github.Client) (*GithubTemplateRegistry, error) {
 	format := fmt.Sprintf("%s;%s", common.VersionedRegistry, common.CollectionRegistry)
 	if service == nil {
 		service = client.Repositories
 	}
 
-	gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), service)
+	gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), httpClient, service)
 	if err != nil {
 		return nil, err
 	}
@@ -212,3 +213,7 @@ func (g GithubTemplateRegistry) MakeRepositoryPath(t Type) (string, error) {
 	}
 	return p + t.Name + "/" + t.GetVersion(), nil
 }
+
+func (g GithubTemplateRegistry) Do(req *http.Request) (resp *http.Response, err error) {
+	return g.httpClient.Do(req)
+}

registry/inmem_credential_provider.go

@@ -38,6 +38,6 @@ func (fcp *InmemCredentialProvider) GetCredential(name string) (*common.Registry
 }
 
 func (fcp *InmemCredentialProvider) SetCredential(name string, credential *common.RegistryCredential) error {
-	fcp.credentials[name] = &common.RegistryCredential{credential.APIToken, credential.BasicAuth}
+	fcp.credentials[name] = &common.RegistryCredential{credential.APIToken, credential.BasicAuth, credential.ServiceAccount}
 	return nil
 }

registry/inmem_registry_service.go

@@ -51,6 +51,14 @@ func NewInmemRegistryService() common.RegistryService {
 		CredentialName: "default",
 	})
 
+	gFormat := fmt.Sprintf("%s", common.CollectionRegistry)
+	rs.Create(&common.Registry{
+		Name:   "charts_gcs",
+		Type:   common.GCSRegistryType,
+		URL:    "gs://helm-charts-test",
+		Format: common.RegistryFormat(gFormat),
+	})
+
 	return rs
 }
 

registry/registry.go

@@ -18,6 +18,7 @@ package registry
 
 import (
 	"github.com/kubernetes/deployment-manager/common"
+	"github.com/kubernetes/deployment-manager/util"
 
 	"fmt"
 	"net/url"
@@ -29,6 +30,9 @@ import (
 // used in a Deployment Manager configuration. There can be multiple
 // registry implementations.
 type Registry interface {
+	// Also handles http.Client.Do method for authenticated File accesses
+	util.HTTPDoer
+
 	// GetRegistryName returns the name of this registry
 	GetRegistryName() string
 	// GetRegistryType returns the type of this registry.
@@ -56,6 +60,13 @@ type GithubRegistry interface {
 	GetRegistryPath() string
 }
 
+// ObjectStorageRegistry abstracts a registry that resides in an Object Storage, for
+// example Google Cloud Storage or AWS S3, etc.
+type ObjectStorageRegistry interface {
+	Registry // An ObjectStorageRegistry is a Registry.
+	GetBucket() string
+}
+
 type Type struct {
 	Collection string
 	Name       string

registry/registryprovider_test.go

@@ -22,7 +22,8 @@ import (
 
 func testUrlConversionDriver(rp RegistryProvider, tests map[string]TestURLAndError, t *testing.T) {
 	for in, expected := range tests {
-		actual, err := GetDownloadURLs(rp, in)
+		// TODO(vaikas): Test to make sure it's the right registry.
+		actual, _, err := GetDownloadURLs(rp, in)
 		if err != expected.Err {
 			t.Fatalf("failed on: %s : expected error %v but got %v", in, expected.Err, err)
 		}
@@ -45,7 +46,8 @@ func TestShortGithubUrlTemplateMapping(t *testing.T) {
 	}
 
 	grp := NewTestGithubRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps)
-	testUrlConversionDriver(NewRegistryProvider(nil, grp, NewInmemCredentialProvider()), tests, t)
+	// TODO(vaikas): XXXX FIXME Add gcsrp
+	testUrlConversionDriver(NewRegistryProvider(nil, grp, nil, NewInmemCredentialProvider()), tests, t)
 }
 
 func TestShortGithubUrlPackageMapping(t *testing.T) {
@@ -60,5 +62,6 @@ func TestShortGithubUrlPackageMapping(t *testing.T) {
 	}
 
 	grp := NewTestGithubRegistryProvider("github.com/helm/charts", githubUrlMaps)
-	testUrlConversionDriver(NewRegistryProvider(nil, grp, NewInmemCredentialProvider()), tests, t)
+	// TODO(vaikas): XXXX FIXME Add gcsrp
+	testUrlConversionDriver(NewRegistryProvider(nil, grp, nil, NewInmemCredentialProvider()), tests, t)
 }

registry/secrets_credential_provider.go

@@ -120,8 +120,6 @@ func (scp *SecretsCredentialProvider) SetCredential(name string, credential *com
 		log.Printf("yaml marshal failed for kubernetes object: %s: %v", name, err)
 		return err
 	}
-	log.Printf("Calling with: %s", string(ko))
-	o, err := scp.k.Create(string(ko))
-	log.Printf("Create returned: %s", o)
+	_, err = scp.k.Create(string(ko))
 	return err
 }

registry/testhelper.go

@@ -23,6 +23,7 @@ import (
 	"github.com/kubernetes/deployment-manager/util"
 
 	"fmt"
+	"net/http"
 	"net/url"
 	"regexp"
 	"strings"
@@ -53,7 +54,7 @@ func NewTestGithubRegistryProvider(shortURL string, responses map[Type]TestURLAn
 func (tgrp testGithubRegistryProvider) GetGithubRegistry(cr common.Registry) (GithubRegistry, error) {
 	trimmed := util.TrimURLScheme(cr.URL)
 	if strings.HasPrefix(trimmed, tgrp.shortURL) {
-		ghr, err := newGithubRegistry(cr.Name, trimmed, cr.Format, nil)
+		ghr, err := newGithubRegistry(cr.Name, trimmed, cr.Format, http.DefaultClient, nil)
 		if err != nil {
 			panic(fmt.Errorf("cannot create a github registry: %s", err))
 		}
@@ -80,3 +81,41 @@ func (tgr testGithubRegistry) GetDownloadURLs(t Type) ([]*url.URL, error) {
 
 	return []*url.URL{URL}, result.Err
 }
+
+func (g testGithubRegistry) Do(req *http.Request) (resp *http.Response, err error) {
+	return nil, fmt.Errorf("Not implemented yet")
+}
+
+type testGCSRegistryProvider struct {
+	shortURL  string
+	responses map[Type]TestURLAndError
+}
+
+type testGCSRegistry struct {
+	GCSRegistry
+	responses map[Type]TestURLAndError
+}
+
+func NewTestGCSRegistryProvider(shortURL string, responses map[Type]TestURLAndError) GCSRegistryProvider {
+	return testGCSRegistryProvider{
+		shortURL:  util.TrimURLScheme(shortURL),
+		responses: responses,
+	}
+}
+
+func (tgrp testGCSRegistryProvider) GetGCSRegistry(cr common.Registry) (ObjectStorageRegistry, error) {
+	trimmed := util.TrimURLScheme(cr.URL)
+	if strings.HasPrefix(trimmed, tgrp.shortURL) {
+		gcsr, err := NewGCSRegistry(cr.Name, trimmed, http.DefaultClient, nil)
+		if err != nil {
+			panic(fmt.Errorf("cannot create gcs registry: %s", err))
+		}
+
+		return &testGCSRegistry{
+			GCSRegistry: *gcsr,
+			responses:   tgrp.responses,
+		}, nil
+	}
+
+	panic(fmt.Errorf("unknown registry: %v", cr))
+}

util/httpclient.go

@@ -6,7 +6,7 @@ you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
     http://www.apache.org/licenses/LICENSE-2.0
- 
+
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -72,6 +72,10 @@ type httpClient struct {
 	sleep   Sleeper
 }
 
+func DefaultHTTPClient() HTTPClient {
+	return NewHTTPClient(3, http.DefaultClient, NewSleeper())
+}
+
 // NewHTTPClient returns a new HTTPClient.
 func NewHTTPClient(retries uint, c HTTPDoer, s Sleeper) HTTPClient {
 	ret := httpClient{}

util/kubernetes_kubectl.go

@@ -126,15 +126,17 @@ func (k *KubernetesKubectl) execute(args []string, input string) (string, error)
 	cmd.Stderr = combined
 
 	if err := cmd.Start(); err != nil {
-		log.Printf("cannot start kubectl %s %#v", combined.String(), err)
-		return combined.String(), err
+		e := fmt.Errorf("cannot start kubectl %s %#v", combined.String(), err)
+		log.Printf("%s", e)
+		return combined.String(), e
 	}
 
 	if err := cmd.Wait(); err != nil {
-		log.Printf("kubectl failed: %s  %#v", combined.String(), err)
-		return combined.String(), err
+		e := fmt.Errorf("kubectl failed %s", combined.String())
+		log.Printf("%s", e)
+		return combined.String(), e
 	}
-	log.Printf("kubectl succeeded: SysTime: %v UserTime: %v\n%v",
-		cmd.ProcessState.SystemTime(), cmd.ProcessState.UserTime(), combined.String())
+	log.Printf("kubectl succeeded: SysTime: %v UserTime: %v",
+		cmd.ProcessState.SystemTime(), cmd.ProcessState.UserTime())
 	return combined.String(), nil
 }