Add GCS as a registry provider. Add a way to get a file using credentials from a…

Add GCS as a registry provider. Add a way to get a file using credentials from a registry to support private repos. Add ability to create / update a registry through manager

common/types.go

@@ -182,11 +182,13 @@ type BasicAuthCredential struct {
 }
 
 type APITokenCredential string
+type JWTTokenCredential string
 
 // Credential used to access the repository
 type RegistryCredential struct {
-	APIToken  APITokenCredential  `json:"apitoken,omitempty"`
-	BasicAuth BasicAuthCredential `json:"basicauth,omitempty"`
+	APIToken       APITokenCredential  `json:"apitoken,omitempty"`
+	BasicAuth      BasicAuthCredential `json:"basicauth,omitempty"`
+	ServiceAccount JWTTokenCredential  `json:"serviceaccount,omitempty"`
 }
 
 // Registry describes a template registry
@@ -204,6 +206,7 @@ type RegistryType string
 
 const (
 	GithubRegistryType RegistryType = "github"
+	GCSRegistryType    RegistryType = "gcs"
 )
 
 // RegistryFormat is a semi-colon delimited string that describes the format

dm/dm.go

@@ -50,6 +50,8 @@ var (
 	username          = flag.String("username", "", "Github user name that overrides GITHUB_USERNAME environment variable")
 	password          = flag.String("password", "", "Github password that overrides GITHUB_PASSWORD environment variable")
 	apitoken          = flag.String("apitoken", "", "Github api token that overrides GITHUB_API_TOKEN environment variable")
+	serviceaccount    = flag.String("serviceaccount", "", "Service account file containing JWT token")
+	registryfile      = flag.String("registryfile", "", "File containing registry specification")
 )
 
 var commands = []string{
@@ -67,6 +69,7 @@ var commands = []string{
 	"describe \t\t Describes the named template in a given template registry",
 	"getcredential \t\t Gets the named credential used by a registry",
 	"setcredential \t\t Sets a credential used by a registry",
+	"createregistry \t\t Creates a registry that holds charts",
 }
 
 var usage = func() {
@@ -86,7 +89,7 @@ var usage = func() {
 	os.Exit(0)
 }
 
-func getGithubCredential() *common.RegistryCredential {
+func getCredential() *common.RegistryCredential {
 	*apitoken = strings.TrimSpace(*apitoken)
 	if *apitoken == "" {
 		*apitoken = strings.TrimSpace(os.Getenv("GITHUB_API_TOKEN"))
@@ -117,6 +120,15 @@ func getGithubCredential() *common.RegistryCredential {
 		}
 	}
 
+	if *serviceaccount != "" {
+		b, err := ioutil.ReadFile(*serviceaccount)
+		if err != nil {
+			log.Fatalf("Unable to read service account file: %v", err)
+		}
+		return &common.RegistryCredential{
+			ServiceAccount: common.JWTTokenCredential(string(b)),
+		}
+	}
 	return nil
 }
 
@@ -124,6 +136,13 @@ func init() {
 	flag.Usage = usage
 }
 
+func getRegistry() ([]byte, error) {
+	if *registryfile == "" {
+		log.Fatalf("No registryfile specified (-registryfile)")
+	}
+	return ioutil.ReadFile(*registryfile)
+}
+
 func main() {
 	defer func() {
 		result := recover()
@@ -145,6 +164,10 @@ func execute() {
 
 	switch args[0] {
 	case "templates":
+		if len(args) < 2 {
+			fmt.Fprintln(os.Stderr, "No registry name supplied")
+			usage()
+		}
 		path := fmt.Sprintf("registries/%s/types", args[1])
 		callService(path, "GET", "list templates", nil)
 	case "describe":
@@ -162,7 +185,7 @@ func execute() {
 		path := fmt.Sprintf("credentials/%s", args[1])
 		callService(path, "GET", "get credential", nil)
 	case "setcredential":
-		c := getGithubCredential()
+		c := getCredential()
 		if c == nil {
 			panic(fmt.Errorf("Failed to create a credential from flags/arguments"))
 		}
@@ -172,7 +195,14 @@ func execute() {
 		}
 
 		path := fmt.Sprintf("credentials/%s", args[1])
-		callService(path, "POST", "get credential", ioutil.NopCloser(bytes.NewReader(y)))
+		callService(path, "POST", "set credential", ioutil.NopCloser(bytes.NewReader(y)))
+	case "createregistry":
+		reg, err := getRegistry()
+		if err != nil {
+			panic(fmt.Errorf("Failed to create a registry from arguments: %#v", err))
+		}
+		path := fmt.Sprintf("registries/%s", args[1])
+		callService(path, "POST", "set registry", ioutil.NopCloser(bytes.NewReader(reg)))
 	case "get":
 		if len(args) < 2 {
 			fmt.Fprintln(os.Stderr, "No deployment name supplied")
@@ -300,13 +330,20 @@ func describeType(args []string) {
 		os.Exit(1)
 	}
 
-	tUrls := getDownloadURLs(args[1])
+	tUrls := getDownloadURLs(url.QueryEscape(args[1]))
 	if len(tUrls) == 0 {
 		panic(fmt.Errorf("Invalid type name, must be a template URL or in the form \"<type-name>:<version>\": %s", args[1]))
 	}
 
-	schemaUrl := tUrls[0] + ".schema"
-	fmt.Println(callHttp(schemaUrl, "GET", "get schema for type ("+tUrls[0]+")", nil))
+	if !strings.Contains(tUrls[0], ".prov") {
+		// It's not a chart, so grab the schema
+		path := fmt.Sprintf("registries/%s/download?file=%s.schema", *template_registry, url.QueryEscape(tUrls[0]))
+		callService(path, "GET", "get schema for type ("+tUrls[0]+")", nil)
+	} else {
+		// It's a chart, so grab the provenance file
+		path := fmt.Sprintf("registries/%s/download?file=%s", *template_registry, url.QueryEscape(tUrls[0]))
+		callService(path, "GET", "get file", nil)
+	}
 }
 
 // getDownloadURLs returns URLs for a type in the given registry

manager/deployments.go

@@ -54,8 +54,10 @@ var deployments = []Route{
 	{"ListTypeInstances", "/types/{type}/instances", "GET", listTypeInstancesHandlerFunc, ""},
 	{"ListRegistries", "/registries", "GET", listRegistriesHandlerFunc, ""},
 	{"GetRegistry", "/registries/{registry}", "GET", getRegistryHandlerFunc, ""},
+	{"CreateRegistry", "/registries/{registry}", "POST", createRegistryHandlerFunc, "JSON"},
 	{"ListRegistryTypes", "/registries/{registry}/types", "GET", listRegistryTypesHandlerFunc, ""},
 	{"GetDownloadURLs", "/registries/{registry}/types/{type}", "GET", getDownloadURLsHandlerFunc, ""},
+	{"GetFile", "/registries/{registry}/download", "GET", getFileHandlerFunc, ""},
 	{"CreateCredential", "/credentials/{credential}", "POST", createCredentialHandlerFunc, "JSON"},
 	{"GetCredential", "/credentials/{credential}", "GET", getCredentialHandlerFunc, ""},
 }
@@ -97,12 +99,12 @@ func init() {
 }
 
 func newManager(cp common.CredentialProvider) manager.Manager {
-	registryProvider := registry.NewDefaultRegistryProvider(cp)
-	resolver := manager.NewTypeResolver(registryProvider)
+	service := registry.NewInmemRegistryService()
+	registryProvider := registry.NewDefaultRegistryProvider(cp, service)
+	resolver := manager.NewTypeResolver(registryProvider, util.DefaultHTTPClient())
 	expander := manager.NewExpander(getServiceURL(*expanderURL, *expanderName), resolver)
 	deployer := manager.NewDeployer(getServiceURL(*deployerURL, *deployerName))
 	r := repository.NewMapBasedRepository()
-	service := registry.NewInmemRegistryService()
 	credentialProvider := cp
 	return manager.NewManager(expander, deployer, r, registryProvider, service, credentialProvider)
 }
@@ -378,6 +380,49 @@ func getRegistryHandlerFunc(w http.ResponseWriter, r *http.Request) {
 	util.LogHandlerExitWithJSON(handler, w, cr, http.StatusOK)
 }
 
+func getRegistry(w http.ResponseWriter, r *http.Request, handler string) *common.Registry {
+	util.LogHandlerEntry(handler, r)
+	j, err := getJsonFromRequest(w, r, handler)
+	if err != nil {
+		return nil
+	}
+
+	t := &common.Registry{}
+	if err := json.Unmarshal(j, t); err != nil {
+		e := fmt.Errorf("%v\n%v", err, string(j))
+		util.LogAndReturnError(handler, http.StatusBadRequest, e, w)
+		return nil
+	}
+
+	return t
+}
+
+func createRegistryHandlerFunc(w http.ResponseWriter, r *http.Request) {
+	handler := "manager: create registry"
+	util.LogHandlerEntry(handler, r)
+	defer r.Body.Close()
+	registryName, err := getPathVariable(w, r, "registry", handler)
+	if err != nil {
+		return
+	}
+
+	reg := getRegistry(w, r, handler)
+	if reg.Name != registryName {
+		e := fmt.Errorf("Registry name does not match %s != %s", reg.Name, registryName)
+		util.LogAndReturnError(handler, http.StatusBadRequest, e, w)
+		return
+	}
+	if reg != nil {
+		err = backend.CreateRegistry(reg)
+		if err != nil {
+			util.LogAndReturnError(handler, http.StatusBadRequest, err, w)
+			return
+		}
+	}
+
+	util.LogHandlerExitWithJSON(handler, w, reg, http.StatusOK)
+}
+
 func listRegistryTypesHandlerFunc(w http.ResponseWriter, r *http.Request) {
 	handler := "manager: list registry types"
 	util.LogHandlerEntry(handler, r)
@@ -437,6 +482,28 @@ func getDownloadURLsHandlerFunc(w http.ResponseWriter, r *http.Request) {
 	util.LogHandlerExitWithJSON(handler, w, urls, http.StatusOK)
 }
 
+func getFileHandlerFunc(w http.ResponseWriter, r *http.Request) {
+	handler := "manager: get file"
+	util.LogHandlerEntry(handler, r)
+	registryName, err := getPathVariable(w, r, "registry", handler)
+	if err != nil {
+		return
+	}
+
+	file := r.FormValue("file")
+	if file == "" {
+		return
+	}
+
+	b, err := backend.GetFile(registryName, file)
+	if err != nil {
+		util.LogAndReturnError(handler, http.StatusBadRequest, err, w)
+		return
+	}
+
+	util.LogHandlerExitWithJSON(handler, w, b, http.StatusOK)
+}
+
 func getCredential(w http.ResponseWriter, r *http.Request, handler string) *common.RegistryCredential {
 	util.LogHandlerEntry(handler, r)
 	j, err := getJsonFromRequest(w, r, handler)

manager/manager/manager.go

@@ -26,6 +26,7 @@ import (
 	"github.com/kubernetes/deployment-manager/common"
 	"github.com/kubernetes/deployment-manager/manager/repository"
 	"github.com/kubernetes/deployment-manager/registry"
+	"github.com/kubernetes/deployment-manager/util"
 )
 
 // Manager manages a persistent set of Deployments.
@@ -55,6 +56,7 @@ type Manager interface {
 	// Registry Types
 	ListRegistryTypes(registryName string, regex *regexp.Regexp) ([]registry.Type, error)
 	GetDownloadURLs(registryName string, t registry.Type) ([]*url.URL, error)
+	GetFile(registryName string, url string) (string, error)
 
 	// Credentials
 	CreateCredential(name string, c *common.RegistryCredential) error
@@ -394,6 +396,21 @@ func (m *manager) GetDownloadURLs(registryName string, t registry.Type) ([]*url.
 	return r.GetDownloadURLs(t)
 }
 
+// GetFile returns a file from the backing registry
+func (m *manager) GetFile(registryName string, url string) (string, error) {
+	r, err := m.registryProvider.GetRegistryByName(registryName)
+	if err != nil {
+		return "", err
+	}
+
+	getter := util.NewHTTPClient(3, r, util.NewSleeper())
+	body, _, err := getter.Get(url)
+	if err != nil {
+		return "", err
+	}
+	return body, nil
+}
+
 // CreateCredential creates a credential that can be used to authenticate to registry
 func (m *manager) CreateCredential(name string, c *common.RegistryCredential) error {
 	return m.credentialProvider.SetCredential(name, c)

manager/manager/manager_test.go

@@ -255,7 +255,7 @@ var testRepository = newRepositoryStub()
 var testDeployer = newDeployerStub()
 var testRegistryService = registry.NewInmemRegistryService()
 var testCredentialProvider = registry.NewInmemCredentialProvider()
-var testProvider = registry.NewRegistryProvider(nil, registry.NewTestGithubRegistryProvider("", nil), testCredentialProvider)
+var testProvider = registry.NewRegistryProvider(nil, registry.NewTestGithubRegistryProvider("", nil), registry.NewTestGCSRegistryProvider("", nil), testCredentialProvider)
 var testManager = NewManager(testExpander, testDeployer, testRepository, testProvider, testRegistryService, testCredentialProvider)
 
 func TestListDeployments(t *testing.T) {

manager/manager/typeresolver.go

@@ -19,7 +19,6 @@ package manager
 import (
 	"fmt"
 	"net/http"
-	"time"
 
 	"github.com/kubernetes/deployment-manager/common"
 	"github.com/kubernetes/deployment-manager/registry"
@@ -40,26 +39,23 @@ type TypeResolver interface {
 }
 
 type typeResolver struct {
-	getter  util.HTTPClient
 	maxUrls int
 	rp      registry.RegistryProvider
+	c       util.HTTPClient
+}
+
+type fetchableURL struct {
+	registry registry.Registry
+	url      string
 }
 
 type fetchUnit struct {
-	urls []string
+	urls []fetchableURL
 }
 
 // NewTypeResolver returns a new initialized TypeResolver.
-func NewTypeResolver(rp registry.RegistryProvider) TypeResolver {
-	ret := &typeResolver{}
-	client := http.DefaultClient
-	//TODO (iantw): Make this a flag
-	timeout, _ := time.ParseDuration("10s")
-	client.Timeout = timeout
-	ret.getter = util.NewHTTPClient(3, client, util.NewSleeper())
-	ret.maxUrls = maxURLImports
-	ret.rp = rp
-	return ret
+func NewTypeResolver(rp registry.RegistryProvider, c util.HTTPClient) TypeResolver {
+	return &typeResolver{maxUrls: maxURLImports, rp: rp, c: c}
 }
 
 func resolverError(c *common.Configuration, err error) error {
@@ -67,7 +63,13 @@ func resolverError(c *common.Configuration, err error) error {
 		c, err)
 }
 
-func performHTTPGet(g util.HTTPClient, u string, allowMissing bool) (content string, err error) {
+func (tr *typeResolver) performHTTPGet(d util.HTTPDoer, u string, allowMissing bool) (content string, err error) {
+	var g util.HTTPClient
+	if d == nil {
+		g = tr.c
+	} else {
+		g = util.NewHTTPClient(3, d, util.NewSleeper())
+	}
 	r, code, err := g.Get(u)
 	if err != nil {
 		return "", err
@@ -100,7 +102,7 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
 	toFetch := make([]*fetchUnit, 0, tr.maxUrls)
 	for _, r := range config.Resources {
 		// Map the type to a fetchable URL (if applicable) or skip it if it's a non-fetchable type (primitive for example).
-		urls, err := registry.GetDownloadURLs(tr.rp, r.Type)
+		urls, registry, err := registry.GetDownloadURLs(tr.rp, r.Type)
 		if err != nil {
 			return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", r.Type, err))
 		}
@@ -108,14 +110,14 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
 			f := &fetchUnit{}
 			for _, u := range urls {
 				if len(u) > 0 {
-					f.urls = append(f.urls, u)
+					f.urls = append(f.urls, fetchableURL{registry, u})
 					// Add to existing map so it is not fetched multiple times.
 					existing[r.Type] = true
 				}
 			}
 			if len(f.urls) > 0 {
 				toFetch = append(toFetch, f)
-				fetched[f.urls[0]] = append(fetched[f.urls[0]], &common.ImportFile{Name: r.Type, Path: f.urls[0]})
+				fetched[f.urls[0].url] = append(fetched[f.urls[0].url], &common.ImportFile{Name: r.Type, Path: f.urls[0].url})
 			}
 		}
 	}
@@ -138,14 +140,14 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
 		templates := []string{}
 		url := toFetch[0].urls[0]
 		for _, u := range toFetch[0].urls {
-			template, err := performHTTPGet(tr.getter, u, false)
+			template, err := tr.performHTTPGet(u.registry, u.url, false)
 			if err != nil {
 				return nil, resolverError(config, err)
 			}
 			templates = append(templates, template)
 		}
 
-		for _, i := range fetched[url] {
+		for _, i := range fetched[url.url] {
 			template, err := parseContent(templates)
 			if err != nil {
 				return nil, resolverError(config, err)
@@ -153,8 +155,8 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
 			i.Content = template
 		}
 
-		schemaURL := url + schemaSuffix
-		sch, err := performHTTPGet(tr.getter, schemaURL, true)
+		schemaURL := url.url + schemaSuffix
+		sch, err := tr.performHTTPGet(url.registry, schemaURL, true)
 		if err != nil {
 			return nil, resolverError(config, err)
 		}
@@ -168,7 +170,7 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
 			for _, v := range s.Imports {
 				i := &common.ImportFile{Name: v.Name}
 				var existingSchema string
-				urls, conversionErr := registry.GetDownloadURLs(tr.rp, v.Path)
+				urls, registry, conversionErr := registry.GetDownloadURLs(tr.rp, v.Path)
 				if conversionErr != nil {
 					return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", v.Path, conversionErr))
 				}
@@ -180,7 +182,7 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
 				for _, u := range urls {
 					if len(fetched[u]) == 0 {
 						// If this import URL is new to us, add it to the URLs to fetch.
-						toFetch = append(toFetch, &fetchUnit{[]string{u}})
+						toFetch = append(toFetch, &fetchUnit{[]fetchableURL{fetchableURL{registry, u}}})
 					} else {
 						// If this is not a new import URL and we've already fetched its contents,
 						// reuse them. Also, check if we also found a schema for that import URL and
@@ -201,7 +203,7 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
 			}
 
 			// Add the schema we've fetched as the schema for any templates which used this URL.
-			for _, i := range fetched[url] {
+			for _, i := range fetched[url.url] {
 				schemaImportName := i.Name + schemaSuffix
 				fetched[schemaURL] = append(fetched[schemaURL],
 					&common.ImportFile{Name: schemaImportName, Content: sch})

manager/manager/typeresolver_test.go

@@ -18,6 +18,7 @@ package manager
 
 import (
 	"errors"
+	"log"
 	"net/http"
 	"reflect"
 	"strings"
@@ -50,19 +51,20 @@ type testGetter struct {
 	test      *testing.T
 }
 
-func (tg *testGetter) Get(url string) (body string, code int, err error) {
+func (tg testGetter) Get(url string) (body string, code int, err error) {
 	tg.count = tg.count + 1
 	ret := tg.responses[url]
-
+	log.Printf("GET RETURNING: '%s' '%d'", ret.resp, tg.count)
 	return ret.resp, ret.code, ret.err
 }
 
 func testDriver(c resolverTestCase, t *testing.T) {
 	g := &testGetter{test: t, responses: c.responses}
+	log.Printf("getter: %#v", g)
 	r := &typeResolver{
-		getter:  g,
 		maxUrls: 5,
 		rp:      c.registryProvider,
+		c:       g,
 	}
 
 	conf := &common.Configuration{}
@@ -74,7 +76,7 @@ func testDriver(c resolverTestCase, t *testing.T) {
 	result, err := r.ResolveTypes(conf, c.imports)
 
 	if g.count != c.urlcount {
-		t.Errorf("Expected %d url GETs but only %d found", c.urlcount, g.count)
+		t.Errorf("Expected %d url GETs but only %d found %#v", c.urlcount, g.count, g)
 	}
 
 	if (err != nil && c.expectedErr == nil) || (err == nil && c.expectedErr != nil) {
@@ -307,13 +309,19 @@ func TestShortGithubUrl(t *testing.T) {
 		registry.NewTypeOrDie("common", "replicatedservice", "v2"): registry.TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py", nil},
 	}
 
+	gcsUrlMaps := map[registry.Type]registry.TestURLAndError{
+		registry.NewTypeOrDie("common", "replicatedservice", "v1"): registry.TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil},
+		registry.NewTypeOrDie("common", "replicatedservice", "v2"): registry.TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py", nil},
+	}
+
 	grp := registry.NewTestGithubRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps)
+	gcsrp := registry.NewTestGCSRegistryProvider("gs://charts", gcsUrlMaps)
 	test := resolverTestCase{
 		config:           templateShortGithubTemplate,
 		importOut:        finalImports,
 		urlcount:         4,
 		responses:        responses,
-		registryProvider: registry.NewRegistryProvider(nil, grp, registry.NewInmemCredentialProvider()),
+		registryProvider: registry.NewRegistryProvider(nil, grp, gcsrp, registry.NewInmemCredentialProvider()),
 	}
 
 	testDriver(test, t)

registry/gcs_registry.go

+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package registry
+
+import (
+	"github.com/kubernetes/deployment-manager/common"
+	"github.com/kubernetes/deployment-manager/util"
+
+	//        "golang.org/x/net/context"
+	//        "golang.org/x/oauth2/google"
+	storage "google.golang.org/api/storage/v1"
+
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+	"regexp"
+)
+
+// GCSRegistry implements the ObbectStorageRegistry interface and implements a
+// Deployment Manager templates registry.
+//
+// A registry root must be a directory that contains all the available charts,
+// one or two files per template.
+// name-version.tgz
+// name-version.prov
+type GCSRegistry struct {
+	name           string
+	shortURL       string
+	bucket         string
+	format         common.RegistryFormat
+	credentialName string
+
+	httpClient *http.Client
+	service    *storage.Service
+}
+
+// RE for GCS storage
+var ChartFormatMatcher = regexp.MustCompile("(.*)-(.*).tgz")
+var URLFormatMatcher = regexp.MustCompile("gs://(.*)")
+
+// NewGithubTemplateRegistry creates a GithubTemplateRegistry.
+func NewGCSRegistry(name, shortURL string, httpClient *http.Client, gcsService *storage.Service) (*GCSRegistry, error) {
+	format := fmt.Sprintf("%s;%s", common.VersionedRegistry, common.OneLevelRegistry)
+	trimmed := util.TrimURLScheme(shortURL)
+	m := URLFormatMatcher.FindStringSubmatch(shortURL)
+	if len(m) != 2 {
+		return nil, fmt.Errorf("URL must be of the form gs://<bucket> was: %s", shortURL)
+	}
+
+	return &GCSRegistry{
+			name:       name,
+			shortURL:   trimmed,
+			format:     common.RegistryFormat(format),
+			httpClient: httpClient,
+			service:    gcsService,
+			bucket:     m[1],
+		},
+		nil
+}
+
+func (g GCSRegistry) GetRegistryName() string {
+	return g.name
+}
+
+func (g GCSRegistry) GetBucket() string {
+	return g.bucket
+}
+
+func (g GCSRegistry) GetRegistryType() common.RegistryType {
+	return common.GCSRegistryType
+}
+
+// ListTypes lists types in this registry whose string values conform to the
+// supplied regular expression, or all types, if the regular expression is nil.
+func (g GCSRegistry) ListTypes(regex *regexp.Regexp) ([]Type, error) {
+	// List all files in the bucket/prefix that contain the
+	types := []Type{}
+
+	// List all objects in a bucket using pagination
+	pageToken := ""
+	for {
+		call := g.service.Objects.List(g.bucket)
+		call.Delimiter("/")
+		if pageToken != "" {
+			call = call.PageToken(pageToken)
+		}
+		res, err := call.Do()
+		if err != nil {
+			return []Type{}, err
+		}
+		for _, object := range res.Items {
+			// Charts should be named bucket/chart-X.Y.Z.tgz, so tease apart the version here
+			m := ChartFormatMatcher.FindStringSubmatch(object.Name)
+			if len(m) != 3 {
+				continue
+			}
+
+			t, err := NewType("", m[1], m[2])
+			if err != nil {
+				return []Type{}, fmt.Errorf("can't create a type type at path %#v", err)
+			}
+			types = append(types, t)
+		}
+		if pageToken = res.NextPageToken; pageToken == "" {
+			break
+		}
+	}
+	return types, nil
+}
+
+func (g GCSRegistry) GetRegistryFormat() common.RegistryFormat {
+	return common.CollectionRegistry
+}
+
+func (g GCSRegistry) GetRegistryShortURL() string {
+	return g.shortURL
+}
+
+// GetDownloadURLs fetches the download URLs for a given Chart
+func (g GCSRegistry) GetDownloadURLs(t Type) ([]*url.URL, error) {
+	call := g.service.Objects.List(g.bucket)
+	call.Delimiter("/")
+	call.Prefix(t.String())
+	res, err := call.Do()
+	ret := []*url.URL{}
+	if err != nil {
+		return ret, err
+	}
+	for _, object := range res.Items {
+		log.Printf("Found: %s", object.Name)
+		u, err := url.Parse(object.MediaLink)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse URL from %s: %s", object.MediaLink, err)
+		}
+		ret = append(ret, u)
+	}
+	return ret, err
+}
+
+func (g GCSRegistry) Do(req *http.Request) (resp *http.Response, err error) {
+	return g.httpClient.Do(req)
+}

registry/github_package_registry.go

@@ -22,6 +22,7 @@ import (
 
 	"fmt"
 	"log"
+	"net/http"
 	"net/url"
 	"regexp"
 	"strings"
@@ -41,7 +42,7 @@ type GithubPackageRegistry struct {
 }
 
 // NewGithubPackageRegistry creates a GithubPackageRegistry.
-func NewGithubPackageRegistry(name, shortURL string, service GithubRepositoryService, client *github.Client) (*GithubPackageRegistry, error) {
+func NewGithubPackageRegistry(name, shortURL string, service GithubRepositoryService, httpClient *http.Client, client *github.Client) (*GithubPackageRegistry, error) {
 	format := fmt.Sprintf("%s;%s", common.UnversionedRegistry, common.OneLevelRegistry)
 	if service == nil {
 		if client == nil {
@@ -51,7 +52,7 @@ func NewGithubPackageRegistry(name, shortURL string, service GithubRepositorySer
 		}
 	}
 
-	gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), service)
+	gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), httpClient, service)
 	if err != nil {
 		return nil, err
 	}
@@ -151,3 +152,7 @@ func (g GithubPackageRegistry) MakeRepositoryPath(t Type) (string, error) {
 	// Construct the return path
 	return t.Name + "/manifests", nil
 }
+
+func (g GithubPackageRegistry) Do(req *http.Request) (resp *http.Response, err error) {
+	return g.httpClient.Do(req)
+}

registry/github_registry.go

@@ -22,6 +22,7 @@ import (
 	"github.com/kubernetes/deployment-manager/util"
 
 	"fmt"
+	"net/http"
 	"strings"
 )
 
@@ -38,6 +39,7 @@ type githubRegistry struct {
 	format         common.RegistryFormat
 	credentialName string
 	service        GithubRepositoryService
+	httpClient     *http.Client
 }
 
 // GithubRepositoryService defines the interface that's defined in github.com/go-github/repos_contents.go GetContents method.
@@ -54,7 +56,7 @@ type GithubRepositoryService interface {
 }
 
 // newGithubRegistry creates a githubRegistry.
-func newGithubRegistry(name, shortURL string, format common.RegistryFormat, service GithubRepositoryService) (*githubRegistry, error) {
+func newGithubRegistry(name, shortURL string, format common.RegistryFormat, httpClient *http.Client, service GithubRepositoryService) (*githubRegistry, error) {
 	trimmed := util.TrimURLScheme(shortURL)
 	owner, repository, path, err := parseGithubShortURL(trimmed)
 	if err != nil {
@@ -69,6 +71,7 @@ func newGithubRegistry(name, shortURL string, format common.RegistryFormat, serv
 		path:       path,
 		format:     format,
 		service:    service,
+		httpClient: httpClient,
 	}, nil
 }
 

registry/github_template_registry.go

@@ -22,6 +22,7 @@ import (
 
 	"fmt"
 	"log"
+	"net/http"
 	"net/url"
 	"regexp"
 	"strings"
@@ -54,13 +55,13 @@ type GithubTemplateRegistry struct {
 }
 
 // NewGithubTemplateRegistry creates a GithubTemplateRegistry.
-func NewGithubTemplateRegistry(name, shortURL string, service GithubRepositoryService, client *github.Client) (*GithubTemplateRegistry, error) {
+func NewGithubTemplateRegistry(name, shortURL string, service GithubRepositoryService, httpClient *http.Client, client *github.Client) (*GithubTemplateRegistry, error) {
 	format := fmt.Sprintf("%s;%s", common.VersionedRegistry, common.CollectionRegistry)
 	if service == nil {
 		service = client.Repositories
 	}
 
-	gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), service)
+	gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), httpClient, service)
 	if err != nil {
 		return nil, err
 	}
@@ -212,3 +213,7 @@ func (g GithubTemplateRegistry) MakeRepositoryPath(t Type) (string, error) {
 	}
 	return p + t.Name + "/" + t.GetVersion(), nil
 }
+
+func (g GithubTemplateRegistry) Do(req *http.Request) (resp *http.Response, err error) {
+	return g.httpClient.Do(req)
+}

registry/inmem_credential_provider.go

@@ -38,6 +38,6 @@ func (fcp *InmemCredentialProvider) GetCredential(name string) (*common.Registry
 }
 
 func (fcp *InmemCredentialProvider) SetCredential(name string, credential *common.RegistryCredential) error {
-	fcp.credentials[name] = &common.RegistryCredential{credential.APIToken, credential.BasicAuth}
+	fcp.credentials[name] = &common.RegistryCredential{credential.APIToken, credential.BasicAuth, credential.ServiceAccount}
 	return nil
 }

registry/inmem_registry_service.go

@@ -51,6 +51,14 @@ func NewInmemRegistryService() common.RegistryService {
 		CredentialName: "default",
 	})
 
+	gFormat := fmt.Sprintf("%s", common.CollectionRegistry)
+	rs.Create(&common.Registry{
+		Name:   "charts_gcs",
+		Type:   common.GCSRegistryType,
+		URL:    "gs://helm-charts-test",
+		Format: common.RegistryFormat(gFormat),
+	})
+
 	return rs
 }
 

registry/registry.go

@@ -18,6 +18,7 @@ package registry
 
 import (
 	"github.com/kubernetes/deployment-manager/common"
+	"github.com/kubernetes/deployment-manager/util"
 
 	"fmt"
 	"net/url"
@@ -29,6 +30,9 @@ import (
 // used in a Deployment Manager configuration. There can be multiple
 // registry implementations.
 type Registry interface {
+	// Also handles http.Client.Do method for authenticated File accesses
+	util.HTTPDoer
+
 	// GetRegistryName returns the name of this registry
 	GetRegistryName() string
 	// GetRegistryType returns the type of this registry.
@@ -56,6 +60,13 @@ type GithubRegistry interface {
 	GetRegistryPath() string
 }
 
+// ObjectStorageRegistry abstracts a registry that resides in an Object Storage, for
+// example Google Cloud Storage or AWS S3, etc.
+type ObjectStorageRegistry interface {
+	Registry // An ObjectStorageRegistry is a Registry.
+	GetBucket() string
+}
+
 type Type struct {
 	Collection string
 	Name       string

registry/registryprovider.go

@@ -21,9 +21,12 @@ import (
 	"github.com/kubernetes/deployment-manager/common"
 	"github.com/kubernetes/deployment-manager/util"
 	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/google"
+	storage "google.golang.org/api/storage/v1"
 
 	"fmt"
 	"log"
+	"net/http"
 	"net/url"
 	"regexp"
 	"strings"
@@ -40,15 +43,16 @@ type registryProvider struct {
 	sync.RWMutex
 	rs         common.RegistryService
 	grp        GithubRegistryProvider
+	gcsrp      GCSRegistryProvider
 	cp         common.CredentialProvider
 	registries map[string]Registry
 }
 
-func NewDefaultRegistryProvider(cp common.CredentialProvider) RegistryProvider {
-	return NewRegistryProvider(nil, NewGithubRegistryProvider(cp), cp)
+func NewDefaultRegistryProvider(cp common.CredentialProvider, rs common.RegistryService) RegistryProvider {
+	return NewRegistryProvider(rs, NewGithubRegistryProvider(cp), NewGCSRegistryProvider(cp), cp)
 }
 
-func NewRegistryProvider(rs common.RegistryService, grp GithubRegistryProvider, cp common.CredentialProvider) RegistryProvider {
+func NewRegistryProvider(rs common.RegistryService, grp GithubRegistryProvider, gcsrp GCSRegistryProvider, cp common.CredentialProvider) RegistryProvider {
 	if rs == nil {
 		rs = NewInmemRegistryService()
 	}
@@ -61,11 +65,27 @@ func NewRegistryProvider(rs common.RegistryService, grp GithubRegistryProvider,
 		grp = NewGithubRegistryProvider(cp)
 	}
 
+	if gcsrp == nil {
+		gcsrp = NewGCSRegistryProvider(cp)
+	}
+
 	registries := make(map[string]Registry)
-	rp := &registryProvider{rs: rs, grp: grp, cp: cp, registries: registries}
+	rp := &registryProvider{rs: rs, grp: grp, gcsrp: gcsrp, cp: cp, registries: registries}
 	return rp
 }
 
+func (rp registryProvider) getRegistry(cr common.Registry) (Registry, error) {
+	switch cr.Type {
+	case common.GithubRegistryType:
+		return rp.grp.GetGithubRegistry(cr)
+	case common.GCSRegistryType:
+		log.Printf("Creating a bigstore client using %#v", rp.gcsrp)
+		return rp.gcsrp.GetGCSRegistry(cr)
+	default:
+		return nil, fmt.Errorf("unknown registry type: %s", cr.Type)
+	}
+}
+
 func (rp registryProvider) GetRegistryByShortURL(URL string) (Registry, error) {
 	rp.RLock()
 	defer rp.RUnlock()
@@ -77,7 +97,7 @@ func (rp registryProvider) GetRegistryByShortURL(URL string) (Registry, error) {
 			return nil, err
 		}
 
-		r, err := rp.grp.GetGithubRegistry(*cr)
+		r, err := rp.getRegistry(*cr)
 		if err != nil {
 			return nil, err
 		}
@@ -111,7 +131,7 @@ func (rp registryProvider) GetRegistryByName(registryName string) (Registry, err
 		return nil, err
 	}
 
-	r, err := rp.grp.GetGithubRegistry(*cr)
+	r, err := rp.getRegistry(*cr)
 	if err != nil {
 		return nil, err
 	}
@@ -143,21 +163,21 @@ type githubRegistryProvider struct {
 // NewGithubRegistryProvider creates a GithubRegistryProvider.
 func NewGithubRegistryProvider(cp common.CredentialProvider) GithubRegistryProvider {
 	if cp == nil {
-		panic(fmt.Errorf("CP IS NIL: %v", cp))
+		panic(fmt.Errorf("cp is nil: %v", cp))
 	}
 	return &githubRegistryProvider{cp: cp}
 }
 
-func (grp githubRegistryProvider) createGithubClient(credentialName string) (*github.Client, error) {
+func (grp githubRegistryProvider) createGithubClient(credentialName string) (*http.Client, *github.Client, error) {
 	if credentialName == "" {
-		return github.NewClient(nil), nil
+		return http.DefaultClient, github.NewClient(nil), nil
 	}
 	c, err := grp.cp.GetCredential(credentialName)
 
 	if err != nil {
 		log.Printf("Failed to fetch credential %s: %v", credentialName, err)
 		log.Print("Trying to use unauthenticated client")
-		return github.NewClient(nil), nil
+		return http.DefaultClient, github.NewClient(nil), nil
 	}
 
 	if c != nil {
@@ -166,44 +186,90 @@ func (grp githubRegistryProvider) createGithubClient(credentialName string) (*gi
 				&oauth2.Token{AccessToken: string(c.APIToken)},
 			)
 			tc := oauth2.NewClient(oauth2.NoContext, ts)
-			return github.NewClient(tc), nil
+			return tc, github.NewClient(tc), nil
 		}
 		if c.BasicAuth.Username != "" && c.BasicAuth.Password != "" {
 			tp := github.BasicAuthTransport{
 				Username: c.BasicAuth.Username,
 				Password: c.BasicAuth.Password,
 			}
-			return github.NewClient(tp.Client()), nil
+			return tp.Client(), github.NewClient(tp.Client()), nil
 		}
 
 	}
-	return nil, fmt.Errorf("No suitable credential found for %s", credentialName)
+	return nil, nil, fmt.Errorf("No suitable credential found for %s", credentialName)
 
 }
 
 // GetGithubRegistry returns a new GithubRegistry. If there's a credential that is specified, will try
 // to fetch it and use it, and if there's no credential found, will fall back to unauthenticated client.
 func (grp githubRegistryProvider) GetGithubRegistry(cr common.Registry) (GithubRegistry, error) {
-	if cr.Type == common.GithubRegistryType {
-		// If there's a credential that we need to use, fetch it and create a client for it.
-		client, err := grp.createGithubClient(cr.CredentialName)
-		if err != nil {
-			return nil, err
-		}
+	// If there's a credential that we need to use, fetch it and create a client for it.
+	httpClient, client, err := grp.createGithubClient(cr.CredentialName)
+	if err != nil {
+		return nil, err
+	}
 
-		fMap := ParseRegistryFormat(cr.Format)
-		if fMap[common.UnversionedRegistry] && fMap[common.OneLevelRegistry] {
-			return NewGithubPackageRegistry(cr.Name, cr.URL, nil, client)
-		}
+	fMap := ParseRegistryFormat(cr.Format)
+	if fMap[common.UnversionedRegistry] && fMap[common.OneLevelRegistry] {
+		return NewGithubPackageRegistry(cr.Name, cr.URL, nil, httpClient, client)
+	}
 
-		if fMap[common.VersionedRegistry] && fMap[common.CollectionRegistry] {
-			return NewGithubTemplateRegistry(cr.Name, cr.URL, nil, client)
-		}
+	if fMap[common.VersionedRegistry] && fMap[common.CollectionRegistry] {
+		return NewGithubTemplateRegistry(cr.Name, cr.URL, nil, httpClient, client)
+	}
+
+	return nil, fmt.Errorf("unknown registry format: %s", cr.Format)
+}
+
+// GCSRegistryProvider is a factory for GCS Registry instances.
+type GCSRegistryProvider interface {
+	GetGCSRegistry(cr common.Registry) (ObjectStorageRegistry, error)
+}
+
+type gcsRegistryProvider struct {
+	cp common.CredentialProvider
+}
+
+// NewGCSRegistryProvider creates a GCSRegistryProvider.
+func NewGCSRegistryProvider(cp common.CredentialProvider) GCSRegistryProvider {
+	if cp == nil {
+		panic(fmt.Errorf("cp is nil: %v", cp))
+	}
+	return &gcsRegistryProvider{cp: cp}
+}
+
+// GetGCSRegistry returns a new Google Cloud Storage . If there's a credential that is specified, will try
+// to fetch it and use it, and if there's no credential found, will fall back to unauthenticated client.
+func (gcsrp gcsRegistryProvider) GetGCSRegistry(cr common.Registry) (ObjectStorageRegistry, error) {
+	// If there's a credential that we need to use, fetch it and create a client for it.
+	if cr.CredentialName == "" {
+		return nil, fmt.Errorf("No CredentialName specified for %s", cr.Name)
+	}
+	client, err := gcsrp.createGCSClient(cr.CredentialName)
+	if err != nil {
+		return nil, err
+	}
+	service, err := storage.New(client)
+	if err != nil {
+		log.Fatalf("Unable to create storage service: %v", err)
+	}
+	return NewGCSRegistry(cr.Name, cr.URL, client, service)
+}
 
-		return nil, fmt.Errorf("unknown registry format: %s", cr.Format)
+func (gcsrp gcsRegistryProvider) createGCSClient(credentialName string) (*http.Client, error) {
+	c, err := gcsrp.cp.GetCredential(credentialName)
+	if err != nil {
+		log.Printf("Failed to fetch credential %s: %v", credentialName, err)
+		return nil, fmt.Errorf("Failed to fetch Credential %s: %s", credentialName, err)
 	}
 
-	return nil, fmt.Errorf("unknown registry type: %s", cr.Type)
+	config, err := google.JWTConfigFromJSON([]byte(c.ServiceAccount), storage.DevstorageReadOnlyScope)
+
+	if err != nil {
+		log.Fatalf("Unable to parse client secret file to config: %v", err)
+	}
+	return config.Client(oauth2.NoContext), nil
 }
 
 // RE for a registry type that does support versions and has collections.
@@ -212,6 +278,9 @@ var TemplateRegistryMatcher = regexp.MustCompile("github.com/(.*)/(.*)/(.*)/(.*)
 // RE for a registry type that does not support versions and does not have collections.
 var PackageRegistryMatcher = regexp.MustCompile("github.com/(.*)/(.*)/(.*)")
 
+// RE for GCS storage
+var GCSRegistryMatcher = regexp.MustCompile("gs://(.*)/(.*)")
+
 // IsGithubShortType returns whether a given type is a type description in a short format to a github repository type.
 // For now, this means using github types:
 // github.com/owner/repo/qualifier/type:version
@@ -231,24 +300,31 @@ func IsGithubShortPackageType(t string) bool {
 	return PackageRegistryMatcher.MatchString(t)
 }
 
+// IsGCSShortType returns whether a given type is a type description in a short format to GCS
+func IsGCSShortType(t string) bool {
+	return strings.HasPrefix(t, "gs://")
+}
+
 // GetDownloadURLs checks a type to see if it is either a short git hub url or a fully specified URL
 // and returns the URLs that should be used to fetch it. If the url is not fetchable (primitive type
 // for example), it returns an empty slice.
-func GetDownloadURLs(rp RegistryProvider, t string) ([]string, error) {
+func GetDownloadURLs(rp RegistryProvider, t string) ([]string, Registry, error) {
 	if IsGithubShortType(t) {
 		return ShortTypeToDownloadURLs(rp, t)
 	} else if IsGithubShortPackageType(t) {
 		return ShortTypeToPackageDownloadURLs(rp, t)
+	} else if IsGCSShortType(t) {
+		return ShortTypeToGCSDownloadUrls(rp, t)
 	} else if util.IsHttpUrl(t) {
 		result, err := url.Parse(t)
 		if err != nil {
-			return nil, fmt.Errorf("cannot parse download URL %s: %s", t, err)
+			return nil, nil, fmt.Errorf("cannot parse download URL %s: %s", t, err)
 		}
 
-		return []string{result.String()}, nil
+		return []string{result.String()}, nil, nil
 	}
 
-	return []string{}, nil
+	return []string{}, nil, nil
 }
 
 // ShortTypeToDownloadURLs converts a github URL into downloadable URL from github.
@@ -256,15 +332,15 @@ func GetDownloadURLs(rp RegistryProvider, t string) ([]string, error) {
 // github.com/owner/repo/qualifier/type:version
 // for example:
 // github.com/kubernetes/application-dm-templates/storage/redis:v1
-func ShortTypeToDownloadURLs(rp RegistryProvider, t string) ([]string, error) {
+func ShortTypeToDownloadURLs(rp RegistryProvider, t string) ([]string, Registry, error) {
 	m := TemplateRegistryMatcher.FindStringSubmatch(t)
 	if len(m) != 6 {
-		return nil, fmt.Errorf("cannot parse short github url: %s", t)
+		return nil, nil, fmt.Errorf("cannot parse short github url: %s", t)
 	}
 
 	r, err := rp.GetRegistryByShortURL(t)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	if r == nil {
@@ -273,15 +349,15 @@ func ShortTypeToDownloadURLs(rp RegistryProvider, t string) ([]string, error) {
 
 	tt, err := NewType(m[3], m[4], m[5])
 	if err != nil {
-		return nil, err
+		return nil, r, err
 	}
 
 	urls, err := r.GetDownloadURLs(tt)
 	if err != nil {
-		return nil, err
+		return nil, r, err
 	}
 
-	return util.ConvertURLsToStrings(urls), err
+	return util.ConvertURLsToStrings(urls), r, err
 }
 
 // ShortTypeToPackageDownloadURLs converts a github URL into downloadable URLs from github.
@@ -289,26 +365,48 @@ func ShortTypeToDownloadURLs(rp RegistryProvider, t string) ([]string, error) {
 // github.com/owner/repo/type
 // for example:
 // github.com/helm/charts/cassandra
-func ShortTypeToPackageDownloadURLs(rp RegistryProvider, t string) ([]string, error) {
+func ShortTypeToPackageDownloadURLs(rp RegistryProvider, t string) ([]string, Registry, error) {
 	m := PackageRegistryMatcher.FindStringSubmatch(t)
 	if len(m) != 4 {
-		return nil, fmt.Errorf("Failed to parse short github url: %s", t)
+		return nil, nil, fmt.Errorf("Failed to parse short github url: %s", t)
 	}
 
 	r, err := rp.GetRegistryByShortURL(t)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	tt, err := NewType("", m[3], "")
 	if err != nil {
-		return nil, err
+		return nil, r, err
 	}
 
 	urls, err := r.GetDownloadURLs(tt)
 	if err != nil {
-		return nil, err
+		return nil, r, err
+	}
+
+	return util.ConvertURLsToStrings(urls), r, err
+}
+
+func ShortTypeToGCSDownloadUrls(rp RegistryProvider, t string) ([]string, Registry, error) {
+	m := GCSRegistryMatcher.FindStringSubmatch(t)
+	if len(m) != 3 {
+		return nil, nil, fmt.Errorf("Failed to parse short gs url: %s", t)
 	}
 
-	return util.ConvertURLsToStrings(urls), err
+	r, err := rp.GetRegistryByShortURL(t)
+	if err != nil {
+		return nil, nil, err
+	}
+	tt, err := NewType(m[1], m[2], "")
+	if err != nil {
+		return nil, r, err
+	}
+
+	urls, err := r.GetDownloadURLs(tt)
+	if err != nil {
+		return nil, r, err
+	}
+	return util.ConvertURLsToStrings(urls), r, err
 }

registry/registryprovider_test.go

@@ -22,7 +22,8 @@ import (
 
 func testUrlConversionDriver(rp RegistryProvider, tests map[string]TestURLAndError, t *testing.T) {
 	for in, expected := range tests {
-		actual, err := GetDownloadURLs(rp, in)
+		// TODO(vaikas): Test to make sure it's the right registry.
+		actual, _, err := GetDownloadURLs(rp, in)
 		if err != expected.Err {
 			t.Fatalf("failed on: %s : expected error %v but got %v", in, expected.Err, err)
 		}
@@ -45,7 +46,8 @@ func TestShortGithubUrlTemplateMapping(t *testing.T) {
 	}
 
 	grp := NewTestGithubRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps)
-	testUrlConversionDriver(NewRegistryProvider(nil, grp, NewInmemCredentialProvider()), tests, t)
+	// TODO(vaikas): XXXX FIXME Add gcsrp
+	testUrlConversionDriver(NewRegistryProvider(nil, grp, nil, NewInmemCredentialProvider()), tests, t)
 }
 
 func TestShortGithubUrlPackageMapping(t *testing.T) {
@@ -60,5 +62,6 @@ func TestShortGithubUrlPackageMapping(t *testing.T) {
 	}
 
 	grp := NewTestGithubRegistryProvider("github.com/helm/charts", githubUrlMaps)
-	testUrlConversionDriver(NewRegistryProvider(nil, grp, NewInmemCredentialProvider()), tests, t)
+	// TODO(vaikas): XXXX FIXME Add gcsrp
+	testUrlConversionDriver(NewRegistryProvider(nil, grp, nil, NewInmemCredentialProvider()), tests, t)
 }

registry/secrets_credential_provider.go

@@ -120,8 +120,6 @@ func (scp *SecretsCredentialProvider) SetCredential(name string, credential *com
 		log.Printf("yaml marshal failed for kubernetes object: %s: %v", name, err)
 		return err
 	}
-	log.Printf("Calling with: %s", string(ko))
-	o, err := scp.k.Create(string(ko))
-	log.Printf("Create returned: %s", o)
+	_, err = scp.k.Create(string(ko))
 	return err
 }

registry/testhelper.go

@@ -23,6 +23,7 @@ import (
 	"github.com/kubernetes/deployment-manager/util"
 
 	"fmt"
+	"net/http"
 	"net/url"
 	"regexp"
 	"strings"
@@ -53,7 +54,7 @@ func NewTestGithubRegistryProvider(shortURL string, responses map[Type]TestURLAn
 func (tgrp testGithubRegistryProvider) GetGithubRegistry(cr common.Registry) (GithubRegistry, error) {
 	trimmed := util.TrimURLScheme(cr.URL)
 	if strings.HasPrefix(trimmed, tgrp.shortURL) {
-		ghr, err := newGithubRegistry(cr.Name, trimmed, cr.Format, nil)
+		ghr, err := newGithubRegistry(cr.Name, trimmed, cr.Format, http.DefaultClient, nil)
 		if err != nil {
 			panic(fmt.Errorf("cannot create a github registry: %s", err))
 		}
@@ -80,3 +81,41 @@ func (tgr testGithubRegistry) GetDownloadURLs(t Type) ([]*url.URL, error) {
 
 	return []*url.URL{URL}, result.Err
 }
+
+func (g testGithubRegistry) Do(req *http.Request) (resp *http.Response, err error) {
+	return nil, fmt.Errorf("Not implemented yet")
+}
+
+type testGCSRegistryProvider struct {
+	shortURL  string
+	responses map[Type]TestURLAndError
+}
+
+type testGCSRegistry struct {
+	GCSRegistry
+	responses map[Type]TestURLAndError
+}
+
+func NewTestGCSRegistryProvider(shortURL string, responses map[Type]TestURLAndError) GCSRegistryProvider {
+	return testGCSRegistryProvider{
+		shortURL:  util.TrimURLScheme(shortURL),
+		responses: responses,
+	}
+}
+
+func (tgrp testGCSRegistryProvider) GetGCSRegistry(cr common.Registry) (ObjectStorageRegistry, error) {
+	trimmed := util.TrimURLScheme(cr.URL)
+	if strings.HasPrefix(trimmed, tgrp.shortURL) {
+		gcsr, err := NewGCSRegistry(cr.Name, trimmed, http.DefaultClient, nil)
+		if err != nil {
+			panic(fmt.Errorf("cannot create gcs registry: %s", err))
+		}
+
+		return &testGCSRegistry{
+			GCSRegistry: *gcsr,
+			responses:   tgrp.responses,
+		}, nil
+	}
+
+	panic(fmt.Errorf("unknown registry: %v", cr))
+}

util/httpclient.go

@@ -6,7 +6,7 @@ you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
     http://www.apache.org/licenses/LICENSE-2.0
- 
+
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -72,6 +72,10 @@ type httpClient struct {
 	sleep   Sleeper
 }
 
+func DefaultHTTPClient() HTTPClient {
+	return NewHTTPClient(3, http.DefaultClient, NewSleeper())
+}
+
 // NewHTTPClient returns a new HTTPClient.
 func NewHTTPClient(retries uint, c HTTPDoer, s Sleeper) HTTPClient {
 	ret := httpClient{}

util/kubernetes_kubectl.go

@@ -126,15 +126,17 @@ func (k *KubernetesKubectl) execute(args []string, input string) (string, error)
 	cmd.Stderr = combined
 
 	if err := cmd.Start(); err != nil {
-		log.Printf("cannot start kubectl %s %#v", combined.String(), err)
-		return combined.String(), err
+		e := fmt.Errorf("cannot start kubectl %s %#v", combined.String(), err)
+		log.Printf("%s", e)
+		return combined.String(), e
 	}
 
 	if err := cmd.Wait(); err != nil {
-		log.Printf("kubectl failed: %s  %#v", combined.String(), err)
-		return combined.String(), err
+		e := fmt.Errorf("kubectl failed %s", combined.String())
+		log.Printf("%s", e)
+		return combined.String(), e
 	}
-	log.Printf("kubectl succeeded: SysTime: %v UserTime: %v\n%v",
-		cmd.ProcessState.SystemTime(), cmd.ProcessState.UserTime(), combined.String())
+	log.Printf("kubectl succeeded: SysTime: %v UserTime: %v",
+		cmd.ProcessState.SystemTime(), cmd.ProcessState.UserTime())
 	return combined.String(), nil
 }