Add GCS as a registry provider. Add a way to get a file using credentials from a…

Add GCS as a registry provider. Add a way to get a file using credentials from a registry to support private repos. Add ability to create / update a registry through manager

common/types.go

@@ -182,11 +182,13 @@ type BasicAuthCredential struct {
 }
 
 type APITokenCredential string
+type JWTTokenCredential string
 
 // Credential used to access the repository
 type RegistryCredential struct {
 	APIToken       APITokenCredential  `json:"apitoken,omitempty"`
 	BasicAuth      BasicAuthCredential `json:"basicauth,omitempty"`
+	ServiceAccount JWTTokenCredential  `json:"serviceaccount,omitempty"`
 }
 
 // Registry describes a template registry
@@ -204,6 +206,7 @@ type RegistryType string
 
 const (
 	GithubRegistryType RegistryType = "github"
+	GCSRegistryType    RegistryType = "gcs"
 )
 
 // RegistryFormat is a semi-colon delimited string that describes the format

dm/dm.go

@@ -50,6 +50,8 @@ var (
 	username          = flag.String("username", "", "Github user name that overrides GITHUB_USERNAME environment variable")
 	password          = flag.String("password", "", "Github password that overrides GITHUB_PASSWORD environment variable")
 	apitoken          = flag.String("apitoken", "", "Github api token that overrides GITHUB_API_TOKEN environment variable")
+	serviceaccount    = flag.String("serviceaccount", "", "Service account file containing JWT token")
+	registryfile      = flag.String("registryfile", "", "File containing registry specification")
 )
 
 var commands = []string{
@@ -67,6 +69,7 @@ var commands = []string{
 	"describe \t\t Describes the named template in a given template registry",
 	"getcredential \t\t Gets the named credential used by a registry",
 	"setcredential \t\t Sets a credential used by a registry",
+	"createregistry \t\t Creates a registry that holds charts",
 }
 
 var usage = func() {
@@ -86,7 +89,7 @@ var usage = func() {
 	os.Exit(0)
 }
 
-func getGithubCredential() *common.RegistryCredential {
+func getCredential() *common.RegistryCredential {
 	*apitoken = strings.TrimSpace(*apitoken)
 	if *apitoken == "" {
 		*apitoken = strings.TrimSpace(os.Getenv("GITHUB_API_TOKEN"))
@@ -117,6 +120,15 @@ func getGithubCredential() *common.RegistryCredential {
 		}
 	}
 
+	if *serviceaccount != "" {
+		b, err := ioutil.ReadFile(*serviceaccount)
+		if err != nil {
+			log.Fatalf("Unable to read service account file: %v", err)
+		}
+		return &common.RegistryCredential{
+			ServiceAccount: common.JWTTokenCredential(string(b)),
+		}
+	}
 	return nil
 }
 
@@ -124,6 +136,13 @@ func init() {
 	flag.Usage = usage
 }
 
+func getRegistry() ([]byte, error) {
+	if *registryfile == "" {
+		log.Fatalf("No registryfile specified (-registryfile)")
+	}
+	return ioutil.ReadFile(*registryfile)
+}
+
 func main() {
 	defer func() {
 		result := recover()
@@ -145,6 +164,10 @@ func execute() {
 
 	switch args[0] {
 	case "templates":
+		if len(args) < 2 {
+			fmt.Fprintln(os.Stderr, "No registry name supplied")
+			usage()
+		}
 		path := fmt.Sprintf("registries/%s/types", args[1])
 		callService(path, "GET", "list templates", nil)
 	case "describe":
@@ -162,7 +185,7 @@ func execute() {
 		path := fmt.Sprintf("credentials/%s", args[1])
 		callService(path, "GET", "get credential", nil)
 	case "setcredential":
-		c := getGithubCredential()
+		c := getCredential()
 		if c == nil {
 			panic(fmt.Errorf("Failed to create a credential from flags/arguments"))
 		}
@@ -172,7 +195,14 @@ func execute() {
 		}
 
 		path := fmt.Sprintf("credentials/%s", args[1])
-		callService(path, "POST", "get credential", ioutil.NopCloser(bytes.NewReader(y)))
+		callService(path, "POST", "set credential", ioutil.NopCloser(bytes.NewReader(y)))
+	case "createregistry":
+		reg, err := getRegistry()
+		if err != nil {
+			panic(fmt.Errorf("Failed to create a registry from arguments: %#v", err))
+		}
+		path := fmt.Sprintf("registries/%s", args[1])
+		callService(path, "POST", "set registry", ioutil.NopCloser(bytes.NewReader(reg)))
 	case "get":
 		if len(args) < 2 {
 			fmt.Fprintln(os.Stderr, "No deployment name supplied")
@@ -300,13 +330,20 @@ func describeType(args []string) {
 		os.Exit(1)
 	}
 
-	tUrls := getDownloadURLs(args[1])
+	tUrls := getDownloadURLs(url.QueryEscape(args[1]))
 	if len(tUrls) == 0 {
 		panic(fmt.Errorf("Invalid type name, must be a template URL or in the form \"<type-name>:<version>\": %s", args[1]))
 	}
 
-	schemaUrl := tUrls[0] + ".schema"
-	fmt.Println(callHttp(schemaUrl, "GET", "get schema for type ("+tUrls[0]+")", nil))
+	if !strings.Contains(tUrls[0], ".prov") {
+		// It's not a chart, so grab the schema
+		path := fmt.Sprintf("registries/%s/download?file=%s.schema", *template_registry, url.QueryEscape(tUrls[0]))
+		callService(path, "GET", "get schema for type ("+tUrls[0]+")", nil)
+	} else {
+		// It's a chart, so grab the provenance file
+		path := fmt.Sprintf("registries/%s/download?file=%s", *template_registry, url.QueryEscape(tUrls[0]))
+		callService(path, "GET", "get file", nil)
+	}
 }
 
 // getDownloadURLs returns URLs for a type in the given registry

manager/deployments.go

@@ -54,8 +54,10 @@ var deployments = []Route{
 	{"ListTypeInstances", "/types/{type}/instances", "GET", listTypeInstancesHandlerFunc, ""},
 	{"ListRegistries", "/registries", "GET", listRegistriesHandlerFunc, ""},
 	{"GetRegistry", "/registries/{registry}", "GET", getRegistryHandlerFunc, ""},
+	{"CreateRegistry", "/registries/{registry}", "POST", createRegistryHandlerFunc, "JSON"},
 	{"ListRegistryTypes", "/registries/{registry}/types", "GET", listRegistryTypesHandlerFunc, ""},
 	{"GetDownloadURLs", "/registries/{registry}/types/{type}", "GET", getDownloadURLsHandlerFunc, ""},
+	{"GetFile", "/registries/{registry}/download", "GET", getFileHandlerFunc, ""},
 	{"CreateCredential", "/credentials/{credential}", "POST", createCredentialHandlerFunc, "JSON"},
 	{"GetCredential", "/credentials/{credential}", "GET", getCredentialHandlerFunc, ""},
 }
@@ -97,12 +99,12 @@ func init() {
 }
 
 func newManager(cp common.CredentialProvider) manager.Manager {
-	registryProvider := registry.NewDefaultRegistryProvider(cp)
-	resolver := manager.NewTypeResolver(registryProvider)
+	service := registry.NewInmemRegistryService()
+	registryProvider := registry.NewDefaultRegistryProvider(cp, service)
+	resolver := manager.NewTypeResolver(registryProvider, util.DefaultHTTPClient())
 	expander := manager.NewExpander(getServiceURL(*expanderURL, *expanderName), resolver)
 	deployer := manager.NewDeployer(getServiceURL(*deployerURL, *deployerName))
 	r := repository.NewMapBasedRepository()
-	service := registry.NewInmemRegistryService()
 	credentialProvider := cp
 	return manager.NewManager(expander, deployer, r, registryProvider, service, credentialProvider)
 }
@@ -378,6 +380,49 @@ func getRegistryHandlerFunc(w http.ResponseWriter, r *http.Request) {
 	util.LogHandlerExitWithJSON(handler, w, cr, http.StatusOK)
 }
 
+func getRegistry(w http.ResponseWriter, r *http.Request, handler string) *common.Registry {
+	util.LogHandlerEntry(handler, r)
+	j, err := getJsonFromRequest(w, r, handler)
+	if err != nil {
+		return nil
+	}
+
+	t := &common.Registry{}
+	if err := json.Unmarshal(j, t); err != nil {
+		e := fmt.Errorf("%v\n%v", err, string(j))
+		util.LogAndReturnError(handler, http.StatusBadRequest, e, w)
+		return nil
+	}
+
+	return t
+}
+
+func createRegistryHandlerFunc(w http.ResponseWriter, r *http.Request) {
+	handler := "manager: create registry"
+	util.LogHandlerEntry(handler, r)
+	defer r.Body.Close()
+	registryName, err := getPathVariable(w, r, "registry", handler)
+	if err != nil {
+		return
+	}
+
+	reg := getRegistry(w, r, handler)
+	if reg.Name != registryName {
+		e := fmt.Errorf("Registry name does not match %s != %s", reg.Name, registryName)
+		util.LogAndReturnError(handler, http.StatusBadRequest, e, w)
+		return
+	}
+	if reg != nil {
+		err = backend.CreateRegistry(reg)
+		if err != nil {
+			util.LogAndReturnError(handler, http.StatusBadRequest, err, w)
+			return
+		}
+	}
+
+	util.LogHandlerExitWithJSON(handler, w, reg, http.StatusOK)
+}
+
 func listRegistryTypesHandlerFunc(w http.ResponseWriter, r *http.Request) {
 	handler := "manager: list registry types"
 	util.LogHandlerEntry(handler, r)
@@ -437,6 +482,28 @@ func getDownloadURLsHandlerFunc(w http.ResponseWriter, r *http.Request) {
 	util.LogHandlerExitWithJSON(handler, w, urls, http.StatusOK)
 }
 
+func getFileHandlerFunc(w http.ResponseWriter, r *http.Request) {
+	handler := "manager: get file"
+	util.LogHandlerEntry(handler, r)
+	registryName, err := getPathVariable(w, r, "registry", handler)
+	if err != nil {
+		return
+	}
+
+	file := r.FormValue("file")
+	if file == "" {
+		return
+	}
+
+	b, err := backend.GetFile(registryName, file)
+	if err != nil {
+		util.LogAndReturnError(handler, http.StatusBadRequest, err, w)
+		return
+	}
+
+	util.LogHandlerExitWithJSON(handler, w, b, http.StatusOK)
+}
+
 func getCredential(w http.ResponseWriter, r *http.Request, handler string) *common.RegistryCredential {
 	util.LogHandlerEntry(handler, r)
 	j, err := getJsonFromRequest(w, r, handler)

manager/manager/manager.go

@@ -26,6 +26,7 @@ import (
 	"github.com/kubernetes/deployment-manager/common"
 	"github.com/kubernetes/deployment-manager/manager/repository"
 	"github.com/kubernetes/deployment-manager/registry"
+	"github.com/kubernetes/deployment-manager/util"
 )
 
 // Manager manages a persistent set of Deployments.
@@ -55,6 +56,7 @@ type Manager interface {
 	// Registry Types
 	ListRegistryTypes(registryName string, regex *regexp.Regexp) ([]registry.Type, error)
 	GetDownloadURLs(registryName string, t registry.Type) ([]*url.URL, error)
+	GetFile(registryName string, url string) (string, error)
 
 	// Credentials
 	CreateCredential(name string, c *common.RegistryCredential) error
@@ -394,6 +396,21 @@ func (m *manager) GetDownloadURLs(registryName string, t registry.Type) ([]*url.
 	return r.GetDownloadURLs(t)
 }
 
+// GetFile returns a file from the backing registry
+func (m *manager) GetFile(registryName string, url string) (string, error) {
+	r, err := m.registryProvider.GetRegistryByName(registryName)
+	if err != nil {
+		return "", err
+	}
+
+	getter := util.NewHTTPClient(3, r, util.NewSleeper())
+	body, _, err := getter.Get(url)
+	if err != nil {
+		return "", err
+	}
+	return body, nil
+}
+
 // CreateCredential creates a credential that can be used to authenticate to registry
 func (m *manager) CreateCredential(name string, c *common.RegistryCredential) error {
 	return m.credentialProvider.SetCredential(name, c)

manager/manager/manager_test.go

@@ -255,7 +255,7 @@ var testRepository = newRepositoryStub()
 var testDeployer = newDeployerStub()
 var testRegistryService = registry.NewInmemRegistryService()
 var testCredentialProvider = registry.NewInmemCredentialProvider()
-var testProvider = registry.NewRegistryProvider(nil, registry.NewTestGithubRegistryProvider("", nil), testCredentialProvider)
+var testProvider = registry.NewRegistryProvider(nil, registry.NewTestGithubRegistryProvider("", nil), registry.NewTestGCSRegistryProvider("", nil), testCredentialProvider)
 var testManager = NewManager(testExpander, testDeployer, testRepository, testProvider, testRegistryService, testCredentialProvider)
 
 func TestListDeployments(t *testing.T) {

manager/manager/typeresolver.go

@@ -19,7 +19,6 @@ package manager
 import (
 	"fmt"
 	"net/http"
-	"time"
 
 	"github.com/kubernetes/deployment-manager/common"
 	"github.com/kubernetes/deployment-manager/registry"
@@ -40,26 +39,23 @@ type TypeResolver interface {
 }
 
 type typeResolver struct {
-	getter  util.HTTPClient
 	maxUrls int
 	rp      registry.RegistryProvider
+	c       util.HTTPClient
+}
+
+type fetchableURL struct {
+	registry registry.Registry
+	url      string
 }
 
 type fetchUnit struct {
-	urls []string
+	urls []fetchableURL
 }
 
 // NewTypeResolver returns a new initialized TypeResolver.
-func NewTypeResolver(rp registry.RegistryProvider) TypeResolver {
-	ret := &typeResolver{}
-	client := http.DefaultClient
-	//TODO (iantw): Make this a flag
-	timeout, _ := time.ParseDuration("10s")
-	client.Timeout = timeout
-	ret.getter = util.NewHTTPClient(3, client, util.NewSleeper())
-	ret.maxUrls = maxURLImports
-	ret.rp = rp
-	return ret
+func NewTypeResolver(rp registry.RegistryProvider, c util.HTTPClient) TypeResolver {
+	return &typeResolver{maxUrls: maxURLImports, rp: rp, c: c}
 }
 
 func resolverError(c *common.Configuration, err error) error {
@@ -67,7 +63,13 @@ func resolverError(c *common.Configuration, err error) error {
 		c, err)
 }
 
-func performHTTPGet(g util.HTTPClient, u string, allowMissing bool) (content string, err error) {
+func (tr *typeResolver) performHTTPGet(d util.HTTPDoer, u string, allowMissing bool) (content string, err error) {
+	var g util.HTTPClient
+	if d == nil {
+		g = tr.c
+	} else {
+		g = util.NewHTTPClient(3, d, util.NewSleeper())
+	}
 	r, code, err := g.Get(u)
 	if err != nil {
 		return "", err
@@ -100,7 +102,7 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
 	toFetch := make([]*fetchUnit, 0, tr.maxUrls)
 	for _, r := range config.Resources {
 		// Map the type to a fetchable URL (if applicable) or skip it if it's a non-fetchable type (primitive for example).
-		urls, err := registry.GetDownloadURLs(tr.rp, r.Type)
+		urls, registry, err := registry.GetDownloadURLs(tr.rp, r.Type)
 		if err != nil {
 			return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", r.Type, err))
 		}
@@ -108,14 +110,14 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
 			f := &fetchUnit{}
 			for _, u := range urls {
 				if len(u) > 0 {
-					f.urls = append(f.urls, u)
+					f.urls = append(f.urls, fetchableURL{registry, u})
 					// Add to existing map so it is not fetched multiple times.
 					existing[r.Type] = true
 				}
 			}
 			if len(f.urls) > 0 {
 				toFetch = append(toFetch, f)
-				fetched[f.urls[0]] = append(fetched[f.urls[0]], &common.ImportFile{Name: r.Type, Path: f.urls[0]})
+				fetched[f.urls[0].url] = append(fetched[f.urls[0].url], &common.ImportFile{Name: r.Type, Path: f.urls[0].url})
 			}
 		}
 	}
@@ -138,14 +140,14 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
 		templates := []string{}
 		url := toFetch[0].urls[0]
 		for _, u := range toFetch[0].urls {
-			template, err := performHTTPGet(tr.getter, u, false)
+			template, err := tr.performHTTPGet(u.registry, u.url, false)
 			if err != nil {
 				return nil, resolverError(config, err)
 			}
 			templates = append(templates, template)
 		}
 
-		for _, i := range fetched[url] {
+		for _, i := range fetched[url.url] {
 			template, err := parseContent(templates)
 			if err != nil {
 				return nil, resolverError(config, err)
@@ -153,8 +155,8 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
 			i.Content = template
 		}
 
-		schemaURL := url + schemaSuffix
-		sch, err := performHTTPGet(tr.getter, schemaURL, true)
+		schemaURL := url.url + schemaSuffix
+		sch, err := tr.performHTTPGet(url.registry, schemaURL, true)
 		if err != nil {
 			return nil, resolverError(config, err)
 		}
@@ -168,7 +170,7 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
 			for _, v := range s.Imports {
 				i := &common.ImportFile{Name: v.Name}
 				var existingSchema string
-				urls, conversionErr := registry.GetDownloadURLs(tr.rp, v.Path)
+				urls, registry, conversionErr := registry.GetDownloadURLs(tr.rp, v.Path)
 				if conversionErr != nil {
 					return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", v.Path, conversionErr))
 				}
@@ -180,7 +182,7 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
 				for _, u := range urls {
 					if len(fetched[u]) == 0 {
 						// If this import URL is new to us, add it to the URLs to fetch.
-						toFetch = append(toFetch, &fetchUnit{[]string{u}})
+						toFetch = append(toFetch, &fetchUnit{[]fetchableURL{fetchableURL{registry, u}}})
 					} else {
 						// If this is not a new import URL and we've already fetched its contents,
 						// reuse them. Also, check if we also found a schema for that import URL and
@@ -201,7 +203,7 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
 			}
 
 			// Add the schema we've fetched as the schema for any templates which used this URL.
-			for _, i := range fetched[url] {
+			for _, i := range fetched[url.url] {
 				schemaImportName := i.Name + schemaSuffix
 				fetched[schemaURL] = append(fetched[schemaURL],
 					&common.ImportFile{Name: schemaImportName, Content: sch})

manager/manager/typeresolver_test.go

@@ -18,6 +18,7 @@ package manager
 
 import (
 	"errors"
+	"log"
 	"net/http"
 	"reflect"
 	"strings"
@@ -50,19 +51,20 @@ type testGetter struct {
 	test      *testing.T
 }
 
-func (tg *testGetter) Get(url string) (body string, code int, err error) {
+func (tg testGetter) Get(url string) (body string, code int, err error) {
 	tg.count = tg.count + 1
 	ret := tg.responses[url]
-
+	log.Printf("GET RETURNING: '%s' '%d'", ret.resp, tg.count)
 	return ret.resp, ret.code, ret.err
 }
 
 func testDriver(c resolverTestCase, t *testing.T) {
 	g := &testGetter{test: t, responses: c.responses}
+	log.Printf("getter: %#v", g)
 	r := &typeResolver{
-		getter:  g,
 		maxUrls: 5,
 		rp:      c.registryProvider,
+		c:       g,
 	}
 
 	conf := &common.Configuration{}
@@ -74,7 +76,7 @@ func testDriver(c resolverTestCase, t *testing.T) {
 	result, err := r.ResolveTypes(conf, c.imports)
 
 	if g.count != c.urlcount {
-		t.Errorf("Expected %d url GETs but only %d found", c.urlcount, g.count)
+		t.Errorf("Expected %d url GETs but only %d found %#v", c.urlcount, g.count, g)
 	}
 
 	if (err != nil && c.expectedErr == nil) || (err == nil && c.expectedErr != nil) {
@@ -307,13 +309,19 @@ func TestShortGithubUrl(t *testing.T) {
 		registry.NewTypeOrDie("common", "replicatedservice", "v2"): registry.TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py", nil},
 	}
 
+	gcsUrlMaps := map[registry.Type]registry.TestURLAndError{
+		registry.NewTypeOrDie("common", "replicatedservice", "v1"): registry.TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil},
+		registry.NewTypeOrDie("common", "replicatedservice", "v2"): registry.TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py", nil},
+	}
+
 	grp := registry.NewTestGithubRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps)
+	gcsrp := registry.NewTestGCSRegistryProvider("gs://charts", gcsUrlMaps)
 	test := resolverTestCase{
 		config:           templateShortGithubTemplate,
 		importOut:        finalImports,
 		urlcount:         4,
 		responses:        responses,
-		registryProvider: registry.NewRegistryProvider(nil, grp, registry.NewInmemCredentialProvider()),
+		registryProvider: registry.NewRegistryProvider(nil, grp, gcsrp, registry.NewInmemCredentialProvider()),
 	}
 
 	testDriver(test, t)

registry/gcs_registry.go

+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package registry
+
+import (
+	"github.com/kubernetes/deployment-manager/common"
+	"github.com/kubernetes/deployment-manager/util"
+
+	//        "golang.org/x/net/context"
+	//        "golang.org/x/oauth2/google"
+	storage "google.golang.org/api/storage/v1"
+
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+	"regexp"
+)
+
+// GCSRegistry implements the ObbectStorageRegistry interface and implements a
+// Deployment Manager templates registry.
+//
+// A registry root must be a directory that contains all the available charts,
+// one or two files per template.
+// name-version.tgz
+// name-version.prov
+type GCSRegistry struct {
+	name           string
+	shortURL       string
+	bucket         string
+	format         common.RegistryFormat
+	credentialName string
+
+	httpClient *http.Client
+	service    *storage.Service
+}
+
+// RE for GCS storage
+var ChartFormatMatcher = regexp.MustCompile("(.*)-(.*).tgz")
+var URLFormatMatcher = regexp.MustCompile("gs://(.*)")
+
+// NewGithubTemplateRegistry creates a GithubTemplateRegistry.
+func NewGCSRegistry(name, shortURL string, httpClient *http.Client, gcsService *storage.Service) (*GCSRegistry, error) {
+	format := fmt.Sprintf("%s;%s", common.VersionedRegistry, common.OneLevelRegistry)
+	trimmed := util.TrimURLScheme(shortURL)
+	m := URLFormatMatcher.FindStringSubmatch(shortURL)
+	if len(m) != 2 {
+		return nil, fmt.Errorf("URL must be of the form gs://<bucket> was: %s", shortURL)
+	}
+
+	return &GCSRegistry{
+			name:       name,
+			shortURL:   trimmed,
+			format:     common.RegistryFormat(format),
+			httpClient: httpClient,
+			service:    gcsService,
+			bucket:     m[1],
+		},
+		nil
+}
+
+func (g GCSRegistry) GetRegistryName() string {
+	return g.name
+}
+
+func (g GCSRegistry) GetBucket() string {
+	return g.bucket
+}
+
+func (g GCSRegistry) GetRegistryType() common.RegistryType {
+	return common.GCSRegistryType
+}
+
+// ListTypes lists types in this registry whose string values conform to the
+// supplied regular expression, or all types, if the regular expression is nil.
+func (g GCSRegistry) ListTypes(regex *regexp.Regexp) ([]Type, error) {
+	// List all files in the bucket/prefix that contain the
+	types := []Type{}
+
+	// List all objects in a bucket using pagination
+	pageToken := ""
+	for {
+		call := g.service.Objects.List(g.bucket)
+		call.Delimiter("/")
+		if pageToken != "" {
+			call = call.PageToken(pageToken)
+		}
+		res, err := call.Do()
+		if err != nil {
+			return []Type{}, err
+		}
+		for _, object := range res.Items {
+			// Charts should be named bucket/chart-X.Y.Z.tgz, so tease apart the version here
+			m := ChartFormatMatcher.FindStringSubmatch(object.Name)
+			if len(m) != 3 {
+				continue
+			}
+
+			t, err := NewType("", m[1], m[2])
+			if err != nil {
+				return []Type{}, fmt.Errorf("can't create a type type at path %#v", err)
+			}
+			types = append(types, t)
+		}
+		if pageToken = res.NextPageToken; pageToken == "" {
+			break
+		}
+	}
+	return types, nil
+}
+
+func (g GCSRegistry) GetRegistryFormat() common.RegistryFormat {
+	return common.CollectionRegistry
+}
+
+func (g GCSRegistry) GetRegistryShortURL() string {
+	return g.shortURL
+}
+
+// GetDownloadURLs fetches the download URLs for a given Chart
+func (g GCSRegistry) GetDownloadURLs(t Type) ([]*url.URL, error) {
+	call := g.service.Objects.List(g.bucket)
+	call.Delimiter("/")
+	call.Prefix(t.String())
+	res, err := call.Do()
+	ret := []*url.URL{}
+	if err != nil {
+		return ret, err
+	}
+	for _, object := range res.Items {
+		log.Printf("Found: %s", object.Name)
+		u, err := url.Parse(object.MediaLink)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse URL from %s: %s", object.MediaLink, err)
+		}
+		ret = append(ret, u)
+	}
+	return ret, err
+}
+
+func (g GCSRegistry) Do(req *http.Request) (resp *http.Response, err error) {
+	return g.httpClient.Do(req)
+}

registry/github_package_registry.go

@@ -22,6 +22,7 @@ import (
 
 	"fmt"
 	"log"
+	"net/http"
 	"net/url"
 	"regexp"
 	"strings"
@@ -41,7 +42,7 @@ type GithubPackageRegistry struct {
 }
 
 // NewGithubPackageRegistry creates a GithubPackageRegistry.
-func NewGithubPackageRegistry(name, shortURL string, service GithubRepositoryService, client *github.Client) (*GithubPackageRegistry, error) {
+func NewGithubPackageRegistry(name, shortURL string, service GithubRepositoryService, httpClient *http.Client, client *github.Client) (*GithubPackageRegistry, error) {
 	format := fmt.Sprintf("%s;%s", common.UnversionedRegistry, common.OneLevelRegistry)
 	if service == nil {
 		if client == nil {
@@ -51,7 +52,7 @@ func NewGithubPackageRegistry(name, shortURL string, service GithubRepositorySer
 		}
 	}
 
-	gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), service)
+	gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), httpClient, service)
 	if err != nil {
 		return nil, err
 	}
@@ -151,3 +152,7 @@ func (g GithubPackageRegistry) MakeRepositoryPath(t Type) (string, error) {
 	// Construct the return path
 	return t.Name + "/manifests", nil
 }
+
+func (g GithubPackageRegistry) Do(req *http.Request) (resp *http.Response, err error) {
+	return g.httpClient.Do(req)
+}

registry/github_registry.go

@@ -22,6 +22,7 @@ import (
 	"github.com/kubernetes/deployment-manager/util"
 
 	"fmt"
+	"net/http"
 	"strings"
 )
 
@@ -38,6 +39,7 @@ type githubRegistry struct {
 	format         common.RegistryFormat
 	credentialName string
 	service        GithubRepositoryService
+	httpClient     *http.Client
 }
 
 // GithubRepositoryService defines the interface that's defined in github.com/go-github/repos_contents.go GetContents method.
@@ -54,7 +56,7 @@ type GithubRepositoryService interface {
 }
 
 // newGithubRegistry creates a githubRegistry.
-func newGithubRegistry(name, shortURL string, format common.RegistryFormat, service GithubRepositoryService) (*githubRegistry, error) {
+func newGithubRegistry(name, shortURL string, format common.RegistryFormat, httpClient *http.Client, service GithubRepositoryService) (*githubRegistry, error) {
 	trimmed := util.TrimURLScheme(shortURL)
 	owner, repository, path, err := parseGithubShortURL(trimmed)
 	if err != nil {
@@ -69,6 +71,7 @@ func newGithubRegistry(name, shortURL string, format common.RegistryFormat, serv
 		path:       path,
 		format:     format,
 		service:    service,
+		httpClient: httpClient,
 	}, nil
 }
 

registry/github_template_registry.go

@@ -22,6 +22,7 @@ import (
 
 	"fmt"
 	"log"
+	"net/http"
 	"net/url"
 	"regexp"
 	"strings"
@@ -54,13 +55,13 @@ type GithubTemplateRegistry struct {
 }
 
 // NewGithubTemplateRegistry creates a GithubTemplateRegistry.
-func NewGithubTemplateRegistry(name, shortURL string, service GithubRepositoryService, client *github.Client) (*GithubTemplateRegistry, error) {
+func NewGithubTemplateRegistry(name, shortURL string, service GithubRepositoryService, httpClient *http.Client, client *github.Client) (*GithubTemplateRegistry, error) {
 	format := fmt.Sprintf("%s;%s", common.VersionedRegistry, common.CollectionRegistry)
 	if service == nil {
 		service = client.Repositories
 	}
 
-	gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), service)
+	gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), httpClient, service)
 	if err != nil {
 		return nil, err
 	}
@@ -212,3 +213,7 @@ func (g GithubTemplateRegistry) MakeRepositoryPath(t Type) (string, error) {
 	}
 	return p + t.Name + "/" + t.GetVersion(), nil
 }
+
+func (g GithubTemplateRegistry) Do(req *http.Request) (resp *http.Response, err error) {
+	return g.httpClient.Do(req)
+}

registry/inmem_credential_provider.go

@@ -38,6 +38,6 @@ func (fcp *InmemCredentialProvider) GetCredential(name string) (*common.Registry
 }
 
 func (fcp *InmemCredentialProvider) SetCredential(name string, credential *common.RegistryCredential) error {
-	fcp.credentials[name] = &common.RegistryCredential{credential.APIToken, credential.BasicAuth}
+	fcp.credentials[name] = &common.RegistryCredential{credential.APIToken, credential.BasicAuth, credential.ServiceAccount}
 	return nil
 }

registry/inmem_registry_service.go

@@ -51,6 +51,14 @@ func NewInmemRegistryService() common.RegistryService {
 		CredentialName: "default",
 	})
 
+	gFormat := fmt.Sprintf("%s", common.CollectionRegistry)
+	rs.Create(&common.Registry{
+		Name:   "charts_gcs",
+		Type:   common.GCSRegistryType,
+		URL:    "gs://helm-charts-test",
+		Format: common.RegistryFormat(gFormat),
+	})
+
 	return rs
 }
 

registry/registry.go

@@ -18,6 +18,7 @@ package registry
 
 import (
 	"github.com/kubernetes/deployment-manager/common"
+	"github.com/kubernetes/deployment-manager/util"
 
 	"fmt"
 	"net/url"
@@ -29,6 +30,9 @@ import (
 // used in a Deployment Manager configuration. There can be multiple
 // registry implementations.
 type Registry interface {
+	// Also handles http.Client.Do method for authenticated File accesses
+	util.HTTPDoer
+
 	// GetRegistryName returns the name of this registry
 	GetRegistryName() string
 	// GetRegistryType returns the type of this registry.
@@ -56,6 +60,13 @@ type GithubRegistry interface {
 	GetRegistryPath() string
 }
 
+// ObjectStorageRegistry abstracts a registry that resides in an Object Storage, for
+// example Google Cloud Storage or AWS S3, etc.
+type ObjectStorageRegistry interface {
+	Registry // An ObjectStorageRegistry is a Registry.
+	GetBucket() string
+}
+
 type Type struct {
 	Collection string
 	Name       string

registry/registryprovider.go

@@ -21,9 +21,12 @@ import (
 	"github.com/kubernetes/deployment-manager/common"
 	"github.com/kubernetes/deployment-manager/util"
 	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/google"
+	storage "google.golang.org/api/storage/v1"
 
 	"fmt"
 	"log"
+	"net/http"
 	"net/url"
 	"regexp"
 	"strings"
@@ -40,15 +43,16 @@ type registryProvider struct {
 	sync.RWMutex
 	rs         common.RegistryService
 	grp        GithubRegistryProvider
+	gcsrp      GCSRegistryProvider
 	cp         common.CredentialProvider
 	registries map[string]Registry
 }
 
-func NewDefaultRegistryProvider(cp common.CredentialProvider) RegistryProvider {
-	return NewRegistryProvider(nil, NewGithubRegistryProvider(cp), cp)
+func NewDefaultRegistryProvider(cp common.CredentialProvider, rs common.RegistryService) RegistryProvider {
+	return NewRegistryProvider(rs, NewGithubRegistryProvider(cp), NewGCSRegistryProvider(cp), cp)
 }
 
-func NewRegistryProvider(rs common.RegistryService, grp GithubRegistryProvider, cp common.CredentialProvider) RegistryProvider {
+func NewRegistryProvider(rs common.RegistryService, grp GithubRegistryProvider, gcsrp GCSRegistryProvider, cp common.CredentialProvider) RegistryProvider {
 	if rs == nil {
 		rs = NewInmemRegistryService()
 	}
@@ -61,11 +65,27 @@ func NewRegistryProvider(rs common.RegistryService, grp GithubRegistryProvider,
 		grp = NewGithubRegistryProvider(cp)
 	}
 
+	if gcsrp == nil {
+		gcsrp = NewGCSRegistryProvider(cp)
+	}
+
 	registries := make(map[string]Registry)
-	rp := &registryProvider{rs: rs, grp: grp, cp: cp, registries: registries}
+	rp := &registryProvider{rs: rs, grp: grp, gcsrp: gcsrp, cp: cp, registries: registries}
 	return rp
 }
 
+func (rp registryProvider) getRegistry(cr common.Registry) (Registry, error) {
+	switch cr.Type {
+	case common.GithubRegistryType:
+		return rp.grp.GetGithubRegistry(cr)
+	case common.GCSRegistryType:
+		log.Printf("Creating a bigstore client using %#v", rp.gcsrp)
+		return rp.gcsrp.GetGCSRegistry(cr)
+	default:
+		return nil, fmt.Errorf("unknown registry type: %s", cr.Type)
+	}
+}
+
 func (rp registryProvider) GetRegistryByShortURL(URL string) (Registry, error) {
 	rp.RLock()
 	defer rp.RUnlock()
@@ -77,7 +97,7 @@ func (rp registryProvider) GetRegistryByShortURL(URL string) (Registry, error) {
 			return nil, err
 		}
 
-		r, err := rp.grp.GetGithubRegistry(*cr)
+		r, err := rp.getRegistry(*cr)
 		if err != nil {
 			return nil, err
 		}
@@ -111,7 +131,7 @@ func (rp registryProvider) GetRegistryByName(registryName string) (Registry, err
 		return nil, err
 	}
 
-	r, err := rp.grp.GetGithubRegistry(*cr)
+	r, err := rp.getRegistry(*cr)
 	if err != nil {
 		return nil, err
 	}
@@ -143,21 +163,21 @@ type githubRegistryProvider struct {
 // NewGithubRegistryProvider creates a GithubRegistryProvider.
 func NewGithubRegistryProvider(cp common.CredentialProvider) GithubRegistryProvider {
 	if cp == nil {
-		panic(fmt.Errorf("CP IS NIL: %v", cp))
+		panic(fmt.Errorf("cp is nil: %v", cp))
 	}
 	return &githubRegistryProvider{cp: cp}
 }
 
-func (grp githubRegistryProvider) createGithubClient(credentialName string) (*github.Client, error) {
+func (grp githubRegistryProvider) createGithubClient(credentialName string) (*http.Client, *github.Client, error) {
 	if credentialName == "" {
-		return github.NewClient(nil), nil
+		return http.DefaultClient, github.NewClient(nil), nil
 	}
 	c, err := grp.cp.GetCredential(credentialName)
 
 	if err != nil {
 		log.Printf("Failed to fetch credential %s: %v", credentialName, err)
 		log.Print("Trying to use unauthenticated client")
-		return github.NewClient(nil), nil
+		return http.DefaultClient, github.NewClient(nil), nil
 	}
 
 	if c != nil {
@@ -166,44 +186,90 @@ func (grp githubRegistryProvider) createGithubClient(credentialName string) (*gi
 				&oauth2.Token{AccessToken: string(c.APIToken)},
 			)
 			tc := oauth2.NewClient(oauth2.NoContext, ts)
-			return github.NewClient(tc), nil
+			return tc, github.NewClient(tc), nil
 		}
 		if c.BasicAuth.Username != "" && c.BasicAuth.Password != "" {
 			tp := github.BasicAuthTransport{
 				Username: c.BasicAuth.Username,
 				Password: c.BasicAuth.Password,
 			}
-			return github.NewClient(tp.Client()), nil
+			return tp.Client(), github.NewClient(tp.Client()), nil
 		}
 
 	}
-	return nil, fmt.Errorf("No suitable credential found for %s", credentialName)
+	return nil, nil, fmt.Errorf("No suitable credential found for %s", credentialName)
 
 }
 
 // GetGithubRegistry returns a new GithubRegistry. If there's a credential that is specified, will try
 // to fetch it and use it, and if there's no credential found, will fall back to unauthenticated client.
 func (grp githubRegistryProvider) GetGithubRegistry(cr common.Registry) (GithubRegistry, error) {
-	if cr.Type == common.GithubRegistryType {
 	// If there's a credential that we need to use, fetch it and create a client for it.
-		client, err := grp.createGithubClient(cr.CredentialName)
+	httpClient, client, err := grp.createGithubClient(cr.CredentialName)
 	if err != nil {
 		return nil, err
 	}
 
 	fMap := ParseRegistryFormat(cr.Format)
 	if fMap[common.UnversionedRegistry] && fMap[common.OneLevelRegistry] {
-			return NewGithubPackageRegistry(cr.Name, cr.URL, nil, client)
+		return NewGithubPackageRegistry(cr.Name, cr.URL, nil, httpClient, client)
 	}
 
 	if fMap[common.VersionedRegistry] && fMap[common.CollectionRegistry] {
-			return NewGithubTemplateRegistry(cr.Name, cr.URL, nil, client)
+		return NewGithubTemplateRegistry(cr.Name, cr.URL, nil, httpClient, client)
 	}
 
 	return nil, fmt.Errorf("unknown registry format: %s", cr.Format)
+}
+
+// GCSRegistryProvider is a factory for GCS Registry instances.
+type GCSRegistryProvider interface {
+	GetGCSRegistry(cr common.Registry) (ObjectStorageRegistry, error)
+}
+
+type gcsRegistryProvider struct {
+	cp common.CredentialProvider
+}
+
+// NewGCSRegistryProvider creates a GCSRegistryProvider.
+func NewGCSRegistryProvider(cp common.CredentialProvider) GCSRegistryProvider {
+	if cp == nil {
+		panic(fmt.Errorf("cp is nil: %v", cp))
 	}
+	return &gcsRegistryProvider{cp: cp}
+}
 
-	return nil, fmt.Errorf("unknown registry type: %s", cr.Type)
+// GetGCSRegistry returns a new Google Cloud Storage . If there's a credential that is specified, will try
+// to fetch it and use it, and if there's no credential found, will fall back to unauthenticated client.
+func (gcsrp gcsRegistryProvider) GetGCSRegistry(cr common.Registry) (ObjectStorageRegistry, error) {
+	// If there's a credential that we need to use, fetch it and create a client for it.
+	if cr.CredentialName == "" {
+		return nil, fmt.Errorf("No CredentialName specified for %s", cr.Name)
+	}
+	client, err := gcsrp.createGCSClient(cr.CredentialName)
+	if err != nil {
+		return nil, err
+	}
+	service, err := storage.New(client)
+	if err != nil {
+		log.Fatalf("Unable to create storage service: %v", err)
+	}
+	return NewGCSRegistry(cr.Name, cr.URL, client, service)
+}
+
+func (gcsrp gcsRegistryProvider) createGCSClient(credentialName string) (*http.Client, error) {
+	c, err := gcsrp.cp.GetCredential(credentialName)
+	if err != nil {
+		log.Printf("Failed to fetch credential %s: %v", credentialName, err)
+		return nil, fmt.Errorf("Failed to fetch Credential %s: %s", credentialName, err)
+	}
+
+	config, err := google.JWTConfigFromJSON([]byte(c.ServiceAccount), storage.DevstorageReadOnlyScope)
+
+	if err != nil {
+		log.Fatalf("Unable to parse client secret file to config: %v", err)
+	}
+	return config.Client(oauth2.NoContext), nil
 }
 
 // RE for a registry type that does support versions and has collections.
@@ -212,6 +278,9 @@ var TemplateRegistryMatcher = regexp.MustCompile("github.com/(.*)/(.*)/(.*)/(.*)
 // RE for a registry type that does not support versions and does not have collections.
 var PackageRegistryMatcher = regexp.MustCompile("github.com/(.*)/(.*)/(.*)")
 
+// RE for GCS storage
+var GCSRegistryMatcher = regexp.MustCompile("gs://(.*)/(.*)")
+
 // IsGithubShortType returns whether a given type is a type description in a short format to a github repository type.
 // For now, this means using github types:
 // github.com/owner/repo/qualifier/type:version
@@ -231,24 +300,31 @@ func IsGithubShortPackageType(t string) bool {
 	return PackageRegistryMatcher.MatchString(t)
 }
 
+// IsGCSShortType returns whether a given type is a type description in a short format to GCS
+func IsGCSShortType(t string) bool {
+	return strings.HasPrefix(t, "gs://")
+}
+
 // GetDownloadURLs checks a type to see if it is either a short git hub url or a fully specified URL
 // and returns the URLs that should be used to fetch it. If the url is not fetchable (primitive type
 // for example), it returns an empty slice.
-func GetDownloadURLs(rp RegistryProvider, t string) ([]string, error) {
+func GetDownloadURLs(rp RegistryProvider, t string) ([]string, Registry, error) {
 	if IsGithubShortType(t) {
 		return ShortTypeToDownloadURLs(rp, t)
 	} else if IsGithubShortPackageType(t) {
 		return ShortTypeToPackageDownloadURLs(rp, t)
+	} else if IsGCSShortType(t) {
+		return ShortTypeToGCSDownloadUrls(rp, t)
 	} else if util.IsHttpUrl(t) {
 		result, err := url.Parse(t)
 		if err != nil {
-			return nil, fmt.Errorf("cannot parse download URL %s: %s", t, err)
+			return nil, nil, fmt.Errorf("cannot parse download URL %s: %s", t, err)
 		}
 
-		return []string{result.String()}, nil
+		return []string{result.String()}, nil, nil
 	}
 
-	return []string{}, nil
+	return []string{}, nil, nil
 }
 
 // ShortTypeToDownloadURLs converts a github URL into downloadable URL from github.
@@ -256,15 +332,15 @@ func GetDownloadURLs(rp RegistryProvider, t string) ([]string, error) {
 // github.com/owner/repo/qualifier/type:version
 // for example:
 // github.com/kubernetes/application-dm-templates/storage/redis:v1
-func ShortTypeToDownloadURLs(rp RegistryProvider, t string) ([]string, error) {
+func ShortTypeToDownloadURLs(rp RegistryProvider, t string) ([]string, Registry, error) {
 	m := TemplateRegistryMatcher.FindStringSubmatch(t)
 	if len(m) != 6 {
-		return nil, fmt.Errorf("cannot parse short github url: %s", t)
+		return nil, nil, fmt.Errorf("cannot parse short github url: %s", t)
 	}
 
 	r, err := rp.GetRegistryByShortURL(t)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	if r == nil {
@@ -273,15 +349,15 @@ func ShortTypeToDownloadURLs(rp RegistryProvider, t string) ([]string, error) {
 
 	tt, err := NewType(m[3], m[4], m[5])
 	if err != nil {
-		return nil, err
+		return nil, r, err
 	}
 
 	urls, err := r.GetDownloadURLs(tt)
 	if err != nil {
-		return nil, err
+		return nil, r, err
 	}
 
-	return util.ConvertURLsToStrings(urls), err
+	return util.ConvertURLsToStrings(urls), r, err
 }
 
 // ShortTypeToPackageDownloadURLs converts a github URL into downloadable URLs from github.
@@ -289,26 +365,48 @@ func ShortTypeToDownloadURLs(rp RegistryProvider, t string) ([]string, error) {
 // github.com/owner/repo/type
 // for example:
 // github.com/helm/charts/cassandra
-func ShortTypeToPackageDownloadURLs(rp RegistryProvider, t string) ([]string, error) {
+func ShortTypeToPackageDownloadURLs(rp RegistryProvider, t string) ([]string, Registry, error) {
 	m := PackageRegistryMatcher.FindStringSubmatch(t)
 	if len(m) != 4 {
-		return nil, fmt.Errorf("Failed to parse short github url: %s", t)
+		return nil, nil, fmt.Errorf("Failed to parse short github url: %s", t)
 	}
 
 	r, err := rp.GetRegistryByShortURL(t)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	tt, err := NewType("", m[3], "")
 	if err != nil {
-		return nil, err
+		return nil, r, err
 	}
 
 	urls, err := r.GetDownloadURLs(tt)
 	if err != nil {
-		return nil, err
+		return nil, r, err
 	}
 
-	return util.ConvertURLsToStrings(urls), err
+	return util.ConvertURLsToStrings(urls), r, err
+}
+
+func ShortTypeToGCSDownloadUrls(rp RegistryProvider, t string) ([]string, Registry, error) {
+	m := GCSRegistryMatcher.FindStringSubmatch(t)
+	if len(m) != 3 {
+		return nil, nil, fmt.Errorf("Failed to parse short gs url: %s", t)
+	}
+
+	r, err := rp.GetRegistryByShortURL(t)
+	if err != nil {
+		return nil, nil, err
+	}
+	tt, err := NewType(m[1], m[2], "")
+	if err != nil {
+		return nil, r, err
+	}
+
+	urls, err := r.GetDownloadURLs(tt)
+	if err != nil {
+		return nil, r, err
+	}
+	return util.ConvertURLsToStrings(urls), r, err
 }

registry/registryprovider_test.go

@@ -22,7 +22,8 @@ import (
 
 func testUrlConversionDriver(rp RegistryProvider, tests map[string]TestURLAndError, t *testing.T) {
 	for in, expected := range tests {
-		actual, err := GetDownloadURLs(rp, in)
+		// TODO(vaikas): Test to make sure it's the right registry.
+		actual, _, err := GetDownloadURLs(rp, in)
 		if err != expected.Err {
 			t.Fatalf("failed on: %s : expected error %v but got %v", in, expected.Err, err)
 		}
@@ -45,7 +46,8 @@ func TestShortGithubUrlTemplateMapping(t *testing.T) {
 	}
 
 	grp := NewTestGithubRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps)
-	testUrlConversionDriver(NewRegistryProvider(nil, grp, NewInmemCredentialProvider()), tests, t)
+	// TODO(vaikas): XXXX FIXME Add gcsrp
+	testUrlConversionDriver(NewRegistryProvider(nil, grp, nil, NewInmemCredentialProvider()), tests, t)
 }
 
 func TestShortGithubUrlPackageMapping(t *testing.T) {
@@ -60,5 +62,6 @@ func TestShortGithubUrlPackageMapping(t *testing.T) {
 	}
 
 	grp := NewTestGithubRegistryProvider("github.com/helm/charts", githubUrlMaps)
-	testUrlConversionDriver(NewRegistryProvider(nil, grp, NewInmemCredentialProvider()), tests, t)
+	// TODO(vaikas): XXXX FIXME Add gcsrp
+	testUrlConversionDriver(NewRegistryProvider(nil, grp, nil, NewInmemCredentialProvider()), tests, t)
 }

registry/secrets_credential_provider.go

@@ -120,8 +120,6 @@ func (scp *SecretsCredentialProvider) SetCredential(name string, credential *com
 		log.Printf("yaml marshal failed for kubernetes object: %s: %v", name, err)
 		return err
 	}
-	log.Printf("Calling with: %s", string(ko))
-	o, err := scp.k.Create(string(ko))
-	log.Printf("Create returned: %s", o)
+	_, err = scp.k.Create(string(ko))
 	return err
 }

registry/testhelper.go

@@ -23,6 +23,7 @@ import (
 	"github.com/kubernetes/deployment-manager/util"
 
 	"fmt"
+	"net/http"
 	"net/url"
 	"regexp"
 	"strings"
@@ -53,7 +54,7 @@ func NewTestGithubRegistryProvider(shortURL string, responses map[Type]TestURLAn
 func (tgrp testGithubRegistryProvider) GetGithubRegistry(cr common.Registry) (GithubRegistry, error) {
 	trimmed := util.TrimURLScheme(cr.URL)
 	if strings.HasPrefix(trimmed, tgrp.shortURL) {
-		ghr, err := newGithubRegistry(cr.Name, trimmed, cr.Format, nil)
+		ghr, err := newGithubRegistry(cr.Name, trimmed, cr.Format, http.DefaultClient, nil)
 		if err != nil {
 			panic(fmt.Errorf("cannot create a github registry: %s", err))
 		}
@@ -80,3 +81,41 @@ func (tgr testGithubRegistry) GetDownloadURLs(t Type) ([]*url.URL, error) {
 
 	return []*url.URL{URL}, result.Err
 }
+
+func (g testGithubRegistry) Do(req *http.Request) (resp *http.Response, err error) {
+	return nil, fmt.Errorf("Not implemented yet")
+}
+
+type testGCSRegistryProvider struct {
+	shortURL  string
+	responses map[Type]TestURLAndError
+}
+
+type testGCSRegistry struct {
+	GCSRegistry
+	responses map[Type]TestURLAndError
+}
+
+func NewTestGCSRegistryProvider(shortURL string, responses map[Type]TestURLAndError) GCSRegistryProvider {
+	return testGCSRegistryProvider{
+		shortURL:  util.TrimURLScheme(shortURL),
+		responses: responses,
+	}
+}
+
+func (tgrp testGCSRegistryProvider) GetGCSRegistry(cr common.Registry) (ObjectStorageRegistry, error) {
+	trimmed := util.TrimURLScheme(cr.URL)
+	if strings.HasPrefix(trimmed, tgrp.shortURL) {
+		gcsr, err := NewGCSRegistry(cr.Name, trimmed, http.DefaultClient, nil)
+		if err != nil {
+			panic(fmt.Errorf("cannot create gcs registry: %s", err))
+		}
+
+		return &testGCSRegistry{
+			GCSRegistry: *gcsr,
+			responses:   tgrp.responses,
+		}, nil
+	}
+
+	panic(fmt.Errorf("unknown registry: %v", cr))
+}

util/httpclient.go

@@ -72,6 +72,10 @@ type httpClient struct {
 	sleep   Sleeper
 }
 
+func DefaultHTTPClient() HTTPClient {
+	return NewHTTPClient(3, http.DefaultClient, NewSleeper())
+}
+
 // NewHTTPClient returns a new HTTPClient.
 func NewHTTPClient(retries uint, c HTTPDoer, s Sleeper) HTTPClient {
 	ret := httpClient{}

util/kubernetes_kubectl.go

@@ -126,15 +126,17 @@ func (k *KubernetesKubectl) execute(args []string, input string) (string, error)
 	cmd.Stderr = combined
 
 	if err := cmd.Start(); err != nil {
-		log.Printf("cannot start kubectl %s %#v", combined.String(), err)
-		return combined.String(), err
+		e := fmt.Errorf("cannot start kubectl %s %#v", combined.String(), err)
+		log.Printf("%s", e)
+		return combined.String(), e
 	}
 
 	if err := cmd.Wait(); err != nil {
-		log.Printf("kubectl failed: %s  %#v", combined.String(), err)
-		return combined.String(), err
+		e := fmt.Errorf("kubectl failed %s", combined.String())
+		log.Printf("%s", e)
+		return combined.String(), e
 	}
-	log.Printf("kubectl succeeded: SysTime: %v UserTime: %v\n%v",
-		cmd.ProcessState.SystemTime(), cmd.ProcessState.UserTime(), combined.String())
+	log.Printf("kubectl succeeded: SysTime: %v UserTime: %v",
+		cmd.ProcessState.SystemTime(), cmd.ProcessState.UserTime())
 	return combined.String(), nil
 }