Skip to content

H
helm3
Commit 4b4d58c8 authored by vaikas-google's avatar vaikas-google
Browse files
Options

Merge pull request #178 from jackgr/helm-integration 

helm integration
parents 3cca3065 2c59a07b
Show whitespace changes
Inline Side-by-side
Showing with 1495 additions and 588 deletions
common/types.go
View file @ 4b4d58c8
......@@ -173,30 +173,76 @@ type BasicAuthCredential struct {
Password string `json:"password"`
}
// Credentials used to access the repository
type APITokenCredential string
// Credential used to access the repository
type RegistryCredential struct {
APIToken APITokenCredential `json:"apitoken,omitempty"`
BasicAuth BasicAuthCredential `json:"basicauth,omitempty"`
}
// Registry describes a template registry
// TODO(jackr): Fix ambiguity re: whether or not URL has a scheme.
type Registry struct {
Name string `json:"name,omitempty"` // Friendly name for the repo
Name string `json:"name,omitempty"` // Friendly name for the registry
Type RegistryType `json:"type,omitempty"` // Technology implementing the registry
URL string `json:"name,omitempty"` // URL to the root of the repo, for example: github.com/helm/charts
URL string `json:"name,omitempty"` // URL to the root of the registry
Format RegistryFormat `json:"format,omitempty"` // Format of the registry
}
// AuthenticatedRegistry describes a type registry with credential.
// Broke this out of Registry, so that we can pass around instances of Registry
// without worrying about secrets.
type AuthenticatedRegistry struct {
Registry
Credential RegistryCredential `json:"credential,omitempty"`
Format RegistryFormat `json:"format,omitempty"`
}
// RegistryType defines the technology that implements the registry
type RegistryType string
const (
Github RegistryType = "github"
GithubRegistryType RegistryType = "github"
)
// RegistryFormat defines the format of the registry
// RegistryFormat is a semi-colon delimited string that describes the format
// of a registry.
type RegistryFormat string
const (
// Versioning.
// If a registry if versioned, then types appear under versions.
VersionedRegistry RegistryFormat = "versioned"
// If a registry is unversioned, then types appear under their names.
UnversionedRegistry RegistryFormat = "unversioned"
// Organization.
// In a collection registry, types are grouped into collections.
CollectionRegistry RegistryFormat = "collection"
// In a one level registry, all types appear at the top level.
OneLevelRegistry RegistryFormat = "onelevel"
)
// RegistryService maintains a set of registries that defines the scope of all
// registry based operations, such as search and type resolution.
type RegistryService interface {
// List all the registries
List() ([]*Registry, error)
// Create a new registry
Create(registry *Registry) error
// Get a registry
Get(name string) (*Registry, error)
// Get a registry with credential.
GetAuthenticatedRegistry(name string) (*AuthenticatedRegistry, error)
// Delete a registry
Delete(name string) error
// Find a registry that backs the given URL
GetByURL(URL string) (*Registry, error)
// GetAuthenticatedRegistryByURL returns an authenticated registry that handles the types for a given URL.
GetAuthenticatedRegistryByURL(URL string) (*AuthenticatedRegistry, error)
// Set the credential for a registry.
// May not be supported by some registry services.
SetCredential(name string, credential RegistryCredential) error
// Get the credential for a registry.
GetCredential(name string) (RegistryCredential, error)
}
dm/dm.go
View file @ 4b4d58c8
......@@ -35,6 +35,8 @@ import (
"net/http"
"net/url"
"os"
"path"
"regexp"
"strconv"
"strings"
"time"
......@@ -48,6 +50,8 @@ var (
service = flag.String("service", "http://localhost:8001/api/v1/proxy/namespaces/dm/services/manager-service:manager", "URL for deployment manager")
binary = flag.String("binary", "../expandybird/expansion/expansion.py", "Path to template expansion binary")
timeout = flag.Int("timeout", 10, "Time in seconds to wait for response")
regex_string = flag.String("regex", "", "Regular expression to filter the templates listed in a template registry")
apitoken = flag.String("apitoken", "", "Github api token that overrides GITHUB_API_TOKEN environment variable")
)
var commands = []string{
......@@ -81,8 +85,58 @@ var usage = func() {
panic("\n")
}
func getGitRegistry() (registry.Registry, error) {
return registry.NewDefaultRegistryProvider().GetRegistry(*template_registry)
// TODO(jackgr): Move all registry related operations to the server side.
var registryProvider registry.RegistryProvider
func getRegistryProvider() registry.RegistryProvider {
if registryProvider == nil {
rs := registry.NewInmemRegistryService()
r, err := rs.GetByURL(*template_registry)
if err != nil {
r := newRegistry(*template_registry)
if err := rs.Create(r); err != nil {
panic(fmt.Errorf("cannot configure registry at %s: %s", r.URL, err))
}
}
if *apitoken == "" {
*apitoken = os.Getenv("DM_GITHUB_API_TOKEN")
}
if *apitoken != "" {
credential := common.RegistryCredential{
APIToken: common.APITokenCredential(*apitoken),
}
if err := rs.SetCredential(r.Name, credential); err != nil {
panic(fmt.Errorf("cannot configure registry at %s: %s", r.Name, err))
}
}
registryProvider = registry.NewRegistryProvider(rs, nil)
}
return registryProvider
}
func newRegistry(URL string) *common.Registry {
tFormat := fmt.Sprintf("%s;%s", common.VersionedRegistry, common.CollectionRegistry)
return &common.Registry{
Name: util.TrimURLScheme(URL),
Type: common.GithubRegistryType,
URL: URL,
Format: common.RegistryFormat(tFormat),
}
}
func getGithubRegistry() registry.Registry {
provider := getRegistryProvider()
git, err := provider.GetRegistryByShortURL(*template_registry)
if err != nil {
panic(fmt.Errorf("cannot open registry %s: %s", *template_registry, err))
}
return git
}
func main() {
......@@ -107,30 +161,29 @@ func execute() {
switch args[0] {
case "templates":
git, err := getGitRegistry()
var regex *regexp.Regexp
if *regex_string != "" {
var err error
regex, err = regexp.Compile(*regex_string)
if err != nil {
panic(fmt.Errorf("Cannot get registry %v", err))
panic(fmt.Errorf("cannot compile regular expression %s: %s", *regex_string, err))
}
templates, err := git.List()
if err != nil {
panic(fmt.Errorf("Cannot list %v", err))
}
fmt.Printf("Templates:\n")
for _, t := range templates {
var typeSpec = ""
if len(t.Collection) > 0 {
typeSpec = t.Collection + "/"
git := getGithubRegistry()
types, err := git.ListTypes(regex)
if err != nil {
panic(fmt.Errorf("cannot list templates in registry %s: %s", *template_registry, err))
}
typeSpec = typeSpec + t.Name
if len(t.Version) > 0 {
typeSpec = typeSpec + ":" + t.Version
for _, t := range types {
fmt.Printf("%s\n", t.String())
urls, err := git.GetDownloadURLs(t)
if err != nil {
panic(fmt.Errorf("cannot get download urls for %s: %s", t, err))
}
fmt.Printf("%s\n", typeSpec)
fmt.Printf("\tshort URL: github.com/%s/%s\n", *template_registry, typeSpec)
fmt.Printf("\tdownload URL(s):\n")
for _, downloadURL := range getDownloadURLs(t) {
for _, downloadURL := range urls {
fmt.Printf("\t%s\n", downloadURL)
}
}
......@@ -197,7 +250,7 @@ func execute() {
usage()
}
tUrls := getTypeURLs(args[1])
tUrls := getDownloadURLs(args[1])
var tUrl = ""
if len(tUrls) == 0 {
// Type is most likely a primitive.
......@@ -268,46 +321,25 @@ func describeType(args []string) {
usage()
}
tUrls := getTypeURLs(args[1])
tUrls := getDownloadURLs(args[1])
if len(tUrls) == 0 {
panic(fmt.Errorf("Invalid type name, must be a template URL or in the form \"<type-name>:<version>\": %s", args[1]))
}
schemaUrl := tUrls[0] + ".schema"
fmt.Println(callHttp(schemaUrl, "GET", "get schema for type ("+tUrls[0]+")", nil))
}
// getTypeURLs returns URLs or empty list if a primitive type.
func getTypeURLs(tName string) []string {
if util.IsHttpUrl(tName) {
// User can pass raw URL to template.
return []string{tName}
}
// User can pass registry type.
t := getRegistryType(tName)
if t == nil {
// Primitive types have no associated URL.
return []string{}
}
return getDownloadURLs(*t)
}
func getDownloadURLs(t registry.Type) []string {
git, err := getGitRegistry()
// getDownloadURLs returns URLs or empty list if a primitive type.
func getDownloadURLs(tName string) []string {
qName := path.Join(*template_registry, tName)
provider := getRegistryProvider()
result, err := registry.GetDownloadURLs(provider, qName)
if err != nil {
panic(fmt.Errorf("Failed to get registry"))
panic(fmt.Errorf("cannot get URLs for %s: %s\n", tName, err))
}
urls, err := git.GetURLs(t)
if err != nil {
panic(fmt.Errorf("Failed to fetch type information for \"%s:%s\": %s", t.Name, t.Version, err))
}
return urls
}
func isHttp(t string) bool {
return strings.HasPrefix(t, "http://") || strings.HasPrefix(t, "https://")
return result
}
func loadTemplate(args []string) *common.Template {
......@@ -343,8 +375,8 @@ func loadTemplate(args []string) *common.Template {
}
} else {
if len(args) < 3 {
if t := getRegistryType(args[1]); t != nil {
template = buildTemplateFromType(*t)
if t, err := registry.ParseType(args[1]); err == nil {
template = buildTemplateFromType(t)
} else {
template, err = expander.NewTemplateFromRootTemplate(args[1])
}
......@@ -365,28 +397,6 @@ func loadTemplate(args []string) *common.Template {
return template
}
// TODO: needs better validation that this is actually a registry type.
func getRegistryType(fullType string) *registry.Type {
tList := strings.Split(fullType, ":")
if len(tList) != 2 {
return nil
}
cList := strings.Split(tList[0], "/")
if len(cList) == 1 {
return &registry.Type{
Name: tList[0],
Version: tList[1],
}
} else {
return &registry.Type{
Collection: cList[0],
Name: cList[1],
Version: tList[1],
}
}
}
func buildTemplateFromType(t registry.Type) *common.Template {
props := make(map[string]interface{})
if *properties != "" {
......@@ -409,11 +419,11 @@ func buildTemplateFromType(t registry.Type) *common.Template {
}
// Name the deployment after the type name.
name := fmt.Sprintf("%s:%s", t.Name, t.Version)
name := fmt.Sprintf("%s:%s", t.Name, t.GetVersion())
config := common.Configuration{Resources: []*common.Resource{&common.Resource{
Name: name,
Type: getDownloadURLs(t)[0],
Type: getDownloadURLs(t.String())[0],
Properties: props,
}}}
......
manager/deployments.go
View file @ 4b4d58c8
......@@ -18,6 +18,7 @@ package main
import (
"encoding/json"
"errors"
"flag"
"fmt"
"io"
......@@ -25,7 +26,9 @@ import (
"log"
"net"
"net/http"
"net/url"
"os"
"regexp"
"strings"
"github.com/ghodss/yaml"
......@@ -50,6 +53,9 @@ var deployments = []Route{
{"ListTypes", "/types", "GET", listTypesHandlerFunc, ""},
{"ListTypeInstances", "/types/{type}/instances", "GET", listTypeInstancesHandlerFunc, ""},
{"ListRegistries", "/registries", "GET", listRegistriesHandlerFunc, ""},
{"GetRegistry", "/registries/{registry}", "GET", getRegistryHandlerFunc, ""},
{"ListRegistryTypes", "/registries/{registry}/types", "GET", listRegistryTypesHandlerFunc, ""},
{"GetDownloadURLs", "/registries/{registry}/types/{type}", "GET", getDownloadURLsHandlerFunc, ""},
}
var (
......@@ -72,11 +78,13 @@ func init() {
}
func newManager() manager.Manager {
expander := manager.NewExpander(getServiceURL(*expanderURL, *expanderName), manager.NewTypeResolver(registry.NewDefaultRegistryProvider()))
provider := registry.NewDefaultRegistryProvider()
resolver := manager.NewTypeResolver(provider)
expander := manager.NewExpander(getServiceURL(*expanderURL, *expanderName), resolver)
deployer := manager.NewDeployer(getServiceURL(*deployerURL, *deployerName))
r := repository.NewMapBasedRepository()
registryService := registry.NewInmemRepositoryService()
return manager.NewManager(expander, deployer, r, registryService)
service := registry.NewInmemRegistryService()
return manager.NewManager(expander, deployer, r, provider, service)
}
func getServiceURL(serviceURL, serviceName string) string {
......@@ -205,13 +213,21 @@ func putDeploymentHandlerFunc(w http.ResponseWriter, r *http.Request) {
func getPathVariable(w http.ResponseWriter, r *http.Request, variable, handler string) (string, error) {
vars := mux.Vars(r)
ret, ok := vars[variable]
escaped, ok := vars[variable]
if !ok {
e := fmt.Errorf("%s parameter not found in URL", variable)
e := errors.New(fmt.Sprintf("%s name not found in URL", variable))
util.LogAndReturnError(handler, http.StatusBadRequest, e, w)
return "", e
}
return ret, nil
unescaped, err := url.QueryUnescape(escaped)
if err != nil {
e := fmt.Errorf("cannot decode name (%v)", variable)
util.LogAndReturnError(handler, http.StatusBadRequest, e, w)
return "", e
}
return unescaped, nil
}
func getTemplate(w http.ResponseWriter, r *http.Request, handler string) *common.Template {
......@@ -342,5 +358,78 @@ func listRegistriesHandlerFunc(w http.ResponseWriter, r *http.Request) {
if err != nil {
return
}
util.LogHandlerExitWithJSON(handler, w, registries, http.StatusOK)
}
func getRegistryHandlerFunc(w http.ResponseWriter, r *http.Request) {
handler := "manager: get registry"
util.LogHandlerEntry(handler, r)
registryName, err := getPathVariable(w, r, "registry", handler)
if err != nil {
return
}
cr, err := backend.GetRegistry(registryName)
if err != nil {
util.LogAndReturnError(handler, http.StatusBadRequest, err, w)
return
}
util.LogHandlerExitWithJSON(handler, w, cr, http.StatusOK)
}
func listRegistryTypesHandlerFunc(w http.ResponseWriter, r *http.Request) {
handler := "manager: list registry types"
util.LogHandlerEntry(handler, r)
registryName, err := getPathVariable(w, r, "registry", handler)
if err != nil {
return
}
var regex *regexp.Regexp
regexString, err := getPathVariable(w, r, "regex", handler)
if err == nil {
regex, err = regexp.Compile(regexString)
if err != nil {
util.LogAndReturnError(handler, http.StatusInternalServerError, err, w)
return
}
}
registryTypes, err := backend.ListRegistryTypes(registryName, regex)
if err != nil {
util.LogAndReturnError(handler, http.StatusInternalServerError, err, w)
return
}
util.LogHandlerExitWithJSON(handler, w, registryTypes, http.StatusOK)
}
func getDownloadURLsHandlerFunc(w http.ResponseWriter, r *http.Request) {
handler := "manager: get download URLs"
util.LogHandlerEntry(handler, r)
registryName, err := getPathVariable(w, r, "registry", handler)
if err != nil {
return
}
typeName, err := getPathVariable(w, r, "type", handler)
if err != nil {
return
}
tt, err := registry.ParseType(typeName)
if err != nil {
util.LogAndReturnError(handler, http.StatusInternalServerError, err, w)
return
}
c, err := backend.GetDownloadURLs(registryName, tt)
if err != nil {
util.LogAndReturnError(handler, http.StatusBadRequest, err, w)
return
}
util.LogHandlerExitWithJSON(handler, w, c, http.StatusOK)
}
manager/manager/expander.go
View file @ 4b4d58c8
......@@ -25,7 +25,6 @@ import (
"github.com/ghodss/yaml"
"github.com/kubernetes/deployment-manager/common"
"github.com/kubernetes/deployment-manager/util"
)
const (
......@@ -101,7 +100,7 @@ func walkLayout(l *common.Layout, imports []*common.ImportFile, toReplace map[st
for len(toVisit) > 0 {
lr := toVisit[0]
nodeKey := lr.Resource.Name + layoutNodeKeySeparator + lr.Resource.Type
if len(lr.Layout.Resources) == 0 && util.IsTemplate(lr.Resource.Type, imports) {
if len(lr.Layout.Resources) == 0 && isTemplate(lr.Resource.Type, imports) {
ret[nodeKey] = lr
} else if toReplace[nodeKey] != nil {
toReplace[nodeKey].Resources = lr.Resources
......@@ -113,6 +112,17 @@ func walkLayout(l *common.Layout, imports []*common.ImportFile, toReplace map[st
return ret
}
// isTemplate returns whether a given type is a template.
func isTemplate(t string, imports []*common.ImportFile) bool {
for _, imp := range imports {
if imp.Name == t {
return true
}
}
return false
}
// ExpandTemplate expands the supplied template, and returns a configuration.
// It will also update the imports in the provided template if any were added
// during type resolution.
......
manager/manager/manager.go
View file @ 4b4d58c8
......@@ -19,6 +19,8 @@ package manager
import (
"fmt"
"log"
"net/url"
"regexp"
"time"
"github.com/kubernetes/deployment-manager/common"
......@@ -28,33 +30,48 @@ import (
// Manager manages a persistent set of Deployments.
type Manager interface {
// Deployments
ListDeployments() ([]common.Deployment, error)
GetDeployment(name string) (*common.Deployment, error)
CreateDeployment(t *common.Template) (*common.Deployment, error)
DeleteDeployment(name string, forget bool) (*common.Deployment, error)
PutDeployment(name string, t *common.Template) (*common.Deployment, error)
// Manifests
ListManifests(deploymentName string) (map[string]*common.Manifest, error)
GetManifest(deploymentName string, manifest string) (*common.Manifest, error)
Expand(t *common.Template) (*common.Manifest, error)
// Types
ListTypes() []string
ListInstances(typeName string) []*common.TypeInstance
// Registry related functions
// Registries
ListRegistries() ([]*common.Registry, error)
CreateRegistry(pr *common.Registry) error
GetRegistry(name string) (*common.Registry, error)
DeleteRegistry(name string) error
// Registry Types
ListRegistryTypes(registryName string, regex *regexp.Regexp) ([]registry.Type, error)
GetDownloadURLs(registryName string, t registry.Type) ([]*url.URL, error)
}
type manager struct {
expander Expander
deployer Deployer
repository repository.Repository
registryService registry.RegistryService
provider registry.RegistryProvider
service common.RegistryService
}
// NewManager returns a new initialized Manager.
func NewManager(expander Expander, deployer Deployer, repository repository.Repository, registryService registry.RegistryService) Manager {
return &manager{expander, deployer, repository, registryService}
func NewManager(expander Expander,
deployer Deployer,
repository repository.Repository,
provider registry.RegistryProvider,
service common.RegistryService) Manager {
return &manager{expander, deployer, repository, provider, service}
}
// ListDeployments returns the list of deployments
......@@ -309,20 +326,32 @@ func (m *manager) ListInstances(typeName string) []*common.TypeInstance {
return m.repository.GetTypeInstances(typeName)
}
// ListRegistries returns the list of registries
func (m *manager) ListRegistries() ([]*common.Registry, error) {
return m.registryService.List()
return m.service.List()
}
func (m *manager) CreateRegistry(pr *common.Registry) error {
return m.registryService.Create(pr)
return m.service.Create(pr)
}
func (m *manager) GetRegistry(name string) (*common.Registry, error) {
return m.registryService.Get(name)
return m.service.Get(name)
}
func (m *manager) DeleteRegistry(name string) error {
return m.registryService.Delete(name)
return m.service.Delete(name)
}
// Set the credential for a registry.
// May not be supported by some registry services.
func (m *manager) SetCredential(name string, credential common.RegistryCredential) error {
return m.service.SetCredential(name, credential)
}
// Get the credential for a registry.
func (m *manager) GetCredential(name string) (common.RegistryCredential, error) {
return m.service.GetCredential(name)
}
func generateManifestName() string {
......@@ -346,3 +375,26 @@ func getResourceErrors(c *common.Configuration) []string {
return errs
}
// ListRegistryTypes lists types in a given registry whose string values
// conform to the supplied regular expression, or all types, if the regular
// expression is nil.
func (m *manager) ListRegistryTypes(registryName string, regex *regexp.Regexp) ([]registry.Type, error) {
r, err := m.provider.GetRegistryByName(registryName)
if err != nil {
return nil, err
}
return r.ListTypes(regex)
}
// GetDownloadURLs returns the URLs required to download the contents
// of a given type in a given registry.
func (m *manager) GetDownloadURLs(registryName string, t registry.Type) ([]*url.URL, error) {
r, err := m.provider.GetRegistryByName(registryName)
if err != nil {
return nil, err
}
return r.GetDownloadURLs(t)
}
manager/manager/manager_test.go
View file @ 4b4d58c8
......@@ -66,7 +66,7 @@ var deploymentList = []common.Deployment{deployment, {Name: "test2"}}
var typeInstMap = map[string][]string{"test": []string{"test"}}
var errTest = errors.New("test")
var errTest = errors.New("test error")
type expanderStub struct{}
......@@ -164,6 +164,7 @@ func (repository *repositoryStub) ListDeployments() ([]common.Deployment, error)
if repository.FailListDeployments {
return deploymentList, errTest
}
return deploymentList, nil
}
......@@ -229,31 +230,32 @@ func (repository *repositoryStub) GetManifest(d string, m string) (*common.Manif
return nil, errTest
}
func (r *repositoryStub) ListTypes() []string {
r.ListTypesCalled = true
func (tgr *repositoryStub) ListTypes() []string {
tgr.ListTypesCalled = true
return []string{}
}
func (r *repositoryStub) GetTypeInstances(t string) []*common.TypeInstance {
r.GetTypeInstancesCalled = true
func (tgr *repositoryStub) GetTypeInstances(t string) []*common.TypeInstance {
tgr.GetTypeInstancesCalled = true
return []*common.TypeInstance{}
}
func (r *repositoryStub) ClearTypeInstances(d string) {
r.TypeInstancesCleared = true
func (tgr *repositoryStub) ClearTypeInstances(d string) {
tgr.TypeInstancesCleared = true
}
func (r *repositoryStub) SetTypeInstances(d string, is map[string][]*common.TypeInstance) {
func (tgr *repositoryStub) SetTypeInstances(d string, is map[string][]*common.TypeInstance) {
for k, _ := range is {
r.TypeInstances[d] = append(r.TypeInstances[d], k)
tgr.TypeInstances[d] = append(tgr.TypeInstances[d], k)
}
}
var testExpander = &expanderStub{}
var testRepository = newRepositoryStub()
var testDeployer = newDeployerStub()
var testRegistryService = registry.NewInmemRepositoryService()
var testManager = NewManager(testExpander, testDeployer, testRepository, testRegistryService)
var testRegistryService = registry.NewInmemRegistryService()
var testProvider = registry.NewRegistryProvider(nil, registry.NewTestGithubRegistryProvider("", nil))
var testManager = NewManager(testExpander, testDeployer, testRepository, testProvider, testRegistryService)
func TestListDeployments(t *testing.T) {
testRepository.reset()
......@@ -523,3 +525,23 @@ func TestListInstances(t *testing.T) {
t.Fatal("expected repository GetTypeInstances() call.")
}
}
// TODO(jackgr): Implement TestListRegistryTypes
func TestListRegistryTypes(t *testing.T) {
/*
types, err := testManager.ListRegistryTypes("", nil)
if err != nil {
t.Fatalf("cannot list registry types: %s", err)
}
*/
}
// TODO(jackgr): Implement TestGetDownloadURLs
func TestGetDownloadURLs(t *testing.T) {
/*
urls, err := testManager.GetDownloadURLs("", registry.Type{})
if err != nil {
t.Fatalf("cannot list get download urls: %s", err)
}
*/
}
manager/manager/typeresolver.go
View file @ 4b4d58c8
......@@ -100,7 +100,7 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
toFetch := make([]*fetchUnit, 0, tr.maxUrls)
for _, r := range config.Resources {
// Map the type to a fetchable URL (if applicable) or skip it if it's a non-fetchable type (primitive for example).
urls, err := tr.MapFetchableURLs(r.Type)
urls, err := registry.GetDownloadURLs(tr.rp, r.Type)
if err != nil {
return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", r.Type, err))
}
......@@ -168,7 +168,7 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
for _, v := range s.Imports {
i := &common.ImportFile{Name: v.Name}
var existingSchema string
urls, conversionErr := tr.MapFetchableURLs(v.Path)
urls, conversionErr := registry.GetDownloadURLs(tr.rp, v.Path)
if conversionErr != nil {
return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", v.Path, conversionErr))
}
......@@ -220,56 +220,6 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
return ret, nil
}
// MapFetchableUrls checks a type to see if it is either a short git hub url or a fully specified URL
// and returns the URL that should be used to fetch it. If the url is not fetchable (primitive type for
// example) will return empty string.
func (tr *typeResolver) MapFetchableURLs(t string) ([]string, error) {
if util.IsGithubShortType(t) {
return tr.ShortTypeToDownloadURLs(t)
} else if util.IsGithubShortPackageType(t) {
return tr.ShortTypeToPackageDownloadURLs(t)
} else if util.IsHttpUrl(t) {
return []string{t}, nil
}
return []string{}, nil
}
// ShortTypeToDownloadURLs converts a github URL into downloadable URL from github.
// Input must be of the type and is assumed to have been validated before this call:
// github.com/owner/repo/qualifier/type:version
// for example:
// github.com/kubernetes/application-dm-templates/storage/redis:v1
func (tr *typeResolver) ShortTypeToDownloadURLs(template string) ([]string, error) {
m := util.TemplateRegistryMatcher.FindStringSubmatch(template)
if len(m) != 6 {
return []string{}, fmt.Errorf("Failed to parse short github url: %s", template)
}
r, err := tr.rp.GetRegistry(template)
if err != nil {
return []string{}, err
}
t := registry.Type{m[3], m[4], m[5]}
return r.GetURLs(t)
}
// ShortTypeToPackageDownloadURLs converts a github URL into downloadable URLs from github.
// Input must be of the type and is assumed to have been validated before this call:
// github.com/owner/repo/type
// for example:
// github.com/helm/charts/cassandra
func (tr *typeResolver) ShortTypeToPackageDownloadURLs(template string) ([]string, error) {
m := util.PackageRegistryMatcher.FindStringSubmatch(template)
if len(m) != 4 {
return []string{}, fmt.Errorf("Failed to parse short github url: %s", template)
}
r, err := tr.rp.GetRegistry(template)
if err != nil {
return []string{}, err
}
t := registry.Type{Name: m[3]}
return r.GetURLs(t)
}
func parseContent(templates []string) (string, error) {
if len(templates) == 1 {
return templates[0], nil
......
manager/manager/typeresolver_test.go
View file @ 4b4d58c8
......@@ -18,7 +18,6 @@ package manager
import (
"errors"
"fmt"
"net/http"
"reflect"
"strings"
......@@ -58,76 +57,6 @@ func (tg *testGetter) Get(url string) (body string, code int, err error) {
return ret.resp, ret.code, ret.err
}
type urlAndError struct {
u string
e error
}
type testRegistryProvider struct {
URLPrefix string
r map[string]registry.Registry
}
func newTestRegistryProvider(URLPrefix string, tests map[registry.Type]urlAndError, count int) registry.RegistryProvider {
r := make(map[string]registry.Registry)
r[URLPrefix] = &testGithubRegistry{tests, count}
return &testRegistryProvider{URLPrefix, r}
}
func (trp *testRegistryProvider) GetRegistry(URL string) (registry.Registry, error) {
for key, r := range trp.r {
if strings.HasPrefix(URL, key) {
return r, nil
}
}
return nil, fmt.Errorf("No registry found for %s", URL)
}
type testGithubRegistry struct {
responses map[registry.Type]urlAndError
count int
}
func (tgr *testGithubRegistry) GetURLs(t registry.Type) ([]string, error) {
tgr.count = tgr.count + 1
ret := tgr.responses[t]
return []string{ret.u}, ret.e
}
func (tgr *testGithubRegistry) List() ([]registry.Type, error) {
return []registry.Type{}, fmt.Errorf("List should not be called in the test")
}
type testGithubPackageRegistry struct {
responses map[registry.Type]urlAndError
count int
}
func (tgr *testGithubPackageRegistry) GetURLs(t registry.Type) ([]string, error) {
tgr.count = tgr.count + 1
ret := tgr.responses[t]
return []string{ret.u}, ret.e
}
func (tgr *testGithubPackageRegistry) List() ([]registry.Type, error) {
return []registry.Type{}, fmt.Errorf("List should not be called in the test")
}
func testUrlConversionDriver(c resolverTestCase, tests map[string]urlAndError, t *testing.T) {
r := &typeResolver{
rp: c.registryProvider,
}
for in, expected := range tests {
actual, err := r.ShortTypeToDownloadURLs(in)
if err != expected.e {
t.Errorf("failed on: %s : expected error %v but got %v", in, expected.e, err)
}
if actual[0] != expected.u {
t.Errorf("failed on: %s : expected %s but got %v", in, expected.u, actual)
}
}
}
func testDriver(c resolverTestCase, t *testing.T) {
g := &testGetter{test: t, responses: c.responses}
r := &typeResolver{
......@@ -299,6 +228,7 @@ func TestTooManyImports(t *testing.T) {
responses: responses,
expectedErr: errors.New("Number of imports exceeds maximum of 5"),
}
testDriver(test, t)
}
......@@ -341,41 +271,8 @@ func TestSharedImport(t *testing.T) {
responses: responses,
importOut: finalImports,
}
testDriver(test, t)
}
func TestShortGithubUrlMapping(t *testing.T) {
githubUrlMaps := map[registry.Type]urlAndError{
registry.Type{"common", "replicatedservice", "v1"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil},
registry.Type{"storage", "redis", "v1"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil},
}
tests := map[string]urlAndError{
"github.com/kubernetes/application-dm-templates/common/replicatedservice:v1": urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil},
"github.com/kubernetes/application-dm-templates/storage/redis:v1": urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil},
}
test := resolverTestCase{
registryProvider: newTestRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps, 2),
}
testUrlConversionDriver(test, tests, t)
}
func TestShortGithubUrlMappingDifferentOwnerAndRepo(t *testing.T) {
githubUrlMaps := map[registry.Type]urlAndError{
registry.Type{"common", "replicatedservice", "v1"}: urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/common/replicatedservice/v1/replicatedservice.py", nil},
registry.Type{"storage", "redis", "v1"}: urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/storage/redis/v1/redis.jinja", nil},
}
tests := map[string]urlAndError{
"github.com/example/mytemplates/common/replicatedservice:v1": urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/common/replicatedservice/v1/replicatedservice.py", nil},
"github.com/example/mytemplates/storage/redis:v1": urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/storage/redis/v1/redis.jinja", nil},
}
test := resolverTestCase{
registryProvider: newTestRegistryProvider("github.com/example/mytemplates", githubUrlMaps, 2),
}
testUrlConversionDriver(test, tests, t)
testDriver(test, t)
}
var templateShortGithubTemplate = `
......@@ -405,17 +302,19 @@ func TestShortGithubUrl(t *testing.T) {
"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py.schema": responseAndError{nil, http.StatusNotFound, ""},
}
githubUrlMaps := map[registry.Type]urlAndError{
registry.Type{"common", "replicatedservice", "v1"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil},
registry.Type{"common", "replicatedservice", "v2"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py", nil},
githubUrlMaps := map[registry.Type]registry.TestURLAndError{
registry.NewTypeOrDie("common", "replicatedservice", "v1"): registry.TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil},
registry.NewTypeOrDie("common", "replicatedservice", "v2"): registry.TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py", nil},
}
grp := registry.NewTestGithubRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps)
test := resolverTestCase{
config: templateShortGithubTemplate,
importOut: finalImports,
urlcount: 4,
responses: responses,
registryProvider: newTestRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps, 2),
registryProvider: registry.NewRegistryProvider(nil, grp),
}
testDriver(test, t)
}
registry/github_package_registry.go
View file @ 4b4d58c8
......@@ -17,14 +17,18 @@ limitations under the License.
package registry
import (
"github.com/google/go-github/github"
"github.com/kubernetes/deployment-manager/common"
"fmt"
"log"
"net/url"
"regexp"
"strings"
"github.com/google/go-github/github"
)
// GithubPackageRegistry implements the Registry interface that talks to github and
// expects packages in helm format without versioning and no qualifier in the path.
// expects packages in helm format without versioning and no collection in the path.
// Format of the directory for a type is like so:
// package/
// Chart.yaml
......@@ -33,22 +37,28 @@ import (
// bar.yaml
// ...
type GithubPackageRegistry struct {
owner string
repository string
client *github.Client
githubRegistry
}
// NewGithubRegistry creates a Registry that can be used to talk to github.
func NewGithubPackageRegistry(owner, repository string, client *github.Client) *GithubPackageRegistry {
return &GithubPackageRegistry{
owner: owner,
repository: repository,
client: client,
// NewGithubPackageRegistry creates a GithubPackageRegistry.
func NewGithubPackageRegistry(name, shortURL string, service RepositoryService) (*GithubPackageRegistry, error) {
format := fmt.Sprintf("%s;%s", common.UnversionedRegistry, common.OneLevelRegistry)
if service == nil {
client := github.NewClient(nil)
service = client.Repositories
}
gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), service)
if err != nil {
return nil, err
}
return &GithubPackageRegistry{githubRegistry: *gr}, nil
}
// List the types from the Registry.
func (g *GithubPackageRegistry) List() ([]Type, error) {
// ListTypes lists types in this registry whose string values conform to the
// supplied regular expression, or all types, if the regular expression is nil.
func (g GithubPackageRegistry) ListTypes(regex *regexp.Regexp) ([]Type, error) {
// Just list all the types at the top level.
types, err := g.getDirs("")
if err != nil {
......@@ -59,7 +69,7 @@ func (g *GithubPackageRegistry) List() ([]Type, error) {
var retTypes []Type
for _, t := range types {
// Check to see if there's a Chart.yaml file in the directory
_, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, t, nil)
_, dc, _, err := g.service.GetContents(g.owner, g.repository, t, nil)
if err != nil {
log.Printf("Failed to list package files at path: %s: %v", t, err)
return nil, err
......@@ -71,33 +81,52 @@ func (g *GithubPackageRegistry) List() ([]Type, error) {
}
}
if regex != nil {
var matchTypes []Type
for _, retType := range retTypes {
if regex.MatchString(retType.String()) {
matchTypes = append(matchTypes, retType)
}
}
return matchTypes, nil
}
return retTypes, nil
}
// GetURLs fetches the download URLs for a given Type.
func (g *GithubPackageRegistry) GetURLs(t Type) ([]string, error) {
// GetDownloadURLs fetches the download URLs for a given Type.
func (g GithubPackageRegistry) GetDownloadURLs(t Type) ([]*url.URL, error) {
path, err := g.MakeRepositoryPath(t)
if err != nil {
return []string{}, err
return nil, err
}
_, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, path, nil)
_, dc, _, err := g.service.GetContents(g.owner, g.repository, path, nil)
if err != nil {
log.Printf("Failed to list package files at path: %s: %v", path, err)
return []string{}, err
return nil, err
}
downloadURLs := []string{}
downloadURLs := []*url.URL{}
for _, f := range dc {
if *f.Type == "file" {
if strings.HasSuffix(*f.Name, ".yaml") {
downloadURLs = append(downloadURLs, *f.DownloadURL)
u, err := url.Parse(*f.DownloadURL)
if err != nil {
return nil, fmt.Errorf("cannot parse URL from %s: %s", *f.DownloadURL, err)
}
downloadURLs = append(downloadURLs, u)
}
}
}
return downloadURLs, nil
}
func (g *GithubPackageRegistry) getDirs(dir string) ([]string, error) {
_, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, dir, nil)
func (g GithubPackageRegistry) getDirs(dir string) ([]string, error) {
_, dc, _, err := g.service.GetContents(g.owner, g.repository, dir, nil)
if err != nil {
log.Printf("Failed to get contents at path: %s: %v", dir, err)
return nil, err
......@@ -116,7 +145,7 @@ func (g *GithubPackageRegistry) getDirs(dir string) ([]string, error) {
// MakeRepositoryPath constructs a github path to a given type based on a repository, and type name.
// The returned repository path will be of the form:
// Type.Name/manifests
func (g *GithubPackageRegistry) MakeRepositoryPath(t Type) (string, error) {
func (g GithubPackageRegistry) MakeRepositoryPath(t Type) (string, error) {
// Construct the return path
return t.Name + "/manifests", nil
}
registry/github_registry.go
View file @ 4b4d58c8
......@@ -18,166 +18,109 @@ package registry
import (
"github.com/google/go-github/github"
"github.com/kubernetes/deployment-manager/common"
"github.com/kubernetes/deployment-manager/util"
"fmt"
"log"
"strings"
)
// GithubRegistry implements the Registry interface that talks to github and
// implements Deployment Manager templates registry.
// A registry root must be a directory that contains all the available templates,
// one directory per template. Each template directory then contains version
// directories, each of which in turn contains all the files necessary for that
// version of the template.
//
// For example, a template registry containing two versions of redis
// (implemented in jinja), and one version of replicatedservice (implemented
// in python) would have a directory structure that looks something like this:
// qualifier [optional] prefix to a virtual root within the repository.
// /redis
// /v1
// redis.jinja
// redis.jinja.schema
// /v2
// redis.jinja
// redis.jinja.schema
// /replicatedservice
// /v1
// replicatedservice.python
// replicatedservice.python.schema
type GithubRegistry struct {
// githubRegistry implements the Registry interface and talks to github.
// The registry short URL and format determine how types are laid out in the
// registry.
type githubRegistry struct {
name string
shortURL string
owner string
repository string
path string
client *github.Client
format common.RegistryFormat
service RepositoryService
}
// NewGithubRegistry creates a Registry that can be used to talk to github.
func NewGithubRegistry(owner, repository, path string, client *github.Client) *GithubRegistry {
return &GithubRegistry{
type RepositoryService interface {
GetContents(
owner, repo, path string,
opt *github.RepositoryContentGetOptions,
) (
fileContent *github.RepositoryContent,
directoryContent []*github.RepositoryContent,
resp *github.Response,
err error,
)
}
// newGithubRegistry creates a githubRegistry.
func newGithubRegistry(name, shortURL string, format common.RegistryFormat, service RepositoryService) (*githubRegistry, error) {
trimmed := util.TrimURLScheme(shortURL)
owner, repository, path, err := parseGithubShortURL(trimmed)
if err != nil {
return nil, fmt.Errorf("cannot create Github template registry %s: %s", name, err)
}
return &githubRegistry{
name: name,
shortURL: trimmed,
owner: owner,
repository: repository,
path: path,
client: client,
}
format: format,
service: service,
}, nil
}
// List the types from the Registry.
func (g *GithubRegistry) List() ([]Type, error) {
// First list all the collections at the top level.
collections, err := g.getDirs("")
if err != nil {
log.Printf("Failed to list qualifiers: %v", err)
return nil, err
func parseGithubShortURL(shortURL string) (string, string, string, error) {
if !strings.HasPrefix(shortURL, "github.com/") {
return "", "", "", fmt.Errorf("invalid Github short URL: %s", shortURL)
}
var retTypes []Type
for _, c := range collections {
// Then we need to fetch the versions (directories for this type)
types, err := g.getDirs(c)
if err != nil {
log.Printf("Failed to fetch types for collection: %s", c)
return nil, err
}
tPath := strings.TrimPrefix(shortURL, "github.com/")
parts := strings.Split(tPath, "/")
for _, t := range types {
// Then we need to fetch the versions (directories for this type)
versions, err := g.getDirs(c + "/" + t)
if err != nil {
log.Printf("Failed to fetch versions for template: %s", t)
return nil, err
}
for _, v := range versions {
retTypes = append(retTypes, Type{Name: t, Version: v, Collection: c})
}
// Handle the case where there's no path after owner and repository.
if len(parts) == 2 {
return parts[0], parts[1], "", nil
}
// Handle the case where there's a path after owner and repository.
if len(parts) == 3 {
return parts[0], parts[1], parts[2], nil
}
return retTypes, nil
return "", "", "", fmt.Errorf("invalid Github short URL: %s", shortURL)
}
// GetURL fetches the download URL for a given Type and checks for existence of a schema file.
func (g *GithubRegistry) GetURLs(t Type) ([]string, error) {
path, err := g.MakeRepositoryPath(t)
if err != nil {
return []string{}, err
}
_, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, path, nil)
if err != nil {
log.Printf("Failed to list versions at path: %s: %v", path, err)
return []string{}, err
}
var downloadURL, typeName, schemaName string
for _, f := range dc {
if *f.Type == "file" {
if *f.Name == t.Name+".jinja" || *f.Name == t.Name+".py" {
typeName = *f.Name
downloadURL = *f.DownloadURL
}
if *f.Name == t.Name+".jinja.schema" || *f.Name == t.Name+".py.schema" {
schemaName = *f.Name
}
}
}
if downloadURL == "" {
return []string{}, fmt.Errorf("Can not find template %s:%s", t.Name, t.Version)
}
if schemaName == typeName+".schema" {
return []string{downloadURL}, nil
}
return []string{}, fmt.Errorf("Can not find schema for %s:%s, expected to find %s", t.Name, t.Version, typeName+".schema")
// GetRegistryName returns the name of this registry
func (g githubRegistry) GetRegistryName() string {
return g.name
}
func (g *GithubRegistry) getDirs(dir string) ([]string, error) {
var path = g.path
if dir != "" {
path = g.path + "/" + dir
}
// GetRegistryType returns the type of this registry.
func (g githubRegistry) GetRegistryType() common.RegistryType {
return common.GithubRegistryType
}
_, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, path, nil)
if err != nil {
log.Printf("Failed to get contents at path: %s: %v", path, err)
return nil, err
}
// GetRegistryShortURL returns the short URL for this registry.
func (g githubRegistry) GetRegistryShortURL() string {
return g.shortURL
}
var dirs []string
for _, entry := range dc {
if *entry.Type == "dir" {
dirs = append(dirs, *entry.Name)
}
}
// GetRegistryFormat returns the format of this registry.
func (g githubRegistry) GetRegistryFormat() common.RegistryFormat {
return g.format
}
return dirs, nil
// GetRegistryOwner returns the owner name for this registry
func (g githubRegistry) GetRegistryOwner() string {
return g.owner
}
func (g *GithubRegistry) mapCollection(collection string) (string, error) {
if strings.ContainsAny(collection, "/") {
return "", fmt.Errorf("collection must not contain slashes, got %s", collection)
}
// TODO(vaikas): Implement lookup from the root metadata file to map collection to a path
return collection, nil
// GetRegistryRepository returns the repository name for this registry.
func (g githubRegistry) GetRegistryRepository() string {
return g.repository
}
// MakeRepositoryPath constructs a github path to a given type based on a repository, and type name and version.
// The returned repository path will be of the form:
// [GithubRegistry.path/][Type.Collection]/Type.Name/Type.Version
// Type.Collection will be mapped using mapCollection in the future, for now it's a straight
// 1:1 mapping (if given)
func (g *GithubRegistry) MakeRepositoryPath(t Type) (string, error) {
// First map the collection
collection, err := g.mapCollection(t.Collection)
if err != nil {
return "", err
}
// Construct the return path
p := ""
if len(g.path) > 0 {
p += g.path + "/"
}
if len(collection) > 0 {
p += collection + "/"
}
return p + t.Name + "/" + t.Version, nil
// GetRegistryName returns the name of this registry
func (g githubRegistry) GetRegistryPath() string {
return g.path
}
registry/github_template_registry.go 0 → 100644
View file @ 4b4d58c8
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry
import (
"github.com/google/go-github/github"
"github.com/kubernetes/deployment-manager/common"
"fmt"
"log"
"net/url"
"regexp"
"strings"
)
// GithubTemplateRegistry implements the Registry interface and implements a
// Deployment Manager templates registry.
// A registry root must be a directory that contains all the available templates,
// one directory per template. Each template directory then contains version
// directories, each of which in turn contains all the files necessary for that
// version of the template.
//
// For example, a template registry containing two versions of redis
// (implemented in jinja), and one version of replicatedservice (implemented
// in python) would have a directory structure that looks something like this:
// qualifier [optional] prefix to a virtual root within the repository.
// /redis
// /v1
// redis.jinja
// redis.jinja.schema
// /v2
// redis.jinja
// redis.jinja.schema
// /replicatedservice
// /v1
// replicatedservice.python
// replicatedservice.python.schema
type GithubTemplateRegistry struct {
githubRegistry
}
// NewGithubTemplateRegistry creates a GithubTemplateRegistry.
func NewGithubTemplateRegistry(name, shortURL string, service RepositoryService) (*GithubTemplateRegistry, error) {
format := fmt.Sprintf("%s;%s", common.VersionedRegistry, common.CollectionRegistry)
if service == nil {
client := github.NewClient(nil)
service = client.Repositories
}
gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), service)
if err != nil {
return nil, err
}
return &GithubTemplateRegistry{githubRegistry: *gr}, nil
}
// ListTypes lists types in this registry whose string values conform to the
// supplied regular expression, or all types, if the regular expression is nil.
func (g GithubTemplateRegistry) ListTypes(regex *regexp.Regexp) ([]Type, error) {
// First list all the collections at the top level.
collections, err := g.getDirs("")
if err != nil {
log.Printf("cannot list qualifiers: %v", err)
return nil, err
}
var retTypes []Type
for _, c := range collections {
// Then we need to fetch the versions (directories for this type)
types, err := g.getDirs(c)
if err != nil {
log.Printf("cannot fetch types for collection: %s", c)
return nil, err
}
for _, t := range types {
path := c + "/" + t
// Then we need to fetch the versions (directories for this type)
versions, err := g.getDirs(path)
if err != nil {
log.Printf("cannot fetch versions at path %s", path)
return nil, err
}
for _, v := range versions {
tt, err := NewType(c, t, v)
if err != nil {
return nil, fmt.Errorf("malformed type at path %s", path)
}
retTypes = append(retTypes, tt)
}
}
}
if regex != nil {
var matchTypes []Type
for _, retType := range retTypes {
if regex.MatchString(retType.String()) {
matchTypes = append(matchTypes, retType)
}
}
return matchTypes, nil
}
return retTypes, nil
}
// GetDownloadURLs fetches the download URLs for a given Type and checks for existence of a schema file.
func (g GithubTemplateRegistry) GetDownloadURLs(t Type) ([]*url.URL, error) {
path, err := g.MakeRepositoryPath(t)
if err != nil {
return nil, err
}
_, dc, _, err := g.service.GetContents(g.owner, g.repository, path, nil)
if err != nil {
return nil, fmt.Errorf("cannot list versions at path %s: %v", path, err)
}
var downloadURL, typeName, schemaName string
for _, f := range dc {
if *f.Type == "file" {
if *f.Name == t.Name+".jinja" || *f.Name == t.Name+".py" {
typeName = *f.Name
downloadURL = *f.DownloadURL
}
if *f.Name == t.Name+".jinja.schema" || *f.Name == t.Name+".py.schema" {
schemaName = *f.Name
}
}
}
if downloadURL == "" {
return nil, fmt.Errorf("cannot find type %s", t.String())
}
if schemaName != typeName+".schema" {
return nil, fmt.Errorf("cannot find schema for %s, expected %s", t.String(), typeName+".schema")
}
result, err := url.Parse(downloadURL)
if err != nil {
return nil, fmt.Errorf("cannot parse URL from %s: %s", downloadURL, err)
}
return []*url.URL{result}, nil
}
func (g GithubTemplateRegistry) getDirs(dir string) ([]string, error) {
var path = g.path
if dir != "" {
path = g.path + "/" + dir
}
_, dc, _, err := g.service.GetContents(g.owner, g.repository, path, nil)
if err != nil {
log.Printf("Failed to get contents at path: %s: %v", path, err)
return nil, err
}
var dirs []string
for _, entry := range dc {
if *entry.Type == "dir" {
dirs = append(dirs, *entry.Name)
}
}
return dirs, nil
}
func (g GithubTemplateRegistry) mapCollection(collection string) (string, error) {
if strings.ContainsAny(collection, "/") {
return "", fmt.Errorf("collection must not contain slashes, got %s", collection)
}
// TODO(vaikas): Implement lookup from the root metadata file to map collection to a path
return collection, nil
}
// MakeRepositoryPath constructs a github path to a given type based on a repository, and type name and version.
// The returned repository path will be of the form:
// [GithubTemplateRegistry.path/][Type.Collection]/Type.Name/Type.Version
// Type.Collection will be mapped using mapCollection in the future, for now it's a straight
// 1:1 mapping (if given)
func (g GithubTemplateRegistry) MakeRepositoryPath(t Type) (string, error) {
// First map the collection
collection, err := g.mapCollection(t.Collection)
if err != nil {
return "", err
}
// Construct the return path
p := ""
if len(g.path) > 0 {
p += g.path + "/"
}
if len(collection) > 0 {
p += collection + "/"
}
return p + t.Name + "/" + t.GetVersion(), nil
}
registry/inmem_registry_service.go 0 → 100644
View file @ 4b4d58c8
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry
import (
"github.com/kubernetes/deployment-manager/common"
"github.com/kubernetes/deployment-manager/util"
"fmt"
"strings"
)
type inmemRegistryService struct {
registries map[string]*common.AuthenticatedRegistry
}
func NewInmemRegistryService() common.RegistryService {
rs := &inmemRegistryService{
registries: make(map[string]*common.AuthenticatedRegistry),
}
pFormat := fmt.Sprintf("%s;%s", common.UnversionedRegistry, common.OneLevelRegistry)
rs.Create(&common.Registry{
Name: "charts",
Type: common.GithubRegistryType,
URL: "github.com/helm/charts",
Format: common.RegistryFormat(pFormat),
})
tFormat := fmt.Sprintf("%s;%s", common.VersionedRegistry, common.CollectionRegistry)
rs.Create(&common.Registry{
Name: "application-dm-templates",
Type: common.GithubRegistryType,
URL: "github.com/kubernetes/application-dm-templates",
Format: common.RegistryFormat(tFormat),
})
return rs
}
// List returns the list of known registries.
func (rs *inmemRegistryService) List() ([]*common.Registry, error) {
ret := []*common.Registry{}
for _, r := range rs.registries {
ret = append(ret, &r.Registry)
}
return ret, nil
}
// Create creates an authenticated registry.
func (rs *inmemRegistryService) Create(registry *common.Registry) error {
rs.registries[registry.Name] = &common.AuthenticatedRegistry{Registry: *registry}
return nil
}
// Get returns a registry with a given name.
func (rs *inmemRegistryService) Get(name string) (*common.Registry, error) {
r, ok := rs.registries[name]
if !ok {
return nil, fmt.Errorf("Failed to find registry named %s", name)
}
return &r.Registry, nil
}
// GetAuthenticatedRegistry returns an authenticated registry with a given name.
func (rs *inmemRegistryService) GetAuthenticatedRegistry(name string) (*common.AuthenticatedRegistry, error) {
r, ok := rs.registries[name]
if !ok {
return nil, fmt.Errorf("Failed to find registry named %s", name)
}
return r, nil
}
// Create deletes the authenticated registry with a given name.
func (rs *inmemRegistryService) Delete(name string) error {
_, ok := rs.registries[name]
if !ok {
return fmt.Errorf("Failed to find registry named %s", name)
}
delete(rs.registries, name)
return nil
}
// GetByURL returns a registry that handles the types for a given URL.
func (rs *inmemRegistryService) GetByURL(URL string) (*common.Registry, error) {
trimmed := util.TrimURLScheme(URL)
for _, r := range rs.registries {
if strings.HasPrefix(trimmed, util.TrimURLScheme(r.URL)) {
return &r.Registry, nil
}
}
return nil, fmt.Errorf("Failed to find registry for url: %s", URL)
}
// GetAuthenticatedRegistryByURL returns an authenticated registry that handles the types for a given URL.
func (rs *inmemRegistryService) GetAuthenticatedRegistryByURL(URL string) (*common.AuthenticatedRegistry, error) {
trimmed := util.TrimURLScheme(URL)
for _, r := range rs.registries {
if strings.HasPrefix(trimmed, util.TrimURLScheme(r.URL)) {
return r, nil
}
}
return nil, fmt.Errorf("Failed to find registry for url: %s", URL)
}
// Set the credential for a registry.
func (rs *inmemRegistryService) SetCredential(name string, credential common.RegistryCredential) error {
r, ok := rs.registries[name]
if !ok {
return fmt.Errorf("Failed to find registry named %s", name)
}
r.Credential = credential
return nil
}
// Get the credential for a registry.
func (rs *inmemRegistryService) GetCredential(name string) (common.RegistryCredential, error) {
r, ok := rs.registries[name]
if !ok {
return common.RegistryCredential{}, fmt.Errorf("Failed to find registry named %s", name)
}
return r.Credential, nil
}
registry/registry.go
View file @ 4b4d58c8
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
......@@ -17,61 +17,126 @@ limitations under the License.
package registry
import (
"strings"
"github.com/kubernetes/deployment-manager/common"
"fmt"
"net/url"
"regexp"
"strings"
)
type RegistryService interface {
// List all the registries
List() ([]*common.Registry, error)
// Create a new registry
Create(repository *common.Registry) error
// Get a registry
Get(name string) (*common.Registry, error)
// Delete a registry
Delete(name string) error
// Find a registry that backs the given URL
GetByURL(URL string) (*common.Registry, error)
// Registry abstracts a registry that holds charts, which can be
// used in a Deployment Manager configuration. There can be multiple
// registry implementations.
type Registry interface {
// GetRegistryName returns the name of this registry
GetRegistryName() string
// GetRegistryType returns the type of this registry.
GetRegistryType() common.RegistryType
// GetRegistryShortURL returns the short URL for this registry.
GetRegistryShortURL() string
// GetRegistryFormat returns the format of this registry.
GetRegistryFormat() common.RegistryFormat
// ListTypes lists types in this registry whose string values conform to the
// supplied regular expression, or all types, if the regular expression is nil.
ListTypes(regex *regexp.Regexp) ([]Type, error)
// GetDownloadURLs returns the URLs required to download the type contents.
GetDownloadURLs(t Type) ([]*url.URL, error)
}
// GithubRegistry abstracts a registry that resides in a Github repository.
type GithubRegistry interface {
Registry // A GithubRegistry is a Registry.
// GetRegistryOwner returns the owner name for this registry
GetRegistryOwner() string
// GetRegistryRepository returns the repository name for this registry.
GetRegistryRepository() string
// GetRegistryPath returns the path to the registry in the repository.
GetRegistryPath() string
}
// Registry abstracts a registry that holds templates, which can be
// used in a Deployment Manager configurations. There can be multiple
// implementations of a registry. Currently we support Deployment Manager
// github.com/kubernetes/application-dm-templates
// and helm packages
// github.com/helm/charts
//
type Type struct {
Collection string
Name string
Version string
version SemVer
}
// ParseType takes a registry name and parses it into a *registry.Type.
func ParseType(name string) *Type {
tt := &Type{}
// NewType initializes a type
func NewType(collection, name, version string) (Type, error) {
result := Type{Collection: collection, Name: name}
err := result.SetVersion(version)
return result, err
}
// NewTypeOrDie initializes a type and panics if initialization fails
func NewTypeOrDie(collection, name, version string) Type {
result, err := NewType(collection, name, version)
if err != nil {
panic(err)
}
return result
}
// Type conforms to the Stringer interface.
func (t Type) String() string {
var result string
if t.Collection != "" {
result = t.Collection + "/"
}
tList := strings.Split(name, ":")
result = result + t.Name
version := t.GetVersion()
if version != "" && version != "v0" {
result = result + ":" + version
}
return result
}
// GetVersion returns the type version with the letter "v" prepended.
func (t Type) GetVersion() string {
var result string
version := t.version.String()
if version != "0" {
result = "v" + version
}
return result
}
// SetVersion strips the letter "v" from version, if present,
// and sets the the version of the type to the result.
func (t *Type) SetVersion(version string) error {
vstring := strings.TrimPrefix(version, "v")
s, err := ParseSemVer(vstring)
if err != nil {
return err
}
t.version = s
return nil
}
// ParseType takes a registry type string and parses it into a *registry.Type.
// TODO: needs better validation that this is actually a registry type.
func ParseType(ts string) (Type, error) {
tt := Type{}
tList := strings.Split(ts, ":")
if len(tList) == 2 {
tt.Version = tList[1]
if err := tt.SetVersion(tList[1]); err != nil {
return tt, fmt.Errorf("malformed type string: %s", ts)
}
}
cList := strings.Split(tList[0], "/")
if len(cList) == 1 {
tt.Name = tList[0]
} else {
tt.Collection = cList[0]
tt.Name = cList[1]
}
return tt
}
// Registry abstracts type interactions.
type Registry interface {
// List all the templates at the given path
List() ([]Type, error)
// Get the download URL(s) for a given type
GetURLs(t Type) ([]string, error)
return tt, nil
}
registry/registry_test.go
View file @ 4b4d58c8
......@@ -4,25 +4,36 @@ import (
"testing"
)
func TestParseType(t *testing.T) {
// TODO: Are there some real-world examples we want to valide here?
tests := map[string]*Type{
"foo": &Type{Name: "foo"},
"foo:v1": &Type{Name: "foo", Version: "v1"},
"github.com/foo": &Type{Name: "foo", Collection: "github.com"},
"github.com/foo:v1.2.3": &Type{Name: "foo", Collection: "github.com", Version: "v1.2.3"},
func TestTypeConversion(t *testing.T) {
// TODO: Are there some real-world examples we want to validate here?
tests := map[string]Type{
"foo": NewTypeOrDie("", "foo", ""),
"foo:v1": NewTypeOrDie("", "foo", "v1"),
"github.com/foo": NewTypeOrDie("github.com", "foo", ""),
"github.com/foo:v1.2.3": NewTypeOrDie("github.com", "foo", "v1.2.3"),
}
for in, expect := range tests {
out := ParseType(in)
out, err := ParseType(in)
if err != nil {
t.Errorf("Error parsing type string %s: $s", in, err)
}
if out.Name != expect.Name {
t.Errorf("Expected name to be %q, got %q", expect.Name, out.Name)
}
if out.Version != expect.Version {
t.Errorf("Expected version to be %q, got %q", expect.Version, out.Version)
if out.GetVersion() != expect.GetVersion() {
t.Errorf("Expected version to be %q, got %q", expect.GetVersion(), out.GetVersion())
}
if out.Collection != expect.Collection {
t.Errorf("Expected collection to be %q, got %q", expect.Collection, out.Collection)
}
svalue := out.String()
if svalue != in {
t.Errorf("Expected string value to be %q, got %q", in, svalue)
}
}
}
registry/registryprovider.go
View file @ 4b4d58c8
......@@ -17,39 +17,249 @@ limitations under the License.
package registry
import (
"fmt"
"github.com/google/go-github/github"
"github.com/kubernetes/deployment-manager/common"
"github.com/kubernetes/deployment-manager/util"
"fmt"
"net/url"
"regexp"
"strings"
"sync"
)
// RegistryProvider returns factories for creating registries for a given RegistryType.
// RegistryProvider is a factory for Registry instances.
type RegistryProvider interface {
GetRegistry(prefix string) (Registry, error)
GetRegistryByShortURL(URL string) (Registry, error)
GetRegistryByName(registryName string) (Registry, error)
}
type registryProvider struct {
sync.RWMutex
rs common.RegistryService
grp GithubRegistryProvider
registries map[string]Registry
}
func NewDefaultRegistryProvider() RegistryProvider {
rs := NewInmemRepositoryService()
return &DefaultRegistryProvider{rs: rs}
return NewRegistryProvider(nil, NewGithubRegistryProvider())
}
func NewRegistryProvider(rs common.RegistryService, grp GithubRegistryProvider) RegistryProvider {
if rs == nil {
rs = NewInmemRegistryService()
}
if grp == nil {
grp = NewGithubRegistryProvider()
}
registries := make(map[string]Registry)
rp := &registryProvider{rs: rs, grp: grp, registries: registries}
return rp
}
type DefaultRegistryProvider struct {
rs RegistryService
func (rp registryProvider) GetRegistryByShortURL(URL string) (Registry, error) {
rp.RLock()
defer rp.RUnlock()
result := rp.findRegistryByShortURL(URL)
if result == nil {
cr, err := rp.rs.GetByURL(URL)
if err != nil {
return nil, err
}
r, err := rp.grp.GetGithubRegistry(*cr)
if err != nil {
return nil, err
}
rp.registries[r.GetRegistryName()] = r
result = r
}
return result, nil
}
func (drp *DefaultRegistryProvider) GetRegistry(URL string) (Registry, error) {
r, err := drp.rs.GetByURL(URL)
// findRegistryByShortURL trims the scheme from both the supplied URL
// and the short URL returned by GetRegistryShortURL.
func (rp registryProvider) findRegistryByShortURL(URL string) Registry {
trimmed := util.TrimURLScheme(URL)
for _, r := range rp.registries {
if strings.HasPrefix(trimmed, util.TrimURLScheme(r.GetRegistryShortURL())) {
return r
}
}
return nil
}
func (rp registryProvider) GetRegistryByName(registryName string) (Registry, error) {
rp.RLock()
defer rp.RUnlock()
result, ok := rp.registries[registryName]
if !ok {
cr, err := rp.rs.Get(registryName)
if err != nil {
return nil, err
}
r, err := rp.grp.GetGithubRegistry(*cr)
if err != nil {
return nil, err
}
if r.Type == common.Github {
if r.Format == common.UnversionedRegistry {
return NewGithubPackageRegistry("helm", "charts", github.NewClient(nil)), nil
rp.registries[r.GetRegistryName()] = r
result = r
}
return result, nil
}
func ParseRegistryFormat(rf common.RegistryFormat) map[common.RegistryFormat]bool {
split := strings.Split(string(rf), ";")
var result = map[common.RegistryFormat]bool{}
for _, format := range split {
result[common.RegistryFormat(format)] = true
}
return result
}
// GithubRegistryProvider is a factory for GithubRegistry instances.
type GithubRegistryProvider interface {
GetGithubRegistry(cr common.Registry) (GithubRegistry, error)
}
type githubRegistryProvider struct {
}
// NewGithubRegistryProvider creates a GithubRegistryProvider.
func NewGithubRegistryProvider() GithubRegistryProvider {
return &githubRegistryProvider{}
}
func (grp githubRegistryProvider) GetGithubRegistry(cr common.Registry) (GithubRegistry, error) {
if cr.Type == common.GithubRegistryType {
fMap := ParseRegistryFormat(cr.Format)
if fMap[common.UnversionedRegistry] && fMap[common.OneLevelRegistry] {
return NewGithubPackageRegistry(cr.Name, cr.URL, nil)
}
if r.Format == common.VersionedRegistry {
return NewGithubRegistry("kubernetes", "application-dm-templates", "", github.NewClient(nil)), nil
if fMap[common.VersionedRegistry] && fMap[common.CollectionRegistry] {
return NewGithubTemplateRegistry(cr.Name, cr.URL, nil)
}
return nil, fmt.Errorf("unknown registry format: %s", cr.Format)
}
return nil, fmt.Errorf("cannot find registry backing url %s", URL)
return nil, fmt.Errorf("unknown registry type: %s", cr.Type)
}
// RE for a registry type that does support versions and has collections.
var TemplateRegistryMatcher = regexp.MustCompile("github.com/(.*)/(.*)/(.*)/(.*):(.*)")
// RE for a registry type that does not support versions and does not have collections.
var PackageRegistryMatcher = regexp.MustCompile("github.com/(.*)/(.*)/(.*)")
// IsGithubShortType returns whether a given type is a type description in a short format to a github repository type.
// For now, this means using github types:
// github.com/owner/repo/qualifier/type:version
// for example:
// github.com/kubernetes/application-dm-templates/storage/redis:v1
func IsGithubShortType(t string) bool {
return TemplateRegistryMatcher.MatchString(t)
}
// IsGithubShortPackageType returns whether a given type is a type description in a short format to a github
// package repository type.
// For now, this means using github types:
// github.com/owner/repo/type
// for example:
// github.com/helm/charts/cassandra
func IsGithubShortPackageType(t string) bool {
return PackageRegistryMatcher.MatchString(t)
}
// GetDownloadURLs checks a type to see if it is either a short git hub url or a fully specified URL
// and returns the URLs that should be used to fetch it. If the url is not fetchable (primitive type
// for example), it returns an empty slice.
func GetDownloadURLs(rp RegistryProvider, t string) ([]string, error) {
if IsGithubShortType(t) {
return ShortTypeToDownloadURLs(rp, t)
} else if IsGithubShortPackageType(t) {
return ShortTypeToPackageDownloadURLs(rp, t)
} else if util.IsHttpUrl(t) {
result, err := url.Parse(t)
if err != nil {
return nil, fmt.Errorf("cannot parse download URL %s: %s", t, err)
}
return []string{result.String()}, nil
}
return []string{}, nil
}
// ShortTypeToDownloadURLs converts a github URL into downloadable URL from github.
// Input must be of the type and is assumed to have been validated before this call:
// github.com/owner/repo/qualifier/type:version
// for example:
// github.com/kubernetes/application-dm-templates/storage/redis:v1
func ShortTypeToDownloadURLs(rp RegistryProvider, t string) ([]string, error) {
m := TemplateRegistryMatcher.FindStringSubmatch(t)
if len(m) != 6 {
return nil, fmt.Errorf("cannot parse short github url: %s", t)
}
r, err := rp.GetRegistryByShortURL(t)
if err != nil {
return nil, err
}
if r == nil {
panic(fmt.Errorf("cannot get github registry for %s", t))
}
tt, err := NewType(m[3], m[4], m[5])
if err != nil {
return nil, err
}
urls, err := r.GetDownloadURLs(tt)
if err != nil {
return nil, err
}
return util.ConvertURLsToStrings(urls), err
}
// ShortTypeToPackageDownloadURLs converts a github URL into downloadable URLs from github.
// Input must be of the type and is assumed to have been validated before this call:
// github.com/owner/repo/type
// for example:
// github.com/helm/charts/cassandra
func ShortTypeToPackageDownloadURLs(rp RegistryProvider, t string) ([]string, error) {
m := PackageRegistryMatcher.FindStringSubmatch(t)
if len(m) != 4 {
return nil, fmt.Errorf("Failed to parse short github url: %s", t)
}
r, err := rp.GetRegistryByShortURL(t)
if err != nil {
return nil, err
}
tt, err := NewType("", m[3], "")
if err != nil {
return nil, err
}
urls, err := r.GetDownloadURLs(tt)
if err != nil {
return nil, err
}
return util.ConvertURLsToStrings(urls), err
}
registry/registryprovider_test.go 0 → 100644
View file @ 4b4d58c8
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry
import (
"testing"
)
func testUrlConversionDriver(rp RegistryProvider, tests map[string]TestURLAndError, t *testing.T) {
for in, expected := range tests {
actual, err := GetDownloadURLs(rp, in)
if err != expected.Err {
t.Fatalf("failed on: %s : expected error %v but got %v", in, expected.Err, err)
}
if actual[0] != expected.URL {
t.Fatalf("failed on: %s : expected %s but got %v", in, expected.URL, actual)
}
}
}
func TestShortGithubUrlTemplateMapping(t *testing.T) {
githubUrlMaps := map[Type]TestURLAndError{
NewTypeOrDie("common", "replicatedservice", "v1"): TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil},
NewTypeOrDie("storage", "redis", "v1"): TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil},
}
tests := map[string]TestURLAndError{
"github.com/kubernetes/application-dm-templates/common/replicatedservice:v1": TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil},
"github.com/kubernetes/application-dm-templates/storage/redis:v1": TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil},
}
grp := NewTestGithubRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps)
testUrlConversionDriver(NewRegistryProvider(nil, grp), tests, t)
}
func TestShortGithubUrlPackageMapping(t *testing.T) {
githubUrlMaps := map[Type]TestURLAndError{
NewTypeOrDie("", "mongodb", ""): TestURLAndError{"https://raw.githubusercontent.com/helm/charts/master/mongodb/manifests/mongodb.yaml", nil},
NewTypeOrDie("", "redis", ""): TestURLAndError{"https://raw.githubusercontent.com/helm/charts/master/redis/manifests/redis.yaml", nil},
}
tests := map[string]TestURLAndError{
"github.com/helm/charts/mongodb": TestURLAndError{"https://raw.githubusercontent.com/helm/charts/master/mongodb/manifests/mongodb.yaml", nil},
"github.com/helm/charts/redis": TestURLAndError{"https://raw.githubusercontent.com/helm/charts/master/redis/manifests/redis.yaml", nil},
}
grp := NewTestGithubRegistryProvider("github.com/helm/charts", githubUrlMaps)
testUrlConversionDriver(NewRegistryProvider(nil, grp), tests, t)
}
registry/semver.go 0 → 100644
View file @ 4b4d58c8
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry
import (
"fmt"
"strconv"
"strings"
)
type SemVer struct {
Major uint
Minor uint
Patch uint
}
func ParseSemVer(version string) (SemVer, error) {
var err error
major, minor, patch := uint64(0), uint64(0), uint64(0)
if len(version) > 0 {
parts := strings.SplitN(version, ".", 3)
if len(parts) > 3 {
return SemVer{}, fmt.Errorf("invalid semantic version: %s", version)
}
if len(parts) < 1 {
return SemVer{}, fmt.Errorf("invalid semantic version: %s", version)
}
if parts[0] != "0" {
major, err = strconv.ParseUint(parts[0], 10, 0)
if err != nil {
return SemVer{}, fmt.Errorf("invalid semantic version: %s", version)
}
}
if len(parts) > 1 {
if parts[1] != "0" {
minor, err = strconv.ParseUint(parts[1], 10, 0)
if err != nil {
return SemVer{}, fmt.Errorf("invalid semantic version: %s", version)
}
}
if len(parts) > 2 {
if parts[2] != "0" {
patch, err = strconv.ParseUint(parts[2], 10, 0)
if err != nil {
return SemVer{}, fmt.Errorf("invalid semantic version: %s", version)
}
}
}
}
}
return SemVer{Major: uint(major), Minor: uint(minor), Patch: uint(patch)}, nil
}
func (s SemVer) IsZero() bool {
return s.Major == 0 && s.Minor == 0 && s.Patch == 0
}
// SemVer conforms to the Stringer interface.
func (s SemVer) String() string {
result := strconv.Itoa(int(s.Major))
if s.Minor != 0 || s.Patch != 0 {
result = result + "." + strconv.Itoa(int(s.Minor))
}
if s.Patch != 0 {
result = result + "." + strconv.Itoa(int(s.Patch))
}
return result
}
registry/semver_test.go 0 → 100644
View file @ 4b4d58c8
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry
import (
"testing"
)
func TestParseInvalidVersionFails(t *testing.T) {
for _, test := range []string{
".",
"..",
"...",
"1.2.3.4",
"notAUnit",
"1.notAUint",
"1.1.notAUint",
"-1",
"1.-1",
"1.1.-1",
"1,1",
"1.1,1",
} {
_, err := ParseSemVer(test)
if err == nil {
t.Errorf("Invalid version parsed successfully: %s\n", test)
}
}
}
func TestParseValidVersionSucceeds(t *testing.T) {
for _, test := range []struct {
String string
Version SemVer
}{
{"", SemVer{0, 0, 0}},
{"0", SemVer{0, 0, 0}},
{"0.0", SemVer{0, 0, 0}},
{"0.0.0", SemVer{0, 0, 0}},
{"1", SemVer{1, 0, 0}},
{"1.0", SemVer{1, 0, 0}},
{"1.0.0", SemVer{1, 0, 0}},
{"1.1", SemVer{1, 1, 0}},
{"1.1.0", SemVer{1, 1, 0}},
{"1.1.1", SemVer{1, 1, 1}},
} {
result, err := ParseSemVer(test.String)
if err != nil {
t.Errorf("Valid version %s did not parse successfully\n", test.String)
}
if result.Major != test.Version.Major ||
result.Minor != test.Version.Minor ||
result.Patch != test.Version.Patch {
t.Errorf("Valid version %s did not parse correctly: %s\n", test.String, test.Version)
}
}
}
func TestConvertSemVerToStringSucceeds(t *testing.T) {
for _, test := range []struct {
String string
Version SemVer
}{
{"0", SemVer{0, 0, 0}},
{"0.1", SemVer{0, 1, 0}},
{"0.0.1", SemVer{0, 0, 1}},
{"1", SemVer{1, 0, 0}},
{"1.1", SemVer{1, 1, 0}},
{"1.1.1", SemVer{1, 1, 1}},
} {
result := test.Version.String()
if result != test.String {
t.Errorf("Valid version %s did not format correctly: %s\n", test.Version, test.String)
}
}
}
registry/inmem_repository_service.go → registry/testhelper.go
View file @ 4b4d58c8
......@@ -16,63 +16,67 @@ limitations under the License.
package registry
// TODO(jackgr): Mock github repository service to test package and template registry implementations.
import (
"github.com/kubernetes/deployment-manager/common"
"github.com/kubernetes/deployment-manager/util"
"fmt"
"net/url"
"regexp"
"strings"
"github.com/kubernetes/deployment-manager/common"
)
type inmemRepositoryService struct {
repositories map[string]*common.Registry
type TestURLAndError struct {
URL string
Err error
}
func NewInmemRepositoryService() RegistryService {
rs := &inmemRepositoryService{
repositories: make(map[string]*common.Registry),
}
rs.Create(&common.Registry{
Name: "charts",
Type: common.Github,
URL: "github.com/helm/charts",
Format: common.UnversionedRegistry,
})
rs.Create(&common.Registry{
Name: "application-dm-templates",
Type: common.Github,
URL: "github.com/kubernetes/application-dm-templates",
Format: common.VersionedRegistry,
})
return rs
type testGithubRegistryProvider struct {
shortURL string
responses map[Type]TestURLAndError
}
func (rs *inmemRepositoryService) List() ([]*common.Registry, error) {
ret := []*common.Registry{}
for _, r := range rs.repositories {
ret = append(ret, r)
}
return ret, nil
type testGithubRegistry struct {
githubRegistry
responses map[Type]TestURLAndError
}
func (rs *inmemRepositoryService) Create(repository *common.Registry) error {
rs.repositories[repository.URL] = repository
return nil
func NewTestGithubRegistryProvider(shortURL string, responses map[Type]TestURLAndError) GithubRegistryProvider {
return testGithubRegistryProvider{
shortURL: util.TrimURLScheme(shortURL),
responses: responses,
}
}
func (rs *inmemRepositoryService) Get(name string) (*common.Registry, error) {
return &common.Registry{}, nil
func (tgrp testGithubRegistryProvider) GetGithubRegistry(cr common.Registry) (GithubRegistry, error) {
trimmed := util.TrimURLScheme(cr.URL)
if strings.HasPrefix(trimmed, tgrp.shortURL) {
ghr, err := newGithubRegistry(cr.Name, trimmed, cr.Format, nil)
if err != nil {
panic(fmt.Errorf("cannot create a github registry: %s", err))
}
return &testGithubRegistry{
githubRegistry: *ghr,
responses: tgrp.responses,
}, nil
}
panic(fmt.Errorf("unknown registry: %v", cr))
}
func (rs *inmemRepositoryService) Delete(name string) error {
return nil
func (tgr testGithubRegistry) ListTypes(regex *regexp.Regexp) ([]Type, error) {
panic(fmt.Errorf("ListTypes should not be called in the test"))
}
// GetByURL returns a registry that handles the types for a given URL.
func (rs *inmemRepositoryService) GetByURL(URL string) (*common.Registry, error) {
for _, r := range rs.repositories {
if strings.HasPrefix(URL, r.URL) {
return r, nil
}
func (tgr testGithubRegistry) GetDownloadURLs(t Type) ([]*url.URL, error) {
result := tgr.responses[t]
URL, err := url.Parse(result.URL)
if err != nil {
panic(err)
}
return nil, fmt.Errorf("Failed to find registry for github url: %s", URL)
return []*url.URL{URL}, result.Err
}
resourcifier/configurations.go
View file @ 4b4d58c8
......@@ -238,14 +238,14 @@ func getConfigurator() *configurator.Configurator {
func getPathVariable(w http.ResponseWriter, r *http.Request, variable, handler string) (string, error) {
vars := mux.Vars(r)
variable, ok := vars[variable]
escaped, ok := vars[variable]
if !ok {
e := errors.New(fmt.Sprintf("%s name not found in URL", variable))
util.LogAndReturnError(handler, http.StatusBadRequest, e, w)
return "", e
}
unescaped, err := url.QueryUnescape(variable)
unescaped, err := url.QueryUnescape(escaped)
if err != nil {
e := fmt.Errorf("cannot decode name (%v)", variable)
util.LogAndReturnError(handler, http.StatusBadRequest, e, w)
......
util/httputil.go
View file @ 4b4d58c8
......@@ -29,6 +29,26 @@ import (
"github.com/ghodss/yaml"
)
// ConvertURLsToStrings converts a slice of *url.URL to a slice of string.
func ConvertURLsToStrings(urls []*url.URL) []string {
var result []string
for _, u := range urls {
result = append(result, u.String())
}
return result
}
// TrimURLScheme removes the scheme, if any, from an URL.
func TrimURLScheme(URL string) string {
parts := strings.SplitAfter(URL, "://")
if len(parts) > 1 {
return parts[1]
}
return URL
}
// A HandlerTester is a function that takes an HTTP method, an URL path, and a
// reader for a request body, creates a request from them, and serves it to the
// handler to which it was bound and returns a response recorder describing the
......
util/templateutil.go deleted 100644 → 0
View file @ 3cca3065
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"regexp"
"github.com/kubernetes/deployment-manager/common"
)
var TemplateRegistryMatcher = regexp.MustCompile("github.com/(.*)/(.*)/(.*)/(.*):(.*)")
// RE for Registry that does not support versions and can have multiple files without imports.
var PackageRegistryMatcher = regexp.MustCompile("github.com/(.*)/(.*)/(.*)")
// IsTemplate returns whether a given type is a template.
func IsTemplate(t string, imports []*common.ImportFile) bool {
for _, imp := range imports {
if imp.Name == t {
return true
}
}
return false
}
// IsGithubShortType returns whether a given type is a type description in a short format to a github repository type.
// For now, this means using github types:
// github.com/owner/repo/qualifier/type:version
// for example:
// github.com/kubernetes/application-dm-templates/storage/redis:v1
func IsGithubShortType(t string) bool {
return TemplateRegistryMatcher.MatchString(t)
}
// IsGithubShortPackageType returns whether a given type is a type description in a short format to a github
// package repository type.
// For now, this means using github types:
// github.com/owner/repo/type
// for example:
// github.com/helm/charts/cassandra
func IsGithubShortPackageType(t string) bool {
return PackageRegistryMatcher.MatchString(t)
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment