Skip to content

H
helm3
Commit e1afffbc authored by jackgr's avatar jackgr
Browse files
Options

First pass removal of TypeResolver

parent 873a8af4
Hide whitespace changes
Inline Side-by-side
Showing with 17 additions and 603 deletions
cmd/manager/manager/expander.go
View file @ e1afffbc
......@@ -43,8 +43,17 @@ type Expander interface {
ExpandTemplate(t *common.Template) (*ExpandedTemplate, error)
}
// TODO: Remove mockResolver when type resolver is completely excised
type mockResolver struct {
}
func (r *mockResolver) ResolveTypes(c *common.Configuration, i []*common.ImportFile) ([]*common.ImportFile, error) {
return nil, nil
}
// NewExpander returns a new initialized Expander.
func NewExpander(url string, tr TypeResolver) Expander {
func NewExpander(url string) Expander {
tr := &mockResolver{}
return &expander{url, tr}
}
......@@ -53,6 +62,12 @@ type expander struct {
typeResolver TypeResolver
}
// TypeResolver finds Types in a Configuration which aren't yet reduceable to an import file
// or primitive, and attempts to replace them with a template from a URL.
type TypeResolver interface {
ResolveTypes(config *common.Configuration, imports []*common.ImportFile) ([]*common.ImportFile, error)
}
func (e *expander) getBaseURL() string {
return fmt.Sprintf("%s/expand", e.expanderURL)
}
......
cmd/manager/manager/expander_test.go
View file @ e1afffbc
......@@ -32,21 +32,6 @@ import (
"github.com/ghodss/yaml"
)
type mockResolver struct {
responses [][]*common.ImportFile
t *testing.T
}
func (r *mockResolver) ResolveTypes(c *common.Configuration, i []*common.ImportFile) ([]*common.ImportFile, error) {
if len(r.responses) < 1 {
return nil, nil
}
ret := r.responses[0]
r.responses = r.responses[1:]
return ret, nil
}
var validTemplateTestCaseData = common.Template{
Name: "TestTemplate",
Content: string(validContentTestCaseData),
......@@ -232,7 +217,6 @@ type ExpanderTestCase struct {
Description string
Error string
Handler func(w http.ResponseWriter, r *http.Request)
Resolver TypeResolver
ValidResponse *ExpandedTemplate
}
......@@ -247,33 +231,21 @@ func TestExpandTemplate(t *testing.T) {
"expect success for ExpandTemplate",
"",
expanderSuccessHandler,
&mockResolver{},
getValidResponse(t, "expect success for ExpandTemplate"),
},
{
"expect error for ExpandTemplate",
"cannot expand template",
expanderErrorHandler,
&mockResolver{},
nil,
},
{
"expect success for ExpandTemplate with two expansions",
"",
roundTripHandler,
&mockResolver{[][]*common.ImportFile{
{},
{&common.ImportFile{Name: "test.py"}},
}, t},
roundTripResponse,
},
}
for _, etc := range tests {
ts := httptest.NewServer(http.HandlerFunc(etc.Handler))
defer ts.Close()
expander := NewExpander(ts.URL, etc.Resolver)
expander := NewExpander(ts.URL)
actualResponse, err := expander.ExpandTemplate(&validTemplateTestCaseData)
if err != nil {
message := err.Error()
......
cmd/manager/manager/typeresolver.go deleted 100644 → 0
View file @ 873a8af4
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package manager
import (
"fmt"
"net/http"
"github.com/kubernetes/helm/pkg/common"
"github.com/kubernetes/helm/pkg/registry"
"github.com/kubernetes/helm/pkg/util"
"github.com/ghodss/yaml"
)
const (
maxURLImports = 100
schemaSuffix = ".schema"
)
// TypeResolver finds Types in a Configuration which aren't yet reduceable to an import file
// or primitive, and attempts to replace them with a template from a URL.
type TypeResolver interface {
ResolveTypes(config *common.Configuration, imports []*common.ImportFile) ([]*common.ImportFile, error)
}
type typeResolver struct {
maxUrls int
rp registry.RegistryProvider
c util.HTTPClient
}
type fetchableURL struct {
reg registry.Registry
url string
}
type fetchUnit struct {
urls []fetchableURL
}
// NewTypeResolver returns a new initialized TypeResolver.
func NewTypeResolver(rp registry.RegistryProvider, c util.HTTPClient) TypeResolver {
return &typeResolver{maxUrls: maxURLImports, rp: rp, c: c}
}
func resolverError(c *common.Configuration, err error) error {
return fmt.Errorf("cannot resolve types in configuration %s due to: \n%s\n",
c, err)
}
func (tr *typeResolver) performHTTPGet(d util.HTTPDoer, u string, allowMissing bool) (content string, err error) {
g := tr.c
if d != nil {
g = util.NewHTTPClient(3, d, util.NewSleeper())
}
r, code, err := g.Get(u)
if err != nil {
return "", err
}
if allowMissing && code == http.StatusNotFound {
return "", nil
}
if code != http.StatusOK {
return "", fmt.Errorf(
"Received status code %d attempting to fetch Type at %s", code, u)
}
return r, nil
}
// ResolveTypes resolves the types in the supplied configuration and returns
// resolved type definitions in t.ImportFiles. Types can be either
// primitive (i.e., built in), resolved (i.e., already t.ImportFiles), or remote
// (i.e., described by a URL that must be fetched to resolve the type).
func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*common.ImportFile) ([]*common.ImportFile, error) {
existing := map[string]bool{}
for _, v := range imports {
existing[v.Name] = true
}
fetched := map[string][]*common.ImportFile{}
// TODO(vaikas): Need to account for multiple URLs being fetched for a given type.
toFetch := make([]*fetchUnit, 0, tr.maxUrls)
for _, r := range config.Resources {
// Map the type to a fetchable URL (if applicable) or skip it if it's a non-fetchable type (primitive for example).
urls, urlRegistry, err := registry.GetDownloadURLs(tr.rp, r.Type)
if err != nil {
return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", r.Type, err))
}
if !existing[r.Type] {
f := &fetchUnit{}
for _, u := range urls {
if len(u) > 0 {
f.urls = append(f.urls, fetchableURL{urlRegistry, u})
// Add to existing map so it is not fetched multiple times.
existing[r.Type] = true
}
}
if len(f.urls) > 0 {
toFetch = append(toFetch, f)
fetched[f.urls[0].url] = append(fetched[f.urls[0].url], &common.ImportFile{Name: r.Type, Path: f.urls[0].url})
}
}
}
count := 0
for len(toFetch) > 0 {
// 1. If short github URL, resolve to a download URL
// 2. Fetch import URL. Exit if no URLs left
// 3. Check/handle HTTP status
// 4. Store results in all ImportFiles from that URL
// 5. Check for the optional schema file at import URL + .schema
// 6. Repeat 2,3 for schema file
// 7. Add each schema import to fetch if not already done
// 8. Mark URL done. Return to 1.
if count >= tr.maxUrls {
return nil, resolverError(config,
fmt.Errorf("Number of imports exceeds maximum of %d", tr.maxUrls))
}
templates := []string{}
url := toFetch[0].urls[0]
for _, u := range toFetch[0].urls {
template, err := tr.performHTTPGet(u.reg, u.url, false)
if err != nil {
return nil, resolverError(config, err)
}
templates = append(templates, template)
}
for _, i := range fetched[url.url] {
template, err := parseContent(templates)
if err != nil {
return nil, resolverError(config, err)
}
i.Content = template
}
schemaURL := url.url + schemaSuffix
sch, err := tr.performHTTPGet(url.reg, schemaURL, true)
if err != nil {
return nil, resolverError(config, err)
}
if sch != "" {
var s common.Schema
if err := yaml.Unmarshal([]byte(sch), &s); err != nil {
return nil, resolverError(config, err)
}
// Here we handle any nested imports in the schema we've just fetched.
for _, v := range s.Imports {
i := &common.ImportFile{Name: v.Name}
var existingSchema string
urls, urlRegistry, conversionErr := registry.GetDownloadURLs(tr.rp, v.Path)
if conversionErr != nil {
return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", v.Path, conversionErr))
}
if len(urls) == 0 {
// If it's not a fetchable URL, we need to use the type name as is, since it is a short name
// for a schema.
urls = []string{v.Path}
}
for _, u := range urls {
if len(fetched[u]) == 0 {
// If this import URL is new to us, add it to the URLs to fetch.
toFetch = append(toFetch, &fetchUnit{[]fetchableURL{{urlRegistry, u}}})
} else {
// If this is not a new import URL and we've already fetched its contents,
// reuse them. Also, check if we also found a schema for that import URL and
// record those contents for re-use as well.
if fetched[u][0].Content != "" {
i.Content = fetched[u][0].Content
if len(fetched[u+schemaSuffix]) > 0 {
existingSchema = fetched[u+schemaSuffix][0].Content
}
}
}
fetched[u] = append(fetched[u], i)
if existingSchema != "" {
fetched[u+schemaSuffix] = append(fetched[u+schemaSuffix],
&common.ImportFile{Name: v.Name + schemaSuffix, Content: existingSchema})
}
}
}
// Add the schema we've fetched as the schema for any templates which used this URL.
for _, i := range fetched[url.url] {
schemaImportName := i.Name + schemaSuffix
fetched[schemaURL] = append(fetched[schemaURL],
&common.ImportFile{Name: schemaImportName, Content: sch})
}
}
count = count + 1
toFetch = toFetch[1:]
}
ret := []*common.ImportFile{}
for _, v := range fetched {
ret = append(ret, v...)
}
return ret, nil
}
func parseContent(templates []string) (string, error) {
if len(templates) == 1 {
return templates[0], nil
}
// If there are multiple URLs that need to be fetched, that implies it's a package
// of raw Kubernetes objects. We need to fetch them all as a unit and create a
// template representing a package out of that below.
fakeConfig := &common.Configuration{}
for _, template := range templates {
o, err := util.ParseKubernetesObject([]byte(template))
if err != nil {
return "", fmt.Errorf("not a kubernetes object: %+v", template)
}
// Looks like a native Kubernetes object, create a configuration out of it
fakeConfig.Resources = append(fakeConfig.Resources, o)
}
marshalled, err := yaml.Marshal(fakeConfig)
if err != nil {
return "", fmt.Errorf("Failed to marshal: %+v", fakeConfig)
}
return string(marshalled), nil
}
cmd/manager/manager/typeresolver_test.go deleted 100644 → 0
View file @ 873a8af4
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package manager
import (
"errors"
"net/http"
"reflect"
"strings"
"testing"
"github.com/ghodss/yaml"
"github.com/kubernetes/helm/pkg/common"
"github.com/kubernetes/helm/pkg/registry"
)
type responseAndError struct {
err error
code int
resp string
}
type resolverTestCase struct {
config string
imports []*common.ImportFile
responses map[string]responseAndError
urlcount int
expectedErr error
importOut []*common.ImportFile
registryProvider registry.RegistryProvider
}
type testGetter struct {
responses map[string]responseAndError
count int
test *testing.T
}
var count = 0
func (tg testGetter) Get(url string) (body string, code int, err error) {
count = count + 1
ret := tg.responses[url]
return ret.resp, ret.code, ret.err
}
func testDriver(c resolverTestCase, t *testing.T) {
g := &testGetter{test: t, responses: c.responses}
count = 0
r := &typeResolver{
maxUrls: 5,
rp: c.registryProvider,
c: g,
}
conf := &common.Configuration{}
dataErr := yaml.Unmarshal([]byte(c.config), conf)
if dataErr != nil {
panic("bad test data")
}
result, err := r.ResolveTypes(conf, c.imports)
if count != c.urlcount {
t.Errorf("Expected %d url GETs but only %d found %#v", c.urlcount, g.count, g)
}
if (err != nil && c.expectedErr == nil) || (err == nil && c.expectedErr != nil) {
t.Errorf("Expected error %s but found %s", c.expectedErr, err)
} else if err != nil && !strings.Contains(err.Error(), c.expectedErr.Error()) {
t.Errorf("Expected error %s but found %s", c.expectedErr, err)
}
resultImport := map[common.ImportFile]bool{}
expectedImport := map[common.ImportFile]bool{}
for _, i := range result {
resultImport[*i] = true
}
for _, i := range c.importOut {
expectedImport[*i] = true
}
if !reflect.DeepEqual(resultImport, expectedImport) {
t.Errorf("Expected imports %+v but found %+v", expectedImport, resultImport)
}
}
var simpleContent = `
resources:
- name: test
type: ReplicationController
`
func TestNoImports(t *testing.T) {
test := resolverTestCase{config: simpleContent}
testDriver(test, t)
}
var includeImport = `
resources:
- name: foo
type: foo.py
`
func TestIncludedImport(t *testing.T) {
imports := []*common.ImportFile{{Name: "foo.py"}}
test := resolverTestCase{
config: includeImport,
imports: imports,
}
testDriver(test, t)
}
var templateSingleURL = `
resources:
- name: foo
type: http://my-fake-url
`
func TestSingleUrl(t *testing.T) {
finalImports := []*common.ImportFile{{Name: "http://my-fake-url", Path: "http://my-fake-url", Content: "my-content"}}
responses := map[string]responseAndError{
"http://my-fake-url": {nil, http.StatusOK, "my-content"},
"http://my-fake-url.schema": {nil, http.StatusNotFound, ""},
}
test := resolverTestCase{
config: templateSingleURL,
importOut: finalImports,
urlcount: 2,
responses: responses,
}
testDriver(test, t)
}
func TestSingleUrlWith500(t *testing.T) {
responses := map[string]responseAndError{
"http://my-fake-url": {nil, http.StatusInternalServerError, "my-content"},
}
test := resolverTestCase{
config: templateSingleURL,
urlcount: 1,
responses: responses,
expectedErr: errors.New("Received status code 500"),
}
testDriver(test, t)
}
var schema1 = `
imports:
- path: my-next-url
name: schema-import
`
func TestSingleUrlWithSchema(t *testing.T) {
finalImports := []*common.ImportFile{
{Name: "http://my-fake-url", Path: "http://my-fake-url", Content: "my-content"},
{Name: "schema-import", Content: "schema-import"},
{Name: "http://my-fake-url.schema", Content: schema1},
}
responses := map[string]responseAndError{
"http://my-fake-url": {nil, http.StatusOK, "my-content"},
"http://my-fake-url.schema": {nil, http.StatusOK, schema1},
"my-next-url": {nil, http.StatusOK, "schema-import"},
"my-next-url.schema": {nil, http.StatusNotFound, ""},
}
test := resolverTestCase{
config: templateSingleURL,
importOut: finalImports,
urlcount: 4,
responses: responses,
}
testDriver(test, t)
}
var templateExceedsMax = `
resources:
- name: foo
type: http://my-fake-url
- name: foo1
type: http://my-fake-url1
- name: foo2
type: http://my-fake-url2
- name: foo3
type: http://my-fake-url3
- name: foo4
type: http://my-fake-url4
- name: foo5
type: http://my-fake-url5
`
func TestTooManyImports(t *testing.T) {
responses := map[string]responseAndError{
"http://my-fake-url": {nil, http.StatusOK, "my-content"},
"http://my-fake-url.schema": {nil, http.StatusNotFound, ""},
"http://my-fake-url1": {nil, http.StatusOK, "my-content"},
"http://my-fake-url1.schema": {nil, http.StatusNotFound, ""},
"http://my-fake-url2": {nil, http.StatusOK, "my-content"},
"http://my-fake-url2.schema": {nil, http.StatusNotFound, ""},
"http://my-fake-url3": {nil, http.StatusOK, "my-content"},
"http://my-fake-url3.schema": {nil, http.StatusNotFound, ""},
"http://my-fake-url4": {nil, http.StatusOK, "my-content"},
"http://my-fake-url4.schema": {nil, http.StatusNotFound, ""},
"http://my-fake-url5": {nil, http.StatusOK, "my-content"},
"http://my-fake-url5.schema": {nil, http.StatusNotFound, ""},
}
test := resolverTestCase{
config: templateExceedsMax,
urlcount: 10,
responses: responses,
expectedErr: errors.New("Number of imports exceeds maximum of 5"),
}
testDriver(test, t)
}
var templateSharesImport = `
resources:
- name: foo
type: http://my-fake-url
- name: foo1
type: http://my-fake-url1
`
var schema2 = `
imports:
- path: my-next-url
name: schema-import-1
`
func TestSharedImport(t *testing.T) {
finalImports := []*common.ImportFile{
{Name: "http://my-fake-url", Path: "http://my-fake-url", Content: "my-content"},
{Name: "http://my-fake-url1", Path: "http://my-fake-url1", Content: "my-content-1"},
{Name: "schema-import", Content: "schema-import"},
{Name: "schema-import-1", Content: "schema-import"},
{Name: "http://my-fake-url.schema", Content: schema1},
{Name: "http://my-fake-url1.schema", Content: schema2},
}
responses := map[string]responseAndError{
"http://my-fake-url": {nil, http.StatusOK, "my-content"},
"http://my-fake-url.schema": {nil, http.StatusOK, schema1},
"http://my-fake-url1": {nil, http.StatusOK, "my-content-1"},
"http://my-fake-url1.schema": {nil, http.StatusOK, schema2},
"my-next-url": {nil, http.StatusOK, "schema-import"},
"my-next-url.schema": {nil, http.StatusNotFound, ""},
}
test := resolverTestCase{
config: templateSharesImport,
urlcount: 6,
responses: responses,
importOut: finalImports,
}
testDriver(test, t)
}
var templateShortGithubTemplate = `
resources:
- name: foo
type: github.com/kubernetes/application-dm-templates/common/replicatedservice:v1
- name: foo1
type: github.com/kubernetes/application-dm-templates/common/replicatedservice:v2
`
func TestShortGithubUrl(t *testing.T) {
finalImports := []*common.ImportFile{
{
Name: "github.com/kubernetes/application-dm-templates/common/replicatedservice:v1",
Path: "https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py",
Content: "my-content"},
{
Name: "github.com/kubernetes/application-dm-templates/common/replicatedservice:v2",
Path: "https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py",
Content: "my-content-2"},
}
downloadResponses := map[string]registry.DownloadResponse{
"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py": {Err: nil, Code: http.StatusOK, Body: "my-content"},
"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py.schema": {Err: nil, Code: http.StatusNotFound, Body: ""},
"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py": {Err: nil, Code: http.StatusOK, Body: "my-content-2"},
"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py.schema": {Err: nil, Code: http.StatusNotFound, Body: ""},
}
githubURLMaps := map[registry.Type]registry.TestURLAndError{
registry.NewTypeOrDie("common", "replicatedservice", "v1"): {URL: "https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", Err: nil},
registry.NewTypeOrDie("common", "replicatedservice", "v2"): {URL: "https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py", Err: nil},
}
gcsURLMaps := map[registry.Type]registry.TestURLAndError{
registry.NewTypeOrDie("common", "replicatedservice", "v1"): {URL: "https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", Err: nil},
registry.NewTypeOrDie("common", "replicatedservice", "v2"): {URL: "https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py", Err: nil},
}
grp := registry.NewTestGithubRegistryProviderWithDownloads("github.com/kubernetes/application-dm-templates", githubURLMaps, downloadResponses)
gcsrp := registry.NewTestGCSRegistryProvider("gs://charts", gcsURLMaps)
test := resolverTestCase{
config: templateShortGithubTemplate,
importOut: finalImports,
urlcount: 0,
responses: map[string]responseAndError{},
registryProvider: registry.NewRegistryProvider(nil, grp, gcsrp, registry.NewInmemCredentialProvider()),
}
testDriver(test, t)
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment