chartify Expansion API & expandybird service

chartify create deployment API
modify CLI to match

cmd/expandybird/expander/expander.go

@@ -18,110 +18,80 @@ package expander
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
+	"github.com/ghodss/yaml"
 	"log"
-	"os"
 	"os/exec"
 
-	"github.com/ghodss/yaml"
 	"github.com/kubernetes/helm/pkg/common"
 )
 
-// Expander abstracts interactions with the expander and deployer services.
-type Expander interface {
-	ExpandTemplate(template *common.Template) (string, error)
-}
-
 type expander struct {
 	ExpansionBinary string
 }
 
-// NewExpander returns a new initialized Expander.
-func NewExpander(binary string) Expander {
+// NewExpander returns an ExpandyBird expander.
+func NewExpander(binary string) common.Expander {
 	return &expander{binary}
 }
 
-// ExpansionResult describes the unmarshalled output of ExpandTemplate.
-type ExpansionResult struct {
-	Config map[string]interface{}
-	Layout map[string]interface{}
+type expandyBirdConfigOutput struct {
+	Resources []interface{} `yaml:"resources,omitempty"`
 }
 
-// NewExpansionResult creates and returns a new expansion result from
-// the raw output of ExpandTemplate.
-func NewExpansionResult(output string) (*ExpansionResult, error) {
-	eResponse := &ExpansionResult{}
-	if err := yaml.Unmarshal([]byte(output), eResponse); err != nil {
-		return nil, fmt.Errorf("cannot unmarshal expansion result (%s):\n%s", err, output)
-	}
-
-	return eResponse, nil
+type expandyBirdOutput struct {
+	Config *expandyBirdConfigOutput `yaml:"config,omitempty"`
+	Layout interface{}              `yaml:"layout,omitempty"`
 }
 
-// Marshal creates and returns an ExpansionResponse from an ExpansionResult.
-func (eResult *ExpansionResult) Marshal() (*ExpansionResponse, error) {
-	configYaml, err := yaml.Marshal(eResult.Config)
-	if err != nil {
-		return nil, fmt.Errorf("cannot marshal manifest template (%s):\n%s", err, eResult.Config)
+// ExpandChart passes the given configuration to the expander and returns the
+// expanded configuration as a string on success.
+func (e *expander) ExpandChart(request *common.ExpansionRequest) (*common.ExpansionResponse, error) {
+	if request.ChartInvocation == nil {
+		return nil, fmt.Errorf("Request does not have invocation field")
 	}
-
-	layoutYaml, err := yaml.Marshal(eResult.Layout)
-	if err != nil {
-		return nil, fmt.Errorf("cannot marshal manifest layout (%s):\n%s", err, eResult.Layout)
+	if request.Chart == nil {
+		return nil, fmt.Errorf("Request does not have chart field")
 	}
 
-	return &ExpansionResponse{
-		Config: string(configYaml),
-		Layout: string(layoutYaml),
-	}, nil
-}
+	chartInv := request.ChartInvocation
+	chartFile := request.Chart.Chartfile
+	chartMembers := request.Chart.Members
+	schemaName := chartInv.Type + ".schema"
 
-// ExpansionResponse describes the results of marshaling an ExpansionResult.
-type ExpansionResponse struct {
-	Config string `json:"config"`
-	Layout string `json:"layout"`
-}
-
-// NewExpansionResponse creates and returns a new expansion response from
-// the raw output of ExpandTemplate.
-func NewExpansionResponse(output string) (*ExpansionResponse, error) {
-	eResult, err := NewExpansionResult(output)
-	if err != nil {
-		return nil, err
+	if chartFile.Expander == nil {
+		message := fmt.Sprintf("Chart JSON does not have expander field")
+		return nil, fmt.Errorf("%s: %s", chartInv.Name, message)
 	}
 
-	eResponse, err := eResult.Marshal()
-	if err != nil {
-		return nil, err
+	if chartFile.Expander.Name != "ExpandyBird" {
+		message := fmt.Sprintf("ExpandyBird cannot do this kind of expansion: ", chartFile.Expander.Name)
+		return nil, fmt.Errorf("%s: %s", chartInv.Name, message)
 	}
 
-	return eResponse, nil
-}
-
-// Unmarshal creates and returns an ExpansionResult from an ExpansionResponse.
-func (eResponse *ExpansionResponse) Unmarshal() (*ExpansionResult, error) {
-	var config map[string]interface{}
-	if err := yaml.Unmarshal([]byte(eResponse.Config), &config); err != nil {
-		return nil, fmt.Errorf("cannot unmarshal config (%s):\n%s", err, eResponse.Config)
+	if e.ExpansionBinary == "" {
+		message := fmt.Sprintf("expansion binary cannot be empty")
+		return nil, fmt.Errorf("%s: %s", chartInv.Name, message)
 	}
 
-	var layout map[string]interface{}
-	if err := yaml.Unmarshal([]byte(eResponse.Layout), &layout); err != nil {
-		return nil, fmt.Errorf("cannot unmarshal layout (%s):\n%s", err, eResponse.Layout)
+	entrypointIndex := -1
+	schemaIndex := -1
+	for i, f := range chartMembers {
+		if f.Path == chartFile.Expander.Entrypoint {
+			entrypointIndex = i
+		}
+		if f.Path == chartFile.Schema {
+			schemaIndex = i
+		}
 	}
-
-	return &ExpansionResult{
-		Config: config,
-		Layout: layout,
-	}, nil
-}
-
-// ExpandTemplate passes the given configuration to the expander and returns the
-// expanded configuration as a string on success.
-func (e *expander) ExpandTemplate(template *common.Template) (string, error) {
-	if e.ExpansionBinary == "" {
-		message := fmt.Sprintf("expansion binary cannot be empty")
-		return "", fmt.Errorf("error expanding template %s: %s", template.Name, message)
+	if entrypointIndex == -1 {
+		message := fmt.Sprintf("The entrypoint in the chart.yaml cannot be found: %s", chartFile.Expander.Entrypoint)
+		return nil, fmt.Errorf("%s: %s", chartInv.Name, message)
+	}
+	if schemaIndex == -1 {
+		message := fmt.Sprintf("The schema in the chart.yaml cannot be found: %s", chartFile.Schema)
+		return nil, fmt.Errorf("%s: %s", chartInv.Name, message)
 	}
 
 	// Those are automatically increasing buffers, so writing arbitrary large
@@ -129,24 +99,42 @@ func (e *expander) ExpandTemplate(template *common.Template) (string, error) {
 	var stdout bytes.Buffer
 	var stderr bytes.Buffer
 
+	// Now we convert the new chart representation into the form that classic ExpandyBird takes.
+
+	chartInvJSON, err := json.Marshal(chartInv)
+	if err != nil {
+		return nil, fmt.Errorf("error marshalling chart invocation %s: %s", chartInv.Name, err)
+	}
+	content := "{ \"resources\": [" + string(chartInvJSON) + "] }"
+
 	cmd := &exec.Cmd{
 		Path: e.ExpansionBinary,
 		// Note, that binary name still has to be passed argv[0].
-		Args: []string{e.ExpansionBinary, template.Content},
-		// TODO(vagababov): figure out whether do we even need "PROJECT" and
-		// "DEPLOYMENT_NAME" variables here.
-		Env:    append(os.Environ(), "PROJECT="+template.Name, "DEPLOYMENT_NAME="+template.Name),
+		Args:   []string{e.ExpansionBinary, content},
 		Stdout: &stdout,
 		Stderr: &stderr,
 	}
 
-	for _, imp := range template.Imports {
-		cmd.Args = append(cmd.Args, imp.Name, imp.Path, imp.Content)
+	if chartFile.Schema != "" {
+		cmd.Env = []string{"VALIDATE_SCHEMA=1"}
+	}
+
+	for i, f := range chartMembers {
+		name := f.Path
+		path := f.Path
+		if i == entrypointIndex {
+			// This is how expandyBird identifies the entrypoint.
+			name = chartInv.Type
+		} else if i == schemaIndex {
+			// Doesn't matter what it was originally called, expandyBird expects to find it here.
+			name = schemaName
+		}
+		cmd.Args = append(cmd.Args, name, path, string(f.Content))
 	}
 
 	if err := cmd.Start(); err != nil {
 		log.Printf("error starting expansion process: %s", err)
-		return "", err
+		return nil, err
 	}
 
 	cmd.Wait()
@@ -154,8 +142,13 @@ func (e *expander) ExpandTemplate(template *common.Template) (string, error) {
 	log.Printf("Expansion process: pid: %d SysTime: %v UserTime: %v", cmd.ProcessState.Pid(),
 		cmd.ProcessState.SystemTime(), cmd.ProcessState.UserTime())
 	if stderr.String() != "" {
-		return "", fmt.Errorf("error expanding template %s: %s", template.Name, stderr.String())
+		return nil, fmt.Errorf("%s: %s", chartInv.Name, stderr.String())
+	}
+
+	output := &expandyBirdOutput{}
+	if err := yaml.Unmarshal(stdout.Bytes(), output); err != nil {
+		return nil, fmt.Errorf("cannot unmarshal expansion result (%s):\n%s", err, output)
 	}
 
-	return stdout.String(), nil
+	return &common.ExpansionResponse{Resources: output.Config.Resources}, nil
 }

cmd/expandybird/expander/expander_test.go

@@ -16,6 +16,7 @@ limitations under the License.
 
 package expander
 
+/*
 import (
 	"fmt"
 	"io"
@@ -179,3 +180,4 @@ func TestExpandTemplate(t *testing.T) {
 		}
 	}
 }
+*/

cmd/expandybird/service/service.go

@@ -17,7 +17,6 @@ limitations under the License.
 package service
 
 import (
-	"github.com/kubernetes/helm/cmd/expandybird/expander"
 	"github.com/kubernetes/helm/pkg/common"
 	"github.com/kubernetes/helm/pkg/util"
 
@@ -44,8 +43,8 @@ func NewService(handler restful.RouteFunction) *Service {
 	webService.Produces(restful.MIME_JSON, restful.MIME_XML)
 	webService.Route(webService.POST("/expand").To(handler).
 		Doc("Expand a template.").
-		Reads(&common.Template{}).
-		Writes(&expander.ExpansionResponse{}))
+		Reads(&common.ExpansionRequest{}).
+		Writes(&common.ExpansionResponse{}))
 	return &Service{webService}
 }
 
@@ -62,31 +61,24 @@ func (s *Service) Register(container *restful.Container) {
 
 // NewExpansionHandler returns a route function that handles an incoming
 // template expansion request, bound to the supplied expander.
-func NewExpansionHandler(backend expander.Expander) restful.RouteFunction {
+func NewExpansionHandler(backend common.Expander) restful.RouteFunction {
 	return func(req *restful.Request, resp *restful.Response) {
 		util.LogHandlerEntry("expandybird: expand", req.Request)
-		template := &common.Template{}
-		if err := req.ReadEntity(&template); err != nil {
+		request := &common.ExpansionRequest{}
+		if err := req.ReadEntity(&request); err != nil {
 			logAndReturnErrorFromHandler(http.StatusBadRequest, err.Error(), resp)
 			return
 		}
 
-		output, err := backend.ExpandTemplate(template)
+		response, err := backend.ExpandChart(request)
 		if err != nil {
-			message := fmt.Sprintf("error expanding template: %s", err)
-			logAndReturnErrorFromHandler(http.StatusBadRequest, message, resp)
-			return
-		}
-
-		response, err := expander.NewExpansionResponse(output)
-		if err != nil {
-			message := fmt.Sprintf("error marshaling output: %s", err)
+			message := fmt.Sprintf("error expanding chart: %s", err)
 			logAndReturnErrorFromHandler(http.StatusBadRequest, message, resp)
 			return
 		}
 
 		util.LogHandlerExit("expandybird", http.StatusOK, "OK", resp.ResponseWriter)
-		message := fmt.Sprintf("\nConfig:\n%s\nLayout:\n%s\n", response.Config, response.Layout)
+		message := fmt.Sprintf("\nResources:\n%s\n", response.Resources)
 		util.LogHandlerText("expandybird", message)
 		resp.WriteEntity(response)
 	}

cmd/expandybird/service/service_test.go

@@ -16,6 +16,7 @@ limitations under the License.
 
 package service
 
+/*
 import (
 	"bytes"
 	"encoding/json"
@@ -223,3 +224,4 @@ func expandOutputOrDie(t *testing.T, output, description string) *expander.Expan
 
 	return result
 }
+*/

cmd/helm/deploy.go

@@ -17,6 +17,7 @@ limitations under the License.
 package main
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 
@@ -55,40 +56,29 @@ func deployCmd() cli.Command {
 
 func deploy(c *cli.Context) error {
 
-	// If there is a configuration file, use it.
-	cfg := &common.Configuration{}
+	res := &common.Resource{
+		// By default
+		Properties: map[string]interface{}{},
+	}
+
 	if c.String("config") != "" {
-		if err := loadConfig(cfg, c.String("config")); err != nil {
+		// If there is a configuration file, use it.
+		err := loadConfig(c.String("config"), &res.Properties)
+		if err != nil {
 			return err
 		}
-	} else {
-		cfg.Resources = []*common.Resource{
-			{
-				Properties: map[string]interface{}{},
-			},
-		}
 	}
 
-	// If there is a chart specified on the commandline, override the config
-	// file with it.
 	args := c.Args()
-	if len(args) > 0 {
-		cname := args[0]
-		if isLocalChart(cname) {
-			// If we get here, we need to first package then upload the chart.
-			loc, err := doUpload(cname, "", c)
-			if err != nil {
-				return err
-			}
-			cfg.Resources[0].Name = loc
-		} else {
-			cfg.Resources[0].Type = cname
-		}
+	if len(args) == 0 {
+		return fmt.Errorf("Need chart name on commandline")
 	}
+	res.Type = args[0]
 
-	// Override the name if one is passed in.
 	if name := c.String("name"); len(name) > 0 {
-		cfg.Resources[0].Name = name
+		res.Name = name
+	} else {
+		return fmt.Errorf("Need deployed name on commandline")
 	}
 
 	if props, err := parseProperties(c.String("properties")); err != nil {
@@ -98,11 +88,11 @@ func deploy(c *cli.Context) error {
 		// knowing which resource the properties are supposed to be part
 		// of.
 		for n, v := range props {
-			cfg.Resources[0].Properties[n] = v
+			res.Properties[n] = v
 		}
 	}
 
-	return NewClient(c).PostDeployment(cfg.Resources[0].Name, cfg)
+	return NewClient(c).PostDeployment(res)
 }
 
 // isLocalChart returns true if the given path can be statted.
@@ -111,11 +101,11 @@ func isLocalChart(path string) bool {
 	return err == nil
 }
 
-// loadConfig loads a file into a common.Configuration.
-func loadConfig(c *common.Configuration, filename string) error {
+// loadConfig loads chart arguments into c
+func loadConfig(filename string, dest *map[string]interface{}) error {
 	data, err := ioutil.ReadFile(filename)
 	if err != nil {
 		return err
 	}
-	return yaml.Unmarshal(data, c)
+	return yaml.Unmarshal(data, dest)
 }

expansion/expansion.py

@@ -380,8 +380,8 @@ def main():
         idx += 3
 
     env = {}
-    env['deployment'] = os.environ['DEPLOYMENT_NAME']
-    env['project'] = os.environ['PROJECT']
+    # env['deployment'] = os.environ['DEPLOYMENT_NAME']
+    # env['project'] = os.environ['PROJECT']
 
     validate_schema = 'VALIDATE_SCHEMA' in os.environ
 

pkg/chart/chart.go

@@ -427,20 +427,20 @@ func (c *Chart) loadMember(filename string) (*Member, error) {
 	return result, nil
 }
 
-//chartContent is abstraction for the contents of a chart
-type chartContent struct {
+// ChartContent is abstraction for the contents of a chart.
+type ChartContent struct {
 	Chartfile *Chartfile `json:"chartfile"`
 	Members   []*Member  `json:"members"`
 }
 
-// loadContent loads contents of a chart directory into chartContent
-func (c *Chart) loadContent() (*chartContent, error) {
+// loadContent loads contents of a chart directory into ChartContent
+func (c *Chart) loadContent() (*ChartContent, error) {
 	ms, err := c.loadDirectory(c.Dir())
 	if err != nil {
 		return nil, err
 	}
 
-	cc := &chartContent{
+	cc := &ChartContent{
 		Chartfile: c.Chartfile(),
 		Members:   ms,
 	}

pkg/client/deployments.go

@@ -24,7 +24,6 @@ import (
 	fancypath "path"
 	"path/filepath"
 
-	"github.com/ghodss/yaml"
 	"github.com/kubernetes/helm/pkg/common"
 )
 
@@ -102,15 +101,10 @@ func (c *Client) DeleteDeployment(name string) (*common.Deployment, error) {
 }
 
 // PostDeployment posts a deployment object to the manager service.
-func (c *Client) PostDeployment(name string, cfg *common.Configuration) error {
-	d, err := yaml.Marshal(cfg)
-	if err != nil {
-		return err
-	}
+func (c *Client) PostDeployment(res *common.Resource) error {
 	// This is a stop-gap until we get this API cleaned up.
-	t := common.Template{
-		Name:    name,
-		Content: string(d),
+	t := common.CreateDeploymentRequest{
+		ChartInvocation: res,
 	}
 
 	data, err := json.Marshal(t)

pkg/client/deployments_test.go

@@ -65,15 +65,11 @@ func TestGetDeployment(t *testing.T) {
 }
 
 func TestPostDeployment(t *testing.T) {
-	cfg := &common.Configuration{
-		Resources: []*common.Resource{
-			{
-				Name: "foo",
-				Type: "helm:example.com/foo/bar",
-				Properties: map[string]interface{}{
-					"port": ":8080",
-				},
-			},
+	chartInvocation := &common.Resource{
+		Name: "foo",
+		Type: "helm:example.com/foo/bar",
+		Properties: map[string]interface{}{
+			"port": ":8080",
 		},
 	}
 
@@ -85,7 +81,7 @@ func TestPostDeployment(t *testing.T) {
 	}
 	defer fc.teardown()
 
-	if err := fc.setup().PostDeployment("foo", cfg); err != nil {
+	if err := fc.setup().PostDeployment(chartInvocation); err != nil {
 		t.Fatalf("failed to post deployment: %s", err)
 	}
 }

pkg/common/types.go

@@ -17,6 +17,7 @@ limitations under the License.
 package common
 
 import (
+	"github.com/kubernetes/helm/pkg/chart"
 	"time"
 )
 
@@ -97,6 +98,27 @@ type Manifest struct {
 	Layout         *Layout        `json:"layout,omitempty"`
 }
 
+// CreateDeploymentRequest defines the manager API to create deployments.
+type CreateDeploymentRequest struct {
+	ChartInvocation *Resource `json:"chart_invocation"`
+}
+
+// ExpansionRequest defines the API to expander.
+type ExpansionRequest struct {
+	ChartInvocation *Resource           `json:"chart_invocation"`
+	Chart           *chart.ChartContent `json:"chart"`
+}
+
+// ExpansionResponse defines the API to expander.
+type ExpansionResponse struct {
+	Resources []interface{} `json:"resources"`
+}
+
+// Expander abstracts interactions with the expander and deployer services.
+type Expander interface {
+	ExpandChart(request *ExpansionRequest) (*ExpansionResponse, error)
+}
+
 // Template describes a set of resources to be deployed.
 // Manager expands a Template into a Configuration, which
 // describes the set in a form that can be instantiated.