Merge pull request #832 from technosophos/feat/helmignore

feat(pkg/ignore): add helmignore library

cmd/helm/create.go

@@ -17,6 +17,9 @@ For example, 'helm create foo' will create a directory structure that looks
 something like this:
 
 	foo/
+	  |
+	  |- .helmignore   # Contains patterns to ignore when packaging Helm charts.
+	  |
 	  |- Chart.yaml    # Information about your chart
 	  |
 	  |- values.yaml   # The default values for your templates

pkg/chartutil/create.go

@@ -18,6 +18,8 @@ const (
 	TemplatesDir = "templates"
 	// ChartsDir is the relative directory name for charts dependencies.
 	ChartsDir = "charts"
+	// IgnorefileName is the name of the Helm ignore file.
+	IgnorefileName = ".helmignore"
 )
 
 const defaultValues = `# Default values for %s.
@@ -26,6 +28,13 @@ const defaultValues = `# Default values for %s.
 # name: value
 `
 
+const defaultIgnore = `# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+.git
+`
+
 // Create creates a new chart in a directory.
 //
 // Inside of dir, this will create a directory based on the name of
@@ -69,6 +78,11 @@ func Create(chartfile *chart.Metadata, dir string) (string, error) {
 		return cdir, err
 	}
 
+	val = []byte(defaultIgnore)
+	if err := ioutil.WriteFile(filepath.Join(cdir, IgnorefileName), val, 0644); err != nil {
+		return cdir, err
+	}
+
 	for _, d := range []string{TemplatesDir, ChartsDir} {
 		if err := os.MkdirAll(filepath.Join(cdir, d), 0755); err != nil {
 			return cdir, err

pkg/chartutil/create_test.go

@@ -42,7 +42,7 @@ func TestCreate(t *testing.T) {
 		}
 	}
 
-	for _, f := range []string{ChartfileName, ValuesfileName} {
+	for _, f := range []string{ChartfileName, ValuesfileName, IgnorefileName} {
 		if fi, err := os.Stat(filepath.Join(dir, f)); err != nil {
 			t.Errorf("Expected %s file: %s", f, err)
 		} else if fi.IsDir() {

pkg/chartutil/load.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/golang/protobuf/ptypes/any"
 
+	"k8s.io/helm/pkg/ignore"
 	"k8s.io/helm/pkg/proto/hapi/chart"
 )
 
@@ -21,6 +22,9 @@ import (
 //
 // This is the preferred way to load a chart. It will discover the chart encoding
 // and hand off to the appropriate chart reader.
+//
+// If a .helmignore file is present, the directory loader will skip loading any files
+// matching it. But .helmignore is not evaluated when reading out of an archive.
 func Load(name string) (*chart.Chart, error) {
 	fi, err := os.Stat(name)
 	if err != nil {
@@ -179,6 +183,17 @@ func LoadDir(dir string) (*chart.Chart, error) {
 	// Just used for errors.
 	c := &chart.Chart{}
 
+	rules := ignore.Empty()
+	ifile := filepath.Join(topdir, ignore.HelmIgnore)
+	fmt.Println(ifile)
+	if _, err := os.Stat(ifile); err == nil {
+		r, err := ignore.ParseFile(ifile)
+		if err != nil {
+			return c, err
+		}
+		rules = r
+	}
+
 	files := []*afile{}
 	topdir += string(filepath.Separator)
 	err = filepath.Walk(topdir, func(name string, fi os.FileInfo, err error) error {
@@ -190,6 +205,11 @@ func LoadDir(dir string) (*chart.Chart, error) {
 			return nil
 		}
 
+		// If a .helmignore file matches, skip this file.
+		if rules.Ignore(n, fi) {
+			return nil
+		}
+
 		data, err := ioutil.ReadFile(name)
 		if err != nil {
 			return fmt.Errorf("error reading %s: %s", n, err)

pkg/ignore/doc.go

+/*Package ignore provides tools for writing ignore files (a la .gitignore).
+
+This provides both an ignore parser and a file-aware processor.
+
+The format of ignore files closely follows, but does not exactly match, the
+format for .gitignore files (https://git-scm.com/docs/gitignore).
+
+The formatting rules are as follows:
+
+	- Parsing is line-by-line
+	- Empty lines are ignored
+	- Lines the begin with # (comments) will be ignored
+	- Leading and trailing spaces are always ignored
+	- Inline comments are NOT supported ('foo* # Any foo' does not contain a comment)
+	- There is no support for multi-line patterns
+	- Shell glob patterns are supported. See Go's "path/filepath".Match
+	- If a pattern begins with a leading !, the match will be negated.
+	- If a pattern begins with a leading /, only paths relatively rooted will match.
+	- If the pattern ends with a trailing /, only directories will match
+	- If a pattern contains no slashes, file basenames are tested (not paths)
+	- The pattern sequence "**", while legal in a glob, will cause an error here
+	  (to indicate incompatibility with .gitignore).
+
+Example:
+
+	# Match any file named foo.txt
+	foo.txt
+
+	# Match any text file
+	*.txt
+
+	# Match only directories named mydir
+	mydir/
+
+	# Match only text files in the top-level directory
+	/*.txt
+
+	# Match only the file foo.txt in the top-level directory
+	/foo.txt
+
+	# Match any file named ab.txt, ac.txt, or ad.txt
+	a[b-d].txt
+
+Notable differences from .gitignore:
+	- The '**' syntax is not supported.
+	- The globbing library is Go's 'filepath.Match', not fnmatch(3)
+	- Trailing spaces are always ignored (there is no supported escape sequence)
+	- The evaluation of escape sequences has not been tested for compatibility
+	- There is no support for '\!' as a special leading sequence.
+*/
+package ignore

pkg/ignore/rules.go

+package ignore
+
+import (
+	"bufio"
+	"errors"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// HelmIgnore default name of an ignorefile.
+const HelmIgnore = ".helmignore"
+
+// Rules is a collection of path matching rules.
+//
+// Parse() and ParseFile() will construct and populate new Rules.
+// Empty() will create an immutable empty ruleset.
+type Rules struct {
+	patterns []*pattern
+}
+
+// Empty builds an empty ruleset.
+func Empty() *Rules {
+	return &Rules{patterns: []*pattern{}}
+}
+
+// ParseFile parses a helmignore file and returns the *Rules.
+func ParseFile(file string) (*Rules, error) {
+	f, err := os.Open(file)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	return Parse(f)
+}
+
+// Parse parses a rules file
+func Parse(file io.Reader) (*Rules, error) {
+	r := &Rules{patterns: []*pattern{}}
+
+	s := bufio.NewScanner(file)
+	for s.Scan() {
+		if err := r.parseRule(s.Text()); err != nil {
+			return r, err
+		}
+	}
+	if err := s.Err(); err != nil {
+		return r, err
+	}
+
+	return r, nil
+}
+
+// Len returns the number of patterns in this rule set.
+func (r *Rules) Len() int {
+	return len(r.patterns)
+}
+
+// Ignore evalutes the file at the given path, and returns true if it should be ignored.
+//
+// Ignore evaluates path against the rules in order. Evaluation stops when a match
+// is found. Matching a negative rule will stop evaluation.
+func (r *Rules) Ignore(path string, fi os.FileInfo) bool {
+	for _, p := range r.patterns {
+		if p.match == nil {
+			log.Printf("ignore: no matcher supplied for %q", p.raw)
+			return false
+		}
+
+		// For negative rules, we need to capture and return non-matches,
+		// and continue for matches.
+		if p.negate {
+			if p.mustDir && !fi.IsDir() {
+				return true
+			}
+			if !p.match(path, fi) {
+				return true
+			}
+			continue
+		}
+
+		if p.mustDir && !fi.IsDir() {
+			return false
+		}
+		if p.match(path, fi) {
+			return true
+		}
+	}
+	return false
+}
+
+// parseRule parses a rule string and creates a pattern, which is then stored in the Rules object.
+func (r *Rules) parseRule(rule string) error {
+	rule = strings.TrimSpace(rule)
+
+	// Ignore blank lines
+	if rule == "" {
+		return nil
+	}
+	// Comment
+	if strings.HasPrefix(rule, "#") {
+		return nil
+	}
+
+	// Fail any rules that contain **
+	if strings.Contains(rule, "**") {
+		return errors.New("double-star (**) syntax is not supported")
+	}
+
+	// Fail any patterns that can't compile. A non-empty string must be
+	// given to Match() to avoid optimization that skips rule evaluation.
+	if _, err := filepath.Match(rule, "abc"); err != nil {
+		return err
+	}
+
+	p := &pattern{raw: rule}
+
+	// Negation is handled at a higher level, so strip the leading ! from the
+	// string.
+	if strings.HasPrefix(rule, "!") {
+		p.negate = true
+		rule = rule[1:]
+	}
+
+	// Directory verification is handled by a higher level, so the trailing /
+	// is removed from the rule. That way, a directory named "foo" matches,
+	// even if the supplied string does not contain a literal slash character.
+	if strings.HasSuffix(rule, "/") {
+		p.mustDir = true
+		rule = strings.TrimSuffix(rule, "/")
+	}
+
+	if strings.HasPrefix(rule, "/") {
+		// Require path matches the root path.
+		p.match = func(n string, fi os.FileInfo) bool {
+			rule = strings.TrimPrefix(rule, "/")
+			ok, err := filepath.Match(rule, n)
+			if err != nil {
+				log.Printf("Failed to compile %q: %s", rule, err)
+				return false
+			}
+			return ok
+		}
+	} else if strings.Contains(rule, "/") {
+		// require structural match.
+		p.match = func(n string, fi os.FileInfo) bool {
+			ok, err := filepath.Match(rule, n)
+			if err != nil {
+				log.Printf("Failed to compile %q: %s", rule, err)
+				return false
+			}
+			return ok
+		}
+	} else {
+		p.match = func(n string, fi os.FileInfo) bool {
+			// When there is no slash in the pattern, we evaluate ONLY the
+			// filename.
+			n = filepath.Base(n)
+			ok, err := filepath.Match(rule, n)
+			if err != nil {
+				log.Printf("Failed to compile %q: %s", rule, err)
+				return false
+			}
+			return ok
+		}
+	}
+
+	r.patterns = append(r.patterns, p)
+	return nil
+}
+
+// matcher is a function capable of computing a match.
+//
+// It returns true if the rule matches.
+type matcher func(name string, fi os.FileInfo) bool
+
+// pattern describes a pattern to be matched in a rule set.
+type pattern struct {
+	// raw is the unparsed string, with nothing stripped.
+	raw string
+	// match is the matcher function.
+	match matcher
+	// negate indicates that the rule's outcome should be negated.
+	negate bool
+	// mustDir indicates that the matched file must be a directory.
+	mustDir bool
+}

pkg/ignore/rules_test.go

+package ignore
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+var testdata = "./testdata"
+
+func TestParse(t *testing.T) {
+	rules := `#ignore
+
+	#ignore
+foo
+bar/*
+baz/bar/foo.txt
+
+one/more
+`
+	r, err := parseString(rules)
+	if err != nil {
+		t.Fatalf("Error parsing rules: %s", err)
+	}
+
+	if len(r.patterns) != 4 {
+		t.Errorf("Expected 4 rules, got %d", len(r.patterns))
+	}
+
+	expects := []string{"foo", "bar/*", "baz/bar/foo.txt", "one/more"}
+	for i, p := range r.patterns {
+		if p.raw != expects[i] {
+			t.Errorf("Expected %q, got %q", expects[i], p.raw)
+		}
+		if p.match == nil {
+			t.Errorf("Expected %s to have a matcher function.", p.raw)
+		}
+	}
+}
+
+func TestParseFail(t *testing.T) {
+	shouldFail := []string{"foo/**/bar", "[z-"}
+	for _, fail := range shouldFail {
+		_, err := parseString(fail)
+		if err == nil {
+			t.Errorf("Rule %q should have failed", fail)
+		}
+	}
+
+}
+
+func TestParseFile(t *testing.T) {
+	f := filepath.Join(testdata, HelmIgnore)
+	if _, err := os.Stat(f); err != nil {
+		t.Fatalf("Fixture %s missing: %s", f, err)
+	}
+
+	r, err := ParseFile(f)
+	if err != nil {
+		t.Fatalf("Failed to parse rules file: %s", err)
+	}
+
+	if len(r.patterns) != 3 {
+		t.Errorf("Expected 3 patterns, got %d", len(r.patterns))
+	}
+}
+
+func TestIgnore(t *testing.T) {
+	// Test table: Given pattern and name, Ignore should return expect.
+	tests := []struct {
+		pattern string
+		name    string
+		expect  bool
+	}{
+		// Glob tests
+		{`helm.txt`, "helm.txt", true},
+		{`helm.*`, "helm.txt", true},
+		{`helm.*`, "rudder.txt", false},
+		{`*.txt`, "tiller.txt", true},
+		{`*.txt`, "cargo/a.txt", true},
+		{`cargo/*.txt`, "cargo/a.txt", true},
+		{`cargo/*.*`, "cargo/a.txt", true},
+		{`cargo/*.txt`, "mast/a.txt", false},
+		{`ru[c-e]?er.txt`, "rudder.txt", true},
+
+		// Directory tests
+		{`cargo/`, "cargo", true},
+		{`cargo/`, "cargo/", true},
+		{`cargo/`, "mast/", false},
+		{`helm.txt/`, "helm.txt", false},
+
+		// Negation tests
+		{`!helm.txt`, "helm.txt", false},
+		{`!helm.txt`, "tiller.txt", true},
+		{`!*.txt`, "cargo", true},
+		{`!cargo/`, "mast/", true},
+
+		// Absolute path tests
+		{`/a.txt`, "a.txt", true},
+		{`/a.txt`, "cargo/a.txt", false},
+		{`/cargo/a.txt`, "cargo/a.txt", true},
+	}
+
+	for _, test := range tests {
+		r, err := parseString(test.pattern)
+		if err != nil {
+			t.Fatalf("Failed to parse: %s", err)
+		}
+		fi, err := os.Stat(filepath.Join(testdata, test.name))
+		if err != nil {
+			t.Fatalf("Fixture missing: %s", err)
+		}
+
+		if r.Ignore(test.name, fi) != test.expect {
+			t.Errorf("Expected %q to be %v for pattern %q", test.name, test.expect, test.pattern)
+		}
+	}
+}
+
+func parseString(str string) (*Rules, error) {
+	b := bytes.NewBuffer([]byte(str))
+	return Parse(b)
+}

pkg/ignore/testdata/.helmignore

+mast/a.txt
+.DS_Store
+.git