Commit 3e8cc7f1 authored by Andrew Gerrand's avatar Andrew Gerrand

misc/dashboard/builder: gobuilder, a continuous build client

R=rsc
CC=golang-dev
https://golang.org/cl/2126042
parent 77a70ddb
# Copyright 2009 The Go Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.
include ../../../src/Make.inc
TARG=gobuilder
GOFILES=\
exec.go\
hg.go\
http.go\
main.go\
include ../../../src/Make.cmd
// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
/*
Go Builder is a continuous build client for the Go project.
It integrates with the Go Dashboard AppEngine application.
Go Builder is intended to run continuously as a background process.
It periodically pulls updates from the Go Mercurial repository.
When a newer revision is found, Go Builder creates a clone of the repository,
runs all.bash, and reports build success or failure to the Go Dashboard.
For a successful build, Go Builder will also run benchmarks
(cd $GOROOT/src/pkg; make bench) and send the results to the Go Dashboard.
For a release revision (a change description that matches "release.YYYY-MM-DD"),
Go Builder will create a tar.gz archive of the GOROOT and deliver it to the
Go Google Code project's downloads section.
Usage:
gobuilder goos-goarch...
Several goos-goarch combinations can be provided, and the builder will
build them in serial.
Optional flags:
-dashboard="godashboard.appspot.com": Go Dashboard Host
The location of the Go Dashboard application to which Go Builder will
report its results.
-bench: Run benchmarks
-release: Build and deliver binary release archive
The key file should be located at $HOME/.gobuilder or, for a builder-specific
key, $HOME/.gobuilder-$BUILDER (eg, $HOME/.gobuilder-linux-amd64).
The build key file is a text file of the format:
godashboard-key
googlecode-username
googlecode-password
If the Google Code credentials are not provided the archival step
will be skipped.
*/
package documentation
package main
import (
"bytes"
"exec"
"os"
)
// run is a simple wrapper for exec.Run/Close
func run(envv []string, dir string, argv ...string) os.Error {
bin, err := exec.LookPath(argv[0])
if err != nil {
return err
}
p, err := exec.Run(bin, argv, envv, dir,
exec.DevNull, exec.DevNull, exec.PassThrough)
if err != nil {
return err
}
return p.Close()
}
// runLog runs a process and returns the combined stdout/stderr
func runLog(envv []string, dir string, argv ...string) (output string, exitStatus int, err os.Error) {
bin, err := exec.LookPath(argv[0])
if err != nil {
return
}
p, err := exec.Run(bin, argv, envv, dir,
exec.DevNull, exec.Pipe, exec.MergeWithStdout)
if err != nil {
return
}
defer p.Close()
b := new(bytes.Buffer)
_, err = b.ReadFrom(p.Stdout)
if err != nil {
return
}
w, err := p.Wait(0)
if err != nil {
return
}
return b.String(), w.WaitStatus.ExitStatus(), nil
}
package main
import (
"os"
"strconv"
"strings"
)
type Commit struct {
num int // mercurial revision number
node string // mercurial hash
parent string // hash of commit's parent
user string // author's Name <email>
date string // date of commit
desc string // description
}
// getCommit returns details about the Commit specified by the revision hash
func getCommit(rev string) (c Commit, err os.Error) {
defer func() {
if err != nil {
err = errf("getCommit: %s: %s", rev, err)
}
}()
parts, err := getCommitParts(rev)
if err != nil {
return
}
num, err := strconv.Atoi(parts[0])
if err != nil {
return
}
parent := ""
if num > 0 {
prev := strconv.Itoa(num - 1)
if pparts, err := getCommitParts(prev); err == nil {
parent = pparts[1]
}
}
user := strings.Replace(parts[2], "&lt;", "<", -1)
user = strings.Replace(user, "&gt;", ">", -1)
return Commit{num, parts[1], parent, user, parts[3], parts[4]}, nil
}
func getCommitParts(rev string) (parts []string, err os.Error) {
const format = "{rev}>{node}>{author|escape}>{date}>{desc}"
s, _, err := runLog(nil, goroot,
"hg", "log", "-r", rev, "-l", "1", "--template", format)
if err != nil {
return
}
return strings.Split(s, ">", 5), nil
}
package main
import (
"bytes"
"encoding/base64"
"encoding/binary"
"fmt"
"http"
"os"
"regexp"
)
// getHighWater returns the current highwater revision hash for this builder
func (b *Builder) getHighWater() (rev string, err os.Error) {
url := fmt.Sprintf("http://%s/hw-get?builder=%s", *dashboard, b.name)
r, _, err := http.Get(url)
if err != nil {
return
}
buf := new(bytes.Buffer)
_, err = buf.ReadFrom(r.Body)
if err != nil {
return
}
r.Body.Close()
return buf.String(), nil
}
// recordResult sends build results to the dashboard
func (b *Builder) recordResult(buildLog string, c Commit) os.Error {
return httpCommand("build", map[string]string{
"builder": b.name,
"key": b.key,
"node": c.node,
"parent": c.parent,
"user": c.user,
"date": c.date,
"desc": c.desc,
"log": buildLog,
})
}
// match lines like: "package.BechmarkFunc 100000 999 ns/op"
var benchmarkRegexp = regexp.MustCompile("([^\n\t ]+)[\t ]+([0-9]+)[\t ]+([0-9]+) ns/op")
// recordBenchmarks sends benchmark results to the dashboard
func (b *Builder) recordBenchmarks(benchLog string, c Commit) os.Error {
results := benchmarkRegexp.FindAllStringSubmatch(benchLog, -1)
var buf bytes.Buffer
b64 := base64.NewEncoder(base64.StdEncoding, &buf)
for _, r := range results {
for _, s := range r[1:] {
binary.Write(b64, binary.BigEndian, uint16(len(s)))
b64.Write([]byte(s))
}
}
b64.Close()
return httpCommand("benchmarks", map[string]string{
"builder": b.name,
"key": b.key,
"node": c.node,
"benchmarkdata": buf.String(),
})
}
func httpCommand(cmd string, args map[string]string) os.Error {
url := fmt.Sprintf("http://%v/%v", *dashboard, cmd)
_, err := http.PostForm(url, args)
return err
}
package main
import (
"container/vector"
"flag"
"fmt"
"io/ioutil"
"log"
"os"
"path"
"regexp"
"strconv"
"strings"
"time"
)
const (
codeProject = "go"
codePyScript = "misc/dashboard/googlecode_upload.py"
hgUrl = "https://go.googlecode.com/hg/"
waitInterval = 10e9 // time to wait before checking for new revs
mkdirPerm = 0750
)
type Builder struct {
name string
goos, goarch string
key string
codeUsername string
codePassword string
}
type BenchRequest struct {
builder *Builder
commit Commit
path string
}
var (
dashboard = flag.String("dashboard", "godashboard.appspot.com", "Go Dashboard Host")
runBenchmarks = flag.Bool("bench", false, "Run benchmarks")
buildRelease = flag.Bool("release", false, "Build and deliver binary release archive")
)
var (
buildroot = path.Join(os.TempDir(), "gobuilder")
goroot = path.Join(buildroot, "goroot")
releaseRegexp = regexp.MustCompile(`^release\.[0-9\-]+`)
benchRequests vector.Vector
)
func main() {
flag.Usage = func() {
fmt.Fprintf(os.Stderr, "usage: %s goos-goarch...\n", os.Args[0])
flag.PrintDefaults()
os.Exit(2)
}
flag.Parse()
if len(flag.Args()) == 0 {
flag.Usage()
}
builders := make([]*Builder, len(flag.Args()))
for i, builder := range flag.Args() {
b, err := NewBuilder(builder)
if err != nil {
log.Exit(err)
}
builders[i] = b
}
if err := os.RemoveAll(buildroot); err != nil {
log.Exitf("Error removing build root (%s): %s", buildroot, err)
}
if err := os.Mkdir(buildroot, mkdirPerm); err != nil {
log.Exitf("Error making build root (%s): %s", buildroot, err)
}
if err := run(nil, buildroot, "hg", "clone", hgUrl, goroot); err != nil {
log.Exit("Error cloning repository:", err)
}
// check for new commits and build them
for {
err := run(nil, goroot, "hg", "pull", "-u")
if err != nil {
log.Stderr("hg pull failed:", err)
time.Sleep(waitInterval)
continue
}
built := false
for _, b := range builders {
if b.build() {
built = true
}
}
// only run benchmarks if we didn't build anything
// so that they don't hold up the builder queue
if !built {
// if we have no benchmarks to do, pause
if benchRequests.Len() == 0 {
time.Sleep(waitInterval)
} else {
runBenchmark(benchRequests.Pop().(BenchRequest))
// after running one benchmark,
// continue to find and build new revisions.
}
}
}
}
func runBenchmark(r BenchRequest) {
// run benchmarks and send to dashboard
pkg := path.Join(r.path, "go", "src", "pkg")
bin := path.Join(r.path, "go", "bin")
env := []string{
"GOOS=" + r.builder.goos,
"GOARCH=" + r.builder.goarch,
"PATH=" + bin + ":" + os.Getenv("PATH"),
}
benchLog, _, err := runLog(env, pkg, "gomake", "bench")
if err != nil {
log.Stderr("%s gomake bench:", r.builder.name, err)
return
}
if err = r.builder.recordBenchmarks(benchLog, r.commit); err != nil {
log.Stderr("recordBenchmarks:", err)
}
}
func NewBuilder(builder string) (*Builder, os.Error) {
b := &Builder{name: builder}
// get goos/goarch from builder string
s := strings.Split(builder, "-", 3)
if len(s) == 2 {
b.goos, b.goarch = s[0], s[1]
} else {
return nil, errf("unsupported builder form: %s", builder)
}
// read keys from keyfile
fn := path.Join(os.Getenv("HOME"), ".gobuildkey")
if s := fn+"-"+b.name; isFile(s) { // builder-specific file
fn = s
}
c, err := ioutil.ReadFile(fn)
if err != nil {
return nil, errf("readKeys %s (%s): %s", b.name, fn, err)
}
v := strings.Split(string(c), "\n", -1)
b.key = v[0]
if len(v) >= 3 {
b.codeUsername, b.codePassword = v[1], v[2]
}
return b, nil
}
// build checks for a new commit for this builder
// and builds it if one is found.
// It returns true if a build was attempted.
func (b *Builder) build() bool {
defer func() {
err := recover()
if err != nil {
log.Stderr("%s build: %s", b.name, err)
}
}()
c, err := b.nextCommit()
if err != nil {
log.Stderr(err)
return false
}
if c == nil {
return false
}
log.Stderrf("%s building %d", b.name, c.num)
err = b.buildCommit(*c)
if err != nil {
log.Stderr(err)
}
return true
}
// nextCommit returns the next unbuilt Commit for this builder
func (b *Builder) nextCommit() (nextC *Commit, err os.Error) {
defer func() {
if err != nil {
err = errf("%s nextCommit: %s", b.name, err)
}
}()
hw, err := b.getHighWater()
if err != nil {
return
}
c, err := getCommit(hw)
if err != nil {
return
}
next := c.num + 1
c, err = getCommit(strconv.Itoa(next))
if err == nil || c.num == next {
return &c, nil
}
return nil, nil
}
func (b *Builder) buildCommit(c Commit) (err os.Error) {
defer func() {
if err != nil {
err = errf("%s buildCommit: %d: %s", b.name, c.num, err)
}
}()
// create place in which to do work
workpath := path.Join(buildroot, b.name+"-"+strconv.Itoa(c.num))
err = os.Mkdir(workpath, mkdirPerm)
if err != nil {
return
}
benchRequested := false
defer func() {
if !benchRequested {
os.RemoveAll(workpath)
}
}()
// clone repo at revision num (new candidate)
err = run(nil, workpath,
"hg", "clone",
"-r", strconv.Itoa(c.num),
goroot, "go")
if err != nil {
return
}
// set up environment for build/bench execution
env := []string{
"GOOS=" + b.goos,
"GOARCH=" + b.goarch,
"GOROOT_FINAL=/usr/local/go",
"PATH=" + os.Getenv("PATH"),
}
srcDir := path.Join(workpath, "go", "src")
// build the release candidate
buildLog, status, err := runLog(env, srcDir, "bash", "all.bash")
if err != nil {
return errf("all.bash: %s", err)
}
if status != 0 {
// record failure
return b.recordResult(buildLog, c)
}
// record success
if err = b.recordResult("", c); err != nil {
return errf("recordResult: %s", err)
}
// send benchmark request if benchmarks are enabled
if *runBenchmarks {
benchRequests.Insert(0, BenchRequest{
builder: b,
commit: c,
path: workpath,
})
benchRequested = true
}
// finish here if codeUsername and codePassword aren't set
if b.codeUsername == "" || b.codePassword == "" || !*buildRelease {
return
}
// if this is a release, create tgz and upload to google code
if release := releaseRegexp.FindString(c.desc); release != "" {
// clean out build state
err = run(env, srcDir, "sh", "clean.bash", "--nopkg")
if err != nil {
return errf("clean.bash: %s", err)
}
// upload binary release
err = b.codeUpload(release)
}
return
}
func (b *Builder) codeUpload(release string) (err os.Error) {
defer func() {
if err != nil {
err = errf("%s codeUpload release: %s: %s", b.name, release, err)
}
}()
fn := fmt.Sprintf("%s.%s-%s.tar.gz", release, b.goos, b.goarch)
err = run(nil, "", "tar", "czf", fn, "go")
if err != nil {
return
}
return run(nil, "", "python",
path.Join(goroot, codePyScript),
"-s", release,
"-p", codeProject,
"-u", b.codeUsername,
"-w", b.codePassword,
"-l", fmt.Sprintf("%s,%s", b.goos, b.goarch),
fn)
}
func isDirectory(name string) bool {
s, err := os.Stat(name)
return err == nil && s.IsDirectory()
}
func isFile(name string) bool {
s, err := os.Stat(name)
return err == nil && (s.IsRegular() || s.IsSymlink())
}
func errf(format string, args ...interface{}) os.Error {
return os.NewError(fmt.Sprintf(format, args))
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment