Skip to content

G
golang
Commit 12c3afc1 authored by Russ Cox's avatar Russ Cox
Browse files
Options

dashboard: fix for branches 

In the new world, one builder runs
        gobuilder -commit
which uploads information about commits to the dashboard,
which then hands the work out to the builders by hash.
There is no assumption anymore that the commit numbers
are consistent across builders.

New builders will need to be deployed.  For now darwin-amd64
is running the new builder to test the code.

The new JSON-based protocol for handing out work via /todo
should be easy to extend if we want to add support for sending
trial CLs to the builders.

This code is already running on godashboard.appspot.com.

R=adg, dave
CC=golang-dev
https://golang.org/cl/4519047
parent 5f6e1cfc
Hide whitespace changes
Inline Side-by-side
Showing with 512 additions and 383 deletions
misc/dashboard/builder/Makefile
View file @ 12c3afc1
......@@ -7,7 +7,6 @@ include ../../../src/Make.inc
TARG=gobuilder
GOFILES=\
exec.go\
hg.go\
http.go\
main.go\
package.go\
......
misc/dashboard/builder/doc.go
View file @ 12c3afc1
......@@ -14,9 +14,6 @@ It periodically pulls updates from the Go Mercurial repository.
When a newer revision is found, Go Builder creates a clone of the repository,
runs all.bash, and reports build success or failure to the Go Dashboard.
For a successful build, Go Builder will also run benchmarks
(cd $GOROOT/src/pkg; make bench) and send the results to the Go Dashboard.
For a release revision (a change description that matches "release.YYYY-MM-DD"),
Go Builder will create a tar.gz archive of the GOROOT and deliver it to the
Go Google Code project's downloads section.
......@@ -34,8 +31,6 @@ Optional flags:
The location of the Go Dashboard application to which Go Builder will
report its results.
-bench: Run benchmarks
-release: Build and deliver binary release archive
-rev=N: Build revision N and exit
......@@ -45,7 +40,7 @@ Optional flags:
-v: Verbose logging
-external: External package builder mode (will not report Go build
state to dashboard, issue releases, or run benchmarks)
state to dashboard or issue releases)
The key file should be located at $HOME/.gobuildkey or, for a builder-specific
key, $HOME/.gobuildkey-$BUILDER (eg, $HOME/.gobuildkey-linux-amd64).
......
misc/dashboard/builder/exec.go
View file @ 12c3afc1
......@@ -18,7 +18,7 @@ func run(envv []string, dir string, argv ...string) os.Error {
if *verbose {
log.Println("run", argv)
}
bin, err := pathLookup(argv[0])
bin, err := lookPath(argv[0])
if err != nil {
return err
}
......@@ -36,7 +36,7 @@ func runLog(envv []string, logfile, dir string, argv ...string) (output string,
if *verbose {
log.Println("runLog", argv)
}
bin, err := pathLookup(argv[0])
bin, err := lookPath(argv[0])
if err != nil {
return
}
......@@ -67,10 +67,10 @@ func runLog(envv []string, logfile, dir string, argv ...string) (output string,
return b.String(), wait.WaitStatus.ExitStatus(), nil
}
// Find bin in PATH if a relative or absolute path hasn't been specified
func pathLookup(s string) (string, os.Error) {
if strings.HasPrefix(s, "/") || strings.HasPrefix(s, "./") || strings.HasPrefix(s, "../") {
return s, nil
// lookPath looks for cmd in $PATH if cmd does not begin with / or ./ or ../.
func lookPath(cmd string) (string, os.Error) {
if strings.HasPrefix(cmd, "/") || strings.HasPrefix(cmd, "./") || strings.HasPrefix(cmd, "../") {
return cmd, nil
}
return exec.LookPath(s)
return exec.LookPath(cmd)
}
misc/dashboard/builder/hg.go deleted 100644 → 0
View file @ 5f6e1cfc
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main
import (
"fmt"
"os"
"regexp"
"strconv"
"strings"
)
type Commit struct {
num int // mercurial revision number
node string // mercurial hash
parent string // hash of commit's parent
user string // author's Name <email>
date string // date of commit
desc string // description
}
// getCommit returns details about the Commit specified by the revision hash
func getCommit(rev string) (c Commit, err os.Error) {
defer func() {
if err != nil {
err = fmt.Errorf("getCommit: %s: %s", rev, err)
}
}()
parts, err := getCommitParts(rev)
if err != nil {
return
}
num, err := strconv.Atoi(parts[0])
if err != nil {
return
}
parent := ""
if num > 0 {
prev := strconv.Itoa(num - 1)
if pparts, err := getCommitParts(prev); err == nil {
parent = pparts[1]
}
}
user := strings.Replace(parts[2], "&lt;", "<", -1)
user = strings.Replace(user, "&gt;", ">", -1)
return Commit{num, parts[1], parent, user, parts[3], parts[4]}, nil
}
func getCommitParts(rev string) (parts []string, err os.Error) {
const format = "{rev}>{node}>{author|escape}>{date}>{desc}"
s, _, err := runLog(nil, "", goroot,
"hg", "log",
"--encoding", "utf-8",
"--rev", rev,
"--limit", "1",
"--template", format,
)
if err != nil {
return
}
return strings.Split(s, ">", 5), nil
}
var revisionRe = regexp.MustCompile(`([0-9]+):[0-9a-f]+$`)
// getTag fetches a Commit by finding the first hg tag that matches re.
func getTag(re *regexp.Regexp) (c Commit, tag string, err os.Error) {
o, _, err := runLog(nil, "", goroot, "hg", "tags")
for _, l := range strings.Split(o, "\n", -1) {
tag = re.FindString(l)
if tag == "" {
continue
}
s := revisionRe.FindStringSubmatch(l)
if s == nil {
err = os.NewError("couldn't find revision number")
return
}
c, err = getCommit(s[1])
return
}
err = os.NewError("no matching tag found")
return
}
misc/dashboard/builder/http.go
View file @ 12c3afc1
......@@ -6,84 +6,104 @@ package main
import (
"bytes"
"encoding/base64"
"encoding/binary"
"fmt"
"http"
"json"
"log"
"os"
"regexp"
"strconv"
)
// getHighWater returns the current highwater revision hash for this builder
func (b *Builder) getHighWater() (rev string, err os.Error) {
url := fmt.Sprintf("http://%s/hw-get?builder=%s", *dashboard, b.name)
r, _, err := http.Get(url)
type param map[string]string
// dash runs the given method and command on the dashboard.
// If args is not nil, it is the query or post parameters.
// If resp is not nil, dash unmarshals the body as JSON into resp.
func dash(meth, cmd string, resp interface{}, args param) os.Error {
var r *http.Response
var err os.Error
if *verbose {
log.Println("dash", cmd, args)
}
cmd = "http://" + *dashboard + "/" + cmd
switch meth {
case "GET":
if args != nil {
m := make(map[string][]string)
for k, v := range args {
m[k] = []string{v}
}
cmd += "?" + http.EncodeQuery(m)
}
r, _, err = http.Get(cmd)
case "POST":
r, err = http.PostForm(cmd, args)
default:
return fmt.Errorf("unknown method %q", meth)
}
if err != nil {
return
return err
}
defer r.Body.Close()
var buf bytes.Buffer
buf.ReadFrom(r.Body)
if resp != nil {
if err = json.Unmarshal(buf.Bytes(), resp); err != nil {
log.Printf("json unmarshal %#q: %s\n", buf.Bytes(), err)
return err
}
}
buf := new(bytes.Buffer)
_, err = buf.ReadFrom(r.Body)
return nil
}
func dashStatus(meth, cmd string, args param) os.Error {
var resp struct {
Status string
Error string
}
err := dash(meth, cmd, &resp, args)
if err != nil {
return err
}
if resp.Status != "OK" {
return os.NewError("/build: " + resp.Error)
}
return nil
}
// todo returns the next hash to build.
func (b *Builder) todo() (rev string, err os.Error) {
var resp []struct{
Hash string
}
if err = dash("GET", "todo", &resp, param{"builder": b.name}); err != nil {
return
}
r.Body.Close()
return buf.String(), nil
if len(resp) > 0 {
rev = resp[0].Hash
}
return
}
// recordResult sends build results to the dashboard
func (b *Builder) recordResult(buildLog string, c Commit) os.Error {
return httpCommand("build", map[string]string{
func (b *Builder) recordResult(buildLog string, hash string) os.Error {
return dash("POST", "build", nil, param{
"builder": b.name,
"key": b.key,
"node": c.node,
"parent": c.parent,
"user": c.user,
"date": c.date,
"desc": c.desc,
"node": hash,
"log": buildLog,
})
}
// match lines like: "package.BechmarkFunc 100000 999 ns/op"
var benchmarkRegexp = regexp.MustCompile("([^\n\t ]+)[\t ]+([0-9]+)[\t ]+([0-9]+) ns/op")
// recordBenchmarks sends benchmark results to the dashboard
func (b *Builder) recordBenchmarks(benchLog string, c Commit) os.Error {
results := benchmarkRegexp.FindAllStringSubmatch(benchLog, -1)
var buf bytes.Buffer
b64 := base64.NewEncoder(base64.StdEncoding, &buf)
for _, r := range results {
for _, s := range r[1:] {
binary.Write(b64, binary.BigEndian, uint16(len(s)))
b64.Write([]byte(s))
}
}
b64.Close()
return httpCommand("benchmarks", map[string]string{
"builder": b.name,
"key": b.key,
"node": c.node,
"benchmarkdata": buf.String(),
})
}
// getPackages fetches a list of package paths from the dashboard
func getPackages() (pkgs []string, err os.Error) {
r, _, err := http.Get(fmt.Sprintf("http://%v/package?fmt=json", *dashboard))
if err != nil {
return
}
defer r.Body.Close()
d := json.NewDecoder(r.Body)
// packages fetches a list of package paths from the dashboard
func packages() (pkgs []string, err os.Error) {
var resp struct {
Packages []struct {
Path string
}
}
if err = d.Decode(&resp); err != nil {
err = dash("GET", "package", &resp, param{"fmt": "json"})
if err != nil {
return
}
for _, p := range resp.Packages {
......@@ -93,24 +113,36 @@ func getPackages() (pkgs []string, err os.Error) {
}
// updatePackage sends package build results and info to the dashboard
func (b *Builder) updatePackage(pkg string, state bool, buildLog, info string, c Commit) os.Error {
args := map[string]string{
func (b *Builder) updatePackage(pkg string, state bool, buildLog, info string, hash string) os.Error {
return dash("POST", "package", nil, param{
"builder": b.name,
"key": b.key,
"path": pkg,
"state": strconv.Btoa(state),
"log": buildLog,
"info": info,
"go_rev": strconv.Itoa(c.num),
}
return httpCommand("package", args)
"go_rev": hash[:12],
})
}
func httpCommand(cmd string, args map[string]string) os.Error {
if *verbose {
log.Println("httpCommand", cmd, args)
// postCommit informs the dashboard of a new commit
func postCommit(key string, l *HgLog) os.Error {
return dashStatus("POST", "commit", param{
"key": key,
"node": l.Hash,
"date": l.Date,
"user": l.Author,
"parent": l.Parent,
"desc": l.Desc,
})
}
// dashboardCommit returns true if the dashboard knows about hash.
func dashboardCommit(hash string) bool {
err := dashStatus("GET", "commit", param{"node": hash})
if err != nil {
log.Printf("check %s: %s", hash, err)
return false
}
url := fmt.Sprintf("http://%v/%v", *dashboard, cmd)
_, err := http.PostForm(url, args)
return err
return true
}
misc/dashboard/builder/main.go
View file @ 12c3afc1
......@@ -5,7 +5,6 @@
package main
import (
"container/vector"
"flag"
"fmt"
"io/ioutil"
......@@ -16,13 +15,14 @@ import (
"strconv"
"strings"
"time"
"xml"
)
const (
codeProject = "go"
codePyScript = "misc/dashboard/googlecode_upload.py"
hgUrl = "https://go.googlecode.com/hg/"
waitInterval = 10e9 // time to wait before checking for new revs
waitInterval = 30e9 // time to wait before checking for new revs
mkdirPerm = 0750
pkgBuildInterval = 1e9 * 60 * 60 * 24 // rebuild packages every 24 hours
)
......@@ -46,16 +46,10 @@ type Builder struct {
codePassword string
}
type BenchRequest struct {
builder *Builder
commit Commit
path string
}
var (
buildroot = flag.String("buildroot", path.Join(os.TempDir(), "gobuilder"), "Directory under which to build")
commitFlag = flag.Bool("commit", false, "upload information about new commits")
dashboard = flag.String("dashboard", "godashboard.appspot.com", "Go Dashboard Host")
runBenchmarks = flag.Bool("bench", false, "Run benchmarks")
buildRelease = flag.Bool("release", false, "Build and upload binary release archives")
buildRevision = flag.String("rev", "", "Build specified revision and exit")
buildCmd = flag.String("cmd", "./all.bash", "Build command (specify absolute or relative to go/src/)")
......@@ -67,7 +61,6 @@ var (
var (
goroot string
releaseRegexp = regexp.MustCompile(`^(release|weekly)\.[0-9\-.]+`)
benchRequests vector.Vector
)
func main() {
......@@ -77,7 +70,7 @@ func main() {
os.Exit(2)
}
flag.Parse()
if len(flag.Args()) == 0 {
if len(flag.Args()) == 0 && !*commitFlag {
flag.Usage()
}
goroot = path.Join(*buildroot, "goroot")
......@@ -100,18 +93,25 @@ func main() {
if err := run(nil, *buildroot, "hg", "clone", hgUrl, goroot); err != nil {
log.Fatal("Error cloning repository:", err)
}
if *commitFlag {
if len(flag.Args()) == 0 {
commitWatcher()
return
}
go commitWatcher()
}
// if specified, build revision and return
if *buildRevision != "" {
c, err := getCommit(*buildRevision)
hash, err := fullHash(*buildRevision)
if err != nil {
log.Fatal("Error finding revision: ", err)
}
for _, b := range builders {
if err := b.buildCommit(c); err != nil {
if err := b.buildHash(hash); err != nil {
log.Println(err)
}
runQueuedBenchmark()
}
return
}
......@@ -134,6 +134,7 @@ func main() {
continue
}
built := false
t := time.Nanoseconds()
if *parallel {
done := make(chan bool)
for _, b := range builders {
......@@ -149,46 +150,15 @@ func main() {
built = b.build() || built
}
}
// only run benchmarks if we didn't build anything
// so that they don't hold up the builder queue
// sleep if there was nothing to build
if !built {
if !runQueuedBenchmark() {
// if we have no benchmarks to do, pause
time.Sleep(waitInterval)
}
// after running one benchmark,
// continue to find and build new revisions.
time.Sleep(waitInterval)
}
// sleep if we're looping too fast.
t1 := time.Nanoseconds() - t
if t1 < waitInterval {
time.Sleep(waitInterval - t1)
}
}
}
func runQueuedBenchmark() bool {
if benchRequests.Len() == 0 {
return false
}
runBenchmark(benchRequests.Pop().(BenchRequest))
return true
}
func runBenchmark(r BenchRequest) {
// run benchmarks and send to dashboard
log.Println(r.builder.name, "benchmarking", r.commit.num)
defer os.RemoveAll(r.path)
pkg := path.Join(r.path, "go", "src", "pkg")
bin := path.Join(r.path, "go", "bin")
env := []string{
"GOOS=" + r.builder.goos,
"GOARCH=" + r.builder.goarch,
"PATH=" + bin + ":" + os.Getenv("PATH"),
}
logfile := path.Join(r.path, "bench.log")
benchLog, _, err := runLog(env, logfile, pkg, "gomake", "bench")
if err != nil {
log.Println(r.builder.name, "gomake bench:", err)
return
}
if err = r.builder.recordBenchmarks(benchLog, r.commit); err != nil {
log.Println("recordBenchmarks:", err)
}
}
......@@ -235,7 +205,7 @@ func (b *Builder) buildExternal() {
log.Println("hg pull failed:", err)
continue
}
c, tag, err := getTag(releaseRegexp)
hash, tag, err := firstTag(releaseRegexp)
if err != nil {
log.Println(err)
continue
......@@ -249,8 +219,8 @@ func (b *Builder) buildExternal() {
if tag == prevTag && time.Nanoseconds() < nextBuild {
continue
}
// buildCommit will also build the packages
if err := b.buildCommit(c); err != nil {
// build will also build the packages
if err := b.buildHash(hash); err != nil {
log.Println(err)
continue
}
......@@ -269,65 +239,37 @@ func (b *Builder) build() bool {
log.Println(b.name, "build:", err)
}
}()
c, err := b.nextCommit()
hash, err := b.todo()
if err != nil {
log.Println(err)
return false
}
if c == nil {
if hash == "" {
return false
}
err = b.buildCommit(*c)
err = b.buildHash(hash)
if err != nil {
log.Println(err)
}
return true
}
// nextCommit returns the next unbuilt Commit for this builder
func (b *Builder) nextCommit() (nextC *Commit, err os.Error) {
func (b *Builder) buildHash(hash string) (err os.Error) {
defer func() {
if err != nil {
err = fmt.Errorf("%s nextCommit: %s", b.name, err)
err = fmt.Errorf("%s build: %s: %s", b.name, hash, err)
}
}()
hw, err := b.getHighWater()
if err != nil {
return
}
c, err := getCommit(hw)
if err != nil {
return
}
next := c.num + 1
c, err = getCommit(strconv.Itoa(next))
if err == nil && c.num == next {
return &c, nil
}
return nil, nil
}
func (b *Builder) buildCommit(c Commit) (err os.Error) {
defer func() {
if err != nil {
err = fmt.Errorf("%s buildCommit: %d: %s", b.name, c.num, err)
}
}()
log.Println(b.name, "building", c.num)
log.Println(b.name, "building", hash)
// create place in which to do work
workpath := path.Join(*buildroot, b.name+"-"+strconv.Itoa(c.num))
workpath := path.Join(*buildroot, b.name+"-"+hash[:12])
err = os.Mkdir(workpath, mkdirPerm)
if err != nil {
return
}
benchRequested := false
defer func() {
if !benchRequested {
os.RemoveAll(workpath)
}
}()
defer os.RemoveAll(workpath)
// clone repo
err = run(nil, workpath, "hg", "clone", goroot, "go")
......@@ -337,7 +279,7 @@ func (b *Builder) buildCommit(c Commit) (err os.Error) {
// update to specified revision
err = run(nil, path.Join(workpath, "go"),
"hg", "update", "-r", strconv.Itoa(c.num))
"hg", "update", hash)
if err != nil {
return
}
......@@ -356,36 +298,27 @@ func (b *Builder) buildCommit(c Commit) (err os.Error) {
if status != 0 {
return os.NewError("go build failed")
}
return b.buildPackages(workpath, c)
return b.buildPackages(workpath, hash)
}
if status != 0 {
// record failure
return b.recordResult(buildLog, c)
return b.recordResult(buildLog, hash)
}
// record success
if err = b.recordResult("", c); err != nil {
if err = b.recordResult("", hash); err != nil {
return fmt.Errorf("recordResult: %s", err)
}
// send benchmark request if benchmarks are enabled
if *runBenchmarks {
benchRequests.Insert(0, BenchRequest{
builder: b,
commit: c,
path: workpath,
})
benchRequested = true
}
// finish here if codeUsername and codePassword aren't set
if b.codeUsername == "" || b.codePassword == "" || !*buildRelease {
return
}
// if this is a release, create tgz and upload to google code
if release := releaseRegexp.FindString(c.desc); release != "" {
releaseHash, release, err := firstTag(releaseRegexp)
if hash == releaseHash {
// clean out build state
err = run(b.envv(), srcDir, "./clean.bash", "--nopkg")
if err != nil {
......@@ -431,3 +364,209 @@ func isFile(name string) bool {
s, err := os.Stat(name)
return err == nil && (s.IsRegular() || s.IsSymlink())
}
// commitWatcher polls hg for new commits and tells the dashboard about them.
func commitWatcher() {
// Create builder just to get master key.
b, err := NewBuilder("mercurial-commit")
if err != nil {
log.Fatal(err)
}
for {
if *verbose {
log.Printf("poll...")
}
commitPoll(b.key)
if *verbose {
log.Printf("sleep...")
}
time.Sleep(60e9)
}
}
// HgLog represents a single Mercurial revision.
type HgLog struct {
Hash string
Author string
Date string
Desc string
Parent string
// Internal metadata
added bool
}
// logByHash is a cache of all Mercurial revisions we know about,
// indexed by full hash.
var logByHash = map[string]*HgLog{}
// xmlLogTemplate is a template to pass to Mercurial to make
// hg log print the log in valid XML for parsing with xml.Unmarshal.
const xmlLogTemplate = `
<log>
<hash>{node|escape}</hash>
<parent>{parent|escape}</parent>
<author>{author|escape}</author>
<date>{date}</date>
<desc>{desc|escape}</desc>
</log>
`
// commitPoll pulls any new revisions from the hg server
// and tells the server about them.
func commitPoll(key string) {
// Catch unexpected panics.
defer func() {
if err := recover(); err != nil {
log.Printf("commitPoll panic: %s", err)
}
}()
if err := run(nil, goroot, "hg", "pull"); err != nil {
log.Printf("hg pull: %v", err)
return
}
const N = 20 // how many revisions to grab
data, _, err := runLog(nil, "", goroot, "hg", "log",
"--encoding=utf-8",
"--limit=" + strconv.Itoa(N),
"--template=" + xmlLogTemplate,
)
if err != nil {
log.Printf("hg log: %v", err)
return
}
var logStruct struct {
Log []HgLog
}
err = xml.Unmarshal(strings.NewReader("<top>" + data + "</top>"), &logStruct)
if err != nil {
log.Printf("unmarshal hg log: %v", err)
return
}
logs := logStruct.Log
// Pass 1. Fill in parents and add new log entries to logsByHash.
// Empty parent means take parent from next log entry.
// Non-empty parent has form 1234:hashhashhash; we weant full hash.
for i := range logs {
l := &logs[i]
log.Printf("hg log: %s < %s\n", l.Hash, l.Parent)
if l.Parent == "" && i+1 < len(logs) {
l.Parent = logs[i+1].Hash
} else if l.Parent != "" {
l.Parent, _ = fullHash(l.Parent)
}
if l.Parent == "" {
// Can't create node without parent.
continue
}
if logByHash[l.Hash] == nil {
// Make copy to avoid pinning entire slice when only one entry is new.
t := *l
logByHash[t.Hash] = &t
}
}
for i := range logs {
l := &logs[i]
if l.Parent == "" {
continue
}
addCommit(l.Hash, key)
}
}
// addCommit adds the commit with the named hash to the dashboard.
// key is the secret key for authentication to the dashboard.
// It avoids duplicate effort.
func addCommit(hash, key string) bool {
l := logByHash[hash]
if l == nil {
return false
}
if l.added {
return true
}
// Check for already added, perhaps in an earlier run.
if dashboardCommit(hash) {
log.Printf("%s already on dashboard\n", hash)
// Record that this hash is on the dashboard,
// as must be all its parents.
for l != nil {
l.added = true
l = logByHash[l.Parent]
}
return true
}
// Create parent first, to maintain some semblance of order.
if !addCommit(l.Parent, key) {
return false
}
// Create commit.
if err := postCommit(key, l); err != nil {
log.Printf("faield to add %s to dashboard: %v", err)
return false
}
return true
}
// fullHash returns the full hash for the given Mercurial revision.
func fullHash(rev string) (hash string, err os.Error) {
defer func() {
if err != nil {
err = fmt.Errorf("fullHash: %s: %s", rev, err)
}
}()
s, _, err := runLog(nil, "", goroot,
"hg", "log",
"--encoding=utf-8",
"--rev="+rev,
"--limit=1",
"--template={node}",
)
if err != nil {
return
}
s = strings.TrimSpace(s)
if s == "" {
return "", fmt.Errorf("cannot find revision")
}
if len(s) != 20 {
return "", fmt.Errorf("hg returned invalid hash " + s)
}
return s, nil
}
var revisionRe = regexp.MustCompile(`^([^ ]+) +[0-9]+:([0-9a-f]+)$`)
// firstTag returns the hash and tag of the most recent tag matching re.
func firstTag(re *regexp.Regexp) (hash string, tag string, err os.Error) {
o, _, err := runLog(nil, "", goroot, "hg", "tags")
for _, l := range strings.Split(o, "\n", -1) {
if l == "" {
continue
}
s := revisionRe.FindStringSubmatch(l)
if s == nil {
err = os.NewError("couldn't find revision number")
return
}
if !re.MatchString(s[1]) {
continue
}
tag = s[1]
hash, err = fullHash(s[3])
return
}
err = os.NewError("no matching tag found")
return
}
misc/dashboard/builder/package.go
View file @ 12c3afc1
......@@ -13,8 +13,8 @@ import (
"path"
)
func (b *Builder) buildPackages(workpath string, c Commit) os.Error {
pkgs, err := getPackages()
func (b *Builder) buildPackages(workpath string, hash string) os.Error {
pkgs, err := packages()
if err != nil {
return err
}
......@@ -32,13 +32,13 @@ func (b *Builder) buildPackages(workpath string, c Commit) os.Error {
built := code != 0
// get doc comment from package source
info, err := getPackageComment(p, path.Join(goroot, "pkg", p))
info, err := packageComment(p, path.Join(goroot, "pkg", p))
if err != nil {
log.Printf("goinstall %v: %v", p, err)
}
// update dashboard with build state + info
err = b.updatePackage(p, built, buildLog, info, c)
err = b.updatePackage(p, built, buildLog, info, hash)
if err != nil {
log.Printf("updatePackage %v: %v", p, err)
}
......@@ -46,7 +46,7 @@ func (b *Builder) buildPackages(workpath string, c Commit) os.Error {
return nil
}
func getPackageComment(pkg, pkgpath string) (info string, err os.Error) {
func packageComment(pkg, pkgpath string) (info string, err os.Error) {
fset := token.NewFileSet()
pkgs, err := parser.ParseDir(fset, pkgpath, nil, parser.PackageClauseOnly|parser.ParseComments)
if err != nil {
......
misc/dashboard/godashboard/app.yaml
View file @ 12c3afc1
application: godashboard
version: 5
version: 6
runtime: python
api_version: 1
......
misc/dashboard/godashboard/gobuild.py
View file @ 12c3afc1
......@@ -5,6 +5,7 @@
# This is the server part of the continuous build system for Go. It must be run
# by AppEngine.
from django.utils import simplejson
from google.appengine.api import mail
from google.appengine.api import memcache
from google.appengine.ext import db
......@@ -50,10 +51,6 @@ class Commit(db.Model):
fail_notification_sent = db.BooleanProperty()
class Cache(db.Model):
data = db.BlobProperty()
expire = db.IntegerProperty()
# A CompressedLog contains the textual build log of a failed build.
# The key name is the hex digest of the SHA256 hash of the contents.
# The contents is bz2 compressed.
......@@ -62,23 +59,6 @@ class CompressedLog(db.Model):
N = 30
def cache_get(key):
c = Cache.get_by_key_name(key)
if c is None or c.expire < time.time():
return None
return c.data
def cache_set(key, val, timeout):
c = Cache(key_name = key)
c.data = val
c.expire = int(time.time() + timeout)
c.put()
def cache_del(key):
c = Cache.get_by_key_name(key)
if c is not None:
c.delete()
def builderInfo(b):
f = b.split('-', 3)
goos = f[0]
......@@ -96,7 +76,7 @@ def builderset():
for c in results:
builders.update(set(parseBuild(build)['builder'] for build in c.builds))
return builders
class MainPage(webapp.RequestHandler):
def get(self):
self.response.headers['Content-Type'] = 'text/html; charset=utf-8'
......@@ -147,7 +127,30 @@ class MainPage(webapp.RequestHandler):
path = os.path.join(os.path.dirname(__file__), 'main.html')
self.response.out.write(template.render(path, values))
class GetHighwater(webapp.RequestHandler):
# A DashboardHandler is a webapp.RequestHandler but provides
# authenticated_post - called by post after authenticating
# json - writes object in json format to response output
class DashboardHandler(webapp.RequestHandler):
def post(self):
if not auth(self.request):
self.response.set_status(403)
return
self.authenticated_post()
def authenticated_post(self):
return
def json(self, obj):
self.response.set_status(200)
simplejson.dump(obj, self.response.out)
return
def auth(req):
k = req.get('key')
return k == hmac.new(key.accessKey, req.get('builder')).hexdigest() or k == key.accessKey
# Todo serves /todo. It tells the builder which commits need to be built.
class Todo(DashboardHandler):
def get(self):
builder = self.request.get('builder')
key = 'todo-%s' % builder
......@@ -155,28 +158,19 @@ class GetHighwater(webapp.RequestHandler):
if response is None:
# Fell out of memcache. Rebuild from datastore results.
# We walk the commit list looking for nodes that have not
# been built by this builder and record the *parents* of those
# nodes, because each builder builds the revision *after* the
# one return (because we might not know about the latest
# revision).
# been built by this builder.
q = Commit.all()
q.order('-__key__')
todo = []
need = False
first = None
for c in q.fetch(N+1):
if first is None:
first = c
if need:
todo.append(c.node)
need = not built(c, builder)
if not todo:
todo.append(first.node)
response = ' '.join(todo)
if not built(c, builder):
todo.append({'Hash': c.node})
response = simplejson.dumps(todo)
memcache.set(key, response, 3600)
self.response.set_status(200)
if self.request.get('all') != 'yes':
response = response.split()[0]
self.response.out.write(response)
def built(c, builder):
......@@ -185,22 +179,8 @@ def built(c, builder):
return True
return False
def auth(req):
k = req.get('key')
return k == hmac.new(key.accessKey, req.get('builder')).hexdigest() or k == key.accessKey
class SetHighwater(webapp.RequestHandler):
def post(self):
if not auth(self.request):
self.response.set_status(403)
return
# Allow for old builders.
# This is a no-op now: we figure out what to build based
# on the current dashboard status.
return
class LogHandler(webapp.RequestHandler):
# Log serves /log/. It retrieves log data by content hash.
class LogHandler(DashboardHandler):
def get(self):
self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
hash = self.request.path[5:]
......@@ -214,12 +194,8 @@ class LogHandler(webapp.RequestHandler):
# Init creates the commit with id 0. Since this commit doesn't have a parent,
# it cannot be created by Build.
class Init(webapp.RequestHandler):
def post(self):
if not auth(self.request):
self.response.set_status(403)
return
class Init(DashboardHandler):
def authenticated_post(self):
date = parseDate(self.request.get('date'))
node = self.request.get('node')
if not validNode(node) or date is None:
......@@ -239,7 +215,86 @@ class Init(webapp.RequestHandler):
self.response.set_status(200)
# Build is the main command: it records the result of a new build.
# The last commit when we switched to using entity groups.
# This is the root of the new commit entity group.
RootCommitKeyName = '00000f26-f32c6f1038207c55d5780231f7484f311020747e'
# CommitHandler serves /commit.
# A GET of /commit retrieves information about the specified commit.
# A POST of /commit creates a node for the given commit.
# If the commit already exists, the POST silently succeeds (like mkdir -p).
class CommitHandler(DashboardHandler):
def get(self):
node = self.request.get('node')
if not validNode(node):
return self.json({'Status': 'FAIL', 'Error': 'malformed node hash'})
n = nodeByHash(node)
if n is None:
return self.json({'Status': 'FAIL', 'Error': 'unknown revision'})
return self.json({'Status': 'OK', 'Node': nodeObj(n)})
def authenticated_post(self):
# Require auth with the master key, not a per-builder key.
if self.request.get('builder'):
self.response.set_status(403)
return
node = self.request.get('node')
date = parseDate(self.request.get('date'))
user = self.request.get('user').encode('utf8')
desc = self.request.get('desc').encode('utf8')
parenthash = self.request.get('parent')
if not validNode(node) or not validNode(parenthash) or date is None:
return self.json({'Status': 'FAIL', 'Error': 'malformed node, parent, or date'})
n = nodeByHash(node)
if n is None:
p = nodeByHash(parenthash)
if p is None:
return self.json({'Status': 'FAIL', 'Error': 'unknown parent'})
# Want to create new node in a transaction so that multiple
# requests creating it do not collide and so that multiple requests
# creating different nodes get different sequence numbers.
# All queries within a transaction must include an ancestor,
# but the original datastore objects we used for the dashboard
# have no common ancestor. Instead, we use a well-known
# root node - the last one before we switched to entity groups -
# as the as the common ancestor.
root = Commit.get_by_key_name(RootCommitKeyName)
def add_commit():
if nodeByHash(node, ancestor=root) is not None:
return
# Determine number for this commit.
# Once we have created one new entry it will be lastRooted.num+1,
# but the very first commit created in this scheme will have to use
# last.num's number instead (last is likely not rooted).
q = Commit.all()
q.order('-__key__')
q.ancestor(root)
last = q.fetch(1)[0]
num = last.num+1
n = Commit(key_name = '%08x-%s' % (num, node), parent = root)
n.num = num
n.node = node
n.parentnode = parenthash
n.user = user
n.date = date
n.desc = desc
n.put()
db.run_in_transaction(add_commit)
n = nodeByHash(node)
if n is None:
return self.json({'Status': 'FAIL', 'Error': 'failed to create commit node'})
return self.json({'Status': 'OK', 'Node': nodeObj(n)})
# Build serves /build.
# A POST to /build records a new build result.
class Build(webapp.RequestHandler):
def post(self):
if not auth(self.request):
......@@ -256,44 +311,33 @@ class Build(webapp.RequestHandler):
l.log = bz2.compress(log)
l.put()
date = parseDate(self.request.get('date'))
user = self.request.get('user').encode('utf8')
desc = self.request.get('desc').encode('utf8')
node = self.request.get('node')
parenthash = self.request.get('parent')
if not validNode(node) or not validNode(parenthash) or date is None:
logging.error("Not valid node ('%s') or bad date (%s %s)", node, date, self.request.get('date'))
if not validNode(node):
logging.error('Invalid node %s' % (node))
self.response.set_status(500)
return
q = Commit.all()
q.filter('node =', parenthash)
parent = q.get()
if parent is None:
logging.error('Cannot find parent %s of node %s' % (parenthash, node))
n = nodeByHash(node)
if n is None:
logging.error('Cannot find node %s' % (node))
self.response.set_status(404)
return
parentnum, _ = parent.key().name().split('-', 1)
nodenum = int(parentnum, 16) + 1
key_name = '%08x-%s' % (nodenum, node)
nn = n
def add_build():
n = Commit.get_by_key_name(key_name)
n = nodeByHash(node, ancestor=nn)
if n is None:
n = Commit(key_name = key_name)
n.num = nodenum
n.node = node
n.parentnode = parenthash
n.user = user
n.date = date
n.desc = desc
logging.error('Cannot find hash in add_build: %s %s' % (builder, node))
return
s = '%s`%s' % (builder, loghash)
for i, b in enumerate(n.builds):
if b.split('`', 1)[0] == builder:
# logging.error('Found result for %s %s already' % (builder, node))
n.builds[i] = s
break
else:
# logging.error('Added result for %s %s' % (builder, node))
n.builds.append(s)
n.put()
......@@ -302,30 +346,7 @@ class Build(webapp.RequestHandler):
key = 'todo-%s' % builder
memcache.delete(key)
def mark_sent():
n = Commit.get_by_key_name(key_name)
n.fail_notification_sent = True
n.put()
n = Commit.get_by_key_name(key_name)
if loghash and not failed(parent, builder) and not n.fail_notification_sent:
subject = const.mail_fail_subject % (builder, desc.split("\n")[0])
path = os.path.join(os.path.dirname(__file__), 'fail-notify.txt')
body = template.render(path, {
"builder": builder,
"node": node[:12],
"user": user,
"desc": desc,
"loghash": loghash
})
mail.send_mail(
sender=const.mail_from,
reply_to=const.mail_fail_reply_to,
to=const.mail_fail_to,
subject=subject,
body=body
)
db.run_in_transaction(mark_sent)
# TODO: Send mail for build breakage.
self.response.set_status(200)
......@@ -342,6 +363,24 @@ def node(num):
n = q.get()
return n
def nodeByHash(hash, ancestor=None):
q = Commit.all()
q.filter('node =', hash)
if ancestor is not None:
q.ancestor(ancestor)
n = q.get()
return n
# nodeObj returns a JSON object (ready to be passed to simplejson.dump) describing node.
def nodeObj(n):
return {
'Hash': n.node,
'ParentHash': n.parentnode,
'User': n.user,
'Date': n.date.strftime('%Y-%m-%d %H:%M %z'),
'Desc': n.desc,
}
class FixedOffset(datetime.tzinfo):
"""Fixed offset in minutes east from UTC."""
......@@ -417,15 +456,20 @@ def toRev(c):
def byBuilder(x, y):
return cmp(x['builder'], y['builder'])
# Give old builders work; otherwise they pound on the web site.
class Hwget(DashboardHandler):
def get(self):
self.response.out.write("8000\n")
# This is the URL map for the server. The first three entries are public, the
# rest are only used by the builders.
application = webapp.WSGIApplication(
[('/', MainPage),
('/hw-get', Hwget),
('/log/.*', LogHandler),
('/hw-get', GetHighwater),
('/hw-set', SetHighwater),
('/commit', CommitHandler),
('/init', Init),
('/todo', Todo),
('/build', Build),
], debug=True)
......
misc/dashboard/godashboard/index.yaml
View file @ 12c3afc1
......@@ -23,6 +23,12 @@ indexes:
- name: __key__
direction: desc
- kind: Commit
ancestor: yes
properties:
- name: __key__
direction: desc
- kind: Project
properties:
- name: approved
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment