Commit 6798adc6 authored by bobbyrullo's avatar bobbyrullo

Merge pull request #67 from bobbyrullo/db_migrate

DB Migrations for Dex
parents c0cf9066 8b6a2699
{
"ImportPath": "github.com/coreos-inc/auth",
"ImportPath": "github.com/coreos/dex",
"GoVersion": "go1.4.2",
"Packages": [
"./..."
......@@ -96,6 +96,10 @@
"ImportPath": "github.com/mbanzon/simplehttp",
"Rev": "04c542e7ac706a25820090f274ea6a4f39a63326"
},
{
"ImportPath": "github.com/rubenv/sql-migrate",
"Rev": "53184e1edfb4f9655b0fa8dd2c23e7763f452bda"
},
{
"ImportPath": "golang.org/x/crypto/bcrypt",
"Rev": "1fbbd62cfec66bd39d91e97749579579d4d3037e"
......@@ -115,6 +119,11 @@
{
"ImportPath": "google.golang.org/api/googleapi",
"Rev": "d3edb0282bde692467788c50070a9211afe75cf3"
},
{
"ImportPath": "gopkg.in/gorp.v1",
"Comment": "v1.7.1",
"Rev": "c87af80f3cc5036b55b83d77171e156791085e2e"
}
]
}
.*.swp
*.test
/sql-migrate/test.db
/test.db
language: go
go:
- 1.2
- 1.3
- 1.4
- tip
services:
- mysql
before_install:
- mysql -e "CREATE DATABASE IF NOT EXISTS test;" -uroot
- psql -c "CREATE DATABASE test;" -U postgres
install:
- go get -t ./...
- go install ./...
script:
- go test -v ./...
- bash test-integration/postgres.sh
- bash test-integration/mysql.sh
- bash test-integration/mysql-flag.sh
- bash test-integration/sqlite.sh
# sql-migrate
> SQL Schema migration tool for [Go](http://golang.org/). Based on [gorp](https://github.com/go-gorp/gorp) and [goose](https://bitbucket.org/liamstask/goose).
[![Build Status](https://travis-ci.org/rubenv/sql-migrate.svg?branch=master)](https://travis-ci.org/rubenv/sql-migrate) [![GoDoc](https://godoc.org/github.com/rubenv/sql-migrate?status.png)](https://godoc.org/github.com/rubenv/sql-migrate)
Using [modl](https://github.com/jmoiron/modl)? Check out [modl-migrate](https://github.com/rubenv/modl-migrate).
## Features
* Usable as a CLI tool or as a library
* Supports SQLite, PostgreSQL, MySQL, MSSQL and Oracle databases (through [gorp](https://github.com/go-gorp/gorp))
* Can embed migrations into your application
* Migrations are defined with SQL for full flexibility
* Atomic migrations
* Up/down migrations to allow rollback
* Supports multiple database types in one project
## Installation
To install the library and command line program, use the following:
```bash
go get github.com/rubenv/sql-migrate/...
```
## Usage
### As a standalone tool
```
$ sql-migrate --help
usage: sql-migrate [--version] [--help] <command> [<args>]
Available commands are:
down Undo a database migration
redo Reapply the last migration
status Show migration status
up Migrates the database to the most recent version available
```
Each command requires a configuration file (which defaults to `dbconfig.yml`, but can be specified with the `-config` flag). This config file should specify one or more environments:
```yml
development:
dialect: sqlite3
datasource: test.db
dir: migrations/sqlite3
production:
dialect: postgres
datasource: dbname=myapp sslmode=disable
dir: migrations/postgres
table: migrations
```
The `table` setting is optional and will default to `gorp_migrations`.
The environment that will be used can be specified with the `-env` flag (defaults to `development`).
Use the `--help` flag in combination with any of the commands to get an overview of its usage:
```
$ sql-migrate up --help
Usage: sql-migrate up [options] ...
Migrates the database to the most recent version available.
Options:
-config=config.yml Configuration file to use.
-env="development" Environment.
-limit=0 Limit the number of migrations (0 = unlimited).
-dryrun Don't apply migrations, just print them.
```
The `up` command applies all available migrations. By contrast, `down` will only apply one migration by default. This behavior can be changed for both by using the `-limit` parameter.
The `redo` command will unapply the last migration and reapply it. This is useful during development, when you're writing migrations.
Use the `status` command to see the state of the applied migrations:
```bash
$ sql-migrate status
+---------------+-----------------------------------------+
| MIGRATION | APPLIED |
+---------------+-----------------------------------------+
| 1_initial.sql | 2014-09-13 08:19:06.788354925 +0000 UTC |
| 2_record.sql | no |
+---------------+-----------------------------------------+
```
### As a library
Import sql-migrate into your application:
```go
import "github.com/rubenv/sql-migrate"
```
Set up a source of migrations, this can be from memory, from a set of files or from bindata (more on that later):
```go
// Hardcoded strings in memory:
migrations := &migrate.MemoryMigrationSource{
Migrations: []*migrate.Migration{
&migrate.Migration{
Id: "123",
Up: []string{"CREATE TABLE people (id int)"},
Down: []string{"DROP TABLE people"},
},
},
}
// OR: Read migrations from a folder:
migrations := &migrate.FileMigrationSource{
Dir: "db/migrations",
}
// OR: Use migrations from bindata:
migrations := &migrate.AssetMigrationSource{
Asset: Asset,
AssetDir: AssetDir,
Dir: "migrations",
}
```
Then use the `Exec` function to upgrade your database:
```go
db, err := sql.Open("sqlite3", filename)
if err != nil {
// Handle errors!
}
n, err := migrate.Exec(db, "sqlite3", migrations, migrate.Up)
if err != nil {
// Handle errors!
}
fmt.Printf("Applied %d migrations!\n", n)
```
Note that `n` can be greater than `0` even if there is an error: any migration that succeeded will remain applied even if a later one fails.
Check [the GoDoc reference](https://godoc.org/github.com/rubenv/sql-migrate) for the full documentation.
## Writing migrations
Migrations are defined in SQL files, which contain a set of SQL statements. Special comments are used to distinguish up and down migrations.
```sql
-- +migrate Up
-- SQL in section 'Up' is executed when this migration is applied
CREATE TABLE people (id int);
-- +migrate Down
-- SQL section 'Down' is executed when this migration is rolled back
DROP TABLE people;
```
You can put multiple statements in each block, as long as you end them with a semicolon (`;`).
If you have complex statements which contain semicolons, use `StatementBegin` and `StatementEnd` to indicate boundaries:
```sql
-- +migrate Up
CREATE TABLE people (id int);
-- +migrate StatementBegin
CREATE OR REPLACE FUNCTION do_something()
returns void AS $$
DECLARE
create_query text;
BEGIN
-- Do something here
END;
$$
language plpgsql;
-- +migrate StatementEnd
-- +migrate Down
DROP FUNCTION do_something();
DROP TABLE people;
```
The order in which migrations are applied is defined through the filename: sql-migrate will sort migrations based on their name. It's recommended to use an increasing version number or a timestamp as the first part of the filename.
## Embedding migrations with [bindata](https://github.com/jteeuwen/go-bindata)
If you like your Go applications self-contained (that is: a single binary): use [bindata](https://github.com/jteeuwen/go-bindata) to embed the migration files.
Just write your migration files as usual, as a set of SQL files in a folder.
Then use bindata to generate a `.go` file with the migrations embedded:
```bash
go-bindata -pkg myapp -o bindata.go db/migrations/
```
The resulting `bindata.go` file will contain your migrations. Remember to regenerate your `bindata.go` file whenever you add/modify a migration (`go generate` will help here, once it arrives).
Use the `AssetMigrationSource` in your application to find the migrations:
```go
migrations := &migrate.AssetMigrationSource{
Asset: Asset,
AssetDir: AssetDir,
Dir: "db/migrations",
}
```
Both `Asset` and `AssetDir` are functions provided by bindata.
Then proceed as usual.
## Extending
Adding a new migration source means implementing `MigrationSource`.
```go
type MigrationSource interface {
FindMigrations() ([]*Migration, error)
}
```
The resulting slice of migrations will be executed in the given order, so it should usually be sorted by the `Id` field.
## License
(The MIT License)
Copyright (C) 2014-2015 by Ruben Vermeersch <ruben@rocketeer.be>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
package migrate
import (
"bytes"
"compress/gzip"
"fmt"
"io"
"strings"
)
func bindata_read(data []byte, name string) ([]byte, error) {
gz, err := gzip.NewReader(bytes.NewBuffer(data))
if err != nil {
return nil, fmt.Errorf("Read %q: %v", name, err)
}
var buf bytes.Buffer
_, err = io.Copy(&buf, gz)
gz.Close()
if err != nil {
return nil, fmt.Errorf("Read %q: %v", name, err)
}
return buf.Bytes(), nil
}
func test_migrations_1_initial_sql() ([]byte, error) {
return bindata_read([]byte{
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x00, 0xff, 0x8c, 0xcd,
0x3d, 0x0e, 0x82, 0x40, 0x10, 0x05, 0xe0, 0x7e, 0x4e, 0xf1, 0x3a, 0x34,
0x86, 0x13, 0x50, 0xa1, 0xd0, 0x91, 0xa8, 0x08, 0x07, 0x40, 0x76, 0x22,
0x13, 0xd7, 0xdd, 0x09, 0xac, 0xc1, 0xe3, 0xbb, 0xc4, 0x68, 0xb4, 0xb3,
0x7c, 0x6f, 0x7e, 0xbe, 0x34, 0xc5, 0xe6, 0x26, 0x97, 0xb1, 0x0b, 0x8c,
0x56, 0x29, 0xc6, 0xd3, 0xb1, 0x82, 0x38, 0x4c, 0xdc, 0x07, 0xf1, 0x0e,
0x49, 0xab, 0x09, 0x64, 0x02, 0x3f, 0xb8, 0xbf, 0x07, 0x36, 0x98, 0x07,
0x76, 0x08, 0x43, 0xac, 0x5e, 0x77, 0xcb, 0x52, 0x0c, 0x9d, 0xaa, 0x15,
0x36, 0xb4, 0xab, 0xcb, 0xbc, 0x29, 0xd1, 0xe4, 0xdb, 0xaa, 0x84, 0xb2,
0x57, 0xcb, 0x58, 0x89, 0x89, 0x2f, 0xc3, 0x3a, 0x23, 0xa2, 0x6f, 0xb0,
0xf0, 0xb3, 0x7b, 0x93, 0x1f, 0x6f, 0x29, 0xff, 0x12, 0x47, 0x6f, 0x6d,
0x9c, 0x9e, 0xbb, 0xfe, 0x4a, 0x45, 0xbd, 0x3f, 0xfc, 0x98, 0x19, 0x3d,
0x03, 0x00, 0x00, 0xff, 0xff, 0x0d, 0x70, 0x5e, 0xf9, 0xda, 0x00, 0x00,
0x00,
},
"test-migrations/1_initial.sql",
)
}
func test_migrations_2_record_sql() ([]byte, error) {
return bindata_read([]byte{
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x00, 0xff, 0xd2, 0xd5,
0x55, 0xd0, 0xce, 0xcd, 0x4c, 0x2f, 0x4a, 0x2c, 0x49, 0x55, 0x08, 0x2d,
0xe0, 0xf2, 0xf4, 0x0b, 0x76, 0x0d, 0x0a, 0x51, 0xf0, 0xf4, 0x0b, 0xf1,
0x57, 0x28, 0x48, 0xcd, 0x2f, 0xc8, 0x49, 0x55, 0xd0, 0xc8, 0x4c, 0xd1,
0x54, 0x08, 0x73, 0xf4, 0x09, 0x75, 0x0d, 0x56, 0xd0, 0x30, 0xd4, 0xb4,
0xe6, 0xe2, 0x42, 0xd6, 0xe3, 0x92, 0x5f, 0x9e, 0xc7, 0xe5, 0xe2, 0xea,
0xe3, 0x1a, 0xe2, 0xaa, 0xe0, 0x16, 0xe4, 0xef, 0x0b, 0xd3, 0x15, 0xee,
0xe1, 0x1a, 0xe4, 0xaa, 0x90, 0x99, 0x62, 0x6b, 0x68, 0xcd, 0x05, 0x08,
0x00, 0x00, 0xff, 0xff, 0xf4, 0x3a, 0x7b, 0xae, 0x64, 0x00, 0x00, 0x00,
},
"test-migrations/2_record.sql",
)
}
// Asset loads and returns the asset for the given name.
// It returns an error if the asset could not be found or
// could not be loaded.
func Asset(name string) ([]byte, error) {
cannonicalName := strings.Replace(name, "\\", "/", -1)
if f, ok := _bindata[cannonicalName]; ok {
return f()
}
return nil, fmt.Errorf("Asset %s not found", name)
}
// AssetNames returns the names of the assets.
func AssetNames() []string {
names := make([]string, 0, len(_bindata))
for name := range _bindata {
names = append(names, name)
}
return names
}
// _bindata is a table, holding each asset generator, mapped to its name.
var _bindata = map[string]func() ([]byte, error){
"test-migrations/1_initial.sql": test_migrations_1_initial_sql,
"test-migrations/2_record.sql": test_migrations_2_record_sql,
}
// AssetDir returns the file names below a certain
// directory embedded in the file by go-bindata.
// For example if you run go-bindata on data/... and data contains the
// following hierarchy:
// data/
// foo.txt
// img/
// a.png
// b.png
// then AssetDir("data") would return []string{"foo.txt", "img"}
// AssetDir("data/img") would return []string{"a.png", "b.png"}
// AssetDir("foo.txt") and AssetDir("notexist") would return an error
// AssetDir("") will return []string{"data"}.
func AssetDir(name string) ([]string, error) {
node := _bintree
if len(name) != 0 {
cannonicalName := strings.Replace(name, "\\", "/", -1)
pathList := strings.Split(cannonicalName, "/")
for _, p := range pathList {
node = node.Children[p]
if node == nil {
return nil, fmt.Errorf("Asset %s not found", name)
}
}
}
if node.Func != nil {
return nil, fmt.Errorf("Asset %s not found", name)
}
rv := make([]string, 0, len(node.Children))
for name := range node.Children {
rv = append(rv, name)
}
return rv, nil
}
type _bintree_t struct {
Func func() ([]byte, error)
Children map[string]*_bintree_t
}
var _bintree = &_bintree_t{nil, map[string]*_bintree_t{
"test-migrations": &_bintree_t{nil, map[string]*_bintree_t{
"1_initial.sql": &_bintree_t{test_migrations_1_initial_sql, map[string]*_bintree_t{
}},
"2_record.sql": &_bintree_t{test_migrations_2_record_sql, map[string]*_bintree_t{
}},
}},
}}
/*
SQL Schema migration tool for Go.
Key features:
* Usable as a CLI tool or as a library
* Supports SQLite, PostgreSQL, MySQL, MSSQL and Oracle databases (through gorp)
* Can embed migrations into your application
* Migrations are defined with SQL for full flexibility
* Atomic migrations
* Up/down migrations to allow rollback
* Supports multiple database types in one project
Installation
To install the library and command line program, use the following:
go get github.com/rubenv/sql-migrate/...
Command-line tool
The main command is called sql-migrate.
$ sql-migrate --help
usage: sql-migrate [--version] [--help] <command> [<args>]
Available commands are:
down Undo a database migration
redo Reapply the last migration
status Show migration status
up Migrates the database to the most recent version available
Each command requires a configuration file (which defaults to dbconfig.yml, but can be specified with the -config flag). This config file should specify one or more environments:
development:
dialect: sqlite3
datasource: test.db
dir: migrations/sqlite3
production:
dialect: postgres
datasource: dbname=myapp sslmode=disable
dir: migrations/postgres
table: migrations
The `table` setting is optional and will default to `gorp_migrations`.
The environment that will be used can be specified with the -env flag (defaults to development).
Use the --help flag in combination with any of the commands to get an overview of its usage:
$ sql-migrate up --help
Usage: sql-migrate up [options] ...
Migrates the database to the most recent version available.
Options:
-config=config.yml Configuration file to use.
-env="development" Environment.
-limit=0 Limit the number of migrations (0 = unlimited).
-dryrun Don't apply migrations, just print them.
The up command applies all available migrations. By contrast, down will only apply one migration by default. This behavior can be changed for both by using the -limit parameter.
The redo command will unapply the last migration and reapply it. This is useful during development, when you're writing migrations.
Use the status command to see the state of the applied migrations:
$ sql-migrate status
+---------------+-----------------------------------------+
| MIGRATION | APPLIED |
+---------------+-----------------------------------------+
| 1_initial.sql | 2014-09-13 08:19:06.788354925 +0000 UTC |
| 2_record.sql | no |
+---------------+-----------------------------------------+
Library
Import sql-migrate into your application:
import "github.com/rubenv/sql-migrate"
Set up a source of migrations, this can be from memory, from a set of files or from bindata (more on that later):
// Hardcoded strings in memory:
migrations := &migrate.MemoryMigrationSource{
Migrations: []*migrate.Migration{
&migrate.Migration{
Id: "123",
Up: []string{"CREATE TABLE people (id int)"},
Down: []string{"DROP TABLE people"},
},
},
}
// OR: Read migrations from a folder:
migrations := &migrate.FileMigrationSource{
Dir: "db/migrations",
}
// OR: Use migrations from bindata:
migrations := &migrate.AssetMigrationSource{
Asset: Asset,
AssetDir: AssetDir,
Dir: "migrations",
}
Then use the Exec function to upgrade your database:
db, err := sql.Open("sqlite3", filename)
if err != nil {
// Handle errors!
}
n, err := migrate.Exec(db, "sqlite3", migrations, migrate.Up)
if err != nil {
// Handle errors!
}
fmt.Printf("Applied %d migrations!\n", n)
Note that n can be greater than 0 even if there is an error: any migration that succeeded will remain applied even if a later one fails.
The full set of capabilities can be found in the API docs below.
Writing migrations
Migrations are defined in SQL files, which contain a set of SQL statements. Special comments are used to distinguish up and down migrations.
-- +migrate Up
-- SQL in section 'Up' is executed when this migration is applied
CREATE TABLE people (id int);
-- +migrate Down
-- SQL section 'Down' is executed when this migration is rolled back
DROP TABLE people;
You can put multiple statements in each block, as long as you end them with a semicolon (;).
If you have complex statements which contain semicolons, use StatementBegin and StatementEnd to indicate boundaries:
-- +migrate Up
CREATE TABLE people (id int);
-- +migrate StatementBegin
CREATE OR REPLACE FUNCTION do_something()
returns void AS $$
DECLARE
create_query text;
BEGIN
-- Do something here
END;
$$
language plpgsql;
-- +migrate StatementEnd
-- +migrate Down
DROP FUNCTION do_something();
DROP TABLE people;
The order in which migrations are applied is defined through the filename: sql-migrate will sort migrations based on their name. It's recommended to use an increasing version number or a timestamp as the first part of the filename.
Embedding migrations with bindata
If you like your Go applications self-contained (that is: a single binary): use bindata (https://github.com/jteeuwen/go-bindata) to embed the migration files.
Just write your migration files as usual, as a set of SQL files in a folder.
Then use bindata to generate a .go file with the migrations embedded:
go-bindata -pkg myapp -o bindata.go db/migrations/
The resulting bindata.go file will contain your migrations. Remember to regenerate your bindata.go file whenever you add/modify a migration (go generate will help here, once it arrives).
Use the AssetMigrationSource in your application to find the migrations:
migrations := &migrate.AssetMigrationSource{
Asset: Asset,
AssetDir: AssetDir,
Dir: "db/migrations",
}
Both Asset and AssetDir are functions provided by bindata.
Then proceed as usual.
Extending
Adding a new migration source means implementing MigrationSource.
type MigrationSource interface {
FindMigrations() ([]*Migration, error)
}
The resulting slice of migrations will be executed in the given order, so it should usually be sorted by the Id field.
*/
package migrate
package migrate
import (
"testing"
. "gopkg.in/check.v1"
)
func Test(t *testing.T) { TestingT(t) }
This diff is collapsed.
package migrate
import (
"database/sql"
"os"
_ "github.com/mattn/go-sqlite3"
. "gopkg.in/check.v1"
"gopkg.in/gorp.v1"
)
var filename = "/tmp/sql-migrate-sqlite.db"
var sqliteMigrations = []*Migration{
&Migration{
Id: "123",
Up: []string{"CREATE TABLE people (id int)"},
Down: []string{"DROP TABLE people"},
},
&Migration{
Id: "124",
Up: []string{"ALTER TABLE people ADD COLUMN first_name text"},
Down: []string{"SELECT 0"}, // Not really supported
},
}
type SqliteMigrateSuite struct {
Db *sql.DB
DbMap *gorp.DbMap
}
var _ = Suite(&SqliteMigrateSuite{})
func (s *SqliteMigrateSuite) SetUpTest(c *C) {
db, err := sql.Open("sqlite3", filename)
c.Assert(err, IsNil)
s.Db = db
s.DbMap = &gorp.DbMap{Db: db, Dialect: &gorp.SqliteDialect{}}
}
func (s *SqliteMigrateSuite) TearDownTest(c *C) {
err := os.Remove(filename)
c.Assert(err, IsNil)
}
func (s *SqliteMigrateSuite) TestRunMigration(c *C) {
migrations := &MemoryMigrationSource{
Migrations: sqliteMigrations[:1],
}
// Executes one migration
n, err := Exec(s.Db, "sqlite3", migrations, Up)
c.Assert(err, IsNil)
c.Assert(n, Equals, 1)
// Can use table now
_, err = s.DbMap.Exec("SELECT * FROM people")
c.Assert(err, IsNil)
// Shouldn't apply migration again
n, err = Exec(s.Db, "sqlite3", migrations, Up)
c.Assert(err, IsNil)
c.Assert(n, Equals, 0)
}
func (s *SqliteMigrateSuite) TestMigrateMultiple(c *C) {
migrations := &MemoryMigrationSource{
Migrations: sqliteMigrations[:2],
}
// Executes two migrations
n, err := Exec(s.Db, "sqlite3", migrations, Up)
c.Assert(err, IsNil)
c.Assert(n, Equals, 2)
// Can use column now
_, err = s.DbMap.Exec("SELECT first_name FROM people")
c.Assert(err, IsNil)
}
func (s *SqliteMigrateSuite) TestMigrateIncremental(c *C) {
migrations := &MemoryMigrationSource{
Migrations: sqliteMigrations[:1],
}
// Executes one migration
n, err := Exec(s.Db, "sqlite3", migrations, Up)
c.Assert(err, IsNil)
c.Assert(n, Equals, 1)
// Execute a new migration
migrations = &MemoryMigrationSource{
Migrations: sqliteMigrations[:2],
}
n, err = Exec(s.Db, "sqlite3", migrations, Up)
c.Assert(err, IsNil)
c.Assert(n, Equals, 1)
// Can use column now
_, err = s.DbMap.Exec("SELECT first_name FROM people")
c.Assert(err, IsNil)
}
func (s *SqliteMigrateSuite) TestFileMigrate(c *C) {
migrations := &FileMigrationSource{
Dir: "test-migrations",
}
// Executes two migrations
n, err := Exec(s.Db, "sqlite3", migrations, Up)
c.Assert(err, IsNil)
c.Assert(n, Equals, 2)
// Has data
id, err := s.DbMap.SelectInt("SELECT id FROM people")
c.Assert(err, IsNil)
c.Assert(id, Equals, int64(1))
}
func (s *SqliteMigrateSuite) TestAssetMigrate(c *C) {
migrations := &AssetMigrationSource{
Asset: Asset,
AssetDir: AssetDir,
Dir: "test-migrations",
}
// Executes two migrations
n, err := Exec(s.Db, "sqlite3", migrations, Up)
c.Assert(err, IsNil)
c.Assert(n, Equals, 2)
// Has data
id, err := s.DbMap.SelectInt("SELECT id FROM people")
c.Assert(err, IsNil)
c.Assert(id, Equals, int64(1))
}
func (s *SqliteMigrateSuite) TestMigrateMax(c *C) {
migrations := &FileMigrationSource{
Dir: "test-migrations",
}
// Executes one migration
n, err := ExecMax(s.Db, "sqlite3", migrations, Up, 1)
c.Assert(err, IsNil)
c.Assert(n, Equals, 1)
id, err := s.DbMap.SelectInt("SELECT COUNT(*) FROM people")
c.Assert(err, IsNil)
c.Assert(id, Equals, int64(0))
}
func (s *SqliteMigrateSuite) TestMigrateDown(c *C) {
migrations := &FileMigrationSource{
Dir: "test-migrations",
}
n, err := Exec(s.Db, "sqlite3", migrations, Up)
c.Assert(err, IsNil)
c.Assert(n, Equals, 2)
// Has data
id, err := s.DbMap.SelectInt("SELECT id FROM people")
c.Assert(err, IsNil)
c.Assert(id, Equals, int64(1))
// Undo the last one
n, err = ExecMax(s.Db, "sqlite3", migrations, Down, 1)
c.Assert(err, IsNil)
c.Assert(n, Equals, 1)
// No more data
id, err = s.DbMap.SelectInt("SELECT COUNT(*) FROM people")
c.Assert(err, IsNil)
c.Assert(id, Equals, int64(0))
// Remove the table.
n, err = ExecMax(s.Db, "sqlite3", migrations, Down, 1)
c.Assert(err, IsNil)
c.Assert(n, Equals, 1)
// Cannot query it anymore
_, err = s.DbMap.SelectInt("SELECT COUNT(*) FROM people")
c.Assert(err, Not(IsNil))
// Nothing left to do.
n, err = ExecMax(s.Db, "sqlite3", migrations, Down, 1)
c.Assert(err, IsNil)
c.Assert(n, Equals, 0)
}
func (s *SqliteMigrateSuite) TestMigrateDownFull(c *C) {
migrations := &FileMigrationSource{
Dir: "test-migrations",
}
n, err := Exec(s.Db, "sqlite3", migrations, Up)
c.Assert(err, IsNil)
c.Assert(n, Equals, 2)
// Has data
id, err := s.DbMap.SelectInt("SELECT id FROM people")
c.Assert(err, IsNil)
c.Assert(id, Equals, int64(1))
// Undo the last one
n, err = Exec(s.Db, "sqlite3", migrations, Down)
c.Assert(err, IsNil)
c.Assert(n, Equals, 2)
// Cannot query it anymore
_, err = s.DbMap.SelectInt("SELECT COUNT(*) FROM people")
c.Assert(err, Not(IsNil))
// Nothing left to do.
n, err = Exec(s.Db, "sqlite3", migrations, Down)
c.Assert(err, IsNil)
c.Assert(n, Equals, 0)
}
func (s *SqliteMigrateSuite) TestMigrateTransaction(c *C) {
migrations := &MemoryMigrationSource{
Migrations: []*Migration{
sqliteMigrations[0],
sqliteMigrations[1],
&Migration{
Id: "125",
Up: []string{"INSERT INTO people (id, first_name) VALUES (1, 'Test')", "SELECT fail"},
Down: []string{}, // Not important here
},
},
}
// Should fail, transaction should roll back the INSERT.
n, err := Exec(s.Db, "sqlite3", migrations, Up)
c.Assert(err, Not(IsNil))
c.Assert(n, Equals, 2)
// INSERT should be rolled back
count, err := s.DbMap.SelectInt("SELECT COUNT(*) FROM people")
c.Assert(err, IsNil)
c.Assert(count, Equals, int64(0))
}
func (s *SqliteMigrateSuite) TestPlanMigration(c *C) {
migrations := &MemoryMigrationSource{
Migrations: []*Migration{
&Migration{
Id: "1_create_table.sql",
Up: []string{"CREATE TABLE people (id int)"},
Down: []string{"DROP TABLE people"},
},
&Migration{
Id: "2_alter_table.sql",
Up: []string{"ALTER TABLE people ADD COLUMN first_name text"},
Down: []string{"SELECT 0"}, // Not really supported
},
&Migration{
Id: "10_add_last_name.sql",
Up: []string{"ALTER TABLE people ADD COLUMN last_name text"},
Down: []string{"ALTER TABLE people DROP COLUMN last_name"},
},
},
}
n, err := Exec(s.Db, "sqlite3", migrations, Up)
c.Assert(err, IsNil)
c.Assert(n, Equals, 3)
migrations.Migrations = append(migrations.Migrations, &Migration{
Id: "11_add_middle_name.sql",
Up: []string{"ALTER TABLE people ADD COLUMN middle_name text"},
Down: []string{"ALTER TABLE people DROP COLUMN middle_name"},
})
plannedMigrations, _, err := PlanMigration(s.Db, "sqlite3", migrations, Up, 0)
c.Assert(err, IsNil)
c.Assert(plannedMigrations, HasLen, 1)
c.Assert(plannedMigrations[0].Migration, Equals, migrations.Migrations[3])
plannedMigrations, _, err = PlanMigration(s.Db, "sqlite3", migrations, Down, 0)
c.Assert(err, IsNil)
c.Assert(plannedMigrations, HasLen, 3)
c.Assert(plannedMigrations[0].Migration, Equals, migrations.Migrations[2])
c.Assert(plannedMigrations[1].Migration, Equals, migrations.Migrations[1])
c.Assert(plannedMigrations[2].Migration, Equals, migrations.Migrations[0])
}
func (s *SqliteMigrateSuite) TestPlanMigrationWithHoles(c *C) {
up := "SELECT 0"
down := "SELECT 1"
migrations := &MemoryMigrationSource{
Migrations: []*Migration{
&Migration{
Id: "1",
Up: []string{up},
Down: []string{down},
},
&Migration{
Id: "3",
Up: []string{up},
Down: []string{down},
},
},
}
n, err := Exec(s.Db, "sqlite3", migrations, Up)
c.Assert(err, IsNil)
c.Assert(n, Equals, 2)
migrations.Migrations = append(migrations.Migrations, &Migration{
Id: "2",
Up: []string{up},
Down: []string{down},
})
migrations.Migrations = append(migrations.Migrations, &Migration{
Id: "4",
Up: []string{up},
Down: []string{down},
})
migrations.Migrations = append(migrations.Migrations, &Migration{
Id: "5",
Up: []string{up},
Down: []string{down},
})
// apply all the missing migrations
plannedMigrations, _, err := PlanMigration(s.Db, "sqlite3", migrations, Up, 0)
c.Assert(err, IsNil)
c.Assert(plannedMigrations, HasLen, 3)
c.Assert(plannedMigrations[0].Migration.Id, Equals, "2")
c.Assert(plannedMigrations[0].Queries[0], Equals, up)
c.Assert(plannedMigrations[1].Migration.Id, Equals, "4")
c.Assert(plannedMigrations[1].Queries[0], Equals, up)
c.Assert(plannedMigrations[2].Migration.Id, Equals, "5")
c.Assert(plannedMigrations[2].Queries[0], Equals, up)
// first catch up to current target state 123, then migrate down 1 step to 12
plannedMigrations, _, err = PlanMigration(s.Db, "sqlite3", migrations, Down, 1)
c.Assert(err, IsNil)
c.Assert(plannedMigrations, HasLen, 2)
c.Assert(plannedMigrations[0].Migration.Id, Equals, "2")
c.Assert(plannedMigrations[0].Queries[0], Equals, up)
c.Assert(plannedMigrations[1].Migration.Id, Equals, "3")
c.Assert(plannedMigrations[1].Queries[0], Equals, down)
// first catch up to current target state 123, then migrate down 2 steps to 1
plannedMigrations, _, err = PlanMigration(s.Db, "sqlite3", migrations, Down, 2)
c.Assert(err, IsNil)
c.Assert(plannedMigrations, HasLen, 3)
c.Assert(plannedMigrations[0].Migration.Id, Equals, "2")
c.Assert(plannedMigrations[0].Queries[0], Equals, up)
c.Assert(plannedMigrations[1].Migration.Id, Equals, "3")
c.Assert(plannedMigrations[1].Queries[0], Equals, down)
c.Assert(plannedMigrations[2].Migration.Id, Equals, "2")
c.Assert(plannedMigrations[2].Queries[0], Equals, down)
}
package migrate
import (
"sort"
. "gopkg.in/check.v1"
)
type SortSuite struct{}
var _ = Suite(&SortSuite{})
func (s *SortSuite) TestSortMigrations(c *C) {
var migrations = byId([]*Migration{
&Migration{Id: "10_abc", Up: nil, Down: nil},
&Migration{Id: "120_cde", Up: nil, Down: nil},
&Migration{Id: "1_abc", Up: nil, Down: nil},
&Migration{Id: "efg", Up: nil, Down: nil},
&Migration{Id: "2_cde", Up: nil, Down: nil},
&Migration{Id: "35_cde", Up: nil, Down: nil},
&Migration{Id: "3_efg", Up: nil, Down: nil},
&Migration{Id: "4_abc", Up: nil, Down: nil},
})
sort.Sort(migrations)
c.Assert(migrations, HasLen, 8)
c.Assert(migrations[0].Id, Equals, "1_abc")
c.Assert(migrations[1].Id, Equals, "2_cde")
c.Assert(migrations[2].Id, Equals, "3_efg")
c.Assert(migrations[3].Id, Equals, "4_abc")
c.Assert(migrations[4].Id, Equals, "10_abc")
c.Assert(migrations[5].Id, Equals, "35_cde")
c.Assert(migrations[6].Id, Equals, "120_cde")
c.Assert(migrations[7].Id, Equals, "efg")
}
package main
import (
"fmt"
"github.com/rubenv/sql-migrate"
)
func ApplyMigrations(dir migrate.MigrationDirection, dryrun bool, limit int) error {
env, err := GetEnvironment()
if err != nil {
return fmt.Errorf("Could not parse config: %s", err)
}
db, dialect, err := GetConnection(env)
if err != nil {
return err
}
source := migrate.FileMigrationSource{
Dir: env.Dir,
}
if dryrun {
migrations, _, err := migrate.PlanMigration(db, dialect, source, dir, limit)
if err != nil {
return fmt.Errorf("Cannot plan migration: %s", err)
}
for _, m := range migrations {
PrintMigration(m, dir)
}
} else {
n, err := migrate.ExecMax(db, dialect, source, dir, limit)
if err != nil {
return fmt.Errorf("Migration failed: %s", err)
}
if n == 1 {
ui.Output("Applied 1 migration")
} else {
ui.Output(fmt.Sprintf("Applied %d migrations", n))
}
}
return nil
}
func PrintMigration(m *migrate.PlannedMigration, dir migrate.MigrationDirection) {
if dir == migrate.Up {
ui.Output(fmt.Sprintf("==> Would apply migration %s (up)", m.Id))
for _, q := range m.Up {
ui.Output(q)
}
} else if dir == migrate.Down {
ui.Output(fmt.Sprintf("==> Would apply migration %s (down)", m.Id))
for _, q := range m.Down {
ui.Output(q)
}
} else {
panic("Not reached")
}
}
package main
import (
"flag"
"strings"
"github.com/rubenv/sql-migrate"
)
type DownCommand struct {
}
func (c *DownCommand) Help() string {
helpText := `
Usage: sql-migrate down [options] ...
Undo a database migration.
Options:
-config=dbconfig.yml Configuration file to use.
-env="development" Environment.
-limit=1 Limit the number of migrations (0 = unlimited).
-dryrun Don't apply migrations, just print them.
`
return strings.TrimSpace(helpText)
}
func (c *DownCommand) Synopsis() string {
return "Undo a database migration"
}
func (c *DownCommand) Run(args []string) int {
var limit int
var dryrun bool
cmdFlags := flag.NewFlagSet("down", flag.ContinueOnError)
cmdFlags.Usage = func() { ui.Output(c.Help()) }
cmdFlags.IntVar(&limit, "limit", 1, "Max number of migrations to apply.")
cmdFlags.BoolVar(&dryrun, "dryrun", false, "Don't apply migrations, just print them.")
ConfigFlags(cmdFlags)
if err := cmdFlags.Parse(args); err != nil {
return 1
}
err := ApplyMigrations(migrate.Down, dryrun, limit)
if err != nil {
ui.Error(err.Error())
return 1
}
return 0
}
package main
import (
"flag"
"fmt"
"strings"
"github.com/rubenv/sql-migrate"
)
type RedoCommand struct {
}
func (c *RedoCommand) Help() string {
helpText := `
Usage: sql-migrate redo [options] ...
Reapply the last migration.
Options:
-config=dbconfig.yml Configuration file to use.
-env="development" Environment.
-dryrun Don't apply migrations, just print them.
`
return strings.TrimSpace(helpText)
}
func (c *RedoCommand) Synopsis() string {
return "Reapply the last migration"
}
func (c *RedoCommand) Run(args []string) int {
var dryrun bool
cmdFlags := flag.NewFlagSet("redo", flag.ContinueOnError)
cmdFlags.Usage = func() { ui.Output(c.Help()) }
cmdFlags.BoolVar(&dryrun, "dryrun", false, "Don't apply migrations, just print them.")
ConfigFlags(cmdFlags)
if err := cmdFlags.Parse(args); err != nil {
return 1
}
env, err := GetEnvironment()
if err != nil {
ui.Error(fmt.Sprintf("Could not parse config: %s", err))
return 1
}
db, dialect, err := GetConnection(env)
if err != nil {
ui.Error(err.Error())
return 1
}
source := migrate.FileMigrationSource{
Dir: env.Dir,
}
migrations, _, err := migrate.PlanMigration(db, dialect, source, migrate.Down, 1)
if len(migrations) == 0 {
ui.Output("Nothing to do!")
return 0
}
if dryrun {
PrintMigration(migrations[0], migrate.Down)
PrintMigration(migrations[0], migrate.Up)
} else {
_, err := migrate.ExecMax(db, dialect, source, migrate.Down, 1)
if err != nil {
ui.Error(fmt.Sprintf("Migration (down) failed: %s", err))
return 1
}
_, err = migrate.ExecMax(db, dialect, source, migrate.Up, 1)
if err != nil {
ui.Error(fmt.Sprintf("Migration (up) failed: %s", err))
return 1
}
ui.Output(fmt.Sprintf("Reapplied migration %s.", migrations[0].Id))
}
return 0
}
package main
import (
"flag"
"fmt"
"os"
"strings"
"time"
"github.com/olekukonko/tablewriter"
"github.com/rubenv/sql-migrate"
)
type StatusCommand struct {
}
func (c *StatusCommand) Help() string {
helpText := `
Usage: sql-migrate status [options] ...
Show migration status.
Options:
-config=dbconfig.yml Configuration file to use.
-env="development" Environment.
`
return strings.TrimSpace(helpText)
}
func (c *StatusCommand) Synopsis() string {
return "Show migration status"
}
func (c *StatusCommand) Run(args []string) int {
cmdFlags := flag.NewFlagSet("status", flag.ContinueOnError)
cmdFlags.Usage = func() { ui.Output(c.Help()) }
ConfigFlags(cmdFlags)
if err := cmdFlags.Parse(args); err != nil {
return 1
}
env, err := GetEnvironment()
if err != nil {
ui.Error(fmt.Sprintf("Could not parse config: %s", err))
return 1
}
db, dialect, err := GetConnection(env)
if err != nil {
ui.Error(err.Error())
return 1
}
source := migrate.FileMigrationSource{
Dir: env.Dir,
}
migrations, err := source.FindMigrations()
if err != nil {
ui.Error(err.Error())
return 1
}
records, err := migrate.GetMigrationRecords(db, dialect)
if err != nil {
ui.Error(err.Error())
return 1
}
table := tablewriter.NewWriter(os.Stdout)
table.SetHeader([]string{"Migration", "Applied"})
table.SetColWidth(60)
rows := make(map[string]*statusRow)
for _, m := range migrations {
rows[m.Id] = &statusRow{
Id: m.Id,
Migrated: false,
}
}
for _, r := range records {
rows[r.Id].Migrated = true
rows[r.Id].AppliedAt = r.AppliedAt
}
for _, m := range migrations {
if rows[m.Id].Migrated {
table.Append([]string{
m.Id,
rows[m.Id].AppliedAt.String(),
})
} else {
table.Append([]string{
m.Id,
"no",
})
}
}
table.Render()
return 0
}
type statusRow struct {
Id string
Migrated bool
AppliedAt time.Time
}
package main
import (
"flag"
"strings"
"github.com/rubenv/sql-migrate"
)
type UpCommand struct {
}
func (c *UpCommand) Help() string {
helpText := `
Usage: sql-migrate up [options] ...
Migrates the database to the most recent version available.
Options:
-config=dbconfig.yml Configuration file to use.
-env="development" Environment.
-limit=0 Limit the number of migrations (0 = unlimited).
-dryrun Don't apply migrations, just print them.
`
return strings.TrimSpace(helpText)
}
func (c *UpCommand) Synopsis() string {
return "Migrates the database to the most recent version available"
}
func (c *UpCommand) Run(args []string) int {
var limit int
var dryrun bool
cmdFlags := flag.NewFlagSet("up", flag.ContinueOnError)
cmdFlags.Usage = func() { ui.Output(c.Help()) }
cmdFlags.IntVar(&limit, "limit", 0, "Max number of migrations to apply.")
cmdFlags.BoolVar(&dryrun, "dryrun", false, "Don't apply migrations, just print them.")
ConfigFlags(cmdFlags)
if err := cmdFlags.Parse(args); err != nil {
return 1
}
err := ApplyMigrations(migrate.Up, dryrun, limit)
if err != nil {
ui.Error(err.Error())
return 1
}
return 0
}
package main
import (
"database/sql"
"errors"
"flag"
"fmt"
"io/ioutil"
"github.com/rubenv/sql-migrate"
"gopkg.in/gorp.v1"
"gopkg.in/yaml.v1"
_ "github.com/go-sql-driver/mysql"
_ "github.com/lib/pq"
_ "github.com/mattn/go-sqlite3"
)
var dialects = map[string]gorp.Dialect{
"sqlite3": gorp.SqliteDialect{},
"postgres": gorp.PostgresDialect{},
"mysql": gorp.MySQLDialect{"InnoDB", "UTF8"},
}
var ConfigFile string
var ConfigEnvironment string
func ConfigFlags(f *flag.FlagSet) {
f.StringVar(&ConfigFile, "config", "dbconfig.yml", "Configuration file to use.")
f.StringVar(&ConfigEnvironment, "env", "development", "Environment to use.")
}
type Environment struct {
Dialect string `yaml:"dialect"`
DataSource string `yaml:"datasource"`
Dir string `yaml:"dir"`
TableName string `yaml:"table"`
SchemaName string `yaml:"schema"`
}
func ReadConfig() (map[string]*Environment, error) {
file, err := ioutil.ReadFile(ConfigFile)
if err != nil {
return nil, err
}
config := make(map[string]*Environment)
err = yaml.Unmarshal(file, config)
if err != nil {
return nil, err
}
return config, nil
}
func GetEnvironment() (*Environment, error) {
config, err := ReadConfig()
if err != nil {
return nil, err
}
env := config[ConfigEnvironment]
if env == nil {
return nil, errors.New("No environment: " + ConfigEnvironment)
}
if env.Dialect == "" {
return nil, errors.New("No dialect specified")
}
if env.DataSource == "" {
return nil, errors.New("No data source specified")
}
if env.Dir == "" {
env.Dir = "migrations"
}
if env.TableName != "" {
migrate.SetTable(env.TableName)
}
if env.SchemaName != "" {
migrate.SetSchema(env.SchemaName)
}
return env, nil
}
func GetConnection(env *Environment) (*sql.DB, string, error) {
db, err := sql.Open(env.Dialect, env.DataSource)
if err != nil {
return nil, "", fmt.Errorf("Cannot connect to database: %s", err)
}
// Make sure we only accept dialects that were compiled in.
_, exists := dialects[env.Dialect]
if !exists {
return nil, "", fmt.Errorf("Unsupported dialect: %s", env.Dialect)
}
return db, env.Dialect, nil
}
package main
import (
"fmt"
"os"
"github.com/mitchellh/cli"
)
func main() {
os.Exit(realMain())
}
var ui cli.Ui
func realMain() int {
ui = &cli.BasicUi{Writer: os.Stdout}
cli := &cli.CLI{
Args: os.Args[1:],
Commands: map[string]cli.CommandFactory{
"up": func() (cli.Command, error) {
return &UpCommand{}, nil
},
"down": func() (cli.Command, error) {
return &DownCommand{}, nil
},
"redo": func() (cli.Command, error) {
return &RedoCommand{}, nil
},
"status": func() (cli.Command, error) {
return &StatusCommand{}, nil
},
},
HelpFunc: cli.BasicHelpFunc("sql-migrate"),
Version: "1.0.0",
}
exitCode, err := cli.Run()
if err != nil {
fmt.Fprintf(os.Stderr, "Error executing CLI: %s\n", err.Error())
return 1
}
return exitCode
}
// +build go1.3
package main
import (
_ "github.com/denisenkom/go-mssqldb"
"gopkg.in/gorp.v1"
)
func init() {
dialects["mssql"] = gorp.SqlServerDialect{}
}
# SQL migration parser
Based on the [goose](https://bitbucket.org/liamstask/goose) migration parser.
## License
(The MIT License)
Copyright (C) 2014 by Ruben Vermeersch <ruben@rocketeer.be>
Copyright (C) 2012-2014 by Liam Staskawicz
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
package sqlparse
import (
"bufio"
"bytes"
"errors"
"io"
"strings"
)
const sqlCmdPrefix = "-- +migrate "
// Checks the line to see if the line has a statement-ending semicolon
// or if the line contains a double-dash comment.
func endsWithSemicolon(line string) bool {
prev := ""
scanner := bufio.NewScanner(strings.NewReader(line))
scanner.Split(bufio.ScanWords)
for scanner.Scan() {
word := scanner.Text()
if strings.HasPrefix(word, "--") {
break
}
prev = word
}
return strings.HasSuffix(prev, ";")
}
// Split the given sql script into individual statements.
//
// The base case is to simply split on semicolons, as these
// naturally terminate a statement.
//
// However, more complex cases like pl/pgsql can have semicolons
// within a statement. For these cases, we provide the explicit annotations
// 'StatementBegin' and 'StatementEnd' to allow the script to
// tell us to ignore semicolons.
func SplitSQLStatements(r io.ReadSeeker, direction bool) ([]string, error) {
_, err := r.Seek(0, 0)
if err != nil {
return nil, err
}
var buf bytes.Buffer
scanner := bufio.NewScanner(r)
// track the count of each section
// so we can diagnose scripts with no annotations
upSections := 0
downSections := 0
statementEnded := false
ignoreSemicolons := false
directionIsActive := false
stmts := make([]string, 0)
for scanner.Scan() {
line := scanner.Text()
// handle any migrate-specific commands
if strings.HasPrefix(line, sqlCmdPrefix) {
cmd := strings.TrimSpace(line[len(sqlCmdPrefix):])
switch cmd {
case "Up":
directionIsActive = (direction == true)
upSections++
break
case "Down":
directionIsActive = (direction == false)
downSections++
break
case "StatementBegin":
if directionIsActive {
ignoreSemicolons = true
}
break
case "StatementEnd":
if directionIsActive {
statementEnded = (ignoreSemicolons == true)
ignoreSemicolons = false
}
break
}
}
if !directionIsActive {
continue
}
if _, err := buf.WriteString(line + "\n"); err != nil {
return nil, err
}
// Wrap up the two supported cases: 1) basic with semicolon; 2) psql statement
// Lines that end with semicolon that are in a statement block
// do not conclude statement.
if (!ignoreSemicolons && endsWithSemicolon(line)) || statementEnded {
statementEnded = false
stmts = append(stmts, buf.String())
buf.Reset()
}
}
if err := scanner.Err(); err != nil {
return nil, err
}
// diagnose likely migration script errors
if ignoreSemicolons {
return nil, errors.New("ERROR: saw '-- +migrate StatementBegin' with no matching '-- +migrate StatementEnd'")
}
if upSections == 0 && downSections == 0 {
return nil, errors.New(`ERROR: no Up/Down annotations found, so no statements were executed.
See https://github.com/rubenv/sql-migrate for details.`)
}
return stmts, nil
}
package sqlparse
import (
"strings"
"testing"
. "gopkg.in/check.v1"
)
func Test(t *testing.T) { TestingT(t) }
type SqlParseSuite struct {
}
var _ = Suite(&SqlParseSuite{})
func (s *SqlParseSuite) TestSemicolons(c *C) {
type testData struct {
line string
result bool
}
tests := []testData{
{
line: "END;",
result: true,
},
{
line: "END; -- comment",
result: true,
},
{
line: "END ; -- comment",
result: true,
},
{
line: "END -- comment",
result: false,
},
{
line: "END -- comment ;",
result: false,
},
{
line: "END \" ; \" -- comment",
result: false,
},
}
for _, test := range tests {
r := endsWithSemicolon(test.line)
c.Assert(r, Equals, test.result)
}
}
func (s *SqlParseSuite) TestSplitStatements(c *C) {
type testData struct {
sql string
direction bool
count int
}
tests := []testData{
{
sql: functxt,
direction: true,
count: 2,
},
{
sql: functxt,
direction: false,
count: 2,
},
{
sql: multitxt,
direction: true,
count: 2,
},
{
sql: multitxt,
direction: false,
count: 2,
},
}
for _, test := range tests {
stmts, err := SplitSQLStatements(strings.NewReader(test.sql), test.direction)
c.Assert(err, IsNil)
c.Assert(stmts, HasLen, test.count)
}
}
var functxt = `-- +migrate Up
CREATE TABLE IF NOT EXISTS histories (
id BIGSERIAL PRIMARY KEY,
current_value varchar(2000) NOT NULL,
created_at timestamp with time zone NOT NULL
);
-- +migrate StatementBegin
CREATE OR REPLACE FUNCTION histories_partition_creation( DATE, DATE )
returns void AS $$
DECLARE
create_query text;
BEGIN
FOR create_query IN SELECT
'CREATE TABLE IF NOT EXISTS histories_'
|| TO_CHAR( d, 'YYYY_MM' )
|| ' ( CHECK( created_at >= timestamp '''
|| TO_CHAR( d, 'YYYY-MM-DD 00:00:00' )
|| ''' AND created_at < timestamp '''
|| TO_CHAR( d + INTERVAL '1 month', 'YYYY-MM-DD 00:00:00' )
|| ''' ) ) inherits ( histories );'
FROM generate_series( $1, $2, '1 month' ) AS d
LOOP
EXECUTE create_query;
END LOOP; -- LOOP END
END; -- FUNCTION END
$$
language plpgsql;
-- +migrate StatementEnd
-- +migrate Down
drop function histories_partition_creation(DATE, DATE);
drop TABLE histories;
`
// test multiple up/down transitions in a single script
var multitxt = `-- +migrate Up
CREATE TABLE post (
id int NOT NULL,
title text,
body text,
PRIMARY KEY(id)
);
-- +migrate Down
DROP TABLE post;
-- +migrate Up
CREATE TABLE fancier_post (
id int NOT NULL,
title text,
body text,
created_on timestamp without time zone,
PRIMARY KEY(id)
);
-- +migrate Down
DROP TABLE fancier_post;
`
postgres:
dialect: postgres
datasource: dbname=test sslmode=disable
dir: test-migrations
mysql:
dialect: mysql
datasource: root@/test?parseTime=true
dir: test-migrations
mysql_noflag:
dialect: mysql
datasource: root@/test
dir: test-migrations
sqlite:
dialect: sqlite3
datasource: test.db
dir: test-migrations
table: migrations
#!/bin/bash
# Tweak PATH for Travis
export PATH=$PATH:$HOME/gopath/bin
OPTIONS="-config=test-integration/dbconfig.yml -env mysql_noflag"
set -ex
sql-migrate status $OPTIONS | grep -q "Make sure that the parseTime option is supplied"
#!/bin/bash
# Tweak PATH for Travis
export PATH=$PATH:$HOME/gopath/bin
OPTIONS="-config=test-integration/dbconfig.yml -env mysql"
set -ex
sql-migrate status $OPTIONS
sql-migrate up $OPTIONS
sql-migrate down $OPTIONS
sql-migrate redo $OPTIONS
sql-migrate status $OPTIONS
#!/bin/bash
# Tweak PATH for Travis
export PATH=$PATH:$HOME/gopath/bin
OPTIONS="-config=test-integration/dbconfig.yml -env postgres"
set -ex
sql-migrate status $OPTIONS
sql-migrate up $OPTIONS
sql-migrate down $OPTIONS
sql-migrate redo $OPTIONS
sql-migrate status $OPTIONS
#!/bin/bash
# Tweak PATH for Travis
export PATH=$PATH:$HOME/gopath/bin
OPTIONS="-config=test-integration/dbconfig.yml -env sqlite"
set -ex
sql-migrate status $OPTIONS
sql-migrate up $OPTIONS
sql-migrate down $OPTIONS
sql-migrate redo $OPTIONS
sql-migrate status $OPTIONS
# Should have used the custom migrations table
sqlite3 test.db "SELECT COUNT(*) FROM migrations"
-- +migrate Up
-- SQL in section 'Up' is executed when this migration is applied
CREATE TABLE people (id int);
-- +migrate Down
-- SQL section 'Down' is executed when this migration is rolled back
DROP TABLE people;
-- +migrate Up
INSERT INTO people (id) VALUES (1);
-- +migrate Down
DELETE FROM people WHERE id=1;
package migrate
import (
"sort"
. "gopkg.in/check.v1"
)
var toapplyMigrations = []*Migration{
&Migration{Id: "abc", Up: nil, Down: nil},
&Migration{Id: "cde", Up: nil, Down: nil},
&Migration{Id: "efg", Up: nil, Down: nil},
}
type ToApplyMigrateSuite struct {
}
var _ = Suite(&ToApplyMigrateSuite{})
func (s *ToApplyMigrateSuite) TestGetAll(c *C) {
toApply := ToApply(toapplyMigrations, "", Up)
c.Assert(toApply, HasLen, 3)
c.Assert(toApply[0], Equals, toapplyMigrations[0])
c.Assert(toApply[1], Equals, toapplyMigrations[1])
c.Assert(toApply[2], Equals, toapplyMigrations[2])
}
func (s *ToApplyMigrateSuite) TestGetAbc(c *C) {
toApply := ToApply(toapplyMigrations, "abc", Up)
c.Assert(toApply, HasLen, 2)
c.Assert(toApply[0], Equals, toapplyMigrations[1])
c.Assert(toApply[1], Equals, toapplyMigrations[2])
}
func (s *ToApplyMigrateSuite) TestGetCde(c *C) {
toApply := ToApply(toapplyMigrations, "cde", Up)
c.Assert(toApply, HasLen, 1)
c.Assert(toApply[0], Equals, toapplyMigrations[2])
}
func (s *ToApplyMigrateSuite) TestGetDone(c *C) {
toApply := ToApply(toapplyMigrations, "efg", Up)
c.Assert(toApply, HasLen, 0)
toApply = ToApply(toapplyMigrations, "zzz", Up)
c.Assert(toApply, HasLen, 0)
}
func (s *ToApplyMigrateSuite) TestDownDone(c *C) {
toApply := ToApply(toapplyMigrations, "", Down)
c.Assert(toApply, HasLen, 0)
}
func (s *ToApplyMigrateSuite) TestDownCde(c *C) {
toApply := ToApply(toapplyMigrations, "cde", Down)
c.Assert(toApply, HasLen, 2)
c.Assert(toApply[0], Equals, toapplyMigrations[1])
c.Assert(toApply[1], Equals, toapplyMigrations[0])
}
func (s *ToApplyMigrateSuite) TestDownAbc(c *C) {
toApply := ToApply(toapplyMigrations, "abc", Down)
c.Assert(toApply, HasLen, 1)
c.Assert(toApply[0], Equals, toapplyMigrations[0])
}
func (s *ToApplyMigrateSuite) TestDownAll(c *C) {
toApply := ToApply(toapplyMigrations, "efg", Down)
c.Assert(toApply, HasLen, 3)
c.Assert(toApply[0], Equals, toapplyMigrations[2])
c.Assert(toApply[1], Equals, toapplyMigrations[1])
c.Assert(toApply[2], Equals, toapplyMigrations[0])
toApply = ToApply(toapplyMigrations, "zzz", Down)
c.Assert(toApply, HasLen, 3)
c.Assert(toApply[0], Equals, toapplyMigrations[2])
c.Assert(toApply[1], Equals, toapplyMigrations[1])
c.Assert(toApply[2], Equals, toapplyMigrations[0])
}
func (s *ToApplyMigrateSuite) TestAlphaNumericMigrations(c *C) {
var migrations = byId([]*Migration{
&Migration{Id: "10_abc", Up: nil, Down: nil},
&Migration{Id: "1_abc", Up: nil, Down: nil},
&Migration{Id: "efg", Up: nil, Down: nil},
&Migration{Id: "2_cde", Up: nil, Down: nil},
&Migration{Id: "35_cde", Up: nil, Down: nil},
})
sort.Sort(migrations)
toApplyUp := ToApply(migrations, "2_cde", Up)
c.Assert(toApplyUp, HasLen, 3)
c.Assert(toApplyUp[0].Id, Equals, "10_abc")
c.Assert(toApplyUp[1].Id, Equals, "35_cde")
c.Assert(toApplyUp[2].Id, Equals, "efg")
toApplyDown := ToApply(migrations, "2_cde", Down)
c.Assert(toApplyDown, HasLen, 2)
c.Assert(toApplyDown[0].Id, Equals, "2_cde")
c.Assert(toApplyDown[1].Id, Equals, "1_abc")
}
_test
_testmain.go
_obj
*~
*.6
6.out
gorptest.bin
tmp
language: go
go:
- 1.1
- tip
services:
- mysql
- postgres
- sqlite3
before_script:
- mysql -e "CREATE DATABASE gorptest;"
- mysql -u root -e "GRANT ALL ON gorptest.* TO gorptest@localhost IDENTIFIED BY 'gorptest'"
- psql -c "CREATE DATABASE gorptest;" -U postgres
- psql -c "CREATE USER "gorptest" WITH SUPERUSER PASSWORD 'gorptest';" -U postgres
- go get github.com/lib/pq
- go get github.com/mattn/go-sqlite3
- go get github.com/ziutek/mymysql/godrv
- go get github.com/go-sql-driver/mysql
script: ./test_all.sh
(The MIT License)
Copyright (c) 2012 James Cooper <james@bitmechanic.com>
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
'Software'), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
include $(GOROOT)/src/Make.inc
TARG = github.com/coopernurse/gorp
GOFILES = gorp.go dialect.go
include $(GOROOT)/src/Make.pkg
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
package gorp
import (
"fmt"
)
// A non-fatal error, when a select query returns columns that do not exist
// as fields in the struct it is being mapped to
type NoFieldInTypeError struct {
TypeName string
MissingColNames []string
}
func (err *NoFieldInTypeError) Error() string {
return fmt.Sprintf("gorp: No fields %+v in type %s", err.MissingColNames, err.TypeName)
}
// returns true if the error is non-fatal (ie, we shouldn't immediately return)
func NonFatalError(err error) bool {
switch err.(type) {
case *NoFieldInTypeError:
return true
default:
return false
}
}
This diff is collapsed.
This diff is collapsed.
#!/bin/sh
# on macs, you may need to:
# export GOBUILDFLAG=-ldflags -linkmode=external
set -e
export GORP_TEST_DSN=gorptest/gorptest/gorptest
export GORP_TEST_DIALECT=mysql
go test $GOBUILDFLAG .
export GORP_TEST_DSN=gorptest:gorptest@/gorptest
export GORP_TEST_DIALECT=gomysql
go test $GOBUILDFLAG .
export GORP_TEST_DSN="user=gorptest password=gorptest dbname=gorptest sslmode=disable"
export GORP_TEST_DIALECT=postgres
go test $GOBUILDFLAG .
export GORP_TEST_DSN=/tmp/gorptest.bin
export GORP_TEST_DIALECT=sqlite
go test $GOBUILDFLAG .
......@@ -3,6 +3,21 @@
export GOPATH=${PWD}/Godeps/_workspace
export GOBIN=${PWD}/bin
if command -v go-bindata &>/dev/null; then
DEX_MIGRATE_FROM_DISK=${DEX_MIGRATE_FROM_DISK:=false}
echo "Turning migrations into ./db/migrations/assets.go"
if [ "$DEX_MIGRATE_FROM_DISK" = true ]; then
echo "Compiling migrations.go: will read migrations from disk."
else
echo "Compiling migrations into migrations.go"
fi
go-bindata -debug=$DEX_MIGRATE_FROM_DISK -modtime=1 -pkg migrations -o ./db/migrations/assets.go ./db/migrations
gofmt -w ./db/migrations/assets.go
else
echo "Could not find go-bindata in path, will not generate migrations"
fi
rm -rf $GOPATH/src/github.com/coreos/dex
mkdir -p $GOPATH/src/github.com/coreos/
ln -s ${PWD} $GOPATH/src/github.com/coreos/dex
......
......@@ -15,6 +15,7 @@ import (
"github.com/coreos/dex/db"
pflag "github.com/coreos/dex/pkg/flag"
"github.com/coreos/dex/pkg/log"
ptime "github.com/coreos/dex/pkg/time"
"github.com/coreos/dex/server"
"github.com/coreos/dex/user"
)
......@@ -29,14 +30,17 @@ func main() {
fs := flag.NewFlagSet("dex-overlord", flag.ExitOnError)
secret := fs.String("key-secret", "", "symmetric key used to encrypt/decrypt signing key data in DB")
dbURL := fs.String("db-url", "", "DSN-formatted database connection string")
dbMigrate := fs.Bool("db-migrate", true, "perform database migrations when starting up overlord. This includes the initial DB objects creation.")
keyPeriod := fs.Duration("key-period", 24*time.Hour, "length of time for-which a given key will be valid")
gcInterval := fs.Duration("gc-interval", time.Hour, "length of time between garbage collection runs")
adminListen := fs.String("admin-listen", "http://0.0.0.0:5557", "scheme, host and port for listening for administrative operation requests ")
localConnectorID := fs.String("local-connector", "local", "ID of the local connector")
logDebug := fs.Bool("log-debug", false, "log debug-level information")
logTimestamps := fs.Bool("log-timestamps", false, "prefix log lines with timestamps")
localConnectorID := fs.String("local-connector", "local", "ID of the local connector")
if err := fs.Parse(os.Args[1:]); err != nil {
fmt.Fprintln(os.Stderr, err.Error())
......@@ -74,6 +78,19 @@ func main() {
log.Fatalf(err.Error())
}
if *dbMigrate {
var sleep time.Duration
for {
if migrations, err := db.MigrateToLatest(dbc); err == nil {
log.Infof("Performed %d db migrations", migrations)
break
}
sleep = ptime.ExpBackoff(sleep, time.Minute)
log.Errorf("Unable to migrate database, retrying in %v: %v", sleep, err)
time.Sleep(sleep)
}
}
userRepo := db.NewUserRepo(dbc)
pwiRepo := db.NewPasswordInfoRepo(dbc)
userManager := user.NewManager(userRepo,
......
......@@ -66,7 +66,7 @@ type clientIdentityModel struct {
ID string `db:"id"`
Secret []byte `db:"secret"`
Metadata string `db:"metadata"`
DexAdmin bool `db:"dexAdmin"`
DexAdmin bool `db:"dex_admin"`
}
func newClientMetadataJSON(cm *oidc.ClientMetadata) *clientMetadataJSON {
......
......@@ -5,13 +5,11 @@ import (
"errors"
"fmt"
"strings"
"time"
"github.com/coopernurse/gorp"
_ "github.com/lib/pq"
"github.com/coreos/dex/pkg/log"
ptime "github.com/coreos/dex/pkg/time"
"github.com/coreos/dex/repo"
)
......@@ -73,16 +71,6 @@ func NewConnection(cfg Config) (*gorp.DbMap, error) {
}
}
var sleep time.Duration
for {
if err = dbm.CreateTablesIfNotExists(); err == nil {
break
}
sleep = ptime.ExpBackoff(sleep, time.Minute)
log.Errorf("Unable to initialize database, retrying in %v: %v", sleep, err)
time.Sleep(sleep)
}
return &dbm, nil
}
......
package db
import (
"fmt"
"github.com/coopernurse/gorp"
"github.com/lib/pq"
migrate "github.com/rubenv/sql-migrate"
"github.com/coreos/dex/db/migrations"
)
const (
migrationDialect = "postgres"
migrationTable = "dex_migrations"
migrationDir = "db/migrations"
)
func init() {
migrate.SetTable(migrationTable)
}
func MigrateToLatest(dbMap *gorp.DbMap) (int, error) {
source := getSource()
return migrate.Exec(dbMap.Db, migrationDialect, source, migrate.Up)
}
func MigrateMaxMigrations(dbMap *gorp.DbMap, max int) (int, error) {
source := getSource()
return migrate.ExecMax(dbMap.Db, migrationDialect, source, migrate.Up, max)
}
func GetPlannedMigrations(dbMap *gorp.DbMap) ([]*migrate.PlannedMigration, error) {
migrations, _, err := migrate.PlanMigration(dbMap.Db, migrationDialect, getSource(), migrate.Up, 0)
return migrations, err
}
func DropMigrationsTable(dbMap *gorp.DbMap) error {
qt := pq.QuoteIdentifier(migrationTable)
_, err := dbMap.Exec(fmt.Sprintf("drop table if exists %s ;", qt))
return err
}
func getSource() migrate.MigrationSource {
return &migrate.AssetMigrationSource{
Dir: migrationDir,
Asset: migrations.Asset,
AssetDir: migrations.AssetDir,
}
}
package db
import (
"fmt"
"os"
"testing"
"github.com/coopernurse/gorp"
)
func initDB(dsn string) *gorp.DbMap {
c, err := NewConnection(Config{DSN: dsn})
if err != nil {
panic(fmt.Sprintf("error making db connection: %q", err))
}
if err = c.DropTablesIfExists(); err != nil {
panic(fmt.Sprintf("Unable to drop database tables: %v", err))
}
return c
}
// TestGetPlannedMigrations is a sanity check, ensuring that at least one
// migration can be found.
func TestGetPlannedMigrations(t *testing.T) {
dsn := os.Getenv("DEX_TEST_DSN")
if dsn == "" {
t.Logf("Test will not run without DEX_TEST_DSN environment variable.")
return
}
dbMap := initDB(dsn)
ms, err := GetPlannedMigrations(dbMap)
if err != nil {
pwd, err := os.Getwd()
t.Logf("pwd: %v", pwd)
t.Fatalf("unexpected err: %q", err)
}
if len(ms) == 0 {
t.Fatalf("expected non-empty migrations")
}
}
-- +migrate Up
CREATE TABLE IF NOT EXISTS "authd_user" (
"id" text not null primary key,
"email" text,
"email_verified" boolean,
"display_name" text,
"admin" boolean) ;
CREATE TABLE IF NOT EXISTS "client_identity" (
"id" text not null primary key,
"secret" bytea,
"metadata" text);
CREATE TABLE IF NOT EXISTS "connector_config" (
"id" text not null primary key,
"type" text, "config" text) ;
CREATE TABLE IF NOT EXISTS "key" (
"value" bytea not null primary key) ;
CREATE TABLE IF NOT EXISTS "password_info" (
"user_id" text not null primary key,
"password" text,
"password_expires" bigint) ;
CREATE TABLE IF NOT EXISTS "session" (
"id" text not null primary key,
"state" text,
"created_at" bigint,
"expires_at" bigint,
"client_id" text,
"client_state" text,
"redirect_url" text, "identity" text,
"connector_id" text,
"user_id" text, "register" boolean) ;
CREATE TABLE IF NOT EXISTS "session_key" (
"key" text not null primary key,
"session_id" text,
"expires_at" bigint,
"stale" boolean) ;
CREATE TABLE IF NOT EXISTS "remote_identity_mapping" (
"connector_id" text not null,
"user_id" text,
"remote_id" text not null,
primary key ("connector_id", "remote_id")) ;
-- +migrate Up
ALTER TABLE client_identity ADD COLUMN "dex_admin" boolean;
-- +migrate Up
ALTER TABLE authd_user ADD COLUMN "created_at" bigint;
-- +migrate Up
ALTER TABLE session ADD COLUMN "nonce" text;
-- +migrate Up
CREATE TABLE refresh_token (
id bigint NOT NULL,
payload_hash text,
user_id text,
client_id text
);
CREATE SEQUENCE refresh_token_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE refresh_token_id_seq OWNED BY refresh_token.id;
ALTER TABLE ONLY refresh_token ALTER COLUMN id SET DEFAULT nextval('refresh_token_id_seq'::regclass);
ALTER TABLE ONLY refresh_token
ADD CONSTRAINT refresh_token_pkey PRIMARY KEY (id);
-- +migrate Up
ALTER TABLE ONLY authd_user
ADD CONSTRAINT authd_user_email_key UNIQUE (email);
This diff is collapsed.
......@@ -14,7 +14,10 @@ import (
)
const (
userTableName = "dex_user"
// This table is named authd_user for historical reasons; namely, that the
// original name of the project was authd, and there are existing tables out
// there that we don't want to have to rename in production.
userTableName = "authd_user"
remoteIdentityMappingTableName = "remote_identity_mapping"
)
......
......@@ -40,10 +40,12 @@ func connect(t *testing.T) *gorp.DbMap {
t.Fatalf("Unable to drop database tables: %v", err)
}
if err = c.CreateTablesIfNotExists(); err != nil {
t.Fatalf("Unable to create database tables: %v", err)
if err = db.DropMigrationsTable(c); err != nil {
panic(fmt.Sprintf("Unable to drop migration table: %v", err))
}
db.MigrateToLatest(c)
return c
}
......
......@@ -18,8 +18,10 @@ func initDB(dsn string) *gorp.DbMap {
panic(fmt.Sprintf("Unable to drop database tables: %v", err))
}
if err = c.CreateTablesIfNotExists(); err != nil {
panic(fmt.Sprintf("Unable to create database tables: %v", err))
if err = db.DropMigrationsTable(c); err != nil {
panic(fmt.Sprintf("Unable to drop migration table: %v", err))
}
db.MigrateToLatest(c)
return c
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment