Commit d96289a8 authored by astaxie's avatar astaxie Committed by GitHub

Merge pull request #2771 from astaxie/develop

v1.9.0
parents cab8458c 4fc95b0d
......@@ -34,7 +34,7 @@ install:
- go get github.com/cloudflare/golz4
- go get github.com/gogo/protobuf/proto
- go get github.com/Knetic/govaluate
- go get github.com/hsluoyz/casbin
- go get github.com/casbin/casbin
- go get -u honnef.co/go/tools/cmd/gosimple
- go get -u github.com/mdempsky/unconvert
- go get -u github.com/gordonklaus/ineffassign
......
......@@ -37,7 +37,7 @@ var beeAdminApp *adminApp
// FilterMonitorFunc is default monitor filter when admin module is enable.
// if this func returns, admin module records qbs for this request by condition of this function logic.
// usage:
// func MyFilterMonitor(method, requestPath string, t time.Duration) bool {
// func MyFilterMonitor(method, requestPath string, t time.Duration, pattern string, statusCode int) bool {
// if method == "POST" {
// return false
// }
......@@ -50,7 +50,7 @@ var beeAdminApp *adminApp
// return true
// }
// beego.FilterMonitorFunc = MyFilterMonitor.
var FilterMonitorFunc func(string, string, time.Duration) bool
var FilterMonitorFunc func(string, string, time.Duration, string, int) bool
func init() {
beeAdminApp = &adminApp{
......@@ -62,7 +62,7 @@ func init() {
beeAdminApp.Route("/healthcheck", healthcheck)
beeAdminApp.Route("/task", taskStatus)
beeAdminApp.Route("/listconf", listConf)
FilterMonitorFunc = func(string, string, time.Duration) bool { return true }
FilterMonitorFunc = func(string, string, time.Duration, string, int) bool { return true }
}
// AdminIndex is the default http.Handler for admin module.
......
......@@ -23,7 +23,7 @@ import (
const (
// VERSION represent beego web framework version.
VERSION = "1.8.3"
VERSION = "1.9.0"
// DEV is for develop
DEV = "dev"
......@@ -40,9 +40,9 @@ var (
// AddAPPStartHook is used to register the hookfunc
// The hookfuncs will run in beego.Run()
// such as sessionInit, middlerware start, buildtemplate, admin start
func AddAPPStartHook(hf hookfunc) {
hooks = append(hooks, hf)
// such as initiating session , starting middleware , building template, starting admin control and so on.
func AddAPPStartHook(hf ...hookfunc) {
hooks = append(hooks, hf...)
}
// Run beego application.
......@@ -69,12 +69,14 @@ func Run(params ...string) {
func initBeforeHTTPRun() {
//init hooks
AddAPPStartHook(registerMime)
AddAPPStartHook(registerDefaultErrorHandler)
AddAPPStartHook(registerSession)
AddAPPStartHook(registerTemplate)
AddAPPStartHook(registerAdmin)
AddAPPStartHook(registerGzip)
AddAPPStartHook(
registerMime,
registerDefaultErrorHandler,
registerSession,
registerTemplate,
registerAdmin,
registerGzip,
)
for _, hk := range hooks {
if err := hk(); err != nil {
......
......@@ -217,26 +217,31 @@ func (bc *MemoryCache) vaccuum() {
if bc.items == nil {
return
}
for name := range bc.items {
bc.itemExpired(name)
if keys := bc.expiredKeys(); len(keys) != 0 {
bc.clearItems(keys)
}
}
}
// itemExpired returns true if an item is expired.
func (bc *MemoryCache) itemExpired(name string) bool {
// expiredKeys returns key list which are expired.
func (bc *MemoryCache) expiredKeys() (keys []string) {
bc.RLock()
defer bc.RUnlock()
for key, itm := range bc.items {
if itm.isExpire() {
keys = append(keys, key)
}
}
return
}
// clearItems removes all the items which key in keys.
func (bc *MemoryCache) clearItems(keys []string) {
bc.Lock()
defer bc.Unlock()
itm, ok := bc.items[name]
if !ok {
return true
for _, key := range keys {
delete(bc.items, key)
}
if itm.isExpire() {
delete(bc.items, name)
return true
}
return false
}
func init() {
......
......@@ -189,16 +189,16 @@ func ParseBool(val interface{}) (value bool, err error) {
return false, nil
}
case int8, int32, int64:
strV := fmt.Sprintf("%s", v)
strV := fmt.Sprintf("%d", v)
if strV == "1" {
return true, nil
} else if strV == "0" {
return false, nil
}
case float64:
if v == 1 {
if v == 1.0 {
return true, nil
} else if v == 0 {
} else if v == 0.0 {
return false, nil
}
}
......
......@@ -16,9 +16,11 @@ package context
import (
"bytes"
"compress/gzip"
"errors"
"io"
"io/ioutil"
"net/http"
"net/url"
"reflect"
"regexp"
......@@ -349,11 +351,22 @@ func (input *BeegoInput) CopyBody(MaxMemory int64) []byte {
if input.Context.Request.Body == nil {
return []byte{}
}
var requestbody []byte
safe := &io.LimitedReader{R: input.Context.Request.Body, N: MaxMemory}
requestbody, _ := ioutil.ReadAll(safe)
if input.Header("Content-Encoding") == "gzip" {
reader, err := gzip.NewReader(safe)
if err != nil {
return nil
}
requestbody, _ = ioutil.ReadAll(reader)
} else {
requestbody, _ = ioutil.ReadAll(safe)
}
input.Context.Request.Body.Close()
bf := bytes.NewBuffer(requestbody)
input.Context.Request.Body = ioutil.NopCloser(bf)
input.Context.Request.Body = http.MaxBytesReader(input.Context.ResponseWriter, ioutil.NopCloser(bf), MaxMemory)
input.RequestBody = requestbody
return requestbody
}
......
......@@ -177,7 +177,7 @@ func jsonRenderer(value interface{}) Renderer {
func errorRenderer(err error) Renderer {
return rendererFunc(func(ctx *Context) {
ctx.Output.SetStatus(500)
ctx.WriteString(err.Error())
ctx.Output.Body([]byte(err.Error()))
})
}
......
......@@ -55,6 +55,13 @@ type ControllerComments struct {
MethodParams []*param.MethodParam
}
// ControllerCommentsSlice implements the sort interface
type ControllerCommentsSlice []ControllerComments
func (p ControllerCommentsSlice) Len() int { return len(p) }
func (p ControllerCommentsSlice) Less(i, j int) bool { return p[i].Router < p[j].Router }
func (p ControllerCommentsSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
// Controller defines some basic http request handler operations, such as
// http context, template and view, session and xsrf.
type Controller struct {
......
......@@ -52,7 +52,7 @@ func TestErrorCode_01(t *testing.T) {
if w.Code != code {
t.Fail()
}
if !strings.Contains(string(w.Body.Bytes()), http.StatusText(code)) {
if !strings.Contains(w.Body.String(), http.StatusText(code)) {
t.Fail()
}
}
......@@ -82,7 +82,7 @@ func TestErrorCode_03(t *testing.T) {
if w.Code != 200 {
t.Fail()
}
if string(w.Body.Bytes()) != parseCodeError {
if w.Body.String() != parseCodeError {
t.Fail()
}
}
......@@ -3,14 +3,17 @@ package grace
import (
"errors"
"net"
"sync"
)
type graceConn struct {
net.Conn
server *Server
m sync.Mutex
closed bool
}
func (c graceConn) Close() (err error) {
func (c *graceConn) Close() (err error) {
defer func() {
if r := recover(); r != nil {
switch x := r.(type) {
......@@ -23,6 +26,14 @@ func (c graceConn) Close() (err error) {
}
}
}()
c.m.Lock()
if c.closed {
c.m.Unlock()
return
}
c.server.wg.Done()
c.closed = true
c.m.Unlock()
return c.Conn.Close()
}
......@@ -37,7 +37,7 @@ func (gl *graceListener) Accept() (c net.Conn, err error) {
tc.SetKeepAlive(true)
tc.SetKeepAlivePeriod(3 * time.Minute)
c = graceConn{
c = &graceConn{
Conn: tc,
server: gl.server,
}
......
......@@ -56,17 +56,20 @@ type fileLogWriter struct {
Perm string `json:"perm"`
RotatePerm string `json:"rotateperm"`
fileNameOnly, suffix string // like "project.log", project is fileNameOnly and .log is suffix
}
// newFileWriter create a FileLogWriter returning as LoggerInterface.
func newFileWriter() Logger {
w := &fileLogWriter{
Daily: true,
MaxDays: 7,
Rotate: true,
Level: LevelTrace,
Perm: "0660",
Daily: true,
MaxDays: 7,
Rotate: true,
RotatePerm: "0440",
Level: LevelTrace,
Perm: "0660",
}
return w
}
......@@ -237,8 +240,12 @@ func (w *fileLogWriter) doRotate(logTime time.Time) error {
// Find the next available number
num := 1
fName := ""
rotatePerm, err := strconv.ParseInt(w.RotatePerm, 8, 64)
if err != nil {
return err
}
_, err := os.Lstat(w.Filename)
_, err = os.Lstat(w.Filename)
if err != nil {
//even if the file is not exist or other ,we should RESTART the logger
goto RESTART_LOGGER
......@@ -271,8 +278,9 @@ func (w *fileLogWriter) doRotate(logTime time.Time) error {
if err != nil {
goto RESTART_LOGGER
}
err = os.Chmod(fName, os.FileMode(0440))
// re-start logger
err = os.Chmod(fName, os.FileMode(rotatePerm))
RESTART_LOGGER:
startLoggerErr := w.startLogger()
......
......@@ -185,11 +185,12 @@ func TestFileRotate_06(t *testing.T) { //test file mode
}
func testFileRotate(t *testing.T, fn1, fn2 string) {
fw := &fileLogWriter{
Daily: true,
MaxDays: 7,
Rotate: true,
Level: LevelTrace,
Perm: "0660",
Daily: true,
MaxDays: 7,
Rotate: true,
Level: LevelTrace,
Perm: "0660",
RotatePerm: "0440",
}
fw.Init(fmt.Sprintf(`{"filename":"%v","maxdays":1}`, fn1))
fw.dailyOpenTime = time.Now().Add(-24 * time.Hour)
......@@ -208,11 +209,12 @@ func testFileRotate(t *testing.T, fn1, fn2 string) {
func testFileDailyRotate(t *testing.T, fn1, fn2 string) {
fw := &fileLogWriter{
Daily: true,
MaxDays: 7,
Rotate: true,
Level: LevelTrace,
Perm: "0660",
Daily: true,
MaxDays: 7,
Rotate: true,
Level: LevelTrace,
Perm: "0660",
RotatePerm: "0440",
}
fw.Init(fmt.Sprintf(`{"filename":"%v","maxdays":1}`, fn1))
fw.dailyOpenTime = time.Now().Add(-24 * time.Hour)
......
This diff is collapsed.
// Package migration enables you to generate migrations back and forth. It generates both migrations.
//
// //Creates a table
// m.CreateTable("tablename","InnoDB","utf8");
//
// //Alter a table
// m.AlterTable("tablename")
//
// Standard Column Methods
// * SetDataType
// * SetNullable
// * SetDefault
// * SetUnsigned (use only on integer types unless produces error)
//
// //Sets a primary column, multiple calls allowed, standard column methods available
// m.PriCol("id").SetAuto(true).SetNullable(false).SetDataType("INT(10)").SetUnsigned(true)
//
// //UniCol Can be used multiple times, allows standard Column methods. Use same "index" string to add to same index
// m.UniCol("index","column")
//
// //Standard Column Initialisation, can call .Remove() after NewCol("") on alter to remove
// m.NewCol("name").SetDataType("VARCHAR(255) COLLATE utf8_unicode_ci").SetNullable(false)
// m.NewCol("value").SetDataType("DOUBLE(8,2)").SetNullable(false)
//
// //Rename Columns , only use with Alter table, doesn't works with Create, prefix standard column methods with "Old" to
// //create a true reversible migration eg: SetOldDataType("DOUBLE(12,3)")
// m.RenameColumn("from","to")...
//
// //Foreign Columns, single columns are only supported, SetOnDelete & SetOnUpdate are available, call appropriately.
// //Supports standard column methods, automatic reverse.
// m.ForeignCol("local_col","foreign_col","foreign_table")
package migration
......@@ -52,6 +52,26 @@ type Migrationer interface {
GetCreated() int64
}
//Migration defines the migrations by either SQL or DDL
type Migration struct {
sqls []string
Created string
TableName string
Engine string
Charset string
ModifyType string
Columns []*Column
Indexes []*Index
Primary []*Column
Uniques []*Unique
Foreigns []*Foreign
Renames []*RenameColumn
RemoveColumns []*Column
RemoveIndexes []*Index
RemoveUniques []*Unique
RemoveForeigns []*Foreign
}
var (
migrationMap map[string]Migrationer
)
......@@ -60,20 +80,34 @@ func init() {
migrationMap = make(map[string]Migrationer)
}
// Migration the basic type which will implement the basic type
type Migration struct {
sqls []string
Created string
}
// Up implement in the Inheritance struct for upgrade
func (m *Migration) Up() {
switch m.ModifyType {
case "reverse":
m.ModifyType = "alter"
case "delete":
m.ModifyType = "create"
}
m.sqls = append(m.sqls, m.GetSQL())
}
// Down implement in the Inheritance struct for down
func (m *Migration) Down() {
switch m.ModifyType {
case "alter":
m.ModifyType = "reverse"
case "create":
m.ModifyType = "delete"
}
m.sqls = append(m.sqls, m.GetSQL())
}
//Migrate adds the SQL to the execution list
func (m *Migration) Migrate(migrationType string) {
m.ModifyType = migrationType
m.sqls = append(m.sqls, m.GetSQL())
}
// SQL add sql want to execute
......
......@@ -94,3 +94,43 @@ func (d *dbBaseOracle) IndexExists(db dbQuerier, table string, name string) bool
row.Scan(&cnt)
return cnt > 0
}
// execute insert sql with given struct and given values.
// insert the given values, not the field values in struct.
func (d *dbBaseOracle) InsertValue(q dbQuerier, mi *modelInfo, isMulti bool, names []string, values []interface{}) (int64, error) {
Q := d.ins.TableQuote()
marks := make([]string, len(names))
for i := range marks {
marks[i] = ":" + names[i]
}
sep := fmt.Sprintf("%s, %s", Q, Q)
qmarks := strings.Join(marks, ", ")
columns := strings.Join(names, sep)
multi := len(values) / len(names)
if isMulti {
qmarks = strings.Repeat(qmarks+"), (", multi-1) + qmarks
}
query := fmt.Sprintf("INSERT INTO %s%s%s (%s%s%s) VALUES (%s)", Q, mi.table, Q, Q, columns, Q, qmarks)
d.ins.ReplaceMarks(&query)
if isMulti || !d.ins.HasReturningID(mi, &query) {
res, err := q.Exec(query, values...)
if err == nil {
if isMulti {
return res.RowsAffected()
}
return res.LastInsertId()
}
return 0, err
}
row := q.QueryRow(query, values...)
var id int64
err := row.Scan(&id)
return id, err
}
......@@ -135,15 +135,16 @@ func parserComments(f *ast.FuncDecl, controllerName, pkgpath string) error {
func buildMethodParams(funcParams []*ast.Field, pc *parsedComment) []*param.MethodParam {
result := make([]*param.MethodParam, 0, len(funcParams))
for _, fparam := range funcParams {
methodParam := buildMethodParam(fparam, pc)
result = append(result, methodParam)
for _, pName := range fparam.Names {
methodParam := buildMethodParam(fparam, pName.Name, pc)
result = append(result, methodParam)
}
}
return result
}
func buildMethodParam(fparam *ast.Field, pc *parsedComment) *param.MethodParam {
func buildMethodParam(fparam *ast.Field, name string, pc *parsedComment) *param.MethodParam {
options := []param.MethodParamOption{}
name := fparam.Names[0].Name
if cparam, ok := pc.params[name]; ok {
//Build param from comment info
name = cparam.name
......@@ -274,6 +275,7 @@ func genRouterCode(pkgRealpath string) {
sort.Strings(sortKey)
for _, k := range sortKey {
cList := genInfoList[k]
sort.Sort(ControllerCommentsSlice(cList))
for _, c := range cList {
allmethod := "nil"
if len(c.AllowHTTPMethods) > 0 {
......
......@@ -17,7 +17,7 @@
// import(
// "github.com/astaxie/beego"
// "github.com/astaxie/beego/plugins/authz"
// "github.com/hsluoyz/casbin"
// "github.com/casbin/casbin"
// )
//
// func main(){
......@@ -42,7 +42,7 @@ package authz
import (
"github.com/astaxie/beego"
"github.com/astaxie/beego/context"
"github.com/hsluoyz/casbin"
"github.com/casbin/casbin"
"net/http"
)
......
......@@ -18,7 +18,7 @@ import (
"github.com/astaxie/beego"
"github.com/astaxie/beego/context"
"github.com/astaxie/beego/plugins/auth"
"github.com/hsluoyz/casbin"
"github.com/casbin/casbin"
"net/http"
"net/http/httptest"
"testing"
......
......@@ -704,7 +704,6 @@ func (p *ControllerRegister) ServeHTTP(rw http.ResponseWriter, r *http.Request)
// User can define RunController and RunMethod in filter
if context.Input.RunController != nil && context.Input.RunMethod != "" {
findRouter = true
isRunnable = true
runMethod = context.Input.RunMethod
runRouter = context.Input.RunController
} else {
......@@ -849,7 +848,15 @@ Admin:
//admin module record QPS
if BConfig.Listen.EnableAdmin {
timeDur := time.Since(startTime)
if FilterMonitorFunc(r.Method, r.URL.Path, timeDur) {
pattern := ""
if routerInfo != nil {
pattern = routerInfo.pattern
}
statusCode := context.ResponseWriter.Status
if statusCode == 0 {
statusCode = 200
}
if FilterMonitorFunc(r.Method, r.URL.Path, timeDur, pattern, statusCode) {
if runRouter != nil {
go toolbox.StatisticsMap.AddStatistics(r.Method, r.URL.Path, runRouter.Name(), timeDur)
} else {
......
......@@ -64,6 +64,9 @@ Struct Tag Use:
func main() {
valid := validation.Validation{}
// ignore empty field valid
// see CanSkipFuncs
// valid := validation.Validation{RequiredFirst:true}
u := user{Name: "test", Age: 40}
b, err := valid.Valid(u)
if err != nil {
......
......@@ -25,6 +25,8 @@ import (
const (
// ValidTag struct tag
ValidTag = "valid"
wordsize = 32 << (^uint(0) >> 32 & 1)
)
var (
......@@ -43,6 +45,8 @@ var (
"Valid": true,
"NoMatch": true,
}
// ErrInt64On32 show 32 bit platform not support int64
ErrInt64On32 = fmt.Errorf("not support int64 on 32-bit platform")
)
func init() {
......@@ -249,16 +253,39 @@ func parseParam(t reflect.Type, s string) (i interface{}, err error) {
switch t.Kind() {
case reflect.Int:
i, err = strconv.Atoi(s)
case reflect.Int64:
if wordsize == 32 {
return nil, ErrInt64On32
}
i, err = strconv.ParseInt(s, 10, 64)
case reflect.Int32:
var v int64
v, err = strconv.ParseInt(s, 10, 32)
if err == nil {
i = int32(v)
}
case reflect.Int16:
var v int64
v, err = strconv.ParseInt(s, 10, 16)
if err == nil {
i = int16(v)
}
case reflect.Int8:
var v int64
v, err = strconv.ParseInt(s, 10, 8)
if err == nil {
i = int8(v)
}
case reflect.String:
i = s
case reflect.Ptr:
if t.Elem().String() != "regexp.Regexp" {
err = fmt.Errorf("does not support %s", t.Elem().String())
err = fmt.Errorf("not support %s", t.Elem().String())
return
}
i, err = regexp.Compile(s)
default:
err = fmt.Errorf("does not support %s", t.Kind().String())
err = fmt.Errorf("not support %s", t.Kind().String())
}
return
}
......
......@@ -106,6 +106,11 @@ func (r *Result) Message(message string, args ...interface{}) *Result {
// A Validation context manages data validation and error messages.
type Validation struct {
// if this field set true, in struct tag valid
// if the struct field vale is empty
// it will skip those valid functions, see CanSkipFuncs
RequiredFirst bool
Errors []*Error
ErrorsMap map[string]*Error
}
......@@ -324,7 +329,19 @@ func (v *Validation) Valid(obj interface{}) (b bool, err error) {
if vfs, err = getValidFuncs(objT.Field(i)); err != nil {
return
}
var hasReuired bool
for _, vf := range vfs {
if vf.Name == "Required" {
hasReuired = true
}
if !hasReuired && v.RequiredFirst && len(objV.Field(i).String()) == 0 {
if _, ok := CanSkipFuncs[vf.Name]; ok {
continue
}
}
if _, err = funcs.Call(vf.Name,
mergeParam(v, objV.Field(i).Interface(), vf.Params)...); err != nil {
return
......
......@@ -391,3 +391,54 @@ func TestRecursiveValid(t *testing.T) {
t.Error("validation should not be passed")
}
}
func TestSkipValid(t *testing.T) {
type User struct {
ID int
Email string `valid:"Email"`
ReqEmail string `valid:"Required;Email"`
IP string `valid:"IP"`
ReqIP string `valid:"Required;IP"`
Mobile string `valid:"Mobile"`
ReqMobile string `valid:"Required;Mobile"`
Tel string `valid:"Tel"`
ReqTel string `valid:"Required;Tel"`
Phone string `valid:"Phone"`
ReqPhone string `valid:"Required;Phone"`
ZipCode string `valid:"ZipCode"`
ReqZipCode string `valid:"Required;ZipCode"`
}
u := User{
ReqEmail: "a@a.com",
ReqIP: "127.0.0.1",
ReqMobile: "18888888888",
ReqTel: "02088888888",
ReqPhone: "02088888888",
ReqZipCode: "510000",
}
valid := Validation{}
b, err := valid.Valid(u)
if err != nil {
t.Fatal(err)
}
if b {
t.Fatal("validation should not be passed")
}
valid = Validation{RequiredFirst: true}
b, err = valid.Valid(u)
if err != nil {
t.Fatal(err)
}
if !b {
t.Fatal("validation should be passed")
}
}
......@@ -23,6 +23,16 @@ import (
"unicode/utf8"
)
// CanSkipFuncs will skip valid if RequiredFirst is true and the struct field's value is empty
var CanSkipFuncs = map[string]struct{}{
"Email": {},
"IP": {},
"Mobile": {},
"Tel": {},
"Phone": {},
"ZipCode": {},
}
// MessageTmpls store commond validate template
var MessageTmpls = map[string]string{
"Required": "Can not be empty",
......@@ -166,12 +176,28 @@ type Min struct {
}
// IsSatisfied judge whether obj is valid
// not support int64 on 32-bit platform
func (m Min) IsSatisfied(obj interface{}) bool {
num, ok := obj.(int)
if ok {
return num >= m.Min
var v int
switch obj.(type) {
case int64:
if wordsize == 32 {
return false
}
v = int(obj.(int64))
case int:
v = obj.(int)
case int32:
v = int(obj.(int32))
case int16:
v = int(obj.(int16))
case int8:
v = int(obj.(int8))
default:
return false
}
return false
return v >= m.Min
}
// DefaultMessage return the default min error message
......@@ -196,12 +222,28 @@ type Max struct {
}
// IsSatisfied judge whether obj is valid
// not support int64 on 32-bit platform
func (m Max) IsSatisfied(obj interface{}) bool {
num, ok := obj.(int)
if ok {
return num <= m.Max
var v int
switch obj.(type) {
case int64:
if wordsize == 32 {
return false
}
v = int(obj.(int64))
case int:
v = obj.(int)
case int32:
v = int(obj.(int32))
case int16:
v = int(obj.(int16))
case int8:
v = int(obj.(int8))
default:
return false
}
return false
return v <= m.Max
}
// DefaultMessage return the default max error message
......@@ -227,6 +269,7 @@ type Range struct {
}
// IsSatisfied judge whether obj is valid
// not support int64 on 32-bit platform
func (r Range) IsSatisfied(obj interface{}) bool {
return r.Min.IsSatisfied(obj) && r.Max.IsSatisfied(obj)
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment