From 22e9965629916efd95e90bffea15a6e3b315e7af Mon Sep 17 00:00:00 2001 From: John Date: Tue, 10 Mar 2020 08:57:37 +0800 Subject: [PATCH 01/32] improve template view instance initialization for instance of ghttp.Server --- .../ghttp/server/template/config/config.go | 17 ++++++ .../ghttp/server/template/config/config.toml | 2 + .example/other/test.go | 56 ++++++++++++++++--- container/gvar/gvar_z_unit_test.go | 9 +++ frame/gins/gins_database.go | 2 +- frame/gins/gins_log.go | 2 +- frame/gins/gins_redis.go | 2 +- frame/gins/gins_server.go | 5 +- frame/gins/gins_view.go | 2 +- 9 files changed, 83 insertions(+), 14 deletions(-) create mode 100644 .example/net/ghttp/server/template/config/config.go create mode 100644 .example/net/ghttp/server/template/config/config.toml diff --git a/.example/net/ghttp/server/template/config/config.go b/.example/net/ghttp/server/template/config/config.go new file mode 100644 index 000000000..f09b8f9ae --- /dev/null +++ b/.example/net/ghttp/server/template/config/config.go @@ -0,0 +1,17 @@ +package main + +import ( + "github.com/gogf/gf/frame/g" + "github.com/gogf/gf/net/ghttp" +) + +func main() { + s := g.Server() + s.BindHandler("/", func(r *ghttp.Request) { + r.Response.WriteTplContent(`${.name}`, g.Map{ + "name": "john", + }) + }) + s.SetPort(8199) + s.Run() +} diff --git a/.example/net/ghttp/server/template/config/config.toml b/.example/net/ghttp/server/template/config/config.toml new file mode 100644 index 000000000..1d5fc0dbb --- /dev/null +++ b/.example/net/ghttp/server/template/config/config.toml @@ -0,0 +1,2 @@ +[viewer] + delimiters = ["${", "}"] \ No newline at end of file diff --git a/.example/other/test.go b/.example/other/test.go index 5db56e11d..267409019 100644 --- a/.example/other/test.go +++ b/.example/other/test.go @@ -2,15 +2,53 @@ package main import ( "fmt" - "github.com/gogf/gf/text/gregex" + "github.com/gogf/gf/container/gvar" + "github.com/gogf/gf/encoding/gparser" + "github.com/gogf/gf/os/gtime" ) -func main() { - data := "@var(.prefix)您收到的验证码为：@var(.code)，请在@var(.expire)内完成验证" - result, err := gregex.ReplaceStringFuncMatch(`(@var\(\.\w+\))`, data, func(match []string) string { - fmt.Println(match) - return "#" - }) - fmt.Println(err) - fmt.Println(result) +type EntityTest struct { + ID int64 + DeviceID string + DeviceType uint8 + ProtoVer uint8 + DataType uint8 + RecordTime *gtime.Time + CreateTime *gtime.Time + RemoteIP string + Voltage int16 + Battery uint8 + CellularVersion uint8 + StateID uint16 + OperatorID uint16 + RegionID uint16 + BaseStationID uint32 + BaseStationSignalStrength int8 + ContainerStatus int8 + PositionX int16 + PositionY int16 + PositionZ int16 + ContainerID string + NetworkRegisterTime int16 + Temperature int8 + Humidity uint8 + SleepMode uint8 + CellularChangeStatus int8 + GPSSearchTime int16 + Latitude float64 + Longitude float64 + Altitude int16 + Speed uint8 + DateStr string + TimeStr string + FrameID uint16 +} + +func main() { + array := []string{"DeviceID", "8134567890ABCDEF", "DeviceType", "16", "ProtoVer", "11", "DataType", "2", "RecordTime", "2020-02-27 15:23:23", "CreateTime", "2020-03-09 19:24:22", "RemoteIP", "127.0.0.1:60471", "Voltage", "3610", "Battery", "90", "CellularVersion", "4", "StateID", "401", "OperatorID", "345", "RegionID", "17716", "BaseStationID", "571479331", "BaseStationSignalStrength", "45", "ContainerStatus", "1", "PositionX", "40", "PositionY", "-25", "PositionZ", "16", "ContainerID", "aAvVDEFPQAA", "DateStr", "270220", "TimeStr", "232323", "NetworkRegisterTime", "60", "Temperature", "25", "Humidity", "70", "SleepMode", "1", "CellularChangeStatus", "15", "GPSSearchTime", "32767", "Latitude", "116.23532", "Longitude", "39.21526", "Altitude", "2800", "Speed", "66", "FrameID", "4660"} + v := gvar.New(array) + o := &EntityTest{} + v.Struct(o) + //b, _ := json.Marshal(o) + fmt.Println(gparser.MustToJsonIndentString(o)) } diff --git a/container/gvar/gvar_z_unit_test.go b/container/gvar/gvar_z_unit_test.go index fcbd427ae..16e38c3c9 100644 --- a/container/gvar/gvar_z_unit_test.go +++ b/container/gvar/gvar_z_unit_test.go @@ -334,6 +334,15 @@ func Test_Struct(t *testing.T) { gtest.Assert(testObj.Test, Kv["Test"]) }) + gtest.Case(t, func() { + type StTest struct { + Test int8 + } + o := &StTest{} + v := gvar.New(g.Slice{"Test", "-25"}) + v.Struct(o) + gtest.Assert(o.Test, -25) + }) } func Test_Json(t *testing.T) { diff --git a/frame/gins/gins_database.go b/frame/gins/gins_database.go index f39a7c16f..23dc4ffbd 100644 --- a/frame/gins/gins_database.go +++ b/frame/gins/gins_database.go @@ -29,7 +29,7 @@ func Database(name ...string) gdb.DB { group = name[0] } instanceKey := fmt.Sprintf("%s.%s", gFRAME_CORE_COMPONENT_NAME_DATABASE, group) - db := instances.GetOrSetFuncLock(instanceKey, func() interface{} { + db := instances.GetOrSetFunc(instanceKey, func() interface{} { // Configuration already exists. if gdb.GetConfig(group) != nil { db, err := gdb.Instance(group) diff --git a/frame/gins/gins_log.go b/frame/gins/gins_log.go index 0b8f21b64..081c39bc2 100644 --- a/frame/gins/gins_log.go +++ b/frame/gins/gins_log.go @@ -24,7 +24,7 @@ func Log(name ...string) *glog.Logger { instanceName = name[0] } instanceKey := fmt.Sprintf("%s.%s", gFRAME_CORE_COMPONENT_NAME_LOGGER, instanceName) - return instances.GetOrSetFuncLock(instanceKey, func() interface{} { + return instances.GetOrSetFunc(instanceKey, func() interface{} { logger := glog.Instance(instanceName) // To avoid file no found error while it's not necessary. if Config().Available() { diff --git a/frame/gins/gins_redis.go b/frame/gins/gins_redis.go index 195308ab2..1f480edc6 100644 --- a/frame/gins/gins_redis.go +++ b/frame/gins/gins_redis.go @@ -24,7 +24,7 @@ func Redis(name ...string) *gredis.Redis { group = name[0] } instanceKey := fmt.Sprintf("%s.%s", gFRAME_CORE_COMPONENT_NAME_REDIS, group) - result := instances.GetOrSetFuncLock(instanceKey, func() interface{} { + result := instances.GetOrSetFunc(instanceKey, func() interface{} { // If already configured, it returns the redis instance. if _, ok := gredis.GetConfig(group); ok { return gredis.Instance(group) diff --git a/frame/gins/gins_server.go b/frame/gins/gins_server.go index 421f157d9..309e516d3 100644 --- a/frame/gins/gins_server.go +++ b/frame/gins/gins_server.go @@ -18,7 +18,7 @@ const ( // Server returns an instance of http server with specified name. func Server(name ...interface{}) *ghttp.Server { instanceKey := fmt.Sprintf("%s.%v", gFRAME_CORE_COMPONENT_NAME_SERVER, name) - return instances.GetOrSetFuncLock(instanceKey, func() interface{} { + return instances.GetOrSetFunc(instanceKey, func() interface{} { s := ghttp.GetServer(name...) // To avoid file no found error while it's not necessary. if Config().Available() { @@ -34,6 +34,9 @@ func Server(name ...interface{}) *ghttp.Server { panic(err) } } + // As it might use template feature, + // it initialize the view instance as well. + View() } return s }).(*ghttp.Server) diff --git a/frame/gins/gins_view.go b/frame/gins/gins_view.go index b7a525cc4..fa14e0e49 100644 --- a/frame/gins/gins_view.go +++ b/frame/gins/gins_view.go @@ -24,7 +24,7 @@ func View(name ...string) *gview.View { instanceName = name[0] } instanceKey := fmt.Sprintf("%s.%s", gFRAME_CORE_COMPONENT_NAME_VIEWER, instanceName) - return instances.GetOrSetFuncLock(instanceKey, func() interface{} { + return instances.GetOrSetFunc(instanceKey, func() interface{} { view := gview.Instance(instanceName) // To avoid file no found error while it's not necessary. if Config().Available() { From 645cecdffb04fb5e8409bf10256c55cdff8c1cf2 Mon Sep 17 00:00:00 2001 From: John Date: Tue, 10 Mar 2020 20:13:36 +0800 Subject: [PATCH 02/32] improve template view instance initialization for instance of ghttp.Server --- frame/gins/gins.go | 17 -------------- frame/gins/gins_database.go | 3 +-- frame/gins/gins_log.go | 2 +- frame/gins/gins_redis.go | 3 +-- frame/gins/gins_server.go | 4 ++-- frame/gins/gins_view.go | 44 ++++++++++++++++++++++--------------- 6 files changed, 31 insertions(+), 42 deletions(-) diff --git a/frame/gins/gins.go b/frame/gins/gins.go index d08ba08d7..5e8ff2e51 100644 --- a/frame/gins/gins.go +++ b/frame/gins/gins.go @@ -8,12 +8,7 @@ package gins import ( - "github.com/gogf/gf/internal/intlog" - "github.com/gogf/gf/os/gfile" - "github.com/gogf/gf/container/gmap" - "github.com/gogf/gf/os/gcfg" - "github.com/gogf/gf/os/gfsnotify" ) var ( @@ -59,15 +54,3 @@ func GetOrSetFuncLock(name string, f func() interface{}) interface{} { func SetIfNotExist(name string, instance interface{}) bool { return instances.SetIfNotExist(name, instance) } - -// addConfigMonitor adds fsnotify monitor for configuration file if it exists. -func addConfigMonitor(key string, config *gcfg.Config) { - if path := config.FilePath(); path != "" && gfile.Exists(path) { - _, err := gfsnotify.Add(path, func(event *gfsnotify.Event) { - instances.Remove(key) - }) - if err != nil { - intlog.Error(err) - } - } -} diff --git a/frame/gins/gins_database.go b/frame/gins/gins_database.go index 23dc4ffbd..92842b178 100644 --- a/frame/gins/gins_database.go +++ b/frame/gins/gins_database.go @@ -29,7 +29,7 @@ func Database(name ...string) gdb.DB { group = name[0] } instanceKey := fmt.Sprintf("%s.%s", gFRAME_CORE_COMPONENT_NAME_DATABASE, group) - db := instances.GetOrSetFunc(instanceKey, func() interface{} { + db := instances.GetOrSetFuncLock(instanceKey, func() interface{} { // Configuration already exists. if gdb.GetConfig(group) != nil { db, err := gdb.Instance(group) @@ -72,7 +72,6 @@ func Database(name ...string) gdb.DB { gdb.SetConfigGroup(gdb.DEFAULT_GROUP_NAME, cg) } } - addConfigMonitor(instanceKey, config) if db, err := gdb.New(name...); err == nil { // Initialize logger for ORM. diff --git a/frame/gins/gins_log.go b/frame/gins/gins_log.go index 081c39bc2..0b8f21b64 100644 --- a/frame/gins/gins_log.go +++ b/frame/gins/gins_log.go @@ -24,7 +24,7 @@ func Log(name ...string) *glog.Logger { instanceName = name[0] } instanceKey := fmt.Sprintf("%s.%s", gFRAME_CORE_COMPONENT_NAME_LOGGER, instanceName) - return instances.GetOrSetFunc(instanceKey, func() interface{} { + return instances.GetOrSetFuncLock(instanceKey, func() interface{} { logger := glog.Instance(instanceName) // To avoid file no found error while it's not necessary. if Config().Available() { diff --git a/frame/gins/gins_redis.go b/frame/gins/gins_redis.go index 1f480edc6..9130ae1a4 100644 --- a/frame/gins/gins_redis.go +++ b/frame/gins/gins_redis.go @@ -24,7 +24,7 @@ func Redis(name ...string) *gredis.Redis { group = name[0] } instanceKey := fmt.Sprintf("%s.%s", gFRAME_CORE_COMPONENT_NAME_REDIS, group) - result := instances.GetOrSetFunc(instanceKey, func() interface{} { + result := instances.GetOrSetFuncLock(instanceKey, func() interface{} { // If already configured, it returns the redis instance. if _, ok := gredis.GetConfig(group); ok { return gredis.Instance(group) @@ -36,7 +36,6 @@ func Redis(name ...string) *gredis.Redis { if err != nil { panic(err) } - addConfigMonitor(instanceKey, config) return gredis.New(redisConfig) } else { panic(fmt.Sprintf(`configuration for redis not found for group "%s"`, group)) diff --git a/frame/gins/gins_server.go b/frame/gins/gins_server.go index 309e516d3..58e7d986c 100644 --- a/frame/gins/gins_server.go +++ b/frame/gins/gins_server.go @@ -18,7 +18,7 @@ const ( // Server returns an instance of http server with specified name. func Server(name ...interface{}) *ghttp.Server { instanceKey := fmt.Sprintf("%s.%v", gFRAME_CORE_COMPONENT_NAME_SERVER, name) - return instances.GetOrSetFunc(instanceKey, func() interface{} { + return instances.GetOrSetFuncLock(instanceKey, func() interface{} { s := ghttp.GetServer(name...) // To avoid file no found error while it's not necessary. if Config().Available() { @@ -36,7 +36,7 @@ func Server(name ...interface{}) *ghttp.Server { } // As it might use template feature, // it initialize the view instance as well. - View() + _ = getViewInstance() } return s }).(*ghttp.Server) diff --git a/frame/gins/gins_view.go b/frame/gins/gins_view.go index fa14e0e49..62ee0f581 100644 --- a/frame/gins/gins_view.go +++ b/frame/gins/gins_view.go @@ -24,23 +24,31 @@ func View(name ...string) *gview.View { instanceName = name[0] } instanceKey := fmt.Sprintf("%s.%s", gFRAME_CORE_COMPONENT_NAME_VIEWER, instanceName) - return instances.GetOrSetFunc(instanceKey, func() interface{} { - view := gview.Instance(instanceName) - // To avoid file no found error while it's not necessary. - if Config().Available() { - var m map[string]interface{} - // It firstly searches the configuration of the instance name. - if m = Config().GetMap(fmt.Sprintf(`%s.%s`, gVIEWER_NODE_NAME, instanceName)); m == nil { - // If the configuration for the instance does not exist, - // it uses the default view configuration. - m = Config().GetMap(gVIEWER_NODE_NAME) - } - if m != nil { - if err := view.SetConfigWithMap(m); err != nil { - panic(err) - } - } - } - return view + return instances.GetOrSetFuncLock(instanceKey, func() interface{} { + return getViewInstance(instanceName) }).(*gview.View) } + +func getViewInstance(name ...string) *gview.View { + instanceName := gview.DEFAULT_NAME + if len(name) > 0 && name[0] != "" { + instanceName = name[0] + } + view := gview.Instance(instanceName) + // To avoid file no found error while it's not necessary. + if Config().Available() { + var m map[string]interface{} + // It firstly searches the configuration of the instance name. + if m = Config().GetMap(fmt.Sprintf(`%s.%s`, gVIEWER_NODE_NAME, name)); m == nil { + // If the configuration for the instance does not exist, + // it uses the default view configuration. + m = Config().GetMap(gVIEWER_NODE_NAME) + } + if m != nil { + if err := view.SetConfigWithMap(m); err != nil { + panic(err) + } + } + } + return view +} From f3bd2b67f705b2e092f2a7e29daaa5fc2b2604c9 Mon Sep 17 00:00:00 2001 From: John Date: Tue, 10 Mar 2020 20:45:22 +0800 Subject: [PATCH 03/32] change attribute Context to context.Context for ghttp.Request --- net/ghttp/ghttp_request.go | 6 +++--- net/ghttp/ghttp_unit_context_test.go | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/net/ghttp/ghttp_request.go b/net/ghttp/ghttp_request.go index 2d3e4160c..7662af053 100644 --- a/net/ghttp/ghttp_request.go +++ b/net/ghttp/ghttp_request.go @@ -7,8 +7,8 @@ package ghttp import ( + "context" "fmt" - "github.com/gogf/gf/container/gmap" "github.com/gogf/gf/os/gres" "github.com/gogf/gf/os/gview" "net/http" @@ -32,7 +32,7 @@ type Request struct { LeaveTime int64 // Request ending time in microseconds. Middleware *Middleware // The middleware manager. StaticFile *StaticFile // Static file object when static file serving. - Context *gmap.StrAnyMap // Custom context map for internal usage purpose. + Context context.Context // Custom context map for internal usage purpose. handlers []*handlerParsedItem // All matched handlers containing handler, hook and middleware for this request . hasHookHandler bool // A bool marking whether there's hook handler in the handlers for performance purpose. hasServeHandler bool // A bool marking whether there's serving handler in the handlers for performance purpose. @@ -68,7 +68,7 @@ func newRequest(s *Server, r *http.Request, w http.ResponseWriter) *Request { Request: r, Response: newResponse(s, w), EnterTime: gtime.TimestampMilli(), - Context: gmap.NewStrAnyMap(), + Context: r.Context(), } request.Cookie = GetCookie(request) request.Session = s.sessionManager.New(request.GetSessionId()) diff --git a/net/ghttp/ghttp_unit_context_test.go b/net/ghttp/ghttp_unit_context_test.go index 4819adc73..d9f0636af 100644 --- a/net/ghttp/ghttp_unit_context_test.go +++ b/net/ghttp/ghttp_unit_context_test.go @@ -7,6 +7,7 @@ package ghttp_test import ( + "context" "fmt" "testing" "time" @@ -21,15 +22,15 @@ func Test_Context(t *testing.T) { s := g.Server(p) s.Group("/", func(group *ghttp.RouterGroup) { group.Middleware(func(r *ghttp.Request) { - r.Context.Set("traceid", 123) + r.Context = context.WithValue(r.Context, "traceid", 123) r.Middleware.Next() }) group.GET("/", func(r *ghttp.Request) { - r.Response.Write(r.Context.Get("traceid")) + r.Response.Write(r.Context.Value("traceid")) }) }) s.SetPort(p) - //s.SetDumpRouterMap(false) + s.SetDumpRouterMap(false) s.Start() defer s.Shutdown() From 2e9be609c8110da4714e21e55fd8a963667ad9fd Mon Sep 17 00:00:00 2001 From: John Date: Wed, 11 Mar 2020 09:22:13 +0800 Subject: [PATCH 04/32] fix issue in unit testing of gins --- frame/gins/gins_view.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/gins/gins_view.go b/frame/gins/gins_view.go index 62ee0f581..05c0d8569 100644 --- a/frame/gins/gins_view.go +++ b/frame/gins/gins_view.go @@ -39,7 +39,7 @@ func getViewInstance(name ...string) *gview.View { if Config().Available() { var m map[string]interface{} // It firstly searches the configuration of the instance name. - if m = Config().GetMap(fmt.Sprintf(`%s.%s`, gVIEWER_NODE_NAME, name)); m == nil { + if m = Config().GetMap(fmt.Sprintf(`%s.%s`, gVIEWER_NODE_NAME, instanceName)); m == nil { // If the configuration for the instance does not exist, // it uses the default view configuration. m = Config().GetMap(gVIEWER_NODE_NAME) From f13a5ad82e2e7dfafea89178df676e520b2912d6 Mon Sep 17 00:00:00 2001 From: fulltimelove Date: Wed, 11 Mar 2020 10:10:00 +0800 Subject: [PATCH 05/32] =?UTF-8?q?=E5=88=86=E7=BB=84=E8=B7=AF=E7=94=B1?= =?UTF-8?q?=E5=AF=B9=E8=B1=A1=E7=BB=91=E5=AE=9A=E6=97=B6=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E5=A4=9A=E6=96=B9=E6=B3=95=E7=BB=91=E5=AE=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- net/ghttp/ghttp_server_router_group.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/net/ghttp/ghttp_server_router_group.go b/net/ghttp/ghttp_server_router_group.go index b0df21c90..9ffb893a3 100644 --- a/net/ghttp/ghttp_server_router_group.go +++ b/net/ghttp/ghttp_server_router_group.go @@ -307,9 +307,17 @@ func (g *RouterGroup) doBind(bindType string, pattern string, object interface{} } else { if len(extras) > 0 { if g.server != nil { - g.server.doBindObjectMethod(pattern, object, extras[0], g.middleware) + if gstr.Contains(extras[0], ",") { + g.server.doBindObject(pattern, object, extras[0], g.middleware) + } else { + g.server.doBindObjectMethod(pattern, object, extras[0], g.middleware) + } } else { - g.domain.doBindObjectMethod(pattern, object, extras[0], g.middleware) + if gstr.Contains(extras[0], ",") { + g.domain.doBindObject(pattern, object, extras[0], g.middleware) + } else { + g.domain.doBindObjectMethod(pattern, object, extras[0], g.middleware) + } } } else { if g.server != nil { From cd7c45c00ca67652de4c2938fa0bd8978dabaf1b Mon Sep 17 00:00:00 2001 From: John Date: Wed, 11 Mar 2020 14:35:23 +0800 Subject: [PATCH 06/32] improve default connection pool configuration for gredis --- .example/other/test.go | 69 ++++++++++++++------------------------- database/gredis/gredis.go | 33 ++++++++++++++----- 2 files changed, 50 insertions(+), 52 deletions(-) diff --git a/.example/other/test.go b/.example/other/test.go index 267409019..84f83a83d 100644 --- a/.example/other/test.go +++ b/.example/other/test.go @@ -1,54 +1,35 @@ package main import ( - "fmt" - "github.com/gogf/gf/container/gvar" - "github.com/gogf/gf/encoding/gparser" + "github.com/gogf/gf/frame/g" + "github.com/gogf/gf/os/glog" "github.com/gogf/gf/os/gtime" + "github.com/gogf/gf/os/gtimer" + "time" ) -type EntityTest struct { - ID int64 - DeviceID string - DeviceType uint8 - ProtoVer uint8 - DataType uint8 - RecordTime *gtime.Time - CreateTime *gtime.Time - RemoteIP string - Voltage int16 - Battery uint8 - CellularVersion uint8 - StateID uint16 - OperatorID uint16 - RegionID uint16 - BaseStationID uint32 - BaseStationSignalStrength int8 - ContainerStatus int8 - PositionX int16 - PositionY int16 - PositionZ int16 - ContainerID string - NetworkRegisterTime int16 - Temperature int8 - Humidity uint8 - SleepMode uint8 - CellularChangeStatus int8 - GPSSearchTime int16 - Latitude float64 - Longitude float64 - Altitude int16 - Speed uint8 - DateStr string - TimeStr string - FrameID uint16 +func GetList() { +START: + for { + res, err := g.Redis().DoVar("RPOP", "mill") + if err != nil { + glog.Debug("Rpop:", err) + break + } + glog.Debug(res) + if res.IsEmpty() { + glog.Debug("nil") + continue START + } + interval := 50 * time.Second + gtimer.AddOnce(interval, func() { + glog.Debug("end------:", res, gtime.Now().Format("Y-m-d H:i:s")) + }) + } } func main() { - array := []string{"DeviceID", "8134567890ABCDEF", "DeviceType", "16", "ProtoVer", "11", "DataType", "2", "RecordTime", "2020-02-27 15:23:23", "CreateTime", "2020-03-09 19:24:22", "RemoteIP", "127.0.0.1:60471", "Voltage", "3610", "Battery", "90", "CellularVersion", "4", "StateID", "401", "OperatorID", "345", "RegionID", "17716", "BaseStationID", "571479331", "BaseStationSignalStrength", "45", "ContainerStatus", "1", "PositionX", "40", "PositionY", "-25", "PositionZ", "16", "ContainerID", "aAvVDEFPQAA", "DateStr", "270220", "TimeStr", "232323", "NetworkRegisterTime", "60", "Temperature", "25", "Humidity", "70", "SleepMode", "1", "CellularChangeStatus", "15", "GPSSearchTime", "32767", "Latitude", "116.23532", "Longitude", "39.21526", "Altitude", "2800", "Speed", "66", "FrameID", "4660"} - v := gvar.New(array) - o := &EntityTest{} - v.Struct(o) - //b, _ := json.Marshal(o) - fmt.Println(gparser.MustToJsonIndentString(o)) + g.Redis().SetMaxActive(2) + //g.Redis().SetMaxIdle(100) + GetList() } diff --git a/database/gredis/gredis.go b/database/gredis/gredis.go index 6bd31378a..f6828d6d2 100644 --- a/database/gredis/gredis.go +++ b/database/gredis/gredis.go @@ -41,10 +41,10 @@ type Config struct { Port int Db int Pass string // Password for AUTH. - MaxIdle int // Maximum number of connections allowed to be idle (default is 0 means no idle connection) - MaxActive int // Maximum number of connections limit (default is 0 means no limit) - IdleTimeout time.Duration // Maximum idle time for connection (default is 60 seconds, not allowed to be set to 0) - MaxConnLifetime time.Duration // Maximum lifetime of the connection (default is 60 seconds, not allowed to be set to 0) + MaxIdle int // Maximum number of connections allowed to be idle (default is 10) + MaxActive int // Maximum number of connections limit (default is 0 means no limit). + IdleTimeout time.Duration // Maximum idle time for connection (default is 10 seconds, not allowed to be set to 0) + MaxConnLifetime time.Duration // Maximum lifetime of the connection (default is 30 seconds, not allowed to be set to 0) ConnectTimeout time.Duration // Dial connection timeout. } @@ -54,8 +54,9 @@ type PoolStats struct { } const ( - gDEFAULT_POOL_IDLE_TIMEOUT = 30 * time.Second + gDEFAULT_POOL_IDLE_TIMEOUT = 10 * time.Second gDEFAULT_POOL_CONN_TIMEOUT = 10 * time.Second + gDEFAULT_POOL_MAX_IDLE = 10 gDEFAULT_POOL_MAX_LIFE_TIME = 30 * time.Second ) @@ -67,6 +68,12 @@ var ( // New creates a redis client object with given configuration. // Redis client maintains a connection pool automatically. func New(config Config) *Redis { + // The MaxIdle is the most important attribute of the connection pool. + // Only if this attribute is set, the created connections from client + // can not exceed the limit of the server. + if config.MaxIdle == 0 { + config.MaxIdle = gDEFAULT_POOL_MAX_IDLE + } if config.IdleTimeout == 0 { config.IdleTimeout = gDEFAULT_POOL_IDLE_TIMEOUT } @@ -132,7 +139,8 @@ func NewFromStr(str string) (*Redis, error) { // It is not necessary to call Close manually. func (r *Redis) Close() error { if r.group != "" { - // If it is an instance object, it needs to remove it from the instance Map. + // If it is an instance object, + // it needs to remove it from the instance Map. instances.Remove(r.group) } pools.Remove(fmt.Sprintf("%v", r.config)) @@ -151,22 +159,31 @@ func (r *Redis) GetConn() *Conn { return r.Conn() } -// SetMaxIdle sets the MaxIdle attribute of the connection pool. +// SetMaxIdle sets the maximum number of idle connections in the pool. func (r *Redis) SetMaxIdle(value int) { r.pool.MaxIdle = value } -// SetMaxActive sets the MaxActive attribute of the connection pool. +// SetMaxActive sets the maximum number of connections allocated by the pool at a given time. +// When zero, there is no limit on the number of connections in the pool. +// +// Note that if the pool is at the MaxActive limit, then all the operations will wait for +// a connection to be returned to the pool before returning. func (r *Redis) SetMaxActive(value int) { r.pool.MaxActive = value } // SetIdleTimeout sets the IdleTimeout attribute of the connection pool. +// It closes connections after remaining idle for this duration. If the value +// is zero, then idle connections are not closed. Applications should set +// the timeout to a value less than the server's timeout. func (r *Redis) SetIdleTimeout(value time.Duration) { r.pool.IdleTimeout = value } // SetMaxConnLifetime sets the MaxConnLifetime attribute of the connection pool. +// It closes connections older than this duration. If the value is zero, then +// the pool does not close connections based on age. func (r *Redis) SetMaxConnLifetime(value time.Duration) { r.pool.MaxConnLifetime = value } From 5a92d7de0d1c12d2969291477e35ab5aff636fff Mon Sep 17 00:00:00 2001 From: John Date: Wed, 11 Mar 2020 15:51:24 +0800 Subject: [PATCH 07/32] fix issue in router group registering for controller --- net/ghttp/ghttp_server_router_group.go | 12 ++++++++-- net/ghttp/ghttp_unit_router_group_test.go | 27 +++++++++++++++++++++-- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/net/ghttp/ghttp_server_router_group.go b/net/ghttp/ghttp_server_router_group.go index 9ffb893a3..64915eedd 100644 --- a/net/ghttp/ghttp_server_router_group.go +++ b/net/ghttp/ghttp_server_router_group.go @@ -293,9 +293,17 @@ func (g *RouterGroup) doBind(bindType string, pattern string, object interface{} } else if g.isController(object) { if len(extras) > 0 { if g.server != nil { - g.server.doBindControllerMethod(pattern, object.(Controller), extras[0], g.middleware) + if gstr.Contains(extras[0], ",") { + g.server.doBindController(pattern, object.(Controller), extras[0], g.middleware) + } else { + g.server.doBindControllerMethod(pattern, object.(Controller), extras[0], g.middleware) + } } else { - g.domain.doBindControllerMethod(pattern, object.(Controller), extras[0], g.middleware) + if gstr.Contains(extras[0], ",") { + g.domain.doBindController(pattern, object.(Controller), extras[0], g.middleware) + } else { + g.domain.doBindControllerMethod(pattern, object.(Controller), extras[0], g.middleware) + } } } else { if g.server != nil { diff --git a/net/ghttp/ghttp_unit_router_group_test.go b/net/ghttp/ghttp_unit_router_group_test.go index b38f95c65..d05a98697 100644 --- a/net/ghttp/ghttp_unit_router_group_test.go +++ b/net/ghttp/ghttp_unit_router_group_test.go @@ -4,7 +4,6 @@ // If a copy of the MIT was not distributed with this file, // You can obtain one at https://github.com/gogf/gf. -// 分组路由测试 package ghttp_test import ( @@ -118,7 +117,7 @@ func Test_Router_GroupBasic1(t *testing.T) { }) } -func Test_Router_Basic2(t *testing.T) { +func Test_Router_GroupBasic2(t *testing.T) { p := ports.PopRand() s := g.Server(p) obj := new(GroupObject) @@ -191,3 +190,27 @@ func Test_Router_GroupBuildInVar(t *testing.T) { gtest.Assert(client.DeleteContent("/api/ThisDoesNotExist"), "Not Found") }) } + +func Test_Router_Group_Mthods(t *testing.T) { + p := ports.PopRand() + s := g.Server(p) + obj := new(GroupObject) + ctl := new(GroupController) + group := s.Group("/") + group.ALL("/obj", obj, "Show, Delete") + group.ALL("/ctl", ctl, "Show, Post") + s.SetPort(p) + s.SetDumpRouterMap(false) + s.Start() + defer s.Shutdown() + + time.Sleep(100 * time.Millisecond) + gtest.Case(t, func() { + client := ghttp.NewClient() + client.SetPrefix(fmt.Sprintf("http://127.0.0.1:%d", p)) + gtest.Assert(client.GetContent("/ctl/show"), "1Controller Show2") + gtest.Assert(client.GetContent("/ctl/post"), "1Controller Post2") + gtest.Assert(client.GetContent("/obj/show"), "1Object Show2") + gtest.Assert(client.GetContent("/obj/delete"), "1Object Delete2") + }) +} From c1cce1793483f7ba0b3d7d557fb1c1af2bf09c10 Mon Sep 17 00:00:00 2001 From: John Date: Wed, 11 Mar 2020 16:08:17 +0800 Subject: [PATCH 08/32] add unit testing cases for gudp --- net/gudp/gudp_conn.go | 2 +- net/gudp/gudp_unit_basic_test.go | 73 ++++++++++++++++++++++++++++++++ net/gudp/gudp_unit_init_test.go | 21 +++++++++ 3 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 net/gudp/gudp_unit_basic_test.go create mode 100644 net/gudp/gudp_unit_init_test.go diff --git a/net/gudp/gudp_conn.go b/net/gudp/gudp_conn.go index 31b1534a6..c9f1a9f3f 100644 --- a/net/gudp/gudp_conn.go +++ b/net/gudp/gudp_conn.go @@ -23,7 +23,7 @@ type Conn struct { const ( gDEFAULT_RETRY_INTERVAL = 100 * time.Millisecond // Retry interval. - gDEFAULT_READ_BUFFER_SIZE = 64 // (KB)Buffer size. + gDEFAULT_READ_BUFFER_SIZE = 64 // (Byte)Buffer size. gRECV_ALL_WAIT_TIMEOUT = time.Millisecond // Default interval for reading buffer. ) diff --git a/net/gudp/gudp_unit_basic_test.go b/net/gudp/gudp_unit_basic_test.go new file mode 100644 index 000000000..4b8448de1 --- /dev/null +++ b/net/gudp/gudp_unit_basic_test.go @@ -0,0 +1,73 @@ +// Copyright 2019 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gudp_test + +import ( + "fmt" + "github.com/gogf/gf/net/gudp" + "github.com/gogf/gf/os/glog" + "github.com/gogf/gf/test/gtest" + "github.com/gogf/gf/util/gconv" + "testing" + "time" +) + +func Test_Basic(t *testing.T) { + p := ports.PopRand() + s := gudp.NewServer(fmt.Sprintf("127.0.0.1:%d", p), func(conn *gudp.Conn) { + defer conn.Close() + for { + data, err := conn.Recv(-1) + if len(data) > 0 { + if err := conn.Send(append([]byte("> "), data...)); err != nil { + glog.Error(err) + } + } + if err != nil { + glog.Error(err) + } + } + }) + go s.Run() + defer s.Close() + time.Sleep(100 * time.Millisecond) + // gudp.Conn.Send + gtest.Case(t, func() { + for i := 0; i < 100; i++ { + conn, err := gudp.NewConn(fmt.Sprintf("127.0.0.1:%d", p)) + gtest.Assert(err, nil) + gtest.Assert(conn.Send([]byte(gconv.String(i))), nil) + conn.Close() + } + }) + // gudp.Conn.SendRecv + gtest.Case(t, func() { + for i := 0; i < 100; i++ { + conn, err := gudp.NewConn(fmt.Sprintf("127.0.0.1:%d", p)) + gtest.Assert(err, nil) + result, err := conn.SendRecv([]byte(gconv.String(i)), -1) + gtest.Assert(err, nil) + gtest.Assert(string(result), fmt.Sprintf(`> %d`, i)) + conn.Close() + } + }) + // gudp.Send + gtest.Case(t, func() { + for i := 0; i < 100; i++ { + err := gudp.Send(fmt.Sprintf("127.0.0.1:%d", p), []byte(gconv.String(i))) + gtest.Assert(err, nil) + } + }) + // gudp.SendRecv + gtest.Case(t, func() { + for i := 0; i < 100; i++ { + result, err := gudp.SendRecv(fmt.Sprintf("127.0.0.1:%d", p), []byte(gconv.String(i)), -1) + gtest.Assert(err, nil) + gtest.Assert(string(result), fmt.Sprintf(`> %d`, i)) + } + }) +} diff --git a/net/gudp/gudp_unit_init_test.go b/net/gudp/gudp_unit_init_test.go new file mode 100644 index 000000000..bc5815922 --- /dev/null +++ b/net/gudp/gudp_unit_init_test.go @@ -0,0 +1,21 @@ +// Copyright 2017 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gudp_test + +import ( + "github.com/gogf/gf/container/garray" +) + +var ( + ports = garray.NewIntArray(true) +) + +func init() { + for i := 9000; i <= 10000; i++ { + ports.Append(i) + } +} From 11f0317e92a4b31bac71577070bb6d7686afc908 Mon Sep 17 00:00:00 2001 From: John Date: Wed, 11 Mar 2020 23:17:41 +0800 Subject: [PATCH 09/32] add Array feature for package gdb --- database/gdb/gdb.go | 1 + database/gdb/gdb_core.go | 10 ++++++ database/gdb/gdb_func.go | 11 ++++-- database/gdb/gdb_model.go | 36 +++++++++++++++++++ database/gdb/gdb_type_result.go | 28 +++++++++++++-- database/gdb/gdb_unit_z_mysql_model_test.go | 26 ++++++++++++++ internal/utils/utils.go | 8 +++++ internal/utils/utils_array.go | 26 ++++++++++++++ .../utilstr.go => utils/utils_str.go} | 3 +- text/gstr/gstr.go | 12 +++---- util/gconv/gconv_map.go | 4 +-- util/gconv/gconv_slice_any.go | 4 +-- util/gconv/gconv_struct.go | 6 ++-- util/gconv/gconv_time.go | 6 ++-- 14 files changed, 158 insertions(+), 23 deletions(-) create mode 100644 internal/utils/utils.go create mode 100644 internal/utils/utils_array.go rename internal/{utilstr/utilstr.go => utils/utils_str.go} (94%) diff --git a/database/gdb/gdb.go b/database/gdb/gdb.go index f2241d756..a37fccdf4 100644 --- a/database/gdb/gdb.go +++ b/database/gdb/gdb.go @@ -47,6 +47,7 @@ type DB interface { GetAll(query string, args ...interface{}) (Result, error) GetOne(query string, args ...interface{}) (Record, error) GetValue(query string, args ...interface{}) (Value, error) + GetArray(query string, args ...interface{}) ([]Value, error) GetCount(query string, args ...interface{}) (int, error) GetStruct(objPointer interface{}, query string, args ...interface{}) error GetStructs(objPointerSlice interface{}, query string, args ...interface{}) error diff --git a/database/gdb/gdb_core.go b/database/gdb/gdb_core.go index 077684a54..4e8cb6c39 100644 --- a/database/gdb/gdb_core.go +++ b/database/gdb/gdb_core.go @@ -177,6 +177,16 @@ func (c *Core) GetOne(query string, args ...interface{}) (Record, error) { return nil, nil } +// GetArray queries and returns data values as slice from database. +// Note that if there're multiple columns in the result, it returns just one column values randomly. +func (c *Core) GetArray(query string, args ...interface{}) ([]Value, error) { + all, err := c.DB.DoGetAll(nil, query, args...) + if err != nil { + return nil, err + } + return all.Array(), nil +} + // GetStruct queries one record from database and converts it to given struct. // The parameter should be a pointer to struct. func (c *Core) GetStruct(pointer interface{}, query string, args ...interface{}) error { diff --git a/database/gdb/gdb_func.go b/database/gdb/gdb_func.go index f60681580..b58f932e0 100644 --- a/database/gdb/gdb_func.go +++ b/database/gdb/gdb_func.go @@ -12,6 +12,7 @@ import ( "errors" "fmt" "github.com/gogf/gf/internal/empty" + "github.com/gogf/gf/internal/utils" "github.com/gogf/gf/os/gtime" "reflect" "regexp" @@ -282,12 +283,18 @@ func formatWhere(db DB, where interface{}, args []interface{}, omitEmpty bool) ( newArgs = append(newArgs, args...) newWhere = buffer.String() if len(newArgs) > 0 { - // It supports formats like: Where/And/Or("uid", 1) , Where/And/Or("uid>=", 1) if gstr.Pos(newWhere, "?") == -1 { if lastOperatorReg.MatchString(newWhere) { + // Eg: Where/And/Or("uid>=", 1) newWhere += "?" } else if gregex.IsMatchString(`^[\w\.\-]+$`, newWhere) { - newWhere += "=?" + if len(newArgs) == 1 && utils.IsArray(newArgs[0]) { + // Eg: Where("id", []int{1,2,3}) + newWhere += " IN (?)" + } else { + // Eg: Where/And/Or("uid", 1) + newWhere += "=?" + } } } } diff --git a/database/gdb/gdb_model.go b/database/gdb/gdb_model.go index 8aac0948d..730483060 100644 --- a/database/gdb/gdb_model.go +++ b/database/gdb/gdb_model.go @@ -804,6 +804,29 @@ func (m *Model) Value(fieldsAndWhere ...interface{}) (Value, error) { return nil, nil } +// Array queries and returns data values as slice from database. +// Note that if there're multiple columns in the result, it returns just one column values randomly. +// +// If the optional parameter is given, the fieldsAndWhere[0] is the selected fields +// and fieldsAndWhere[1:] is treated as where condition fields. +// Also see Model.Fields and Model.Where functions. +func (m *Model) Array(fieldsAndWhere ...interface{}) ([]Value, error) { + if len(fieldsAndWhere) > 0 { + if len(fieldsAndWhere) > 2 { + return m.Fields(gconv.String(fieldsAndWhere[0])).Where(fieldsAndWhere[1], fieldsAndWhere[2:]...).Array() + } else if len(fieldsAndWhere) == 2 { + return m.Fields(gconv.String(fieldsAndWhere[0])).Where(fieldsAndWhere[1]).Array() + } else { + return m.Fields(gconv.String(fieldsAndWhere[0])).Array() + } + } + all, err := m.All() + if err != nil { + return nil, err + } + return all.Array(), nil +} + // Struct retrieves one record from table and converts it into given struct. // The parameter should be type of *struct/**struct. If type **struct is given, // it can create the struct internally during converting. @@ -954,6 +977,19 @@ func (m *Model) FindValue(fieldsAndWhere ...interface{}) (Value, error) { return m.Value() } +// FindArray queries and returns data values as slice from database. +// Note that if there're multiple columns in the result, it returns just one column values randomly. +// Also see Model.WherePri and Model.Value. +func (m *Model) FindArray(fieldsAndWhere ...interface{}) ([]Value, error) { + if len(fieldsAndWhere) >= 2 { + return m.WherePri(fieldsAndWhere[1], fieldsAndWhere[2:]...).Fields(gconv.String(fieldsAndWhere[0])).Array() + } + if len(fieldsAndWhere) == 1 { + return m.Fields(gconv.String(fieldsAndWhere[0])).Array() + } + return m.Array() +} + // FindCount retrieves and returns the record number by Model.WherePri and Model.Count. // Also see Model.WherePri and Model.Count. func (m *Model) FindCount(where ...interface{}) (int, error) { diff --git a/database/gdb/gdb_type_result.go b/database/gdb/gdb_type_result.go index b10d00b83..b73711a87 100644 --- a/database/gdb/gdb_type_result.go +++ b/database/gdb/gdb_type_result.go @@ -28,11 +28,33 @@ func (r Result) Xml(rootTag ...string) string { // List converts to a List. func (r Result) List() List { - l := make(List, len(r)) + list := make(List, len(r)) for k, v := range r { - l[k] = v.Map() + list[k] = v.Map() } - return l + return list +} + +// Array retrieves and returns specified column values as slice. +// The parameter is optional is the column field is only one. +func (r Result) Array(field ...string) []Value { + array := make([]Value, len(r)) + if len(r) == 0 { + return array + } + key := "" + if len(field) > 0 && field[0] != "" { + key = field[0] + } else { + for k, _ := range r[0] { + key = k + break + } + } + for k, v := range r { + array[k] = v[key] + } + return array } // MapKeyStr converts to a map[string]Map of which key is specified by . diff --git a/database/gdb/gdb_unit_z_mysql_model_test.go b/database/gdb/gdb_unit_z_mysql_model_test.go index f25fd1d39..7b0b16360 100644 --- a/database/gdb/gdb_unit_z_mysql_model_test.go +++ b/database/gdb/gdb_unit_z_mysql_model_test.go @@ -534,6 +534,32 @@ func Test_Model_Value(t *testing.T) { }) } +func Test_Model_Array(t *testing.T) { + table := createInitTable() + defer dropTable(table) + + gtest.Case(t, func() { + array, err := db.Table(table).Fields("nickname").Where("id", g.Slice{1, 2, 3}).Array() + gtest.Assert(err, nil) + gtest.Assert(array, g.Slice{"name_1", "name_2", "name_3"}) + }) + gtest.Case(t, func() { + array, err := db.Table(table).Array("nickname", "id", g.Slice{1, 2, 3}) + gtest.Assert(err, nil) + gtest.Assert(array, g.Slice{"name_1", "name_2", "name_3"}) + }) + gtest.Case(t, func() { + array, err := db.Table(table).FindArray("nickname", "id", g.Slice{1, 2, 3}) + gtest.Assert(err, nil) + gtest.Assert(array, g.Slice{"name_1", "name_2", "name_3"}) + }) + gtest.Case(t, func() { + array, err := db.Table(table).FindArray("nickname", g.Slice{1, 2, 3}) + gtest.Assert(err, nil) + gtest.Assert(array, g.Slice{"name_1", "name_2", "name_3"}) + }) +} + func Test_Model_FindValue(t *testing.T) { table := createInitTable() defer dropTable(table) diff --git a/internal/utils/utils.go b/internal/utils/utils.go new file mode 100644 index 000000000..05d3fa6af --- /dev/null +++ b/internal/utils/utils.go @@ -0,0 +1,8 @@ +// Copyright 2019 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +// Package utils provides some utility functions for internal usage. +package utils diff --git a/internal/utils/utils_array.go b/internal/utils/utils_array.go new file mode 100644 index 000000000..eae9b96b6 --- /dev/null +++ b/internal/utils/utils_array.go @@ -0,0 +1,26 @@ +// Copyright 2019 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package utils + +import "reflect" + +// IsArray checks whether given value is array/slice. +// Note that it uses reflect internally implementing this feature. +func IsArray(value interface{}) bool { + rv := reflect.ValueOf(value) + kind := rv.Kind() + if kind == reflect.Ptr { + rv = rv.Elem() + kind = rv.Kind() + } + switch kind { + case reflect.Array, reflect.Slice: + return true + default: + return false + } +} diff --git a/internal/utilstr/utilstr.go b/internal/utils/utils_str.go similarity index 94% rename from internal/utilstr/utilstr.go rename to internal/utils/utils_str.go index 8c80c08b6..ff56aa0e0 100644 --- a/internal/utilstr/utilstr.go +++ b/internal/utils/utils_str.go @@ -4,8 +4,7 @@ // If a copy of the MIT was not distributed with this file, // You can obtain one at https://github.com/gogf/gf. -// Package utilstr provides some string functions for internal usage. -package utilstr +package utils import "strings" diff --git a/text/gstr/gstr.go b/text/gstr/gstr.go index 27b61f6d5..e225f423a 100644 --- a/text/gstr/gstr.go +++ b/text/gstr/gstr.go @@ -16,7 +16,7 @@ import ( "unicode" "unicode/utf8" - "github.com/gogf/gf/internal/utilstr" + "github.com/gogf/gf/internal/utils" "github.com/gogf/gf/util/gconv" @@ -98,7 +98,7 @@ func ReplaceIByArray(origin string, array []string) string { // ReplaceByMap returns a copy of , // which is replaced by a map in unordered way, case-sensitively. func ReplaceByMap(origin string, replaces map[string]string) string { - return utilstr.ReplaceByMap(origin, replaces) + return utils.ReplaceByMap(origin, replaces) } // ReplaceIByMap returns a copy of , @@ -122,7 +122,7 @@ func ToUpper(s string) string { // UcFirst returns a copy of the string s with the first letter mapped to its upper case. func UcFirst(s string) string { - return utilstr.UcFirst(s) + return utils.UcFirst(s) } // LcFirst returns a copy of the string s with the first letter mapped to its lower case. @@ -143,17 +143,17 @@ func UcWords(str string) string { // IsLetterLower tests whether the given byte b is in lower case. func IsLetterLower(b byte) bool { - return utilstr.IsLetterLower(b) + return utils.IsLetterLower(b) } // IsLetterUpper tests whether the given byte b is in upper case. func IsLetterUpper(b byte) bool { - return utilstr.IsLetterUpper(b) + return utils.IsLetterUpper(b) } // IsNumeric tests whether the given string s is numeric. func IsNumeric(s string) bool { - return utilstr.IsNumeric(s) + return utils.IsNumeric(s) } // SubStr returns a portion of string specified by the and parameters. diff --git a/util/gconv/gconv_map.go b/util/gconv/gconv_map.go index af23daef6..663b0d65f 100644 --- a/util/gconv/gconv_map.go +++ b/util/gconv/gconv_map.go @@ -13,7 +13,7 @@ import ( "strings" "github.com/gogf/gf/internal/empty" - "github.com/gogf/gf/internal/utilstr" + "github.com/gogf/gf/internal/utils" ) // apiMapStrAny is the interface support for converting struct parameter to map. @@ -176,7 +176,7 @@ func doMapConvert(value interface{}, recursive bool, tags ...string) map[string] rvField = rv.Field(i) // Only convert the public attributes. fieldName := rtField.Name - if !utilstr.IsLetterUpper(fieldName[0]) { + if !utils.IsLetterUpper(fieldName[0]) { continue } name = "" diff --git a/util/gconv/gconv_slice_any.go b/util/gconv/gconv_slice_any.go index 1bce42c75..dd52d09fb 100644 --- a/util/gconv/gconv_slice_any.go +++ b/util/gconv/gconv_slice_any.go @@ -7,7 +7,7 @@ package gconv import ( - "github.com/gogf/gf/internal/utilstr" + "github.com/gogf/gf/internal/utils" "reflect" ) @@ -121,7 +121,7 @@ func Interfaces(i interface{}) []interface{} { array = make([]interface{}, 0) for i := 0; i < rv.NumField(); i++ { // Only public attributes. - if !utilstr.IsLetterUpper(rt.Field(i).Name[0]) { + if !utils.IsLetterUpper(rt.Field(i).Name[0]) { continue } array = append(array, rv.Field(i).Interface()) diff --git a/util/gconv/gconv_struct.go b/util/gconv/gconv_struct.go index 46fdeea40..7f57f0735 100644 --- a/util/gconv/gconv_struct.go +++ b/util/gconv/gconv_struct.go @@ -15,7 +15,7 @@ import ( "strings" "github.com/gogf/gf/internal/structs" - "github.com/gogf/gf/internal/utilstr" + "github.com/gogf/gf/internal/utils" ) // apiUnmarshalValue is the interface for custom defined types customizing value assignment. @@ -122,7 +122,7 @@ func Struct(params interface{}, pointer interface{}, mapping ...map[string]strin tempName := "" for i := 0; i < elem.NumField(); i++ { // Only do converting to public attributes. - if !utilstr.IsLetterUpper(elemType.Field(i).Name[0]) { + if !utils.IsLetterUpper(elemType.Field(i).Name[0]) { continue } tempName = elemType.Field(i).Name @@ -193,7 +193,7 @@ func StructDeep(params interface{}, pointer interface{}, mapping ...map[string]s rt := rv.Type() for i := 0; i < rv.NumField(); i++ { // Only do converting to public attributes. - if !utilstr.IsLetterUpper(rt.Field(i).Name[0]) { + if !utils.IsLetterUpper(rt.Field(i).Name[0]) { continue } trv := rv.Field(i) diff --git a/util/gconv/gconv_time.go b/util/gconv/gconv_time.go index ecba490d1..faa25492b 100644 --- a/util/gconv/gconv_time.go +++ b/util/gconv/gconv_time.go @@ -9,7 +9,7 @@ package gconv import ( "time" - "github.com/gogf/gf/internal/utilstr" + "github.com/gogf/gf/internal/utils" "github.com/gogf/gf/os/gtime" ) @@ -26,7 +26,7 @@ func Time(i interface{}, format ...string) time.Time { // If is numeric, then it converts as nanoseconds. func Duration(i interface{}) time.Duration { s := String(i) - if !utilstr.IsNumeric(s) { + if !utils.IsNumeric(s) { d, _ := time.ParseDuration(s) return d } @@ -50,7 +50,7 @@ func GTime(i interface{}, format ...string) *gtime.Time { t, _ := gtime.StrToTimeFormat(s, format[0]) return t } - if utilstr.IsNumeric(s) { + if utils.IsNumeric(s) { return gtime.NewFromTimeStamp(Int64(s)) } else { t, _ := gtime.StrToTime(s) From 13ab139afc19a4a28e3f8069c44ca6192b82f810 Mon Sep 17 00:00:00 2001 From: John Date: Wed, 11 Mar 2020 23:24:19 +0800 Subject: [PATCH 10/32] add more unit testing case for package gdb --- database/gdb/gdb_unit_z_mysql_model_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/database/gdb/gdb_unit_z_mysql_model_test.go b/database/gdb/gdb_unit_z_mysql_model_test.go index 7b0b16360..90a673732 100644 --- a/database/gdb/gdb_unit_z_mysql_model_test.go +++ b/database/gdb/gdb_unit_z_mysql_model_test.go @@ -538,6 +538,12 @@ func Test_Model_Array(t *testing.T) { table := createInitTable() defer dropTable(table) + gtest.Case(t, func() { + all, err := db.Table(table).Where("id", g.Slice{1, 2, 3}).All() + gtest.Assert(err, nil) + gtest.Assert(all.Array("id"), g.Slice{1, 2, 3}) + gtest.Assert(all.Array("nickname"), g.Slice{"name_1", "name_2", "name_3"}) + }) gtest.Case(t, func() { array, err := db.Table(table).Fields("nickname").Where("id", g.Slice{1, 2, 3}).Array() gtest.Assert(err, nil) From f44c19868cf1c4d453bfe838c600f63e2ac073b0 Mon Sep 17 00:00:00 2001 From: John Date: Wed, 11 Mar 2020 23:54:35 +0800 Subject: [PATCH 11/32] improve package gudp --- net/gudp/gudp_conn.go | 67 +++++++------------------------- net/gudp/gudp_func.go | 11 ------ net/gudp/gudp_unit_basic_test.go | 35 ++++++++++++++++- 3 files changed, 48 insertions(+), 65 deletions(-) diff --git a/net/gudp/gudp_conn.go b/net/gudp/gudp_conn.go index c9f1a9f3f..059775413 100644 --- a/net/gudp/gudp_conn.go +++ b/net/gudp/gudp_conn.go @@ -23,7 +23,7 @@ type Conn struct { const ( gDEFAULT_RETRY_INTERVAL = 100 * time.Millisecond // Retry interval. - gDEFAULT_READ_BUFFER_SIZE = 64 // (Byte)Buffer size. + gDEFAULT_READ_BUFFER_SIZE = 1024 // (Byte)Buffer size. gRECV_ALL_WAIT_TIMEOUT = time.Millisecond // Default interval for reading buffer. ) @@ -83,68 +83,32 @@ func (c *Conn) Send(data []byte, retry ...Retry) (err error) { } // Recv receives data from remote address. +// The parameter is used for customizing the receiving buffer size. The default +// buffer size is 1024 byte. // -// Note that, -// 1. There's package border in UDP protocol, so we can receive a complete package if it specifies length < 0. -// 2. If length = 0, it means it receives the data from current buffer and returns immediately. -// 3. If length > 0, it means it blocks reading data from connection until length size was received. -func (c *Conn) Recv(length int, retry ...Retry) ([]byte, error) { +// There's package border in UDP protocol, we can receive a complete package if specified +// buffer size is big enough. VERY NOTE that we should receive the complete package in once +// or else the leftover package data would be dropped. +func (c *Conn) Recv(buffer int, retry ...Retry) ([]byte, error) { var err error // Reading error. var size int // Reading size. - var index int // Received size. - var buffer []byte // Buffer object. - var bufferWait bool // Whether buffer reading timeout set. + var data []byte // Buffer object. var remoteAddr *net.UDPAddr // Current remote address for reading. - - if length > 0 { - buffer = make([]byte, length) + if buffer > 0 { + data = make([]byte, buffer) } else { - buffer = make([]byte, gDEFAULT_READ_BUFFER_SIZE) + data = make([]byte, gDEFAULT_READ_BUFFER_SIZE) } - for { - if length < 0 && index > 0 { - bufferWait = true - if err = c.SetReadDeadline(time.Now().Add(c.recvBufferWait)); err != nil { - return nil, err - } - } - size, remoteAddr, err = c.ReadFromUDP(buffer[index:]) + size, remoteAddr, err = c.ReadFromUDP(data) if err == nil { c.remoteAddr = remoteAddr } - if size > 0 { - index += size - if length > 0 { - // It reads til size if is specified. - if index == length { - break - } - } else { - if index >= gDEFAULT_READ_BUFFER_SIZE { - // If it exceeds the buffer size, it then automatically increases its buffer size. - buffer = append(buffer, make([]byte, gDEFAULT_READ_BUFFER_SIZE)...) - } else { - // It returns immediately if received size is lesser than buffer size. - if !bufferWait { - break - } - } - } - } if err != nil { // Connection closed. if err == io.EOF { break } - // Re-set the timeout when reading data. - if bufferWait && isTimeout(err) { - if err = c.SetReadDeadline(c.recvDeadline); err != nil { - return nil, err - } - err = nil - break - } if len(retry) > 0 { // It fails even it retried. if retry[0].Count == 0 { @@ -159,12 +123,9 @@ func (c *Conn) Recv(length int, retry ...Retry) ([]byte, error) { } break } - // Just read once from buffer. - if length == 0 { - break - } + break } - return buffer[:index], err + return data[:size], err } // SendRecv writes data to connection and blocks reading response. diff --git a/net/gudp/gudp_func.go b/net/gudp/gudp_func.go index 0e8042c8c..a4d08fe6d 100644 --- a/net/gudp/gudp_func.go +++ b/net/gudp/gudp_func.go @@ -52,14 +52,3 @@ func SendRecv(address string, data []byte, receive int, retry ...Retry) ([]byte, defer conn.Close() return conn.SendRecv(data, receive, retry...) } - -// isTimeout checks whether given is a timeout error. -func isTimeout(err error) bool { - if err == nil { - return false - } - if netErr, ok := err.(net.Error); ok && netErr.Timeout() { - return true - } - return false -} diff --git a/net/gudp/gudp_unit_basic_test.go b/net/gudp/gudp_unit_basic_test.go index 4b8448de1..956d36702 100644 --- a/net/gudp/gudp_unit_basic_test.go +++ b/net/gudp/gudp_unit_basic_test.go @@ -28,7 +28,7 @@ func Test_Basic(t *testing.T) { } } if err != nil { - glog.Error(err) + break } } }) @@ -71,3 +71,36 @@ func Test_Basic(t *testing.T) { } }) } + +// If the read buffer size is less than the sent package size, +// the rest data would be dropped. +func Test_Buffer(t *testing.T) { + p := ports.PopRand() + s := gudp.NewServer(fmt.Sprintf("127.0.0.1:%d", p), func(conn *gudp.Conn) { + defer conn.Close() + for { + data, err := conn.Recv(1) + if len(data) > 0 { + if err := conn.Send(data); err != nil { + glog.Error(err) + } + } + if err != nil { + break + } + } + }) + go s.Run() + defer s.Close() + time.Sleep(100 * time.Millisecond) + gtest.Case(t, func() { + result, err := gudp.SendRecv(fmt.Sprintf("127.0.0.1:%d", p), []byte("123"), -1) + gtest.Assert(err, nil) + gtest.Assert(string(result), "1") + }) + gtest.Case(t, func() { + result, err := gudp.SendRecv(fmt.Sprintf("127.0.0.1:%d", p), []byte("456"), -1) + gtest.Assert(err, nil) + gtest.Assert(string(result), "4") + }) +} From 707b08c58565753e7cd81ec3aa7a7e21867ba54d Mon Sep 17 00:00:00 2001 From: John Date: Wed, 11 Mar 2020 23:56:58 +0800 Subject: [PATCH 12/32] improve package gudp --- net/gudp/gudp_conn.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/net/gudp/gudp_conn.go b/net/gudp/gudp_conn.go index 059775413..d6d77474d 100644 --- a/net/gudp/gudp_conn.go +++ b/net/gudp/gudp_conn.go @@ -83,8 +83,8 @@ func (c *Conn) Send(data []byte, retry ...Retry) (err error) { } // Recv receives data from remote address. -// The parameter is used for customizing the receiving buffer size. The default -// buffer size is 1024 byte. +// The parameter is used for customizing the receiving buffer size. If <= 0, +// it uses the default buffer size, which is 1024 byte. // // There's package border in UDP protocol, we can receive a complete package if specified // buffer size is big enough. VERY NOTE that we should receive the complete package in once From 99f1d9d0ed76a5fa96b2bc0e4c9bb9dfced57557 Mon Sep 17 00:00:00 2001 From: John Date: Wed, 11 Mar 2020 23:59:43 +0800 Subject: [PATCH 13/32] improve package gudp --- net/gtcp/gtcp_conn.go | 2 +- net/gudp/gudp_conn.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/net/gtcp/gtcp_conn.go b/net/gtcp/gtcp_conn.go index aedc235cf..673c34658 100644 --- a/net/gtcp/gtcp_conn.go +++ b/net/gtcp/gtcp_conn.go @@ -94,7 +94,7 @@ func (c *Conn) Send(data []byte, retry ...Retry) error { } } -// Recv receives data from the connection. +// Recv receives and returns data from the connection. // // Note that, // 1. If length = 0, which means it receives the data from current buffer and returns immediately. diff --git a/net/gudp/gudp_conn.go b/net/gudp/gudp_conn.go index d6d77474d..d9fdf317f 100644 --- a/net/gudp/gudp_conn.go +++ b/net/gudp/gudp_conn.go @@ -82,7 +82,7 @@ func (c *Conn) Send(data []byte, retry ...Retry) (err error) { } } -// Recv receives data from remote address. +// Recv receives and returns data from remote address. // The parameter is used for customizing the receiving buffer size. If <= 0, // it uses the default buffer size, which is 1024 byte. // From b702d98700ce0f3847e601a57181c0914705fa96 Mon Sep 17 00:00:00 2001 From: John Date: Thu, 12 Mar 2020 10:05:38 +0800 Subject: [PATCH 14/32] version updates --- version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.go b/version.go index 97cfb7fbc..504db0668 100644 --- a/version.go +++ b/version.go @@ -1,4 +1,4 @@ package gf -const VERSION = "v1.11.5" +const VERSION = "v1.11.6" const AUTHORS = "john" From e513cd10ede029859ebca1ad7360d842aaa09baa Mon Sep 17 00:00:00 2001 From: John Date: Fri, 13 Mar 2020 17:21:30 +0800 Subject: [PATCH 15/32] add Lock* functions for gdb.Model and improve Data/Where functions for gdb.Model --- database/gdb/gdb.go | 2 +- database/gdb/gdb_batch_result.go | 25 - database/gdb/gdb_core.go | 6 +- database/gdb/gdb_driver_oracle.go | 14 +- database/gdb/gdb_func.go | 58 +- database/gdb/gdb_model.go | 1041 +------------------ database/gdb/gdb_model_cache.go | 36 + database/gdb/gdb_model_condition.go | 170 +++ database/gdb/gdb_model_delete.go | 27 + database/gdb/gdb_model_fields.go | 94 ++ database/gdb/gdb_model_insert.go | 213 ++++ database/gdb/gdb_model_join.go | 30 + database/gdb/gdb_model_lock.go | 21 + database/gdb/gdb_model_option.go | 27 + database/gdb/gdb_model_select.go | 315 ++++++ database/gdb/gdb_model_update.go | 45 + database/gdb/gdb_model_utility.go | 205 ++++ database/gdb/gdb_result.go | 47 + database/gdb/gdb_unit_z_mysql_model_test.go | 50 + frame/gmvc/model.go | 10 +- internal/empty/empty.go | 14 +- 21 files changed, 1352 insertions(+), 1098 deletions(-) delete mode 100644 database/gdb/gdb_batch_result.go create mode 100644 database/gdb/gdb_model_cache.go create mode 100644 database/gdb/gdb_model_condition.go create mode 100644 database/gdb/gdb_model_delete.go create mode 100644 database/gdb/gdb_model_fields.go create mode 100644 database/gdb/gdb_model_insert.go create mode 100644 database/gdb/gdb_model_join.go create mode 100644 database/gdb/gdb_model_lock.go create mode 100644 database/gdb/gdb_model_option.go create mode 100644 database/gdb/gdb_model_select.go create mode 100644 database/gdb/gdb_model_update.go create mode 100644 database/gdb/gdb_model_utility.go create mode 100644 database/gdb/gdb_result.go diff --git a/database/gdb/gdb.go b/database/gdb/gdb.go index a37fccdf4..40280f893 100644 --- a/database/gdb/gdb.go +++ b/database/gdb/gdb.go @@ -364,7 +364,7 @@ func (c *Core) getSqlDb(master bool, schema ...string) (sqlDb *sql.DB, err error n.Name = nodeSchema node = &n } - // Cache the underlying connection object by node. + // Cache the underlying connection pool object by node. v := c.cache.GetOrSetFuncLock(node.String(), func() interface{} { sqlDb, err = c.DB.Open(node) if err != nil { diff --git a/database/gdb/gdb_batch_result.go b/database/gdb/gdb_batch_result.go deleted file mode 100644 index 256716edc..000000000 --- a/database/gdb/gdb_batch_result.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2019 gf Author(https://github.com/gogf/gf). All Rights Reserved. -// -// This Source Code Form is subject to the terms of the MIT License. -// If a copy of the MIT was not distributed with this file, -// You can obtain one at https://github.com/gogf/gf. - -package gdb - -import "database/sql" - -// batchSqlResult is execution result for batch operations. -type batchSqlResult struct { - rowsAffected int64 - lastResult sql.Result -} - -// see sql.Result.RowsAffected -func (r *batchSqlResult) RowsAffected() (int64, error) { - return r.rowsAffected, nil -} - -// see sql.Result.LastInsertId -func (r *batchSqlResult) LastInsertId() (int64, error) { - return r.lastResult.LastInsertId() -} diff --git a/database/gdb/gdb_core.go b/database/gdb/gdb_core.go index 4e8cb6c39..92e2ead29 100644 --- a/database/gdb/gdb_core.go +++ b/database/gdb/gdb_core.go @@ -504,7 +504,7 @@ func (c *Core) DoBatchInsert(link Link, table string, list interface{}, option i holders = append(holders, "?") } // Prepare the batch result pointer. - batchResult := new(batchSqlResult) + batchResult := new(SqlResult) charL, charR := c.DB.GetChars() keysStr := charL + strings.Join(keys, charR+","+charL) + charR valueHolderStr := "(" + strings.Join(holders, ",") + ")" @@ -555,8 +555,8 @@ func (c *Core) DoBatchInsert(link Link, table string, list interface{}, option i if n, err := r.RowsAffected(); err != nil { return r, err } else { - batchResult.lastResult = r - batchResult.rowsAffected += n + batchResult.result = r + batchResult.affected += n } params = params[:0] values = values[:0] diff --git a/database/gdb/gdb_driver_oracle.go b/database/gdb/gdb_driver_oracle.go index fbd48c339..374abffc3 100644 --- a/database/gdb/gdb_driver_oracle.go +++ b/database/gdb/gdb_driver_oracle.go @@ -377,7 +377,7 @@ func (d *DriverOracle) DoBatchInsert(link Link, table string, list interface{}, keys = append(keys, k) holders = append(holders, "?") } - batchResult := new(batchSqlResult) + batchResult := new(SqlResult) charL, charR := d.DB.GetChars() keyStr := charL + strings.Join(keys, charL+","+charR) + charR valueHolderStr := strings.Join(holders, ",") @@ -393,8 +393,8 @@ func (d *DriverOracle) DoBatchInsert(link Link, table string, list interface{}, if n, err := r.RowsAffected(); err != nil { return r, err } else { - batchResult.lastResult = r - batchResult.rowsAffected += n + batchResult.result = r + batchResult.affected += n } } return batchResult, nil @@ -421,8 +421,8 @@ func (d *DriverOracle) DoBatchInsert(link Link, table string, list interface{}, if n, err := r.RowsAffected(); err != nil { return r, err } else { - batchResult.lastResult = r - batchResult.rowsAffected += n + batchResult.result = r + batchResult.affected += n } params = params[:0] intoStr = intoStr[:0] @@ -437,8 +437,8 @@ func (d *DriverOracle) DoBatchInsert(link Link, table string, list interface{}, if n, err := r.RowsAffected(); err != nil { return r, err } else { - batchResult.lastResult = r - batchResult.rowsAffected += n + batchResult.result = r + batchResult.affected += n } } return batchResult, nil diff --git a/database/gdb/gdb_func.go b/database/gdb/gdb_func.go index b58f932e0..cead92cc0 100644 --- a/database/gdb/gdb_func.go +++ b/database/gdb/gdb_func.go @@ -288,12 +288,19 @@ func formatWhere(db DB, where interface{}, args []interface{}, omitEmpty bool) ( // Eg: Where/And/Or("uid>=", 1) newWhere += "?" } else if gregex.IsMatchString(`^[\w\.\-]+$`, newWhere) { - if len(newArgs) == 1 && utils.IsArray(newArgs[0]) { - // Eg: Where("id", []int{1,2,3}) - newWhere += " IN (?)" - } else { - // Eg: Where/And/Or("uid", 1) - newWhere += "=?" + newWhere = db.QuoteString(newWhere) + if len(newArgs) > 0 { + if utils.IsArray(newArgs[0]) { + // Eg: Where("id", []int{1,2,3}) + newWhere += " IN (?)" + } else if empty.IsNil(newArgs[0]) { + // Eg: Where("id", nil) + newWhere += " IS NULL" + newArgs = nil + } else { + // Eg: Where/And/Or("uid", 1) + newWhere += "=?" + } } } } @@ -323,7 +330,7 @@ func formatWhereInterfaces(db DB, where []interface{}, buffer *bytes.Buffer, new // formatWhereKeyValue handles each key-value pair of the parameter map. func formatWhereKeyValue(db DB, buffer *bytes.Buffer, newArgs []interface{}, key string, value interface{}) []interface{} { - key = db.QuoteWord(key) + quotedKey := db.QuoteWord(key) if buffer.Len() > 0 { buffer.WriteString(" AND ") } @@ -331,36 +338,43 @@ func formatWhereKeyValue(db DB, buffer *bytes.Buffer, newArgs []interface{}, key // the key string, it automatically adds '?' holder chars according to its arguments count // and converts it to "IN" statement. rv := reflect.ValueOf(value) - switch rv.Kind() { + kind := rv.Kind() + switch kind { case reflect.Slice, reflect.Array: - count := gstr.Count(key, "?") + count := gstr.Count(quotedKey, "?") if count == 0 { - buffer.WriteString(key + " IN(?)") + buffer.WriteString(quotedKey + " IN(?)") newArgs = append(newArgs, value) } else if count != rv.Len() { - buffer.WriteString(key) + buffer.WriteString(quotedKey) newArgs = append(newArgs, value) } else { - buffer.WriteString(key) + buffer.WriteString(quotedKey) newArgs = append(newArgs, gconv.Interfaces(value)...) } default: - if value == nil { - buffer.WriteString(key) + if value == nil || empty.IsNil(rv) { + if gregex.IsMatchString(`^[\w\.\-]+$`, key) { + // The key is a single field name. + buffer.WriteString(quotedKey + " IS NULL") + } else { + // The key may have operation chars. + buffer.WriteString(quotedKey) + } } else { // It also supports "LIKE" statement, which we considers it an operator. - key = gstr.Trim(key) - if gstr.Pos(key, "?") == -1 { + quotedKey = gstr.Trim(quotedKey) + if gstr.Pos(quotedKey, "?") == -1 { like := " like" - if len(key) > len(like) && gstr.Equal(key[len(key)-len(like):], like) { - buffer.WriteString(key + " ?") - } else if lastOperatorReg.MatchString(key) { - buffer.WriteString(key + " ?") + if len(quotedKey) > len(like) && gstr.Equal(quotedKey[len(quotedKey)-len(like):], like) { + buffer.WriteString(quotedKey + " ?") + } else if lastOperatorReg.MatchString(quotedKey) { + buffer.WriteString(quotedKey + " ?") } else { - buffer.WriteString(key + "=?") + buffer.WriteString(quotedKey + "=?") } } else { - buffer.WriteString(key) + buffer.WriteString(quotedKey) } newArgs = append(newArgs, value) } diff --git a/database/gdb/gdb_model.go b/database/gdb/gdb_model.go index 730483060..0e0f76084 100644 --- a/database/gdb/gdb_model.go +++ b/database/gdb/gdb_model.go @@ -7,18 +7,9 @@ package gdb import ( - "database/sql" - "errors" - "fmt" - "github.com/gogf/gf/container/garray" - "github.com/gogf/gf/container/gmap" - "reflect" "time" - "github.com/gogf/gf/container/gset" "github.com/gogf/gf/text/gstr" - - "github.com/gogf/gf/util/gconv" ) // Model is the DAO for ORM. @@ -31,7 +22,7 @@ type Model struct { tables string // Operation table names, which can be more than one table names and aliases, like: "user", "user u", "user u, user_detail ud". fields string // Operation fields, multiple fields joined using char ','. fieldsEx string // Excluded operation fields, multiple fields joined using char ','. - whereArgs []interface{} // Arguments for where operation. + extraArgs []interface{} // Extra custom arguments for sql. whereHolder []*whereHolder // Condition strings for where operation. groupBy string // Used for "group by" statement. orderBy string // Used for "order by" statement. @@ -42,6 +33,7 @@ type Model struct { data interface{} // Data for operation, which can be type of map/[]map/struct/*struct/string, etc. batch int // Batch number for batch Insert/Replace/Save operations. filter bool // Filter data and where key-value pairs according to the fields of the table. + lockInfo string // Lock for update or in shared lock. cacheEnabled bool // Enable sql result cache feature. cacheDuration time.Duration // Cache TTL duration. cacheName string // Cache name for custom operation. @@ -167,9 +159,9 @@ func (m *Model) Clone() *Model { } *newModel = *m // Deep copy slice attributes. - if n := len(m.whereArgs); n > 0 { - newModel.whereArgs = make([]interface{}, n) - copy(newModel.whereArgs, m.whereArgs) + if n := len(m.extraArgs); n > 0 { + newModel.extraArgs = make([]interface{}, n) + copy(newModel.extraArgs, m.extraArgs) } if n := len(m.whereHolder); n > 0 { newModel.whereHolder = make([]*whereHolder, n) @@ -203,1026 +195,3 @@ func (m *Model) Safe(safe ...bool) *Model { } return m } - -// getModel creates and returns a cloned model of current model if is true, or else it returns -// the current model. -func (m *Model) getModel() *Model { - if !m.safe { - return m - } else { - return m.Clone() - } -} - -// LeftJoin does "LEFT JOIN ... ON ..." statement on the model. -func (m *Model) LeftJoin(table string, on string) *Model { - model := m.getModel() - model.tables += fmt.Sprintf(" LEFT JOIN %s ON (%s)", m.db.QuotePrefixTableName(table), on) - return model -} - -// RightJoin does "RIGHT JOIN ... ON ..." statement on the model. -func (m *Model) RightJoin(table string, on string) *Model { - model := m.getModel() - model.tables += fmt.Sprintf(" RIGHT JOIN %s ON (%s)", m.db.QuotePrefixTableName(table), on) - return model -} - -// InnerJoin does "INNER JOIN ... ON ..." statement on the model. -func (m *Model) InnerJoin(table string, on string) *Model { - model := m.getModel() - model.tables += fmt.Sprintf(" INNER JOIN %s ON (%s)", m.db.QuotePrefixTableName(table), on) - return model -} - -// Fields sets the operation fields of the model, multiple fields joined using char ','. -func (m *Model) Fields(fields string) *Model { - model := m.getModel() - model.fields = fields - return model -} - -// FieldsEx sets the excluded operation fields of the model, multiple fields joined using char ','. -func (m *Model) FieldsEx(fields string) *Model { - if gstr.Contains(m.tables, " ") { - panic("function FieldsEx supports only single table operations") - } - model := m.getModel() - model.fieldsEx = fields - fieldsExSet := gset.NewStrSetFrom(gstr.SplitAndTrim(fields, ",")) - if m, err := m.db.TableFields(m.tables); err == nil { - model.fields = "" - for k, _ := range m { - if fieldsExSet.Contains(k) { - continue - } - if len(model.fields) > 0 { - model.fields += "," - } - model.fields += k - } - } - return model -} - -// FieldsStr retrieves and returns all fields from the table, joined with char ','. -// The optional parameter specifies the prefix for each field, eg: FieldsStr("u."). -func (m *Model) FieldsStr(prefix ...string) string { - prefixStr := "" - if len(prefix) > 0 { - prefixStr = prefix[0] - } - if m, err := m.db.TableFields(m.tables); err == nil { - fieldsArray := garray.NewStrArraySize(len(m), len(m)) - for _, field := range m { - fieldsArray.Set(field.Index, prefixStr+field.Name) - } - return fieldsArray.Join(",") - } - return "" -} - -// FieldsExStr retrieves and returns fields which are not in parameter from the table, -// joined with char ','. -// The parameter specifies the fields that are excluded. -// The optional parameter specifies the prefix for each field, eg: FieldsExStr("id", "u."). -func (m *Model) FieldsExStr(fields string, prefix ...string) string { - prefixStr := "" - if len(prefix) > 0 { - prefixStr = prefix[0] - } - if m, err := m.db.TableFields(m.tables); err == nil { - fieldsArray := garray.NewStrArraySize(len(m), len(m)) - fieldsExSet := gset.NewStrSetFrom(gstr.SplitAndTrim(fields, ",")) - for _, field := range m { - if fieldsExSet.Contains(field.Name) { - continue - } - fieldsArray.Set(field.Index, prefixStr+field.Name) - } - fieldsArray.FilterEmpty() - return fieldsArray.Join(",") - } - return "" -} - -// Option adds extra operation option for the model. -func (m *Model) Option(option int) *Model { - model := m.getModel() - model.option = model.option | option - return model -} - -// OptionOmitEmpty sets OPTION_OMITEMPTY option for the model, which automatically filers -// the data and where attributes for empty values. -// Deprecated, use OmitEmpty instead. -func (m *Model) OptionOmitEmpty() *Model { - return m.Option(OPTION_OMITEMPTY) -} - -// OmitEmpty sets OPTION_OMITEMPTY option for the model, which automatically filers -// the data and where attributes for empty values. -func (m *Model) OmitEmpty() *Model { - return m.Option(OPTION_OMITEMPTY) -} - -// Filter marks filtering the fields which does not exist in the fields of the operated table. -func (m *Model) Filter() *Model { - if gstr.Contains(m.tables, " ") { - panic("function Filter supports only single table operations") - } - model := m.getModel() - model.filter = true - return model -} - -// Where sets the condition statement for the model. The parameter can be type of -// string/map/gmap/slice/struct/*struct, etc. Note that, if it's called more than one times, -// multiple conditions will be joined into where statement using "AND". -// Eg: -// Where("uid=10000") -// Where("uid", 10000) -// Where("money>? AND name like ?", 99999, "vip_%") -// Where("uid", 1).Where("name", "john") -// Where("status IN (?)", g.Slice{1,2,3}) -// Where("age IN(?,?)", 18, 50) -// Where(User{ Id : 1, UserName : "john"}) -func (m *Model) Where(where interface{}, args ...interface{}) *Model { - model := m.getModel() - if model.whereHolder == nil { - model.whereHolder = make([]*whereHolder, 0) - } - model.whereHolder = append(model.whereHolder, &whereHolder{ - operator: gWHERE_HOLDER_WHERE, - where: where, - args: args, - }) - return model -} - -// WherePri does the same logic as Model.Where except that if the parameter -// is a single condition like int/string/float/slice, it treats the condition as the primary -// key value. That is, if primary key is "id" and given parameter as "123", the -// WherePri function treats it as "id=123", but Model.Where treats it as string "123". -func (m *Model) WherePri(where interface{}, args ...interface{}) *Model { - if len(args) > 0 { - return m.Where(where, args...) - } - newWhere := GetPrimaryKeyCondition(m.getPrimaryKey(), where) - return m.Where(newWhere[0], newWhere[1:]...) -} - -// And adds "AND" condition to the where statement. -func (m *Model) And(where interface{}, args ...interface{}) *Model { - model := m.getModel() - if model.whereHolder == nil { - model.whereHolder = make([]*whereHolder, 0) - } - model.whereHolder = append(model.whereHolder, &whereHolder{ - operator: gWHERE_HOLDER_AND, - where: where, - args: args, - }) - return model -} - -// Or adds "OR" condition to the where statement. -func (m *Model) Or(where interface{}, args ...interface{}) *Model { - model := m.getModel() - if model.whereHolder == nil { - model.whereHolder = make([]*whereHolder, 0) - } - model.whereHolder = append(model.whereHolder, &whereHolder{ - operator: gWHERE_HOLDER_OR, - where: where, - args: args, - }) - return model -} - -// Group sets the "GROUP BY" statement for the model. -func (m *Model) Group(groupBy string) *Model { - model := m.getModel() - model.groupBy = m.db.QuoteString(groupBy) - return model -} - -// GroupBy is alias of Model.Group. -// See Model.Group. -// Deprecated. -func (m *Model) GroupBy(groupBy string) *Model { - return m.Group(groupBy) -} - -// Order sets the "ORDER BY" statement for the model. -func (m *Model) Order(orderBy string) *Model { - model := m.getModel() - model.orderBy = m.db.QuoteString(orderBy) - return model -} - -// OrderBy is alias of Model.Order. -// See Model.Order. -// Deprecated. -func (m *Model) OrderBy(orderBy string) *Model { - return m.Order(orderBy) -} - -// Limit sets the "LIMIT" statement for the model. -// The parameter can be either one or two number, if passed two number is passed, -// it then sets "LIMIT limit[0],limit[1]" statement for the model, or else it sets "LIMIT limit[0]" -// statement. -func (m *Model) Limit(limit ...int) *Model { - model := m.getModel() - switch len(limit) { - case 1: - model.limit = limit[0] - case 2: - model.start = limit[0] - model.limit = limit[1] - } - return model -} - -// Offset sets the "OFFSET" statement for the model. -// It only makes sense for some databases like SQLServer, PostgreSQL, etc. -func (m *Model) Offset(offset int) *Model { - model := m.getModel() - model.offset = offset - return model -} - -// Page sets the paging number for the model. -// The parameter is started from 1 for paging. -// Note that, it differs that the Limit function start from 0 for "LIMIT" statement. -func (m *Model) Page(page, limit int) *Model { - model := m.getModel() - if page <= 0 { - page = 1 - } - model.start = (page - 1) * limit - model.limit = limit - return model -} - -// ForPage is alias of Model.Page. -// See Model.Page. -// Deprecated. -func (m *Model) ForPage(page, limit int) *Model { - return m.Page(page, limit) -} - -// Batch sets the batch operation number for the model. -func (m *Model) Batch(batch int) *Model { - model := m.getModel() - model.batch = batch - return model -} - -// Cache sets the cache feature for the model. It caches the result of the sql, which means -// if there's another same sql request, it just reads and returns the result from cache, it -// but not committed and executed into the database. -// -// If the parameter < 0, which means it clear the cache with given . -// If the parameter = 0, which means it never expires. -// If the parameter > 0, which means it expires after . -// -// The optional parameter is used to bind a name to the cache, which means you can later -// control the cache like changing the or clearing the cache with specified . -// -// Note that, the cache feature is disabled if the model is operating on a transaction. -func (m *Model) Cache(duration time.Duration, name ...string) *Model { - model := m.getModel() - model.cacheDuration = duration - if len(name) > 0 { - model.cacheName = name[0] - } - // It does not support cache on transaction. - if model.tx == nil { - model.cacheEnabled = true - } - return model -} - -// Data sets the operation data for the model. -// The parameter can be type of string/map/gmap/slice/struct/*struct, etc. -// Eg: -// Data("uid=10000") -// Data("uid", 10000) -// Data(g.Map{"uid": 10000, "name":"john"}) -// Data(g.Slice{g.Map{"uid": 10000, "name":"john"}, g.Map{"uid": 20000, "name":"smith"}) -func (m *Model) Data(data ...interface{}) *Model { - model := m.getModel() - if len(data) > 1 { - m := make(map[string]interface{}) - for i := 0; i < len(data); i += 2 { - m[gconv.String(data[i])] = data[i+1] - } - model.data = m - } else { - switch params := data[0].(type) { - case Result: - model.data = params.List() - case Record: - model.data = params.Map() - case List: - model.data = params - case Map: - model.data = params - default: - rv := reflect.ValueOf(params) - kind := rv.Kind() - if kind == reflect.Ptr { - rv = rv.Elem() - kind = rv.Kind() - } - switch kind { - case reflect.Slice, reflect.Array: - list := make(List, rv.Len()) - for i := 0; i < rv.Len(); i++ { - list[i] = DataToMapDeep(rv.Index(i).Interface()) - } - model.data = list - case reflect.Map, reflect.Struct: - model.data = DataToMapDeep(data[0]) - default: - model.data = data[0] - } - } - } - return model -} - -// Insert does "INSERT INTO ..." statement for the model. -// The optional parameter is the same as the parameter of Model.Data function, -// see Model.Data. -func (m *Model) Insert(data ...interface{}) (result sql.Result, err error) { - return m.doInsertWithOption(gINSERT_OPTION_DEFAULT, data...) -} - -// InsertIgnore does "INSERT IGNORE INTO ..." statement for the model. -// The optional parameter is the same as the parameter of Model.Data function, -// see Model.Data. -func (m *Model) InsertIgnore(data ...interface{}) (result sql.Result, err error) { - return m.doInsertWithOption(gINSERT_OPTION_IGNORE, data...) -} - -// doInsertWithOption inserts data with option parameter. -func (m *Model) doInsertWithOption(option int, data ...interface{}) (result sql.Result, err error) { - if len(data) > 0 { - return m.Data(data...).Insert() - } - defer func() { - if err == nil { - m.checkAndRemoveCache() - } - }() - if m.data == nil { - return nil, errors.New("inserting into table with empty data") - } - if list, ok := m.data.(List); ok { - // Batch insert. - batch := 10 - if m.batch > 0 { - batch = m.batch - } - return m.db.DoBatchInsert( - m.getLink(true), - m.tables, - m.filterDataForInsertOrUpdate(list), - option, - batch, - ) - } else if data, ok := m.data.(Map); ok { - // Single insert. - return m.db.DoInsert( - m.getLink(true), - m.tables, - m.filterDataForInsertOrUpdate(data), - option, - ) - } - return nil, errors.New("inserting into table with invalid data type") -} - -// Replace does "REPLACE INTO ..." statement for the model. -// The optional parameter is the same as the parameter of Model.Data function, -// see Model.Data. -func (m *Model) Replace(data ...interface{}) (result sql.Result, err error) { - if len(data) > 0 { - return m.Data(data...).Replace() - } - defer func() { - if err == nil { - m.checkAndRemoveCache() - } - }() - if m.data == nil { - return nil, errors.New("replacing into table with empty data") - } - if list, ok := m.data.(List); ok { - // Batch replace. - batch := 10 - if m.batch > 0 { - batch = m.batch - } - return m.db.DoBatchInsert( - m.getLink(true), - m.tables, - m.filterDataForInsertOrUpdate(list), - gINSERT_OPTION_REPLACE, - batch, - ) - } else if data, ok := m.data.(Map); ok { - // Single insert. - return m.db.DoInsert( - m.getLink(true), - m.tables, - m.filterDataForInsertOrUpdate(data), - gINSERT_OPTION_REPLACE, - ) - } - return nil, errors.New("replacing into table with invalid data type") -} - -// Save does "INSERT INTO ... ON DUPLICATE KEY UPDATE..." statement for the model. -// The optional parameter is the same as the parameter of Model.Data function, -// see Model.Data. -// -// It updates the record if there's primary or unique index in the saving data, -// or else it inserts a new record into the table. -func (m *Model) Save(data ...interface{}) (result sql.Result, err error) { - if len(data) > 0 { - return m.Data(data...).Save() - } - defer func() { - if err == nil { - m.checkAndRemoveCache() - } - }() - if m.data == nil { - return nil, errors.New("saving into table with empty data") - } - if list, ok := m.data.(List); ok { - // Batch save. - batch := gDEFAULT_BATCH_NUM - if m.batch > 0 { - batch = m.batch - } - return m.db.DoBatchInsert( - m.getLink(true), - m.tables, - m.filterDataForInsertOrUpdate(list), - gINSERT_OPTION_SAVE, - batch, - ) - } else if data, ok := m.data.(Map); ok { - // Single save. - return m.db.DoInsert( - m.getLink(true), - m.tables, - m.filterDataForInsertOrUpdate(data), - gINSERT_OPTION_SAVE, - ) - } - return nil, errors.New("saving into table with invalid data type") -} - -// Update does "UPDATE ... " statement for the model. -// -// If the optional parameter is given, the dataAndWhere[0] is the updated data field, -// and dataAndWhere[1:] is treated as where condition fields. -// Also see Model.Data and Model.Where functions. -func (m *Model) Update(dataAndWhere ...interface{}) (result sql.Result, err error) { - if len(dataAndWhere) > 0 { - if len(dataAndWhere) > 2 { - return m.Data(dataAndWhere[0]).Where(dataAndWhere[1], dataAndWhere[2:]...).Update() - } else if len(dataAndWhere) == 2 { - return m.Data(dataAndWhere[0]).Where(dataAndWhere[1]).Update() - } else { - return m.Data(dataAndWhere[0]).Update() - } - } - defer func() { - if err == nil { - m.checkAndRemoveCache() - } - }() - if m.data == nil { - return nil, errors.New("updating table with empty data") - } - condition, conditionArgs := m.formatCondition(false) - return m.db.DoUpdate( - m.getLink(true), - m.tables, - m.filterDataForInsertOrUpdate(m.data), - condition, - conditionArgs..., - ) -} - -// Delete does "DELETE FROM ... " statement for the model. -// The optional parameter is the same as the parameter of Model.Where function, -// see Model.Where. -func (m *Model) Delete(where ...interface{}) (result sql.Result, err error) { - if len(where) > 0 { - return m.Where(where[0], where[1:]...).Delete() - } - defer func() { - if err == nil { - m.checkAndRemoveCache() - } - }() - condition, conditionArgs := m.formatCondition(false) - return m.db.DoDelete(m.getLink(true), m.tables, condition, conditionArgs...) -} - -// Select is alias of Model.All. -// See Model.All. -// Deprecated. -func (m *Model) Select(where ...interface{}) (Result, error) { - return m.All(where...) -} - -// All does "SELECT FROM ..." statement for the model. -// It retrieves the records from table and returns the result as slice type. -// It returns nil if there's no record retrieved with the given conditions from table. -// -// The optional parameter is the same as the parameter of Model.Where function, -// see Model.Where. -func (m *Model) All(where ...interface{}) (Result, error) { - if len(where) > 0 { - return m.Where(where[0], where[1:]...).All() - } - condition, conditionArgs := m.formatCondition(false) - return m.getAll(fmt.Sprintf("SELECT %s FROM %s%s", m.fields, m.tables, condition), conditionArgs...) -} - -// One retrieves one record from table and returns the result as map type. -// It returns nil if there's no record retrieved with the given conditions from table. -// -// The optional parameter is the same as the parameter of Model.Where function, -// see Model.Where. -func (m *Model) One(where ...interface{}) (Record, error) { - if len(where) > 0 { - return m.Where(where[0], where[1:]...).One() - } - condition, conditionArgs := m.formatCondition(true) - all, err := m.getAll(fmt.Sprintf("SELECT %s FROM %s%s", m.fields, m.tables, condition), conditionArgs...) - if err != nil { - return nil, err - } - if len(all) > 0 { - return all[0], nil - } - return nil, nil -} - -// Value retrieves a specified record value from table and returns the result as interface type. -// It returns nil if there's no record found with the given conditions from table. -// -// If the optional parameter is given, the fieldsAndWhere[0] is the selected fields -// and fieldsAndWhere[1:] is treated as where condition fields. -// Also see Model.Fields and Model.Where functions. -func (m *Model) Value(fieldsAndWhere ...interface{}) (Value, error) { - if len(fieldsAndWhere) > 0 { - if len(fieldsAndWhere) > 2 { - return m.Fields(gconv.String(fieldsAndWhere[0])).Where(fieldsAndWhere[1], fieldsAndWhere[2:]...).Value() - } else if len(fieldsAndWhere) == 2 { - return m.Fields(gconv.String(fieldsAndWhere[0])).Where(fieldsAndWhere[1]).Value() - } else { - return m.Fields(gconv.String(fieldsAndWhere[0])).Value() - } - } - one, err := m.One() - if err != nil { - return nil, err - } - for _, v := range one { - return v, nil - } - return nil, nil -} - -// Array queries and returns data values as slice from database. -// Note that if there're multiple columns in the result, it returns just one column values randomly. -// -// If the optional parameter is given, the fieldsAndWhere[0] is the selected fields -// and fieldsAndWhere[1:] is treated as where condition fields. -// Also see Model.Fields and Model.Where functions. -func (m *Model) Array(fieldsAndWhere ...interface{}) ([]Value, error) { - if len(fieldsAndWhere) > 0 { - if len(fieldsAndWhere) > 2 { - return m.Fields(gconv.String(fieldsAndWhere[0])).Where(fieldsAndWhere[1], fieldsAndWhere[2:]...).Array() - } else if len(fieldsAndWhere) == 2 { - return m.Fields(gconv.String(fieldsAndWhere[0])).Where(fieldsAndWhere[1]).Array() - } else { - return m.Fields(gconv.String(fieldsAndWhere[0])).Array() - } - } - all, err := m.All() - if err != nil { - return nil, err - } - return all.Array(), nil -} - -// Struct retrieves one record from table and converts it into given struct. -// The parameter should be type of *struct/**struct. If type **struct is given, -// it can create the struct internally during converting. -// -// The optional parameter is the same as the parameter of Model.Where function, -// see Model.Where. -// -// Note that it returns sql.ErrNoRows if there's no record retrieved with the given conditions -// from table. -// -// Eg: -// user := new(User) -// err := db.Table("user").Where("id", 1).Struct(user) -// -// user := (*User)(nil) -// err := db.Table("user").Where("id", 1).Struct(&user) -func (m *Model) Struct(pointer interface{}, where ...interface{}) error { - one, err := m.One(where...) - if err != nil { - return err - } - if len(one) == 0 { - return sql.ErrNoRows - } - return one.Struct(pointer) -} - -// Structs retrieves records from table and converts them into given struct slice. -// The parameter should be type of *[]struct/*[]*struct. It can create and fill the struct -// slice internally during converting. -// -// The optional parameter is the same as the parameter of Model.Where function, -// see Model.Where. -// -// Note that it returns sql.ErrNoRows if there's no record retrieved with the given conditions -// from table. -// -// Eg: -// users := ([]User)(nil) -// err := db.Table("user").Structs(&users) -// -// users := ([]*User)(nil) -// err := db.Table("user").Structs(&users) -func (m *Model) Structs(pointer interface{}, where ...interface{}) error { - all, err := m.All(where...) - if err != nil { - return err - } - if len(all) == 0 { - return sql.ErrNoRows - } - return all.Structs(pointer) -} - -// Scan automatically calls Struct or Structs function according to the type of parameter . -// It calls function Struct if is type of *struct/**struct. -// It calls function Structs if is type of *[]struct/*[]*struct. -// -// The optional parameter is the same as the parameter of Model.Where function, -// see Model.Where. -// -// Note that it returns sql.ErrNoRows if there's no record retrieved with the given conditions -// from table. -// -// Eg: -// user := new(User) -// err := db.Table("user").Where("id", 1).Struct(user) -// -// user := (*User)(nil) -// err := db.Table("user").Where("id", 1).Struct(&user) -// -// users := ([]User)(nil) -// err := db.Table("user").Structs(&users) -// -// users := ([]*User)(nil) -// err := db.Table("user").Structs(&users) -func (m *Model) Scan(pointer interface{}, where ...interface{}) error { - t := reflect.TypeOf(pointer) - k := t.Kind() - if k != reflect.Ptr { - return fmt.Errorf("params should be type of pointer, but got: %v", k) - } - switch t.Elem().Kind() { - case reflect.Array: - case reflect.Slice: - return m.Structs(pointer, where...) - default: - return m.Struct(pointer, where...) - } - return nil -} - -// Count does "SELECT COUNT(x) FROM ..." statement for the model. -// The optional parameter is the same as the parameter of Model.Where function, -// see Model.Where. -func (m *Model) Count(where ...interface{}) (int, error) { - if len(where) > 0 { - return m.Where(where[0], where[1:]...).Count() - } - countFields := "COUNT(1)" - if m.fields != "" && m.fields != "*" { - countFields = fmt.Sprintf(`COUNT(%s)`, m.fields) - } - condition, conditionArgs := m.formatCondition(false) - s := fmt.Sprintf("SELECT %s FROM %s %s", countFields, m.tables, condition) - if len(m.groupBy) > 0 { - s = fmt.Sprintf("SELECT COUNT(1) FROM (%s) count_alias", s) - } - list, err := m.getAll(s, conditionArgs...) - if err != nil { - return 0, err - } - if len(list) > 0 { - for _, v := range list[0] { - return v.Int(), nil - } - } - return 0, nil -} - -// FindOne retrieves and returns a single Record by Model.WherePri and Model.One. -// Also see Model.WherePri and Model.One. -func (m *Model) FindOne(where ...interface{}) (Record, error) { - if len(where) > 0 { - return m.WherePri(where[0], where[1:]...).One() - } - return m.One() -} - -// FindAll retrieves and returns Result by by Model.WherePri and Model.All. -// Also see Model.WherePri and Model.All. -func (m *Model) FindAll(where ...interface{}) (Result, error) { - if len(where) > 0 { - return m.WherePri(where[0], where[1:]...).All() - } - return m.All() -} - -// FindValue retrieves and returns single field value by Model.WherePri and Model.Value. -// Also see Model.WherePri and Model.Value. -func (m *Model) FindValue(fieldsAndWhere ...interface{}) (Value, error) { - if len(fieldsAndWhere) >= 2 { - return m.WherePri(fieldsAndWhere[1], fieldsAndWhere[2:]...).Fields(gconv.String(fieldsAndWhere[0])).Value() - } - if len(fieldsAndWhere) == 1 { - return m.Fields(gconv.String(fieldsAndWhere[0])).Value() - } - return m.Value() -} - -// FindArray queries and returns data values as slice from database. -// Note that if there're multiple columns in the result, it returns just one column values randomly. -// Also see Model.WherePri and Model.Value. -func (m *Model) FindArray(fieldsAndWhere ...interface{}) ([]Value, error) { - if len(fieldsAndWhere) >= 2 { - return m.WherePri(fieldsAndWhere[1], fieldsAndWhere[2:]...).Fields(gconv.String(fieldsAndWhere[0])).Array() - } - if len(fieldsAndWhere) == 1 { - return m.Fields(gconv.String(fieldsAndWhere[0])).Array() - } - return m.Array() -} - -// FindCount retrieves and returns the record number by Model.WherePri and Model.Count. -// Also see Model.WherePri and Model.Count. -func (m *Model) FindCount(where ...interface{}) (int, error) { - if len(where) > 0 { - return m.WherePri(where[0], where[1:]...).Count() - } - return m.Count() -} - -// FindScan retrieves and returns the record/records by Model.WherePri and Model.Scan. -// Also see Model.WherePri and Model.Scan. -func (m *Model) FindScan(pointer interface{}, where ...interface{}) error { - if len(where) > 0 { - return m.WherePri(where[0], where[1:]...).Scan(pointer) - } - return m.Scan(pointer) -} - -// Chunk iterates the table with given size and callback function. -func (m *Model) Chunk(limit int, callback func(result Result, err error) bool) { - page := m.start - if page == 0 { - page = 1 - } - model := m - for { - model = model.Page(page, limit) - data, err := model.All() - if err != nil { - callback(nil, err) - break - } - if len(data) == 0 { - break - } - if callback(data, err) == false { - break - } - if len(data) < limit { - break - } - page++ - } -} - -// filterDataForInsertOrUpdate does filter feature with data for inserting/updating operations. -// Note that, it does not filter list item, which is also type of map, for "omit empty" feature. -func (m *Model) filterDataForInsertOrUpdate(data interface{}) interface{} { - if list, ok := m.data.(List); ok { - for k, item := range list { - list[k] = m.doFilterDataMapForInsertOrUpdate(item, false) - } - return list - } else if item, ok := m.data.(Map); ok { - return m.doFilterDataMapForInsertOrUpdate(item, true) - } - return data -} - -// doFilterDataMapForInsertOrUpdate does the filter features for map. -// Note that, it does not filter list item, which is also type of map, for "omit empty" feature. -func (m *Model) doFilterDataMapForInsertOrUpdate(data Map, allowOmitEmpty bool) Map { - if m.filter { - data = m.db.filterFields(m.schema, m.tables, data) - } - // Remove key-value pairs of which the value is empty. - if allowOmitEmpty && m.option&OPTION_OMITEMPTY > 0 { - m := gmap.NewStrAnyMapFrom(data) - m.FilterEmpty() - data = m.Map() - } - - if len(m.fields) > 0 && m.fields != "*" { - // Keep specified fields. - set := gset.NewStrSetFrom(gstr.SplitAndTrim(m.fields, ",")) - for k := range data { - if !set.Contains(k) { - delete(data, k) - } - } - } else if len(m.fieldsEx) > 0 { - // Filter specified fields. - for _, v := range gstr.SplitAndTrim(m.fieldsEx, ",") { - delete(data, v) - } - } - return data -} - -// getLink returns the underlying database link object with configured attribute. -// The parameter specifies whether using the master node if master-slave configured. -func (m *Model) getLink(master bool) Link { - if m.tx != nil { - return m.tx.tx - } - linkType := m.linkType - if linkType == 0 { - if master { - linkType = gLINK_TYPE_MASTER - } else { - linkType = gLINK_TYPE_SLAVE - } - } - switch linkType { - case gLINK_TYPE_MASTER: - link, err := m.db.GetMaster(m.schema) - if err != nil { - panic(err) - } - return link - case gLINK_TYPE_SLAVE: - link, err := m.db.GetSlave(m.schema) - if err != nil { - panic(err) - } - return link - } - return nil -} - -// getAll does the query from database. -func (m *Model) getAll(query string, args ...interface{}) (result Result, err error) { - cacheKey := "" - // Retrieve from cache. - if m.cacheEnabled { - cacheKey = m.cacheName - if len(cacheKey) == 0 { - cacheKey = query + "/" + gconv.String(args) - } - if v := m.db.GetCache().Get(cacheKey); v != nil { - return v.(Result), nil - } - } - result, err = m.db.DoGetAll(m.getLink(false), query, args...) - // Cache the result. - if len(cacheKey) > 0 && err == nil { - if m.cacheDuration < 0 { - m.db.GetCache().Remove(cacheKey) - } else { - m.db.GetCache().Set(cacheKey, result, m.cacheDuration) - } - } - return result, err -} - -// getPrimaryKey retrieves and returns the primary key name of the model table. -// It parses m.tables to retrieve the primary table name, supporting m.tables like: -// "user", "user u", "user as u, user_detail as ud". -func (m *Model) getPrimaryKey() string { - table := gstr.SplitAndTrim(m.tables, " ")[0] - tableFields, err := m.db.TableFields(table) - if err != nil { - return "" - } - for name, field := range tableFields { - if gstr.ContainsI(field.Key, "pri") { - return name - } - } - return "" -} - -// checkAndRemoveCache checks and remove the cache if necessary. -func (m *Model) checkAndRemoveCache() { - if m.cacheEnabled && m.cacheDuration < 0 && len(m.cacheName) > 0 { - m.db.GetCache().Remove(m.cacheName) - } -} - -// formatCondition formats where arguments of the model and returns a new condition sql and its arguments. -// Note that this function does not change any attribute value of the . -// -// The parameter specifies whether limits querying only one record if m.limit is not set. -func (m *Model) formatCondition(limit bool) (condition string, conditionArgs []interface{}) { - var where string - if len(m.whereHolder) > 0 { - for _, v := range m.whereHolder { - switch v.operator { - case gWHERE_HOLDER_WHERE: - if where == "" { - newWhere, newArgs := formatWhere(m.db, v.where, v.args, m.option&OPTION_OMITEMPTY > 0) - if len(newWhere) > 0 { - where = newWhere - conditionArgs = newArgs - } - continue - } - fallthrough - - case gWHERE_HOLDER_AND: - newWhere, newArgs := formatWhere(m.db, v.where, v.args, m.option&OPTION_OMITEMPTY > 0) - if len(newWhere) > 0 { - if where[0] == '(' { - where = fmt.Sprintf(`%s AND (%s)`, where, newWhere) - } else { - where = fmt.Sprintf(`(%s) AND (%s)`, where, newWhere) - } - conditionArgs = append(conditionArgs, newArgs...) - } - - case gWHERE_HOLDER_OR: - newWhere, newArgs := formatWhere(m.db, v.where, v.args, m.option&OPTION_OMITEMPTY > 0) - if len(newWhere) > 0 { - if where[0] == '(' { - where = fmt.Sprintf(`%s OR (%s)`, where, newWhere) - } else { - where = fmt.Sprintf(`(%s) OR (%s)`, where, newWhere) - } - conditionArgs = append(conditionArgs, newArgs...) - } - } - } - } - if where != "" { - condition += " WHERE " + where - } - if m.groupBy != "" { - condition += " GROUP BY " + m.groupBy - } - if m.orderBy != "" { - condition += " ORDER BY " + m.orderBy - } - if m.limit != 0 { - if m.start >= 0 { - condition += fmt.Sprintf(" LIMIT %d,%d", m.start, m.limit) - } else { - condition += fmt.Sprintf(" LIMIT %d", m.limit) - } - } else if limit { - condition += " LIMIT 1" - } - if m.offset >= 0 { - condition += fmt.Sprintf(" OFFSET %d", m.offset) - } - return -} diff --git a/database/gdb/gdb_model_cache.go b/database/gdb/gdb_model_cache.go new file mode 100644 index 000000000..016c0b945 --- /dev/null +++ b/database/gdb/gdb_model_cache.go @@ -0,0 +1,36 @@ +// Copyright 2017 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gdb + +import ( + "time" +) + +// Cache sets the cache feature for the model. It caches the result of the sql, which means +// if there's another same sql request, it just reads and returns the result from cache, it +// but not committed and executed into the database. +// +// If the parameter < 0, which means it clear the cache with given . +// If the parameter = 0, which means it never expires. +// If the parameter > 0, which means it expires after . +// +// The optional parameter is used to bind a name to the cache, which means you can later +// control the cache like changing the or clearing the cache with specified . +// +// Note that, the cache feature is disabled if the model is operating on a transaction. +func (m *Model) Cache(duration time.Duration, name ...string) *Model { + model := m.getModel() + model.cacheDuration = duration + if len(name) > 0 { + model.cacheName = name[0] + } + // It does not support cache on transaction. + if model.tx == nil { + model.cacheEnabled = true + } + return model +} diff --git a/database/gdb/gdb_model_condition.go b/database/gdb/gdb_model_condition.go new file mode 100644 index 000000000..66705a25b --- /dev/null +++ b/database/gdb/gdb_model_condition.go @@ -0,0 +1,170 @@ +// Copyright 2017 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gdb + +import "github.com/gogf/gf/util/gconv" + +// Where sets the condition statement for the model. The parameter can be type of +// string/map/gmap/slice/struct/*struct, etc. Note that, if it's called more than one times, +// multiple conditions will be joined into where statement using "AND". +// Eg: +// Where("uid=10000") +// Where("uid", 10000) +// Where("money>? AND name like ?", 99999, "vip_%") +// Where("uid", 1).Where("name", "john") +// Where("status IN (?)", g.Slice{1,2,3}) +// Where("age IN(?,?)", 18, 50) +// Where(User{ Id : 1, UserName : "john"}) +func (m *Model) Where(where interface{}, args ...interface{}) *Model { + model := m.getModel() + if model.whereHolder == nil { + model.whereHolder = make([]*whereHolder, 0) + } + model.whereHolder = append(model.whereHolder, &whereHolder{ + operator: gWHERE_HOLDER_WHERE, + where: where, + args: args, + }) + return model +} + +// WherePri does the same logic as Model.Where except that if the parameter +// is a single condition like int/string/float/slice, it treats the condition as the primary +// key value. That is, if primary key is "id" and given parameter as "123", the +// WherePri function treats it as "id=123", but Model.Where treats it as string "123". +func (m *Model) WherePri(where interface{}, args ...interface{}) *Model { + if len(args) > 0 { + return m.Where(where, args...) + } + newWhere := GetPrimaryKeyCondition(m.getPrimaryKey(), where) + return m.Where(newWhere[0], newWhere[1:]...) +} + +// And adds "AND" condition to the where statement. +func (m *Model) And(where interface{}, args ...interface{}) *Model { + model := m.getModel() + if model.whereHolder == nil { + model.whereHolder = make([]*whereHolder, 0) + } + model.whereHolder = append(model.whereHolder, &whereHolder{ + operator: gWHERE_HOLDER_AND, + where: where, + args: args, + }) + return model +} + +// Or adds "OR" condition to the where statement. +func (m *Model) Or(where interface{}, args ...interface{}) *Model { + model := m.getModel() + if model.whereHolder == nil { + model.whereHolder = make([]*whereHolder, 0) + } + model.whereHolder = append(model.whereHolder, &whereHolder{ + operator: gWHERE_HOLDER_OR, + where: where, + args: args, + }) + return model +} + +// Group sets the "GROUP BY" statement for the model. +func (m *Model) Group(groupBy string) *Model { + model := m.getModel() + model.groupBy = m.db.QuoteString(groupBy) + return model +} + +// GroupBy is alias of Model.Group. +// See Model.Group. +// Deprecated. +func (m *Model) GroupBy(groupBy string) *Model { + return m.Group(groupBy) +} + +// Order sets the "ORDER BY" statement for the model. +func (m *Model) Order(orderBy string) *Model { + model := m.getModel() + model.orderBy = m.db.QuoteString(orderBy) + return model +} + +// OrderBy is alias of Model.Order. +// See Model.Order. +// Deprecated. +func (m *Model) OrderBy(orderBy string) *Model { + return m.Order(orderBy) +} + +// Limit sets the "LIMIT" statement for the model. +// The parameter can be either one or two number, if passed two number is passed, +// it then sets "LIMIT limit[0],limit[1]" statement for the model, or else it sets "LIMIT limit[0]" +// statement. +func (m *Model) Limit(limit ...int) *Model { + model := m.getModel() + switch len(limit) { + case 1: + model.limit = limit[0] + case 2: + model.start = limit[0] + model.limit = limit[1] + } + return model +} + +// Offset sets the "OFFSET" statement for the model. +// It only makes sense for some databases like SQLServer, PostgreSQL, etc. +func (m *Model) Offset(offset int) *Model { + model := m.getModel() + model.offset = offset + return model +} + +// Page sets the paging number for the model. +// The parameter is started from 1 for paging. +// Note that, it differs that the Limit function start from 0 for "LIMIT" statement. +func (m *Model) Page(page, limit int) *Model { + model := m.getModel() + if page <= 0 { + page = 1 + } + model.start = (page - 1) * limit + model.limit = limit + return model +} + +// ForPage is alias of Model.Page. +// See Model.Page. +// Deprecated. +func (m *Model) ForPage(page, limit int) *Model { + return m.Page(page, limit) +} + +// getAll does the query from database. +func (m *Model) getAll(query string, args ...interface{}) (result Result, err error) { + cacheKey := "" + // Retrieve from cache. + if m.cacheEnabled { + cacheKey = m.cacheName + if len(cacheKey) == 0 { + cacheKey = query + "/" + gconv.String(args) + } + if v := m.db.GetCache().Get(cacheKey); v != nil { + return v.(Result), nil + } + } + result, err = m.db.DoGetAll(m.getLink(false), query, m.mergeArguments(args)...) + // Cache the result. + if len(cacheKey) > 0 && err == nil { + if m.cacheDuration < 0 { + m.db.GetCache().Remove(cacheKey) + } else { + m.db.GetCache().Set(cacheKey, result, m.cacheDuration) + } + } + return result, err +} diff --git a/database/gdb/gdb_model_delete.go b/database/gdb/gdb_model_delete.go new file mode 100644 index 000000000..e0638a8f7 --- /dev/null +++ b/database/gdb/gdb_model_delete.go @@ -0,0 +1,27 @@ +// Copyright 2017 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gdb + +import ( + "database/sql" +) + +// Delete does "DELETE FROM ... " statement for the model. +// The optional parameter is the same as the parameter of Model.Where function, +// see Model.Where. +func (m *Model) Delete(where ...interface{}) (result sql.Result, err error) { + if len(where) > 0 { + return m.Where(where[0], where[1:]...).Delete() + } + defer func() { + if err == nil { + m.checkAndRemoveCache() + } + }() + condition, conditionArgs := m.formatCondition(false) + return m.db.DoDelete(m.getLink(true), m.tables, condition, conditionArgs...) +} diff --git a/database/gdb/gdb_model_fields.go b/database/gdb/gdb_model_fields.go new file mode 100644 index 000000000..163385377 --- /dev/null +++ b/database/gdb/gdb_model_fields.go @@ -0,0 +1,94 @@ +// Copyright 2017 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gdb + +import ( + "github.com/gogf/gf/container/garray" + "github.com/gogf/gf/container/gset" + "github.com/gogf/gf/text/gstr" +) + +// Filter marks filtering the fields which does not exist in the fields of the operated table. +func (m *Model) Filter() *Model { + if gstr.Contains(m.tables, " ") { + panic("function Filter supports only single table operations") + } + model := m.getModel() + model.filter = true + return model +} + +// Fields sets the operation fields of the model, multiple fields joined using char ','. +func (m *Model) Fields(fields string) *Model { + model := m.getModel() + model.fields = fields + return model +} + +// FieldsEx sets the excluded operation fields of the model, multiple fields joined using char ','. +func (m *Model) FieldsEx(fields string) *Model { + if gstr.Contains(m.tables, " ") { + panic("function FieldsEx supports only single table operations") + } + model := m.getModel() + model.fieldsEx = fields + fieldsExSet := gset.NewStrSetFrom(gstr.SplitAndTrim(fields, ",")) + if m, err := m.db.TableFields(m.tables); err == nil { + model.fields = "" + for k, _ := range m { + if fieldsExSet.Contains(k) { + continue + } + if len(model.fields) > 0 { + model.fields += "," + } + model.fields += k + } + } + return model +} + +// FieldsStr retrieves and returns all fields from the table, joined with char ','. +// The optional parameter specifies the prefix for each field, eg: FieldsStr("u."). +func (m *Model) FieldsStr(prefix ...string) string { + prefixStr := "" + if len(prefix) > 0 { + prefixStr = prefix[0] + } + if m, err := m.db.TableFields(m.tables); err == nil { + fieldsArray := garray.NewStrArraySize(len(m), len(m)) + for _, field := range m { + fieldsArray.Set(field.Index, prefixStr+field.Name) + } + return fieldsArray.Join(",") + } + return "" +} + +// FieldsExStr retrieves and returns fields which are not in parameter from the table, +// joined with char ','. +// The parameter specifies the fields that are excluded. +// The optional parameter specifies the prefix for each field, eg: FieldsExStr("id", "u."). +func (m *Model) FieldsExStr(fields string, prefix ...string) string { + prefixStr := "" + if len(prefix) > 0 { + prefixStr = prefix[0] + } + if m, err := m.db.TableFields(m.tables); err == nil { + fieldsArray := garray.NewStrArraySize(len(m), len(m)) + fieldsExSet := gset.NewStrSetFrom(gstr.SplitAndTrim(fields, ",")) + for _, field := range m { + if fieldsExSet.Contains(field.Name) { + continue + } + fieldsArray.Set(field.Index, prefixStr+field.Name) + } + fieldsArray.FilterEmpty() + return fieldsArray.Join(",") + } + return "" +} diff --git a/database/gdb/gdb_model_insert.go b/database/gdb/gdb_model_insert.go new file mode 100644 index 000000000..2f0b67b02 --- /dev/null +++ b/database/gdb/gdb_model_insert.go @@ -0,0 +1,213 @@ +// Copyright 2017 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gdb + +import ( + "database/sql" + "errors" + "github.com/gogf/gf/text/gstr" + "github.com/gogf/gf/util/gconv" + "reflect" +) + +// Batch sets the batch operation number for the model. +func (m *Model) Batch(batch int) *Model { + model := m.getModel() + model.batch = batch + return model +} + +// Data sets the operation data for the model. +// The parameter can be type of string/map/gmap/slice/struct/*struct, etc. +// Eg: +// Data("uid=10000") +// Data("uid", 10000) +// Data("uid=? AND name=?", 10000, "john") +// Data(g.Map{"uid": 10000, "name":"john"}) +// Data(g.Slice{g.Map{"uid": 10000, "name":"john"}, g.Map{"uid": 20000, "name":"smith"}) +func (m *Model) Data(data ...interface{}) *Model { + model := m.getModel() + if len(data) > 1 { + s := gconv.String(data[0]) + if gstr.Contains(s, "?") { + model.data = s + model.extraArgs = data[1:] + } else { + m := make(map[string]interface{}) + for i := 0; i < len(data); i += 2 { + m[gconv.String(data[i])] = data[i+1] + } + model.data = m + } + } else { + switch params := data[0].(type) { + case Result: + model.data = params.List() + case Record: + model.data = params.Map() + case List: + model.data = params + case Map: + model.data = params + default: + rv := reflect.ValueOf(params) + kind := rv.Kind() + if kind == reflect.Ptr { + rv = rv.Elem() + kind = rv.Kind() + } + switch kind { + case reflect.Slice, reflect.Array: + list := make(List, rv.Len()) + for i := 0; i < rv.Len(); i++ { + list[i] = DataToMapDeep(rv.Index(i).Interface()) + } + model.data = list + case reflect.Map, reflect.Struct: + model.data = DataToMapDeep(data[0]) + default: + model.data = data[0] + } + } + } + return model +} + +// Insert does "INSERT INTO ..." statement for the model. +// The optional parameter is the same as the parameter of Model.Data function, +// see Model.Data. +func (m *Model) Insert(data ...interface{}) (result sql.Result, err error) { + return m.doInsertWithOption(gINSERT_OPTION_DEFAULT, data...) +} + +// InsertIgnore does "INSERT IGNORE INTO ..." statement for the model. +// The optional parameter is the same as the parameter of Model.Data function, +// see Model.Data. +func (m *Model) InsertIgnore(data ...interface{}) (result sql.Result, err error) { + return m.doInsertWithOption(gINSERT_OPTION_IGNORE, data...) +} + +// doInsertWithOption inserts data with option parameter. +func (m *Model) doInsertWithOption(option int, data ...interface{}) (result sql.Result, err error) { + if len(data) > 0 { + return m.Data(data...).Insert() + } + defer func() { + if err == nil { + m.checkAndRemoveCache() + } + }() + if m.data == nil { + return nil, errors.New("inserting into table with empty data") + } + if list, ok := m.data.(List); ok { + // Batch insert. + batch := 10 + if m.batch > 0 { + batch = m.batch + } + return m.db.DoBatchInsert( + m.getLink(true), + m.tables, + m.filterDataForInsertOrUpdate(list), + option, + batch, + ) + } else if data, ok := m.data.(Map); ok { + // Single insert. + return m.db.DoInsert( + m.getLink(true), + m.tables, + m.filterDataForInsertOrUpdate(data), + option, + ) + } + return nil, errors.New("inserting into table with invalid data type") +} + +// Replace does "REPLACE INTO ..." statement for the model. +// The optional parameter is the same as the parameter of Model.Data function, +// see Model.Data. +func (m *Model) Replace(data ...interface{}) (result sql.Result, err error) { + if len(data) > 0 { + return m.Data(data...).Replace() + } + defer func() { + if err == nil { + m.checkAndRemoveCache() + } + }() + if m.data == nil { + return nil, errors.New("replacing into table with empty data") + } + if list, ok := m.data.(List); ok { + // Batch replace. + batch := 10 + if m.batch > 0 { + batch = m.batch + } + return m.db.DoBatchInsert( + m.getLink(true), + m.tables, + m.filterDataForInsertOrUpdate(list), + gINSERT_OPTION_REPLACE, + batch, + ) + } else if data, ok := m.data.(Map); ok { + // Single insert. + return m.db.DoInsert( + m.getLink(true), + m.tables, + m.filterDataForInsertOrUpdate(data), + gINSERT_OPTION_REPLACE, + ) + } + return nil, errors.New("replacing into table with invalid data type") +} + +// Save does "INSERT INTO ... ON DUPLICATE KEY UPDATE..." statement for the model. +// The optional parameter is the same as the parameter of Model.Data function, +// see Model.Data. +// +// It updates the record if there's primary or unique index in the saving data, +// or else it inserts a new record into the table. +func (m *Model) Save(data ...interface{}) (result sql.Result, err error) { + if len(data) > 0 { + return m.Data(data...).Save() + } + defer func() { + if err == nil { + m.checkAndRemoveCache() + } + }() + if m.data == nil { + return nil, errors.New("saving into table with empty data") + } + if list, ok := m.data.(List); ok { + // Batch save. + batch := gDEFAULT_BATCH_NUM + if m.batch > 0 { + batch = m.batch + } + return m.db.DoBatchInsert( + m.getLink(true), + m.tables, + m.filterDataForInsertOrUpdate(list), + gINSERT_OPTION_SAVE, + batch, + ) + } else if data, ok := m.data.(Map); ok { + // Single save. + return m.db.DoInsert( + m.getLink(true), + m.tables, + m.filterDataForInsertOrUpdate(data), + gINSERT_OPTION_SAVE, + ) + } + return nil, errors.New("saving into table with invalid data type") +} diff --git a/database/gdb/gdb_model_join.go b/database/gdb/gdb_model_join.go new file mode 100644 index 000000000..d3d156c93 --- /dev/null +++ b/database/gdb/gdb_model_join.go @@ -0,0 +1,30 @@ +// Copyright 2017 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gdb + +import "fmt" + +// LeftJoin does "LEFT JOIN ... ON ..." statement on the model. +func (m *Model) LeftJoin(table string, on string) *Model { + model := m.getModel() + model.tables += fmt.Sprintf(" LEFT JOIN %s ON (%s)", m.db.QuotePrefixTableName(table), on) + return model +} + +// RightJoin does "RIGHT JOIN ... ON ..." statement on the model. +func (m *Model) RightJoin(table string, on string) *Model { + model := m.getModel() + model.tables += fmt.Sprintf(" RIGHT JOIN %s ON (%s)", m.db.QuotePrefixTableName(table), on) + return model +} + +// InnerJoin does "INNER JOIN ... ON ..." statement on the model. +func (m *Model) InnerJoin(table string, on string) *Model { + model := m.getModel() + model.tables += fmt.Sprintf(" INNER JOIN %s ON (%s)", m.db.QuotePrefixTableName(table), on) + return model +} diff --git a/database/gdb/gdb_model_lock.go b/database/gdb/gdb_model_lock.go new file mode 100644 index 000000000..36dce9ee0 --- /dev/null +++ b/database/gdb/gdb_model_lock.go @@ -0,0 +1,21 @@ +// Copyright 2019 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gdb + +// LockUpdate sets the lock for update for current operation. +func (m *Model) LockUpdate() *Model { + model := m.getModel() + model.lockInfo = "FOR UPDATE" + return model +} + +// LockShared sets the lock in share mode for current operation. +func (m *Model) LockShared() *Model { + model := m.getModel() + model.lockInfo = "LOCK IN SHARE MODE" + return model +} diff --git a/database/gdb/gdb_model_option.go b/database/gdb/gdb_model_option.go new file mode 100644 index 000000000..8d8d2f6f9 --- /dev/null +++ b/database/gdb/gdb_model_option.go @@ -0,0 +1,27 @@ +// Copyright 2017 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gdb + +// Option adds extra operation option for the model. +func (m *Model) Option(option int) *Model { + model := m.getModel() + model.option = model.option | option + return model +} + +// OptionOmitEmpty sets OPTION_OMITEMPTY option for the model, which automatically filers +// the data and where attributes for empty values. +// Deprecated, use OmitEmpty instead. +func (m *Model) OptionOmitEmpty() *Model { + return m.Option(OPTION_OMITEMPTY) +} + +// OmitEmpty sets OPTION_OMITEMPTY option for the model, which automatically filers +// the data and where attributes for empty values. +func (m *Model) OmitEmpty() *Model { + return m.Option(OPTION_OMITEMPTY) +} diff --git a/database/gdb/gdb_model_select.go b/database/gdb/gdb_model_select.go new file mode 100644 index 000000000..c3eefcb01 --- /dev/null +++ b/database/gdb/gdb_model_select.go @@ -0,0 +1,315 @@ +// Copyright 2017 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gdb + +import ( + "database/sql" + "fmt" + "github.com/gogf/gf/util/gconv" + "reflect" +) + +// Select is alias of Model.All. +// See Model.All. +// Deprecated. +func (m *Model) Select(where ...interface{}) (Result, error) { + return m.All(where...) +} + +// All does "SELECT FROM ..." statement for the model. +// It retrieves the records from table and returns the result as slice type. +// It returns nil if there's no record retrieved with the given conditions from table. +// +// The optional parameter is the same as the parameter of Model.Where function, +// see Model.Where. +func (m *Model) All(where ...interface{}) (Result, error) { + if len(where) > 0 { + return m.Where(where[0], where[1:]...).All() + } + condition, conditionArgs := m.formatCondition(false) + return m.getAll( + fmt.Sprintf("SELECT %s FROM %s%s", m.fields, m.tables, condition), + conditionArgs..., + ) +} + +// Chunk iterates the query result with given size and callback function. +func (m *Model) Chunk(limit int, callback func(result Result, err error) bool) { + page := m.start + if page == 0 { + page = 1 + } + model := m + for { + model = model.Page(page, limit) + data, err := model.All() + if err != nil { + callback(nil, err) + break + } + if len(data) == 0 { + break + } + if callback(data, err) == false { + break + } + if len(data) < limit { + break + } + page++ + } +} + +// One retrieves one record from table and returns the result as map type. +// It returns nil if there's no record retrieved with the given conditions from table. +// +// The optional parameter is the same as the parameter of Model.Where function, +// see Model.Where. +func (m *Model) One(where ...interface{}) (Record, error) { + if len(where) > 0 { + return m.Where(where[0], where[1:]...).One() + } + condition, conditionArgs := m.formatCondition(true) + all, err := m.getAll(fmt.Sprintf("SELECT %s FROM %s%s", m.fields, m.tables, condition), conditionArgs...) + if err != nil { + return nil, err + } + if len(all) > 0 { + return all[0], nil + } + return nil, nil +} + +// Value retrieves a specified record value from table and returns the result as interface type. +// It returns nil if there's no record found with the given conditions from table. +// +// If the optional parameter is given, the fieldsAndWhere[0] is the selected fields +// and fieldsAndWhere[1:] is treated as where condition fields. +// Also see Model.Fields and Model.Where functions. +func (m *Model) Value(fieldsAndWhere ...interface{}) (Value, error) { + if len(fieldsAndWhere) > 0 { + if len(fieldsAndWhere) > 2 { + return m.Fields(gconv.String(fieldsAndWhere[0])).Where(fieldsAndWhere[1], fieldsAndWhere[2:]...).Value() + } else if len(fieldsAndWhere) == 2 { + return m.Fields(gconv.String(fieldsAndWhere[0])).Where(fieldsAndWhere[1]).Value() + } else { + return m.Fields(gconv.String(fieldsAndWhere[0])).Value() + } + } + one, err := m.One() + if err != nil { + return nil, err + } + for _, v := range one { + return v, nil + } + return nil, nil +} + +// Array queries and returns data values as slice from database. +// Note that if there're multiple columns in the result, it returns just one column values randomly. +// +// If the optional parameter is given, the fieldsAndWhere[0] is the selected fields +// and fieldsAndWhere[1:] is treated as where condition fields. +// Also see Model.Fields and Model.Where functions. +func (m *Model) Array(fieldsAndWhere ...interface{}) ([]Value, error) { + if len(fieldsAndWhere) > 0 { + if len(fieldsAndWhere) > 2 { + return m.Fields(gconv.String(fieldsAndWhere[0])).Where(fieldsAndWhere[1], fieldsAndWhere[2:]...).Array() + } else if len(fieldsAndWhere) == 2 { + return m.Fields(gconv.String(fieldsAndWhere[0])).Where(fieldsAndWhere[1]).Array() + } else { + return m.Fields(gconv.String(fieldsAndWhere[0])).Array() + } + } + all, err := m.All() + if err != nil { + return nil, err + } + return all.Array(), nil +} + +// Struct retrieves one record from table and converts it into given struct. +// The parameter should be type of *struct/**struct. If type **struct is given, +// it can create the struct internally during converting. +// +// The optional parameter is the same as the parameter of Model.Where function, +// see Model.Where. +// +// Note that it returns sql.ErrNoRows if there's no record retrieved with the given conditions +// from table. +// +// Eg: +// user := new(User) +// err := db.Table("user").Where("id", 1).Struct(user) +// +// user := (*User)(nil) +// err := db.Table("user").Where("id", 1).Struct(&user) +func (m *Model) Struct(pointer interface{}, where ...interface{}) error { + one, err := m.One(where...) + if err != nil { + return err + } + if len(one) == 0 { + return sql.ErrNoRows + } + return one.Struct(pointer) +} + +// Structs retrieves records from table and converts them into given struct slice. +// The parameter should be type of *[]struct/*[]*struct. It can create and fill the struct +// slice internally during converting. +// +// The optional parameter is the same as the parameter of Model.Where function, +// see Model.Where. +// +// Note that it returns sql.ErrNoRows if there's no record retrieved with the given conditions +// from table. +// +// Eg: +// users := ([]User)(nil) +// err := db.Table("user").Structs(&users) +// +// users := ([]*User)(nil) +// err := db.Table("user").Structs(&users) +func (m *Model) Structs(pointer interface{}, where ...interface{}) error { + all, err := m.All(where...) + if err != nil { + return err + } + if len(all) == 0 { + return sql.ErrNoRows + } + return all.Structs(pointer) +} + +// Scan automatically calls Struct or Structs function according to the type of parameter . +// It calls function Struct if is type of *struct/**struct. +// It calls function Structs if is type of *[]struct/*[]*struct. +// +// The optional parameter is the same as the parameter of Model.Where function, +// see Model.Where. +// +// Note that it returns sql.ErrNoRows if there's no record retrieved with the given conditions +// from table. +// +// Eg: +// user := new(User) +// err := db.Table("user").Where("id", 1).Struct(user) +// +// user := (*User)(nil) +// err := db.Table("user").Where("id", 1).Struct(&user) +// +// users := ([]User)(nil) +// err := db.Table("user").Structs(&users) +// +// users := ([]*User)(nil) +// err := db.Table("user").Structs(&users) +func (m *Model) Scan(pointer interface{}, where ...interface{}) error { + t := reflect.TypeOf(pointer) + k := t.Kind() + if k != reflect.Ptr { + return fmt.Errorf("params should be type of pointer, but got: %v", k) + } + switch t.Elem().Kind() { + case reflect.Array: + case reflect.Slice: + return m.Structs(pointer, where...) + default: + return m.Struct(pointer, where...) + } + return nil +} + +// Count does "SELECT COUNT(x) FROM ..." statement for the model. +// The optional parameter is the same as the parameter of Model.Where function, +// see Model.Where. +func (m *Model) Count(where ...interface{}) (int, error) { + if len(where) > 0 { + return m.Where(where[0], where[1:]...).Count() + } + countFields := "COUNT(1)" + if m.fields != "" && m.fields != "*" { + countFields = fmt.Sprintf(`COUNT(%s)`, m.fields) + } + condition, conditionArgs := m.formatCondition(false) + s := fmt.Sprintf("SELECT %s FROM %s %s", countFields, m.tables, condition) + if len(m.groupBy) > 0 { + s = fmt.Sprintf("SELECT COUNT(1) FROM (%s) count_alias", s) + } + list, err := m.getAll(s, conditionArgs...) + if err != nil { + return 0, err + } + if len(list) > 0 { + for _, v := range list[0] { + return v.Int(), nil + } + } + return 0, nil +} + +// FindOne retrieves and returns a single Record by Model.WherePri and Model.One. +// Also see Model.WherePri and Model.One. +func (m *Model) FindOne(where ...interface{}) (Record, error) { + if len(where) > 0 { + return m.WherePri(where[0], where[1:]...).One() + } + return m.One() +} + +// FindAll retrieves and returns Result by by Model.WherePri and Model.All. +// Also see Model.WherePri and Model.All. +func (m *Model) FindAll(where ...interface{}) (Result, error) { + if len(where) > 0 { + return m.WherePri(where[0], where[1:]...).All() + } + return m.All() +} + +// FindValue retrieves and returns single field value by Model.WherePri and Model.Value. +// Also see Model.WherePri and Model.Value. +func (m *Model) FindValue(fieldsAndWhere ...interface{}) (Value, error) { + if len(fieldsAndWhere) >= 2 { + return m.WherePri(fieldsAndWhere[1], fieldsAndWhere[2:]...).Fields(gconv.String(fieldsAndWhere[0])).Value() + } + if len(fieldsAndWhere) == 1 { + return m.Fields(gconv.String(fieldsAndWhere[0])).Value() + } + return m.Value() +} + +// FindArray queries and returns data values as slice from database. +// Note that if there're multiple columns in the result, it returns just one column values randomly. +// Also see Model.WherePri and Model.Value. +func (m *Model) FindArray(fieldsAndWhere ...interface{}) ([]Value, error) { + if len(fieldsAndWhere) >= 2 { + return m.WherePri(fieldsAndWhere[1], fieldsAndWhere[2:]...).Fields(gconv.String(fieldsAndWhere[0])).Array() + } + if len(fieldsAndWhere) == 1 { + return m.Fields(gconv.String(fieldsAndWhere[0])).Array() + } + return m.Array() +} + +// FindCount retrieves and returns the record number by Model.WherePri and Model.Count. +// Also see Model.WherePri and Model.Count. +func (m *Model) FindCount(where ...interface{}) (int, error) { + if len(where) > 0 { + return m.WherePri(where[0], where[1:]...).Count() + } + return m.Count() +} + +// FindScan retrieves and returns the record/records by Model.WherePri and Model.Scan. +// Also see Model.WherePri and Model.Scan. +func (m *Model) FindScan(pointer interface{}, where ...interface{}) error { + if len(where) > 0 { + return m.WherePri(where[0], where[1:]...).Scan(pointer) + } + return m.Scan(pointer) +} diff --git a/database/gdb/gdb_model_update.go b/database/gdb/gdb_model_update.go new file mode 100644 index 000000000..a8cdf7019 --- /dev/null +++ b/database/gdb/gdb_model_update.go @@ -0,0 +1,45 @@ +// Copyright 2017 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gdb + +import ( + "database/sql" + "errors" +) + +// Update does "UPDATE ... " statement for the model. +// +// If the optional parameter is given, the dataAndWhere[0] is the updated data field, +// and dataAndWhere[1:] is treated as where condition fields. +// Also see Model.Data and Model.Where functions. +func (m *Model) Update(dataAndWhere ...interface{}) (result sql.Result, err error) { + if len(dataAndWhere) > 0 { + if len(dataAndWhere) > 2 { + return m.Data(dataAndWhere[0]).Where(dataAndWhere[1], dataAndWhere[2:]...).Update() + } else if len(dataAndWhere) == 2 { + return m.Data(dataAndWhere[0]).Where(dataAndWhere[1]).Update() + } else { + return m.Data(dataAndWhere[0]).Update() + } + } + defer func() { + if err == nil { + m.checkAndRemoveCache() + } + }() + if m.data == nil { + return nil, errors.New("updating table with empty data") + } + condition, conditionArgs := m.formatCondition(false) + return m.db.DoUpdate( + m.getLink(true), + m.tables, + m.filterDataForInsertOrUpdate(m.data), + condition, + m.mergeArguments(conditionArgs)..., + ) +} diff --git a/database/gdb/gdb_model_utility.go b/database/gdb/gdb_model_utility.go new file mode 100644 index 000000000..6e0052574 --- /dev/null +++ b/database/gdb/gdb_model_utility.go @@ -0,0 +1,205 @@ +// Copyright 2017 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gdb + +import ( + "fmt" + "github.com/gogf/gf/container/gmap" + "github.com/gogf/gf/container/gset" + "github.com/gogf/gf/text/gstr" +) + +// getModel creates and returns a cloned model of current model if is true, or else it returns +// the current model. +func (m *Model) getModel() *Model { + if !m.safe { + return m + } else { + return m.Clone() + } +} + +// filterDataForInsertOrUpdate does filter feature with data for inserting/updating operations. +// Note that, it does not filter list item, which is also type of map, for "omit empty" feature. +func (m *Model) filterDataForInsertOrUpdate(data interface{}) interface{} { + if list, ok := m.data.(List); ok { + for k, item := range list { + list[k] = m.doFilterDataMapForInsertOrUpdate(item, false) + } + return list + } else if item, ok := m.data.(Map); ok { + return m.doFilterDataMapForInsertOrUpdate(item, true) + } + return data +} + +// doFilterDataMapForInsertOrUpdate does the filter features for map. +// Note that, it does not filter list item, which is also type of map, for "omit empty" feature. +func (m *Model) doFilterDataMapForInsertOrUpdate(data Map, allowOmitEmpty bool) Map { + if m.filter { + data = m.db.filterFields(m.schema, m.tables, data) + } + // Remove key-value pairs of which the value is empty. + if allowOmitEmpty && m.option&OPTION_OMITEMPTY > 0 { + m := gmap.NewStrAnyMapFrom(data) + m.FilterEmpty() + data = m.Map() + } + + if len(m.fields) > 0 && m.fields != "*" { + // Keep specified fields. + set := gset.NewStrSetFrom(gstr.SplitAndTrim(m.fields, ",")) + for k := range data { + if !set.Contains(k) { + delete(data, k) + } + } + } else if len(m.fieldsEx) > 0 { + // Filter specified fields. + for _, v := range gstr.SplitAndTrim(m.fieldsEx, ",") { + delete(data, v) + } + } + return data +} + +// getLink returns the underlying database link object with configured attribute. +// The parameter specifies whether using the master node if master-slave configured. +func (m *Model) getLink(master bool) Link { + if m.tx != nil { + return m.tx.tx + } + linkType := m.linkType + if linkType == 0 { + if master { + linkType = gLINK_TYPE_MASTER + } else { + linkType = gLINK_TYPE_SLAVE + } + } + switch linkType { + case gLINK_TYPE_MASTER: + link, err := m.db.GetMaster(m.schema) + if err != nil { + panic(err) + } + return link + case gLINK_TYPE_SLAVE: + link, err := m.db.GetSlave(m.schema) + if err != nil { + panic(err) + } + return link + } + return nil +} + +// getPrimaryKey retrieves and returns the primary key name of the model table. +// It parses m.tables to retrieve the primary table name, supporting m.tables like: +// "user", "user u", "user as u, user_detail as ud". +func (m *Model) getPrimaryKey() string { + table := gstr.SplitAndTrim(m.tables, " ")[0] + tableFields, err := m.db.TableFields(table) + if err != nil { + return "" + } + for name, field := range tableFields { + if gstr.ContainsI(field.Key, "pri") { + return name + } + } + return "" +} + +// checkAndRemoveCache checks and remove the cache if necessary. +func (m *Model) checkAndRemoveCache() { + if m.cacheEnabled && m.cacheDuration < 0 && len(m.cacheName) > 0 { + m.db.GetCache().Remove(m.cacheName) + } +} + +// formatCondition formats where arguments of the model and returns a new condition sql and its arguments. +// Note that this function does not change any attribute value of the . +// +// The parameter specifies whether limits querying only one record if m.limit is not set. +func (m *Model) formatCondition(limit bool) (condition string, conditionArgs []interface{}) { + var where string + if len(m.whereHolder) > 0 { + for _, v := range m.whereHolder { + switch v.operator { + case gWHERE_HOLDER_WHERE: + if where == "" { + newWhere, newArgs := formatWhere(m.db, v.where, v.args, m.option&OPTION_OMITEMPTY > 0) + if len(newWhere) > 0 { + where = newWhere + conditionArgs = newArgs + } + continue + } + fallthrough + + case gWHERE_HOLDER_AND: + newWhere, newArgs := formatWhere(m.db, v.where, v.args, m.option&OPTION_OMITEMPTY > 0) + if len(newWhere) > 0 { + if where[0] == '(' { + where = fmt.Sprintf(`%s AND (%s)`, where, newWhere) + } else { + where = fmt.Sprintf(`(%s) AND (%s)`, where, newWhere) + } + conditionArgs = append(conditionArgs, newArgs...) + } + + case gWHERE_HOLDER_OR: + newWhere, newArgs := formatWhere(m.db, v.where, v.args, m.option&OPTION_OMITEMPTY > 0) + if len(newWhere) > 0 { + if where[0] == '(' { + where = fmt.Sprintf(`%s OR (%s)`, where, newWhere) + } else { + where = fmt.Sprintf(`(%s) OR (%s)`, where, newWhere) + } + conditionArgs = append(conditionArgs, newArgs...) + } + } + } + } + if where != "" { + condition += " WHERE " + where + } + if m.groupBy != "" { + condition += " GROUP BY " + m.groupBy + } + if m.orderBy != "" { + condition += " ORDER BY " + m.orderBy + } + if m.limit != 0 { + if m.start >= 0 { + condition += fmt.Sprintf(" LIMIT %d,%d", m.start, m.limit) + } else { + condition += fmt.Sprintf(" LIMIT %d", m.limit) + } + } else if limit { + condition += " LIMIT 1" + } + if m.offset >= 0 { + condition += fmt.Sprintf(" OFFSET %d", m.offset) + } + if m.lockInfo != "" { + condition += " " + m.lockInfo + } + return +} + +// mergeArguments creates and returns new arguments by merging and given . +func (m *Model) mergeArguments(args []interface{}) []interface{} { + if len(m.extraArgs) > 0 { + newArgs := make([]interface{}, len(m.extraArgs)+len(args)) + copy(newArgs, m.extraArgs) + copy(newArgs[len(m.extraArgs):], args) + return newArgs + } + return args +} diff --git a/database/gdb/gdb_result.go b/database/gdb/gdb_result.go new file mode 100644 index 000000000..2ebfc55fe --- /dev/null +++ b/database/gdb/gdb_result.go @@ -0,0 +1,47 @@ +// Copyright 2019 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gdb + +import "database/sql" + +// SqlResult is execution result for sql operations. +// It also supports batch operation result for rowsAffected. +type SqlResult struct { + result sql.Result + affected int64 +} + +// MustGetAffected returns the affected rows count, if any error occurs, it panics. +func (r *SqlResult) MustGetAffected() int64 { + rows, err := r.RowsAffected() + if err != nil { + panic(err) + } + return rows +} + +// MustGetInsertId returns the last insert id, if any error occurs, it panics. +func (r *SqlResult) MustGetInsertId() int64 { + id, err := r.LastInsertId() + if err != nil { + panic(err) + } + return id +} + +// see sql.Result.RowsAffected +func (r *SqlResult) RowsAffected() (int64, error) { + if r.affected > 0 { + return r.affected, nil + } + return r.result.RowsAffected() +} + +// see sql.Result.LastInsertId +func (r *SqlResult) LastInsertId() (int64, error) { + return r.result.LastInsertId() +} diff --git a/database/gdb/gdb_unit_z_mysql_model_test.go b/database/gdb/gdb_unit_z_mysql_model_test.go index 90a673732..e484e58e3 100644 --- a/database/gdb/gdb_unit_z_mysql_model_test.go +++ b/database/gdb/gdb_unit_z_mysql_model_test.go @@ -936,6 +936,18 @@ func Test_Model_GroupBy(t *testing.T) { }) } +func Test_Model_Data(t *testing.T) { + table := createInitTable() + defer dropTable(table) + + gtest.Case(t, func() { + result, err := db.Table(table).Data("nickname=?", "test").Where("id=?", 3).Update() + gtest.Assert(err, nil) + n, _ := result.RowsAffected() + gtest.Assert(n, 1) + }) +} + func Test_Model_Where(t *testing.T) { table := createInitTable() defer dropTable(table) @@ -1179,6 +1191,44 @@ func Test_Model_Where(t *testing.T) { }) } +func Test_Model_Where_ISNULL_1(t *testing.T) { + table := createInitTable() + defer dropTable(table) + + gtest.Case(t, func() { + //db.SetDebug(true) + result, err := db.Table(table).Data("nickname", nil).Where("id", 2).Update() + gtest.Assert(err, nil) + n, _ := result.RowsAffected() + gtest.Assert(n, 1) + + one, err := db.Table(table).Where("nickname", nil).One() + gtest.Assert(err, nil) + gtest.Assert(one.IsEmpty(), false) + gtest.Assert(one["id"], 2) + }) +} + +func Test_Model_Where_ISNULL_2(t *testing.T) { + table := createInitTable() + defer dropTable(table) + + // complicated one. + gtest.Case(t, func() { + //db.SetDebug(true) + conditions := g.Map{ + "nickname like ?": "%name%", + "id between ? and ?": g.Slice{1, 3}, + "id > 0": nil, + "create_time > 0": nil, + "id": g.Slice{1, 2, 3}, + } + result, err := db.Table(table).WherePri(conditions).Order("id asc").All() + gtest.Assert(err, nil) + gtest.Assert(len(result), 3) + gtest.Assert(result[0]["id"].Int(), 1) + }) +} func Test_Model_WherePri(t *testing.T) { table := createInitTable() defer dropTable(table) diff --git a/frame/gmvc/model.go b/frame/gmvc/model.go index 870ec79da..81a0ea2a5 100644 --- a/frame/gmvc/model.go +++ b/frame/gmvc/model.go @@ -6,5 +6,11 @@ package gmvc -// Model is the base struct for model. -type Model struct{} +import "github.com/gogf/gf/database/gdb" + +// M is alias for Model, +// just for short write purpose. +type M = Model + +// Model is alias for *gdb.Model. +type Model = *gdb.Model diff --git a/internal/empty/empty.go b/internal/empty/empty.go index aa859c266..675317138 100644 --- a/internal/empty/empty.go +++ b/internal/empty/empty.go @@ -51,7 +51,12 @@ func IsEmpty(value interface{}) bool { return len(value) == 0 default: // Finally using reflect. - rv := reflect.ValueOf(value) + var rv reflect.Value + if v, ok := value.(reflect.Value); ok { + rv = v + } else { + rv = reflect.ValueOf(value) + } switch rv.Kind() { case reflect.Chan, reflect.Map, @@ -77,7 +82,12 @@ func IsNil(value interface{}) bool { if value == nil { return true } - rv := reflect.ValueOf(value) + var rv reflect.Value + if v, ok := value.(reflect.Value); ok { + rv = v + } else { + rv = reflect.ValueOf(value) + } switch rv.Kind() { case reflect.Chan, reflect.Map, From 0a92df691b3f6aef825f74a69953248b763c987c Mon Sep 17 00:00:00 2001 From: John Date: Sat, 14 Mar 2020 17:12:44 +0800 Subject: [PATCH 16/32] improve package gpool/gfpool; donator updates --- .example/container/gpool/gpool.go | 2 +- .example/container/gpool/gpool_expirefunc.go | 2 +- .example/os/gfpool/gfpool.go | 2 +- DONATOR.MD | 17 ++- container/gpool/gpool.go | 94 +++++++++----- container/gpool/gpool_bench_test.go | 3 +- container/gpool/gpool_z_unit_test.go | 2 +- container/gtype/z_bench_basic_test.go | 6 + net/ghttp/ghttp_request.go | 14 +-- net/gtcp/gtcp_pool.go | 8 +- os/gfpool/gfpool.go | 124 +++++++++++-------- os/gfpool/gfpool_z_unit_test.go | 4 +- os/glog/glog_logger.go | 2 +- 13 files changed, 178 insertions(+), 102 deletions(-) diff --git a/.example/container/gpool/gpool.go b/.example/container/gpool/gpool.go index e0e215e8f..771c12edf 100644 --- a/.example/container/gpool/gpool.go +++ b/.example/container/gpool/gpool.go @@ -9,7 +9,7 @@ import ( func main() { // 创建一个对象池，过期时间为1000毫秒 - p := gpool.New(1000, nil) + p := gpool.New(1000*time.Millisecond, nil) // 从池中取一个对象，返回nil及错误信息 fmt.Println(p.Get()) diff --git a/.example/container/gpool/gpool_expirefunc.go b/.example/container/gpool/gpool_expirefunc.go index 39ced098f..cc490b1d4 100644 --- a/.example/container/gpool/gpool_expirefunc.go +++ b/.example/container/gpool/gpool_expirefunc.go @@ -11,7 +11,7 @@ import ( func main() { // 创建对象复用池，对象过期时间为3000毫秒，并给定创建及销毁方法 - p := gpool.New(3000, func() (interface{}, error) { + p := gpool.New(3000*time.Millisecond, func() (interface{}, error) { return gtcp.NewConn("www.baidu.com:80") }, func(i interface{}) { glog.Println("expired") diff --git a/.example/os/gfpool/gfpool.go b/.example/os/gfpool/gfpool.go index c7f1570f8..48090606e 100644 --- a/.example/os/gfpool/gfpool.go +++ b/.example/os/gfpool/gfpool.go @@ -11,7 +11,7 @@ import ( func main() { for { time.Sleep(time.Second) - if f, err := gfpool.Open("/home/john/temp/log.log", os.O_RDONLY, 0666, 60000000*1000); err == nil { + if f, err := gfpool.Open("/home/john/temp/log.log", os.O_RDONLY, 0666, time.Hour); err == nil { fmt.Println(f.Name()) f.Close() } else { diff --git a/DONATOR.MD b/DONATOR.MD index d5327b5a9..acd6ba0e6 100644 --- a/DONATOR.MD +++ b/DONATOR.MD @@ -40,7 +40,7 @@ We currently accept donation by Alipay/WechatPay, please note your github/gitee |R*s|wechat|￥18.88| 谢谢GF！辛苦了！ |粟*e|wechat|￥50.00| |[李超](https://github.com/effortlee)|wechat|￥124.00| -|张炳贤|wechat+qq|￥600.00| +|soidea666|wechat+qq|￥800.00| |[王哈哈](https://gitee.com/develop1024)|wechat|￥6.66| 希望gf越来越好 |夕景|alipay+qq|￥9.96+3.57| |struggler|alipay|￥18.80| @@ -51,6 +51,21 @@ We currently accept donation by Alipay/WechatPay, please note your github/gitee |[Zeroing-ZY](https://gitee.com/yunjieg)|gitee|￥20.00| 感谢您的开源项目！ |[katydid酱](https://gitee.com/katydid2005)|gitee|￥50.00| 感谢您的开源项目！框架给予了很大的帮助！谢谢大佬！ |[李海峰](https://gitee.com/dlhf)|gitee|￥10.00| 希望GF越来越好，框架很牛逼！ +|陆昱天|alipay|￥100.00| +|[Dockercore](https://github.com/dockercore)|wechat|￥200.00| 非常喜欢！简洁好用！文档超级全！ +|🚶|wechat|￥6.88| 喝杯冰阔落 +|a*l|wechat|￥10.00| gf +|[wxkj](https://gitee.com/wxkj)|wechat|￥10.00| +|*包|wechat|￥9.99| +|重庆宝尔威科技|wechat|￥6.66| +|琦玉-QPT|wechat|￥6.66| +|sailsea|wechat|￥11.00| +|[seny0929](https://gitee.com/seny0929)|wechat|￥99.90| +|*华|wechat|￥6.66| 感谢郭强的热心 +|[Playhi](https://github.com/Playhi)|alipay|￥10.00| +|北京京纬互动科技有限公司|alipay|￥200.00| +|米司特包|wechat|￥99.99| +|金毛|alipay|￥100.00| diff --git a/container/gpool/gpool.go b/container/gpool/gpool.go index ed9424c10..93df5b3ba 100644 --- a/container/gpool/gpool.go +++ b/container/gpool/gpool.go @@ -17,22 +17,31 @@ import ( "github.com/gogf/gf/os/gtimer" ) -// Object-Reusable Pool. +// Pool is an Object-Reusable Pool. type Pool struct { - list *glist.List // Available/idle list. - closed *gtype.Bool // Whether the pool is closed. - Expire int64 // Max idle time(ms), after which it is recycled. - NewFunc func() (interface{}, error) // Callback function to create item. - ExpireFunc func(interface{}) // Expired destruction function for objects. - // This function needs to be defined when the pool object - // needs to perform additional destruction operations. + // Available/idle items list. + list *glist.List + + // Whether the pool is closed. + closed *gtype.Bool + + // Time To Live for pool items. + TTL time.Duration + + // Callback function to create pool item. + NewFunc func() (interface{}, error) + + // ExpireFunc is the for expired items destruction. + // This function needs to be defined when the pool items + // need to perform additional destruction operations. // Eg: net.Conn, os.File, etc. + ExpireFunc func(interface{}) } // Pool item. type poolItem struct { - expire int64 // Expire time(millisecond). - value interface{} // Value. + expire int64 // Expire timestamp in milliseconds. + value interface{} // Item value. } // Creation function for object. @@ -41,47 +50,62 @@ type NewFunc func() (interface{}, error) // Destruction function for object. type ExpireFunc func(interface{}) -// New returns a new object pool. +// New creates and returns a new object pool. // To ensure execution efficiency, the expiration time cannot be modified once it is set. // -// Expiration logic: -// expire = 0 : not expired; -// expire < 0 : immediate expired after use; -// expire > 0 : timeout expired; -// Note that the expiration time unit is ** milliseconds **. -func New(expire int, newFunc NewFunc, expireFunc ...ExpireFunc) *Pool { +// Note the expiration logic: +// ttl = 0 : not expired; +// ttl < 0 : immediate expired after use; +// ttl > 0 : timeout expired; +func New(ttl time.Duration, newFunc NewFunc, expireFunc ...ExpireFunc) *Pool { r := &Pool{ list: glist.New(true), closed: gtype.NewBool(), - Expire: int64(expire), + TTL: ttl, NewFunc: newFunc, } if len(expireFunc) > 0 { r.ExpireFunc = expireFunc[0] } - gtimer.AddSingleton(time.Second, r.checkExpire) + gtimer.AddSingleton(time.Second, r.checkExpireItems) return r } // Put puts an item to pool. -func (p *Pool) Put(value interface{}) { +func (p *Pool) Put(value interface{}) error { + if p.closed.Val() { + return errors.New("pool is closed") + } item := &poolItem{ value: value, } - if p.Expire == 0 { + if p.TTL == 0 { item.expire = 0 } else { - item.expire = gtime.TimestampMilli() + p.Expire + item.expire = gtime.TimestampMilli() + p.TTL.Milliseconds() } p.list.PushBack(item) + return nil } // Clear clears pool, which means it will remove all items from pool. func (p *Pool) Clear() { - p.list.RemoveAll() + if p.ExpireFunc != nil { + for { + if r := p.list.PopFront(); r != nil { + p.ExpireFunc(r.(*poolItem).value) + } else { + break + } + } + } else { + p.list.RemoveAll() + } + } -// Get picks an item from pool. +// Get picks and returns an item from pool. If the pool is empty and NewFunc is defined, +// it creates and returns one from NewFunc. func (p *Pool) Get() (interface{}, error) { for !p.closed.Val() { if r := p.list.PopFront(); r != nil { @@ -106,12 +130,13 @@ func (p *Pool) Size() int { // Close closes the pool. If

has ExpireFunc, // then it automatically closes all items using this function before it's closed. +// Commonly you do not need call this function manually. func (p *Pool) Close() { p.closed.Set(true) } -// checkExpire removes expired items from pool every second. -func (p *Pool) checkExpire() { +// checkExpire removes expired items from pool in every second. +func (p *Pool) checkExpireItems() { if p.closed.Val() { // If p has ExpireFunc, // then it must close all items using this function. @@ -126,11 +151,24 @@ func (p *Pool) checkExpire() { } gtimer.Exit() } + // All items do not expire. + if p.TTL == 0 { + return + } + // The latest item expire timestamp in milliseconds. + var latestExpire int64 = -1 + // Retrieve the current timestamp in milliseconds, it expires the items + // by comparing with this timestamp. It is not accurate comparison for + // every items expired, but high performance. + var timestampMilli = gtime.TimestampMilli() for { - // TODO Do not use Pop and Push mechanism, which is not graceful. + if latestExpire > timestampMilli { + break + } if r := p.list.PopFront(); r != nil { item := r.(*poolItem) - if item.expire == 0 || item.expire > gtime.TimestampMilli() { + latestExpire = item.expire + if item.expire > timestampMilli { p.list.PushFront(item) break } diff --git a/container/gpool/gpool_bench_test.go b/container/gpool/gpool_bench_test.go index b45ba19f3..4dc1346cf 100644 --- a/container/gpool/gpool_bench_test.go +++ b/container/gpool/gpool_bench_test.go @@ -11,11 +11,12 @@ package gpool_test import ( "sync" "testing" + "time" "github.com/gogf/gf/container/gpool" ) -var pool = gpool.New(99999999, nil) +var pool = gpool.New(time.Hour, nil) var syncp = sync.Pool{} func BenchmarkGPoolPut(b *testing.B) { diff --git a/container/gpool/gpool_z_unit_test.go b/container/gpool/gpool_z_unit_test.go index 4f691d335..c27487c2c 100644 --- a/container/gpool/gpool_z_unit_test.go +++ b/container/gpool/gpool_z_unit_test.go @@ -62,7 +62,7 @@ func Test_Gpool(t *testing.T) { gtest.Case(t, func() { // //expire > 0 - p2 := gpool.New(2000, nil, ef) + p2 := gpool.New(2*time.Second, nil, ef) for index := 0; index < 10; index++ { p2.Put(index) } diff --git a/container/gtype/z_bench_basic_test.go b/container/gtype/z_bench_basic_test.go index 67c6ce3ca..8a0d6f932 100644 --- a/container/gtype/z_bench_basic_test.go +++ b/container/gtype/z_bench_basic_test.go @@ -157,6 +157,12 @@ func BenchmarkBool_Val(b *testing.B) { } } +func BenchmarkBool_Cas(b *testing.B) { + for i := 0; i < b.N; i++ { + bl.Cas(false, true) + } +} + func BenchmarkString_Set(b *testing.B) { for i := 0; i < b.N; i++ { str.Set(strconv.Itoa(i)) diff --git a/net/ghttp/ghttp_request.go b/net/ghttp/ghttp_request.go index 7662af053..e0ffb322b 100644 --- a/net/ghttp/ghttp_request.go +++ b/net/ghttp/ghttp_request.go @@ -23,23 +23,23 @@ import ( // Request is the context object for a request. type Request struct { *http.Request - Server *Server // Parent server. + Server *Server // Server. Cookie *Cookie // Cookie. Session *gsession.Session // Session. Response *Response // Corresponding Response of this request. - Router *Router // Matched Router for this request. Note that it's only available in HTTP handler, not in HOOK or MiddleWare. + Router *Router // Matched Router for this request. Note that it's not available in HOOK handler. EnterTime int64 // Request starting time in microseconds. LeaveTime int64 // Request ending time in microseconds. - Middleware *Middleware // The middleware manager. - StaticFile *StaticFile // Static file object when static file serving. - Context context.Context // Custom context map for internal usage purpose. - handlers []*handlerParsedItem // All matched handlers containing handler, hook and middleware for this request . + Middleware *Middleware // Middleware manager. + StaticFile *StaticFile // Static file object for static file serving. + Context context.Context // Custom context for internal usage purpose. + handlers []*handlerParsedItem // All matched handlers containing handler, hook and middleware for this request. hasHookHandler bool // A bool marking whether there's hook handler in the handlers for performance purpose. hasServeHandler bool // A bool marking whether there's serving handler in the handlers for performance purpose. parsedQuery bool // A bool marking whether the GET parameters parsed. parsedBody bool // A bool marking whether the request body parsed. parsedForm bool // A bool marking whether request Form parsed for HTTP method PUT, POST, PATCH. - paramsMap map[string]interface{} // Custom parameters. + paramsMap map[string]interface{} // Custom parameters map. routerMap map[string]string // Router parameters map, which might be nil if there're no router parameters. queryMap map[string]interface{} // Query parameters map, which is nil if there's no query string. formMap map[string]interface{} // Form parameters map, which is nil if there's no form data from client. diff --git a/net/gtcp/gtcp_pool.go b/net/gtcp/gtcp_pool.go index c421dc2e2..815c8a8bf 100644 --- a/net/gtcp/gtcp_pool.go +++ b/net/gtcp/gtcp_pool.go @@ -23,10 +23,10 @@ type PoolConn struct { } const ( - gDEFAULT_POOL_EXPIRE = 10000 // (Millisecond) Default TTL for connection in the pool. - gCONN_STATUS_UNKNOWN = 0 // Means it is unknown it's connective or not. - gCONN_STATUS_ACTIVE = 1 // Means it is now connective. - gCONN_STATUS_ERROR = 2 // Means it should be closed and removed from pool. + gDEFAULT_POOL_EXPIRE = 10 * time.Second // Default TTL for connection in the pool. + gCONN_STATUS_UNKNOWN = 0 // Means it is unknown it's connective or not. + gCONN_STATUS_ACTIVE = 1 // Means it is now connective. + gCONN_STATUS_ERROR = 2 // Means it should be closed and removed from pool. ) var ( diff --git a/os/gfpool/gfpool.go b/os/gfpool/gfpool.go index caae3486e..9dd92af0c 100644 --- a/os/gfpool/gfpool.go +++ b/os/gfpool/gfpool.go @@ -9,8 +9,9 @@ package gfpool import ( "fmt" + "github.com/gogf/gf/os/gfile" "os" - "sync" + "time" "github.com/gogf/gf/container/gmap" "github.com/gogf/gf/container/gpool" @@ -20,71 +21,84 @@ import ( // File pointer pool. type Pool struct { - id *gtype.Int // 指针池ID，用以识别指针池是否需要重建 - pool *gpool.Pool // 底层对象池 - inited *gtype.Bool // 是否初始化(在执行第一次执行File方法后初始化，主要用于文件监听的添加，但是只能添加一次) - expire int // 过期时间 + id *gtype.Int // Pool id, which is used to mark this pool whether recreated. + pool *gpool.Pool // Underlying pool. + init *gtype.Bool // Whether initialized, used for marking this file added to fsnotify, and it can only be added just once. + ttl time.Duration // Time to live for file pointer items. } -// 文件指针池指针 +// File is an item in the pool. type File struct { - *os.File // 底层文件指针 - mu sync.RWMutex // 互斥锁 - pool *Pool // 所属池 - poolid int // 所属池ID，如果池ID不同表示池已经重建，那么该文件指针也应当销毁，不能重新丢到原有的池中 - flag int // 打开标志 - perm os.FileMode // 打开权限 - path string // 绝对路径 + *os.File // Underlying file pointer. + pid int // Belonging pool id, which is set when file pointer created. It's used to check whether the pool is recreated. + pool *Pool // Belonging ool. + flag int // Flash for opening file. + perm os.FileMode // Permission for opening file. + path string // Absolute path of the file. } var ( - // 全局文件指针池Map, 不过期 + // Global file pointer pool. pools = gmap.NewStrAnyMap(true) ) -// 获得文件对象，并自动创建指针池(过期时间单位：毫秒) -func Open(path string, flag int, perm os.FileMode, expire ...int) (file *File, err error) { - fpExpire := 0 - if len(expire) > 0 { - fpExpire = expire[0] +// Open creates and returns a file item with given file path, flag and opening permission. +// It automatically creates an associated file pointer pool internally when it's called first time. +// It retrieves a file item from the file pointer pool after then. +func Open(path string, flag int, perm os.FileMode, ttl ...time.Duration) (file *File, err error) { + var fpTTL time.Duration + if len(ttl) > 0 { + fpTTL = ttl[0] } - pool := pools.GetOrSetFuncLock(fmt.Sprintf("%s&%d&%d&%d", path, flag, expire, perm), func() interface{} { - return New(path, flag, perm, fpExpire) - }).(*Pool) + path, err = gfile.Search(path) + if err != nil { + return nil, err + } + pool := pools.GetOrSetFuncLock( + fmt.Sprintf("%s&%d&%d&%d", path, flag, fpTTL, perm), + func() interface{} { + return New(path, flag, perm, fpTTL) + }, + ).(*Pool) return pool.File() } -// 创建一个文件指针池，expire = 0表示不过期，expire < 0表示使用完立即回收，expire > 0表示超时回收，默认值为0表示不过期。 -// 注意过期时间单位为：毫秒。 -func New(path string, flag int, perm os.FileMode, expire ...int) *Pool { - fpExpire := 0 - if len(expire) > 0 { - fpExpire = expire[0] +// New creates and returns a file pointer pool with given file path, flag and opening permission. +// +// Note the expiration logic: +// ttl = 0 : not expired; +// ttl < 0 : immediate expired after use; +// ttl > 0 : timeout expired; +// It is not expired in default. +func New(path string, flag int, perm os.FileMode, ttl ...time.Duration) *Pool { + var fpTTL time.Duration + if len(ttl) > 0 { + fpTTL = ttl[0] } p := &Pool{ - id: gtype.NewInt(), - expire: fpExpire, - inited: gtype.NewBool(), + id: gtype.NewInt(), + ttl: fpTTL, + init: gtype.NewBool(), } - p.pool = newFilePool(p, path, flag, perm, fpExpire) + p.pool = newFilePool(p, path, flag, perm, fpTTL) return p } -// 创建文件指针池 -func newFilePool(p *Pool, path string, flag int, perm os.FileMode, expire int) *gpool.Pool { - pool := gpool.New(expire, func() (interface{}, error) { +// newFilePool creates and returns a file pointer pool with given file path, flag and opening permission. +func newFilePool(p *Pool, path string, flag int, perm os.FileMode, ttl time.Duration) *gpool.Pool { + pool := gpool.New(ttl, func() (interface{}, error) { file, err := os.OpenFile(path, flag, perm) if err != nil { return nil, err } return &File{ - File: file, - pool: p, - poolid: p.id.Val(), - flag: flag, - perm: perm, - path: path, + File: file, + pool: p, + pid: p.id.Val(), + flag: flag, + perm: perm, + path: path, }, nil }, func(i interface{}) { _ = i.(*File).File.Close() @@ -92,7 +106,10 @@ func newFilePool(p *Pool, path string, flag int, perm os.FileMode, expire int) * return pool } -// 获得一个文件打开指针 +// File retrieves file item from the file pointer pool and returns it. It creates one if +// the file pointer pool is empty. +// Note that it should be closed when it will never be used. When it's closed, it is not +// really closed the underlying file pointer but put back to the file pinter pool. func (p *Pool) File() (*File, error) { if v, err := p.pool.Get(); err != nil { return nil, err @@ -127,18 +144,17 @@ func (p *Pool) File() (*File, error) { return nil, err } } - // 优先使用 !p.inited.Val() 原子读取操作判断，保证判断操作的效率； - // p.inited.Set(true) == false 使用原子写入操作，保证该操作的原子性； - if !p.inited.Val() && p.inited.Set(true) == false { + // It firstly checks using !p.init.Val() for performance purpose. + if !p.init.Val() && p.init.Cas(false, true) { _, _ = gfsnotify.Add(f.path, func(event *gfsnotify.Event) { - // 如果文件被删除或者重命名，立即重建指针池 + // If teh file is removed or renamed, recreates the pool by increasing the pool id. if event.IsRemove() || event.IsRename() { - // 原有的指针都不要了 + // It drops the old pool. p.id.Add(1) - // Clear相当于重建指针池 + // Clears the pool items staying in the pool. p.pool.Clear() - // 为保证原子操作，但又不想加锁， - // 这里再执行一次原子Add，将在两次Add中间可能分配出去的文件指针丢弃掉 + // It uses another adding to drop the file items between the two adding. + // Whenever the pool id changes, the pool will be recreated. p.id.Add(1) } }, false) @@ -147,15 +163,15 @@ func (p *Pool) File() (*File, error) { } } -// 关闭指针池 +// Close closes current file pointer pool. func (p *Pool) Close() { p.pool.Close() } -// 获得底层文件指针(返回error是标准库io.ReadWriteCloser接口实现) +// Close puts the file pointer back to the file pointer pool. func (f *File) Close() error { - if f.poolid == f.pool.id.Val() { - f.pool.pool.Put(f) + if f.pid == f.pool.id.Val() { + return f.pool.pool.Put(f) } return nil } diff --git a/os/gfpool/gfpool_z_unit_test.go b/os/gfpool/gfpool_z_unit_test.go index 3bfa7f7c0..af3d11130 100644 --- a/os/gfpool/gfpool_z_unit_test.go +++ b/os/gfpool/gfpool_z_unit_test.go @@ -79,12 +79,12 @@ func TestOpenExpire(t *testing.T) { testFile := start("TestOpenExpire.txt") gtest.Case(t, func() { - f, err := gfpool.Open(testFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_APPEND, 0666, 100) + f, err := gfpool.Open(testFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_APPEND, 0666, 100*time.Millisecond) gtest.AssertEQ(err, nil) f.Close() time.Sleep(150 * time.Millisecond) - f2, err1 := gfpool.Open(testFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_APPEND, 0666, 100) + f2, err1 := gfpool.Open(testFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_APPEND, 0666, 100*time.Millisecond) gtest.AssertEQ(err1, nil) //gtest.AssertNE(f, f2) f2.Close() diff --git a/os/glog/glog_logger.go b/os/glog/glog_logger.go index f990c0057..781bb277a 100644 --- a/os/glog/glog_logger.go +++ b/os/glog/glog_logger.go @@ -34,7 +34,7 @@ const ( gDEFAULT_FILE_FORMAT = `{Y-m-d}.log` gDEFAULT_FILE_POOL_FLAGS = os.O_CREATE | os.O_WRONLY | os.O_APPEND gDEFAULT_FPOOL_PERM = os.FileMode(0666) - gDEFAULT_FPOOL_EXPIRE = 60000 + gDEFAULT_FPOOL_EXPIRE = time.Minute gPATH_FILTER_KEY = "/os/glog/glog" ) From e9fba5a166024e3b23399e864997181e5d684727 Mon Sep 17 00:00:00 2001 From: John Date: Sat, 14 Mar 2020 17:15:25 +0800 Subject: [PATCH 17/32] readme updates --- README_ZH.MD | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README_ZH.MD b/README_ZH.MD index 42cbe14d0..cec8c3b2e 100644 --- a/README_ZH.MD +++ b/README_ZH.MD @@ -8,11 +8,12 @@ [English](README.MD) | 简体中文 -`GF(Go Frame)`是一款模块化、高性能、生产级的Go基础开发框架。实现了比较完善的基础设施建设，包括常用的核心开发组件， -如：缓存、日志、文件、时间、队列、数组、集合、字符串、定时器、命令行、文件锁、内存锁、对象池、连接池、资源管理、数据校验、数据编码、文件监控、 -定时任务、数据库ORM、TCP/UDP组件、进程管理/通信、并发安全容器等等。 -并提供了Web服务开发的系列核心组件，如：Router、Cookie、Session、Middleware、服务注册、配置管理、模板引擎等等， -支持热重启、热更新、多域名、多端口、多服务、HTTPS、Rewrite等特性。 +`GF(Go Frame)`是一款模块化、高性能、生产级的Go基础开发框架。 +实现了比较完善的基础设施建设以及开发工具链，提供了常用的基础开发模块， +如：缓存、日志、队列、数组、集合、容器、定时器、命令行、内存锁、对象池、 +配置管理、资源管理、数据校验、数据编码、定时任务、数据库ORM、TCP/UDP组件、进程管理/通信等等。 +并提供了Web服务开发的系列核心组件，如：Router、Cookie、Session、Middleware、服务注册、模板引擎等等， +支持热重启、热更新、域名绑定、TLS/HTTPS、Rewrite等特性。 # 特点 From 74be9fac1863b7fbb389db3182aaafa6a3c261b5 Mon Sep 17 00:00:00 2001 From: John Date: Sun, 15 Mar 2020 19:32:26 +0800 Subject: [PATCH 18/32] add file rotation feature for package glog; improve gpool/gfpool; fix issue in gfile.MTimeMillisecond --- container/garray/garray_sorted_any.go | 6 +- database/gdb/gdb_core.go | 2 +- database/gdb/gdb_unit_init_test.go | 1 + encoding/gcompress/gcompress_gzip.go | 60 ++++++- .../gcompress/gcompress_z_unit_gzip_test.go | 42 ++++- ...gcompress_zip_file.go => gcompress_zip.go} | 0 encoding/gcompress/testdata/gzip/file.txt | 1 + os/gfile/gfile.go | 4 +- os/gfile/gfile_time.go | 2 +- os/gfpool/gfpool.go | 145 +-------------- os/gfpool/gfpool_file.go | 55 ++++++ os/gfpool/gfpool_pool.go | 121 +++++++++++++ os/gfpool/gfpool_z_bench_test.go | 42 +++-- os/glog/glog_logger.go | 39 +++- os/glog/glog_logger_config.go | 45 +++-- os/glog/glog_logger_rotate.go | 168 ++++++++++++++++++ os/glog/glog_z_unit_rotate_test.go | 56 ++++++ 17 files changed, 593 insertions(+), 196 deletions(-) rename encoding/gcompress/{gcompress_zip_file.go => gcompress_zip.go} (100%) create mode 100644 encoding/gcompress/testdata/gzip/file.txt create mode 100644 os/gfpool/gfpool_file.go create mode 100644 os/gfpool/gfpool_pool.go create mode 100644 os/glog/glog_logger_rotate.go create mode 100644 os/glog/glog_z_unit_rotate_test.go diff --git a/container/garray/garray_sorted_any.go b/container/garray/garray_sorted_any.go index 32439e8f8..b55dca332 100644 --- a/container/garray/garray_sorted_any.go +++ b/container/garray/garray_sorted_any.go @@ -33,9 +33,9 @@ type SortedArray struct { // NewSortedArray creates and returns an empty sorted array. // The parameter is used to specify whether using array in concurrent-safety, which is false in default. // The parameter used to compare values to sort in array, -// if it returns value < 0, means v1 < v2; -// if it returns value = 0, means v1 = v2; -// if it returns value > 0, means v1 > v2; +// if it returns value < 0, means v1 < v2; the v1 will be inserted before v2; +// if it returns value = 0, means v1 = v2; the v1 will be replaced by v2; +// if it returns value > 0, means v1 > v2; the v1 will be inserted after v2; func NewSortedArray(comparator func(a, b interface{}) int, safe ...bool) *SortedArray { return NewSortedArraySize(0, comparator, safe...) } diff --git a/database/gdb/gdb_core.go b/database/gdb/gdb_core.go index 92e2ead29..7770f2cdd 100644 --- a/database/gdb/gdb_core.go +++ b/database/gdb/gdb_core.go @@ -721,7 +721,7 @@ func (c *Core) MarshalJSON() ([]byte, error) { // writeSqlToLogger outputs the sql object to logger. // It is enabled when configuration "debug" is true. func (c *Core) writeSqlToLogger(v *Sql) { - s := fmt.Sprintf("[%d ms] %s", v.End-v.Start, v.Format) + s := fmt.Sprintf("[%3d ms] %s", v.End-v.Start, v.Format) if v.Error != nil { s += "\nError: " + v.Error.Error() c.logger.StackWithFilter(gPATH_FILTER_KEY).Error(s) diff --git a/database/gdb/gdb_unit_init_test.go b/database/gdb/gdb_unit_init_test.go index 2879fe6ab..957529bd1 100644 --- a/database/gdb/gdb_unit_init_test.go +++ b/database/gdb/gdb_unit_init_test.go @@ -62,6 +62,7 @@ func init() { } else { db = r } + db.SetDebug(true) schemaTemplate := "CREATE DATABASE IF NOT EXISTS `%s` CHARACTER SET UTF8" if _, err := db.Exec(fmt.Sprintf(schemaTemplate, SCHEMA1)); err != nil { gtest.Error(err) diff --git a/encoding/gcompress/gcompress_gzip.go b/encoding/gcompress/gcompress_gzip.go index 43bd05c43..fd5cc90a3 100644 --- a/encoding/gcompress/gcompress_gzip.go +++ b/encoding/gcompress/gcompress_gzip.go @@ -9,10 +9,11 @@ package gcompress import ( "bytes" "compress/gzip" + "github.com/gogf/gf/os/gfile" "io" ) -// Gzip compresses with gzip algorithm. +// Gzip compresses using gzip algorithm. // The optional parameter specifies the compression level from // 1 to 9 which means from none to the best compression. // @@ -38,6 +39,38 @@ func Gzip(data []byte, level ...int) ([]byte, error) { return buf.Bytes(), nil } +// GzipFile compresses the file to using gzip algorithm. +func GzipFile(src, dst string, level ...int) error { + var writer *gzip.Writer + var err error + srcFile, err := gfile.Open(src) + if err != nil { + return err + } + defer srcFile.Close() + dstFile, err := gfile.Create(dst) + if err != nil { + return err + } + defer dstFile.Close() + + if len(level) > 0 { + writer, err = gzip.NewWriterLevel(dstFile, level[0]) + if err != nil { + return err + } + } else { + writer = gzip.NewWriter(dstFile) + } + defer writer.Close() + + _, err = io.Copy(writer, srcFile) + if err != nil { + return err + } + return nil +} + // UnGzip decompresses with gzip algorithm. func UnGzip(data []byte) ([]byte, error) { var buf bytes.Buffer @@ -53,3 +86,28 @@ func UnGzip(data []byte) ([]byte, error) { } return buf.Bytes(), nil } + +// UnGzip decompresses file to using gzip algorithm. +func UnGzipFile(src, dst string) error { + srcFile, err := gfile.Open(src) + if err != nil { + return err + } + defer srcFile.Close() + dstFile, err := gfile.Create(dst) + if err != nil { + return err + } + defer dstFile.Close() + + reader, err := gzip.NewReader(srcFile) + if err != nil { + return err + } + defer reader.Close() + + if _, err = io.Copy(dstFile, reader); err != nil { + return err + } + return nil +} diff --git a/encoding/gcompress/gcompress_z_unit_gzip_test.go b/encoding/gcompress/gcompress_z_unit_gzip_test.go index 712261a03..14ffab44a 100644 --- a/encoding/gcompress/gcompress_z_unit_gzip_test.go +++ b/encoding/gcompress/gcompress_z_unit_gzip_test.go @@ -7,6 +7,9 @@ package gcompress_test import ( + "github.com/gogf/gf/debug/gdebug" + "github.com/gogf/gf/os/gfile" + "github.com/gogf/gf/os/gtime" "testing" "github.com/gogf/gf/encoding/gcompress" @@ -26,14 +29,37 @@ func Test_Gzip_UnGzip(t *testing.T) { 0x24, 0xa8, 0xd1, 0x0d, 0x00, 0x00, 0x00, } + gtest.Case(t, func() { + arr := []byte(src) + data, _ := gcompress.Gzip(arr) + gtest.Assert(data, gzip) - arr := []byte(src) - data, _ := gcompress.Gzip(arr) - gtest.Assert(data, gzip) + data, _ = gcompress.UnGzip(gzip) + gtest.Assert(data, arr) - data, _ = gcompress.UnGzip(gzip) - gtest.Assert(data, arr) - - data, _ = gcompress.UnGzip(gzip[1:]) - gtest.Assert(data, nil) + data, _ = gcompress.UnGzip(gzip[1:]) + gtest.Assert(data, nil) + }) +} + +func Test_Gzip_UnGzip_File(t *testing.T) { + srcPath := gfile.Join(gdebug.CallerDirectory(), "testdata", "gzip", "file.txt") + dstPath1 := gfile.Join(gfile.TempDir(), gtime.TimestampNanoStr(), "gzip.zip") + dstPath2 := gfile.Join(gfile.TempDir(), gtime.TimestampNanoStr(), "file.txt") + + // Compress. + gtest.Case(t, func() { + err := gcompress.GzipFile(srcPath, dstPath1, 9) + gtest.Assert(err, nil) + defer gfile.Remove(dstPath1) + gtest.Assert(gfile.Exists(dstPath1), true) + + // Decompress. + err = gcompress.UnGzipFile(dstPath1, dstPath2) + gtest.Assert(err, nil) + defer gfile.Remove(dstPath2) + gtest.Assert(gfile.Exists(dstPath2), true) + + gtest.Assert(gfile.GetContents(srcPath), gfile.GetContents(dstPath2)) + }) } diff --git a/encoding/gcompress/gcompress_zip_file.go b/encoding/gcompress/gcompress_zip.go similarity index 100% rename from encoding/gcompress/gcompress_zip_file.go rename to encoding/gcompress/gcompress_zip.go diff --git a/encoding/gcompress/testdata/gzip/file.txt b/encoding/gcompress/testdata/gzip/file.txt new file mode 100644 index 000000000..14a9d9b49 --- /dev/null +++ b/encoding/gcompress/testdata/gzip/file.txt @@ -0,0 +1 @@ +This is a test file for gzip compression. \ No newline at end of file diff --git a/os/gfile/gfile.go b/os/gfile/gfile.go index d905f09d4..f230bbad0 100644 --- a/os/gfile/gfile.go +++ b/os/gfile/gfile.go @@ -312,7 +312,7 @@ func SelfDir() string { return filepath.Dir(SelfPath()) } -// Basename returns the last element of path. +// Basename returns the last element of path, which contains file extension. // Trailing path separators are removed before extracting the last element. // If the path is empty, Base returns ".". // If the path consists entirely of separators, Basename returns a single separator. @@ -320,7 +320,7 @@ func Basename(path string) string { return filepath.Base(path) } -// Name returns the last element of path without extension. +// Name returns the last element of path without file extension. func Name(path string) string { base := filepath.Base(path) if i := strings.LastIndexByte(base, '.'); i != -1 { diff --git a/os/gfile/gfile_time.go b/os/gfile/gfile_time.go index b468bc712..8cacb7677 100644 --- a/os/gfile/gfile_time.go +++ b/os/gfile/gfile_time.go @@ -25,5 +25,5 @@ func MTimeMillisecond(path string) int64 { if e != nil { return 0 } - return int64(s.ModTime().Nanosecond() / 1000000) + return s.ModTime().UnixNano() / 1000000 } diff --git a/os/gfpool/gfpool.go b/os/gfpool/gfpool.go index 9dd92af0c..db4e7ecd9 100644 --- a/os/gfpool/gfpool.go +++ b/os/gfpool/gfpool.go @@ -1,4 +1,4 @@ -// Copyright 2017-2019 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// Copyright 2017-2020 gf Author(https://github.com/gogf/gf). All Rights Reserved. // // This Source Code Form is subject to the terms of the MIT License. // If a copy of the MIT was not distributed with this file, @@ -8,15 +8,11 @@ package gfpool import ( - "fmt" - "github.com/gogf/gf/os/gfile" - "os" - "time" - "github.com/gogf/gf/container/gmap" "github.com/gogf/gf/container/gpool" "github.com/gogf/gf/container/gtype" - "github.com/gogf/gf/os/gfsnotify" + "os" + "time" ) // File pointer pool. @@ -30,6 +26,7 @@ type Pool struct { // File is an item in the pool. type File struct { *os.File // Underlying file pointer. + stat os.FileInfo // State of current file pointer. pid int // Belonging pool id, which is set when file pointer created. It's used to check whether the pool is recreated. pool *Pool // Belonging ool. flag int // Flash for opening file. @@ -41,137 +38,3 @@ var ( // Global file pointer pool. pools = gmap.NewStrAnyMap(true) ) - -// Open creates and returns a file item with given file path, flag and opening permission. -// It automatically creates an associated file pointer pool internally when it's called first time. -// It retrieves a file item from the file pointer pool after then. -func Open(path string, flag int, perm os.FileMode, ttl ...time.Duration) (file *File, err error) { - var fpTTL time.Duration - if len(ttl) > 0 { - fpTTL = ttl[0] - } - path, err = gfile.Search(path) - if err != nil { - return nil, err - } - pool := pools.GetOrSetFuncLock( - fmt.Sprintf("%s&%d&%d&%d", path, flag, fpTTL, perm), - func() interface{} { - return New(path, flag, perm, fpTTL) - }, - ).(*Pool) - - return pool.File() -} - -// New creates and returns a file pointer pool with given file path, flag and opening permission. -// -// Note the expiration logic: -// ttl = 0 : not expired; -// ttl < 0 : immediate expired after use; -// ttl > 0 : timeout expired; -// It is not expired in default. -func New(path string, flag int, perm os.FileMode, ttl ...time.Duration) *Pool { - var fpTTL time.Duration - if len(ttl) > 0 { - fpTTL = ttl[0] - } - p := &Pool{ - id: gtype.NewInt(), - ttl: fpTTL, - init: gtype.NewBool(), - } - p.pool = newFilePool(p, path, flag, perm, fpTTL) - return p -} - -// newFilePool creates and returns a file pointer pool with given file path, flag and opening permission. -func newFilePool(p *Pool, path string, flag int, perm os.FileMode, ttl time.Duration) *gpool.Pool { - pool := gpool.New(ttl, func() (interface{}, error) { - file, err := os.OpenFile(path, flag, perm) - if err != nil { - return nil, err - } - return &File{ - File: file, - pool: p, - pid: p.id.Val(), - flag: flag, - perm: perm, - path: path, - }, nil - }, func(i interface{}) { - _ = i.(*File).File.Close() - }) - return pool -} - -// File retrieves file item from the file pointer pool and returns it. It creates one if -// the file pointer pool is empty. -// Note that it should be closed when it will never be used. When it's closed, it is not -// really closed the underlying file pointer but put back to the file pinter pool. -func (p *Pool) File() (*File, error) { - if v, err := p.pool.Get(); err != nil { - return nil, err - } else { - f := v.(*File) - stat, err := os.Stat(f.path) - if f.flag&os.O_CREATE > 0 { - if os.IsNotExist(err) { - if file, err := os.OpenFile(f.path, f.flag, f.perm); err != nil { - return nil, err - } else { - f.File = file - if stat, err = f.Stat(); err != nil { - return nil, err - } - } - } - } - if f.flag&os.O_TRUNC > 0 { - if stat.Size() > 0 { - if err := f.Truncate(0); err != nil { - return nil, err - } - } - } - if f.flag&os.O_APPEND > 0 { - if _, err := f.Seek(0, 2); err != nil { - return nil, err - } - } else { - if _, err := f.Seek(0, 0); err != nil { - return nil, err - } - } - // It firstly checks using !p.init.Val() for performance purpose. - if !p.init.Val() && p.init.Cas(false, true) { - _, _ = gfsnotify.Add(f.path, func(event *gfsnotify.Event) { - // If teh file is removed or renamed, recreates the pool by increasing the pool id. - if event.IsRemove() || event.IsRename() { - // It drops the old pool. - p.id.Add(1) - // Clears the pool items staying in the pool. - p.pool.Clear() - // It uses another adding to drop the file items between the two adding. - // Whenever the pool id changes, the pool will be recreated. - p.id.Add(1) - } - }, false) - } - return f, nil - } -} - -// Close closes current file pointer pool. -func (p *Pool) Close() { - p.pool.Close() -} - -// Close puts the file pointer back to the file pointer pool. -func (f *File) Close() error { - if f.pid == f.pool.id.Val() { - return f.pool.pool.Put(f) - } - return nil -} diff --git a/os/gfpool/gfpool_file.go b/os/gfpool/gfpool_file.go new file mode 100644 index 000000000..b985722b1 --- /dev/null +++ b/os/gfpool/gfpool_file.go @@ -0,0 +1,55 @@ +// Copyright 2017-2020 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gfpool + +import ( + "errors" + "fmt" + "os" + "time" +) + +// Open creates and returns a file item with given file path, flag and opening permission. +// It automatically creates an associated file pointer pool internally when it's called first time. +// It retrieves a file item from the file pointer pool after then. +func Open(path string, flag int, perm os.FileMode, ttl ...time.Duration) (file *File, err error) { + var fpTTL time.Duration + if len(ttl) > 0 { + fpTTL = ttl[0] + } + // DO NOT search the path here wasting performance! + // Leave following codes just for warning you. + // + //path, err = gfile.Search(path) + //if err != nil { + // return nil, err + //} + pool := pools.GetOrSetFuncLock( + fmt.Sprintf("%s&%d&%d&%d", path, flag, fpTTL, perm), + func() interface{} { + return New(path, flag, perm, fpTTL) + }, + ).(*Pool) + + return pool.File() +} + +// Stat returns the FileInfo structure describing file. +func (f *File) Stat() (os.FileInfo, error) { + if f.stat == nil { + return nil, errors.New("file stat is empty") + } + return f.stat, nil +} + +// Close puts the file pointer back to the file pointer pool. +func (f *File) Close() error { + if f.pid == f.pool.id.Val() { + return f.pool.pool.Put(f) + } + return nil +} diff --git a/os/gfpool/gfpool_pool.go b/os/gfpool/gfpool_pool.go new file mode 100644 index 000000000..e83abf6aa --- /dev/null +++ b/os/gfpool/gfpool_pool.go @@ -0,0 +1,121 @@ +// Copyright 2017-2020 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gfpool + +import ( + "os" + "time" + + "github.com/gogf/gf/container/gpool" + "github.com/gogf/gf/container/gtype" + "github.com/gogf/gf/os/gfsnotify" +) + +// New creates and returns a file pointer pool with given file path, flag and opening permission. +// +// Note the expiration logic: +// ttl = 0 : not expired; +// ttl < 0 : immediate expired after use; +// ttl > 0 : timeout expired; +// It is not expired in default. +func New(path string, flag int, perm os.FileMode, ttl ...time.Duration) *Pool { + var fpTTL time.Duration + if len(ttl) > 0 { + fpTTL = ttl[0] + } + p := &Pool{ + id: gtype.NewInt(), + ttl: fpTTL, + init: gtype.NewBool(), + } + p.pool = newFilePool(p, path, flag, perm, fpTTL) + return p +} + +// newFilePool creates and returns a file pointer pool with given file path, flag and opening permission. +func newFilePool(p *Pool, path string, flag int, perm os.FileMode, ttl time.Duration) *gpool.Pool { + pool := gpool.New(ttl, func() (interface{}, error) { + file, err := os.OpenFile(path, flag, perm) + if err != nil { + return nil, err + } + return &File{ + File: file, + pid: p.id.Val(), + pool: p, + flag: flag, + perm: perm, + path: path, + }, nil + }, func(i interface{}) { + _ = i.(*File).File.Close() + }) + return pool +} + +// File retrieves file item from the file pointer pool and returns it. It creates one if +// the file pointer pool is empty. +// Note that it should be closed when it will never be used. When it's closed, it is not +// really closed the underlying file pointer but put back to the file pinter pool. +func (p *Pool) File() (*File, error) { + if v, err := p.pool.Get(); err != nil { + return nil, err + } else { + var err error + f := v.(*File) + f.stat, err = os.Stat(f.path) + if f.flag&os.O_CREATE > 0 { + if os.IsNotExist(err) { + if f.File, err = os.OpenFile(f.path, f.flag, f.perm); err != nil { + return nil, err + } else { + // Retrieve the state of the new created file. + if f.stat, err = f.File.Stat(); err != nil { + return nil, err + } + } + } + } + if f.flag&os.O_TRUNC > 0 { + if f.stat.Size() > 0 { + if err = f.Truncate(0); err != nil { + return nil, err + } + } + } + if f.flag&os.O_APPEND > 0 { + if _, err = f.Seek(0, 2); err != nil { + return nil, err + } + } else { + if _, err = f.Seek(0, 0); err != nil { + return nil, err + } + } + // It firstly checks using !p.init.Val() for performance purpose. + if !p.init.Val() && p.init.Cas(false, true) { + _, _ = gfsnotify.Add(f.path, func(event *gfsnotify.Event) { + // If teh file is removed or renamed, recreates the pool by increasing the pool id. + if event.IsRemove() || event.IsRename() { + // It drops the old pool. + p.id.Add(1) + // Clears the pool items staying in the pool. + p.pool.Clear() + // It uses another adding to drop the file items between the two adding. + // Whenever the pool id changes, the pool will be recreated. + p.id.Add(1) + } + }, false) + } + return f, nil + } +} + +// Close closes current file pointer pool. +func (p *Pool) Close() { + p.pool.Close() +} diff --git a/os/gfpool/gfpool_z_bench_test.go b/os/gfpool/gfpool_z_bench_test.go index da59aa54f..025865033 100644 --- a/os/gfpool/gfpool_z_bench_test.go +++ b/os/gfpool/gfpool_z_bench_test.go @@ -5,44 +5,62 @@ import ( "testing" ) -func Benchmark_os_Open_Close_ALLFlags(b *testing.B) { +func Benchmark_OS_Open_Close_ALLFlags(b *testing.B) { for i := 0; i < b.N; i++ { - f, _ := os.OpenFile("/tmp/bench-test", os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_APPEND, 0666) + f, err := os.OpenFile("/tmp/bench-test", os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_APPEND, 0666) + if err != nil { + panic(err) + } f.Close() } } -func Benchmark_gfpool_Open_Close_ALLFlags(b *testing.B) { +func Benchmark_GFPool_Open_Close_ALLFlags(b *testing.B) { for i := 0; i < b.N; i++ { - f, _ := Open("/tmp/bench-test", os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_APPEND, 0666) + f, err := Open("/tmp/bench-test", os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_APPEND, 0666) + if err != nil { + panic(err) + } f.Close() } } -func Benchmark_os_Open_Close_RDWR(b *testing.B) { +func Benchmark_OS_Open_Close_RDWR(b *testing.B) { for i := 0; i < b.N; i++ { - f, _ := os.OpenFile("/tmp/bench-test", os.O_RDWR, 0666) + f, err := os.OpenFile("/tmp/bench-test", os.O_RDWR, 0666) + if err != nil { + panic(err) + } f.Close() } } -func Benchmark_gfpool_Open_Close_RDWR(b *testing.B) { +func Benchmark_GFPool_Open_Close_RDWR(b *testing.B) { for i := 0; i < b.N; i++ { - f, _ := Open("/tmp/bench-test", os.O_RDWR, 0666) + f, err := Open("/tmp/bench-test", os.O_RDWR, 0666) + if err != nil { + panic(err) + } f.Close() } } -func Benchmark_os_Open_Close_RDONLY(b *testing.B) { +func Benchmark_OS_Open_Close_RDONLY(b *testing.B) { for i := 0; i < b.N; i++ { - f, _ := os.OpenFile("/tmp/bench-test", os.O_RDONLY, 0666) + f, err := os.OpenFile("/tmp/bench-test", os.O_RDONLY, 0666) + if err != nil { + panic(err) + } f.Close() } } -func Benchmark_gfpool_Open_Close_RDONLY(b *testing.B) { +func Benchmark_GFPool_Open_Close_RDONLY(b *testing.B) { for i := 0; i < b.N; i++ { - f, _ := Open("/tmp/bench-test", os.O_RDONLY, 0666) + f, err := Open("/tmp/bench-test", os.O_RDONLY, 0666) + if err != nil { + panic(err) + } f.Close() } } diff --git a/os/glog/glog_logger.go b/os/glog/glog_logger.go index 781bb277a..0575ed436 100644 --- a/os/glog/glog_logger.go +++ b/os/glog/glog_logger.go @@ -10,9 +10,11 @@ import ( "bytes" "fmt" "github.com/gogf/gf/internal/intlog" + "github.com/gogf/gf/os/gtimer" "io" "os" "strings" + "sync" "time" "github.com/gogf/gf/debug/gdebug" @@ -26,8 +28,9 @@ import ( // Logger is the struct for logging management. type Logger struct { - parent *Logger // Parent logger. - config Config // Logger configuration. + mu sync.Mutex // Mutex is not for common logging, but for file rotation feature. + parent *Logger // Parent logger. + config Config // Logger configuration. } const ( @@ -50,9 +53,11 @@ const ( // New creates and returns a custom logger. func New() *Logger { - return &Logger{ + logger := &Logger{ config: DefaultConfig(), } + gtimer.AddOnce(time.Second, logger.rotateChecks) + return logger } // NewWithWriter creates and returns a custom logger with io.Writer. @@ -63,6 +68,7 @@ func NewWithWriter(writer io.Writer) *Logger { } // Clone returns a new logger, which is the clone the current logger. +// It's commonly used for chaining operations. func (l *Logger) Clone() *Logger { logger := Logger{} logger = *l @@ -74,10 +80,6 @@ func (l *Logger) Clone() *Logger { // It returns nil if file logging is disabled, or file opening fails. func (l *Logger) getFilePointer() *gfpool.File { if path := l.config.Path; path != "" { - // Content containing "{}" in the file name is formatted using gtime. - file, _ := gregex.ReplaceStringFunc(`{.+?}`, l.config.File, func(s string) string { - return gtime.Now().Format(strings.Trim(s, "{}")) - }) // Create path if it does not exist. if !gfile.Exists(path) { if err := gfile.Mkdir(path); err != nil { @@ -86,7 +88,7 @@ func (l *Logger) getFilePointer() *gfpool.File { } } if fp, err := gfpool.Open( - path+gfile.Separator+file, + l.getFilePath(), gDEFAULT_FILE_POOL_FLAGS, gDEFAULT_FPOOL_PERM, gDEFAULT_FPOOL_EXPIRE); err == nil { @@ -98,6 +100,15 @@ func (l *Logger) getFilePointer() *gfpool.File { return nil } +// getFilePath returns the logging file path. +func (l *Logger) getFilePath() string { + // Content containing "{}" in the file name is formatted using gtime. + file, _ := gregex.ReplaceStringFunc(`{.+?}`, l.config.File, func(s string) string { + return gtime.Now().Format(strings.Trim(s, "{}")) + }) + return gfile.Join(l.config.Path, file) +} + // print prints to defined writer, logging file or passed . func (l *Logger) print(std io.Writer, lead string, value ...interface{}) { buffer := bytes.NewBuffer(nil) @@ -183,6 +194,18 @@ func (l *Logger) printToWriter(std io.Writer, buffer *bytes.Buffer) { if l.config.Writer == nil { if f := l.getFilePointer(); f != nil { defer f.Close() + // Rotation file size checks. + if l.config.RotateSize > 0 { + state, err := f.Stat() + if err != nil { + panic(err) + } + if state.Size() > l.config.RotateSize { + l.rotateFile() + l.printToWriter(std, buffer) + return + } + } if _, err := io.WriteString(f, buffer.String()); err != nil { fmt.Fprintln(os.Stderr, err.Error()) } diff --git a/os/glog/glog_logger_config.go b/os/glog/glog_logger_config.go index c66836d9f..7b8ee2d56 100644 --- a/os/glog/glog_logger_config.go +++ b/os/glog/glog_logger_config.go @@ -14,34 +14,41 @@ import ( "github.com/gogf/gf/util/gutil" "io" "strings" + "time" ) // Config is the configuration object for logger. type Config struct { - Writer io.Writer // Customized io.Writer. - Flags int // Extra flags for logging output features. - Path string // Logging directory path. - File string // Format for logging file. - Level int // Output level. - Prefix string // Prefix string for every logging content. - StSkip int // Skip count for stack. - StStatus int // Stack status(1: enabled - default; 0: disabled) - StFilter string // Stack string filter. - HeaderPrint bool `c:"header"` // Print header or not(true in default). - StdoutPrint bool `c:"stdout"` // Output to stdout or not(true in default). - LevelPrefixes map[int]string // Logging level to its prefix string mapping. + Writer io.Writer // Customized io.Writer. + Flags int // Extra flags for logging output features. + Path string // Logging directory path. + File string // Format for logging file. + Level int // Output level. + Prefix string // Prefix string for every logging content. + StSkip int // Skip count for stack. + StStatus int // Stack status(1: enabled - default; 0: disabled) + StFilter string // Stack string filter. + HeaderPrint bool `c:"header"` // Print header or not(true in default). + StdoutPrint bool `c:"stdout"` // Output to stdout or not(true in default). + LevelPrefixes map[int]string // Logging level to its prefix string mapping. + RotateSize int64 // Enables the rotate feature by set the size > 0 in bytes. + RotateBackups int // Max backups for rotated files, default is 0, means no backups. + RotateExpire time.Duration // Max expire age for rotated files. It's 0 in default, means no expiration. + RotateCompress int // Compress level for rotated files using gzip algorithm. It's 0 in default, means no compression. + RotateInterval time.Duration // Asynchronizely checks the backups and expiration at intervals. It's 1 minute in default. } // DefaultConfig returns the default configuration for logger. func DefaultConfig() Config { c := Config{ - File: gDEFAULT_FILE_FORMAT, - Flags: F_TIME_STD, - Level: LEVEL_ALL, - StStatus: 1, - HeaderPrint: true, - StdoutPrint: true, - LevelPrefixes: make(map[int]string, len(defaultLevelPrefixes)), + File: gDEFAULT_FILE_FORMAT, + Flags: F_TIME_STD, + Level: LEVEL_ALL, + StStatus: 1, + HeaderPrint: true, + StdoutPrint: true, + LevelPrefixes: make(map[int]string, len(defaultLevelPrefixes)), + RotateInterval: time.Minute, } for k, v := range defaultLevelPrefixes { c.LevelPrefixes[k] = v diff --git a/os/glog/glog_logger_rotate.go b/os/glog/glog_logger_rotate.go new file mode 100644 index 000000000..e61d8a952 --- /dev/null +++ b/os/glog/glog_logger_rotate.go @@ -0,0 +1,168 @@ +// Copyright 2020 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package glog + +import ( + "fmt" + "github.com/gogf/gf/container/garray" + "github.com/gogf/gf/encoding/gcompress" + "github.com/gogf/gf/internal/intlog" + "github.com/gogf/gf/os/gfile" + "github.com/gogf/gf/os/gtime" + "github.com/gogf/gf/os/gtimer" + "github.com/gogf/gf/text/gregex" +) + +// rotateFile rotates the current logging file. +func (l *Logger) rotateFile() { + // Rotation feature is not enabled as rotation file size is zero. + if l.config.RotateSize == 0 { + return + } + l.mu.Lock() + defer l.mu.Unlock() + filePath := l.getFilePath() + // No backups, it then just removes the current logging file. + if l.config.RotateBackups == 0 { + if err := gfile.Remove(filePath); err != nil { + intlog.Print(err) + } + intlog.Printf(`%d size exceeds, no backups set, remove original logging file: %s`, l.config.RotateSize, filePath) + return + } + // Else it creates new backup files. + var ( + dirPath = gfile.Dir(filePath) + fileName = gfile.Name(filePath) + fileExt = gfile.Ext(filePath) + newFilePath = "" + ) + for { + // Rename the logging file by adding extra time information to milliseconds, like: + // access -> access.20200102190000899 + // access.log -> access.20200102190000899.log + // access.20200102.log -> access.20200102.20200102190000899.log + newFilePath = gfile.Join( + dirPath, + fmt.Sprintf(`%s.%s%s`, fileName, gtime.Now().Format("YmdHisu"), fileExt), + ) + if !gfile.Exists(newFilePath) { + break + } + } + if err := gfile.Rename(filePath, newFilePath); err != nil { + panic(err) + } +} + +// rotateChecks timely checks the backups expiration and the compression. +func (l *Logger) rotateChecks() { + defer func() { + gtimer.AddOnce(l.config.RotateInterval, l.rotateChecks) + }() + + // Checks whether file rotation not enabled. + if l.config.RotateSize == 0 || l.config.RotateBackups == 0 { + return + } + files, _ := gfile.ScanDirFile(l.config.Path, "*.*", true) + intlog.Printf("logging rotation start checks: %+v", files) + // Compression. + needCompressFileArray := garray.NewStrArray() + if l.config.RotateCompress > 0 { + for _, file := range files { + // Eg: access.20200102190000899.gz + if gfile.ExtName(file) == "gz" { + continue + } + // Eg: + // access.20200102190000899 + // access.20200102190000899.log + if gregex.IsMatchString(`.+\.\d{14,}`, file) { + needCompressFileArray.Append(file) + } + } + if needCompressFileArray.Len() > 0 { + needCompressFileArray.Iterator(func(_ int, path string) bool { + err := gcompress.GzipFile(path, path+".gz") + if err == nil { + intlog.Printf(`compressed done, remove original logging file: %s`, path) + if err = gfile.Remove(path); err != nil { + intlog.Print(err) + } + } else { + intlog.Print(err) + } + return true + }) + // Update the files array. + files, _ = gfile.ScanDirFile(l.config.Path, "*.*", true) + } + } + // Backups count limit and expiration checks. + var ( + backupFilesMap = make(map[string]*garray.SortedArray) + originalLoggingFilePath = "" + ) + if l.config.RotateBackups > 0 || l.config.RotateExpire > 0 { + for _, file := range files { + originalLoggingFilePath, _ = gregex.ReplaceString(`\.\d{14,}`, "", file) + if backupFilesMap[originalLoggingFilePath] == nil { + backupFilesMap[originalLoggingFilePath] = garray.NewSortedArray(func(a, b interface{}) int { + // Sorted by backup file mtime. + // The old backup file is put in the head of array. + file1 := a.(string) + file2 := b.(string) + result := gfile.MTimeMillisecond(file1) - gfile.MTimeMillisecond(file2) + if result <= 0 { + return -1 + } + return 1 + }) + } + if gregex.IsMatchString(`.+\.\d{14,}`, file) { + backupFilesMap[originalLoggingFilePath].Add(file) + } + } + intlog.Printf(`calculated backup files map: %+v`, backupFilesMap) + for _, array := range backupFilesMap { + for i := 0; i < array.Len()-l.config.RotateBackups; i++ { + path := array.PopLeft().(string) + intlog.Printf(`remove exceeded backup file: %s`, path) + if err := gfile.Remove(path); err != nil { + intlog.Print(err) + } + } + } + // Expiration checks. + if l.config.RotateExpire > 0 { + nowTimestampMilli := gtime.TimestampMilli() + expireMillisecond := l.config.RotateExpire.Milliseconds() + for _, array := range backupFilesMap { + array.Iterator(func(_ int, v interface{}) bool { + path := v.(string) + mtime := gfile.MTimeMillisecond(path) + differ := nowTimestampMilli - mtime + if differ > expireMillisecond { + intlog.Printf( + `%d - %d = %d > %d, remove expired backup file: %s`, + nowTimestampMilli, mtime, differ, + expireMillisecond, + path, + ) + if err := gfile.Remove(path); err != nil { + intlog.Print(err) + } + return true + } else { + return false + } + }) + } + } + } +} diff --git a/os/glog/glog_z_unit_rotate_test.go b/os/glog/glog_z_unit_rotate_test.go new file mode 100644 index 000000000..4d51ea816 --- /dev/null +++ b/os/glog/glog_z_unit_rotate_test.go @@ -0,0 +1,56 @@ +// Copyright 2020 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package glog_test + +import ( + "github.com/gogf/gf/frame/g" + "github.com/gogf/gf/os/gfile" + "github.com/gogf/gf/os/glog" + "github.com/gogf/gf/os/gtime" + "github.com/gogf/gf/test/gtest" + "github.com/gogf/gf/text/gstr" + "testing" + "time" +) + +func Test_Rotate(t *testing.T) { + gtest.Case(t, func() { + l := glog.New() + p := gfile.Join(gfile.TempDir(), gtime.TimestampNanoStr()) + err := l.SetConfigWithMap(g.Map{ + "Path": p, + "File": "access.log", + "StdoutPrint": false, + "RotateSize": 10, + "RotateBackups": 2, + "RotateExpire": 5 * time.Second, + "RotateCompress": 9, + "RotateInterval": time.Second, // For unit testing only. + }) + gtest.Assert(err, nil) + defer gfile.Remove(p) + + s := "1234567890abcdefg" + for i := 0; i < 10; i++ { + l.Print(s) + } + + time.Sleep(time.Second * 3) + + files, err := gfile.ScanDirFile(p, "*.gz") + gtest.Assert(err, nil) + gtest.Assert(len(files), 2) + + content := gfile.GetContents(gfile.Join(p, "access.log")) + gtest.Assert(gstr.Count(content, s), 1) + + time.Sleep(time.Second * 4) + files, err = gfile.ScanDirFile(p, "*.gz") + gtest.Assert(err, nil) + gtest.Assert(len(files), 0) + }) +} From 855a4ddb2c67f4db9bd9be437303c95f91c253cb Mon Sep 17 00:00:00 2001 From: John Date: Sun, 15 Mar 2020 19:32:29 +0800 Subject: [PATCH 19/32] add file rotation feature for package glog; improve gpool/gfpool; fix issue in gfile.MTimeMillisecond --- database/gdb/gdb_unit_init_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/database/gdb/gdb_unit_init_test.go b/database/gdb/gdb_unit_init_test.go index 957529bd1..2879fe6ab 100644 --- a/database/gdb/gdb_unit_init_test.go +++ b/database/gdb/gdb_unit_init_test.go @@ -62,7 +62,6 @@ func init() { } else { db = r } - db.SetDebug(true) schemaTemplate := "CREATE DATABASE IF NOT EXISTS `%s` CHARACTER SET UTF8" if _, err := db.Exec(fmt.Sprintf(schemaTemplate, SCHEMA1)); err != nil { gtest.Error(err) From d716037caa02f2c99a1dcc4ea9e028e809104ab8 Mon Sep 17 00:00:00 2001 From: John Date: Sun, 15 Mar 2020 19:56:07 +0800 Subject: [PATCH 20/32] improve package gcfg adding default configuration file searching for instance --- os/gcfg/gcfg.go | 24 +++++++++++------------- os/gcfg/gcfg_instance.go | 14 +++++++++++--- os/gcfg/gcfg_z_unit_test.go | 9 +++++++++ os/gcfg/testdata/c1.toml | 2 ++ os/gcfg/testdata/folder1/c1.toml | 2 ++ 5 files changed, 35 insertions(+), 16 deletions(-) create mode 100644 os/gcfg/testdata/c1.toml create mode 100644 os/gcfg/testdata/folder1/c1.toml diff --git a/os/gcfg/gcfg.go b/os/gcfg/gcfg.go index d3a16be53..e7aac4378 100644 --- a/os/gcfg/gcfg.go +++ b/os/gcfg/gcfg.go @@ -17,7 +17,6 @@ import ( "github.com/gogf/gf/container/garray" "github.com/gogf/gf/container/gmap" - "github.com/gogf/gf/container/gtype" "github.com/gogf/gf/encoding/gjson" "github.com/gogf/gf/internal/cmdenv" "github.com/gogf/gf/os/gfile" @@ -33,10 +32,10 @@ const ( // Configuration struct. type Config struct { - name *gtype.String // Default configuration file name. + name string // Default configuration file name. paths *garray.StrArray // Searching path array. jsons *gmap.StrAnyMap // The pared JSON objects for configuration files. - vc *gtype.Bool // Whether do violence check in value index searching. It affects the performance when set true(false in default). + vc bool // Whether do violence check in value index searching. It affects the performance when set true(false in default). } var ( @@ -51,10 +50,9 @@ func New(file ...string) *Config { name = file[0] } c := &Config{ - name: gtype.NewString(name), + name: name, paths: garray.NewStrArray(true), jsons: gmap.NewStrAnyMap(true), - vc: gtype.NewBool(), } // Customized dir path from env/cmd. if envPath := cmdenv.Get("gf.gcfg.path").String(); envPath != "" { @@ -82,7 +80,7 @@ func New(file ...string) *Config { // filePath returns the absolute configuration file path for the given filename by . func (c *Config) filePath(file ...string) (path string) { - name := c.name.Val() + name := c.name if len(file) > 0 { name = file[0] } @@ -182,7 +180,7 @@ func (c *Config) SetPath(path string) error { // and it is not recommended to allow separators in the key names. // It is best to avoid this on the application side. func (c *Config) SetViolenceCheck(check bool) { - c.vc.Set(check) + c.vc = check c.Clear() } @@ -250,7 +248,7 @@ func (c *Config) AddPath(path string) error { // If the specified configuration file does not exist, // an empty string is returned. func (c *Config) FilePath(file ...string) (path string) { - name := c.name.Val() + name := c.name if len(file) > 0 { name = file[0] } @@ -290,13 +288,13 @@ func (c *Config) FilePath(file ...string) (path string) { // SetFileName sets the default configuration file name. func (c *Config) SetFileName(name string) *Config { - c.name.Set(name) + c.name = name return c } // GetFileName returns the default configuration file name. func (c *Config) GetFileName() string { - return c.name.Val() + return c.name } // Available checks and returns whether configuration of given is available. @@ -305,7 +303,7 @@ func (c *Config) Available(file ...string) bool { if len(file) > 0 && file[0] != "" { name = file[0] } else { - name = c.name.Val() + name = c.name } if c.FilePath(name) != "" { return true @@ -323,7 +321,7 @@ func (c *Config) getJson(file ...string) *gjson.Json { if len(file) > 0 && file[0] != "" { name = file[0] } else { - name = c.name.Val() + name = c.name } r := c.jsons.GetOrSetFuncLock(name, func() interface{} { content := "" @@ -340,7 +338,7 @@ func (c *Config) getJson(file ...string) *gjson.Json { } } if j, err := gjson.LoadContent(content, true); err == nil { - j.SetViolenceCheck(c.vc.Val()) + j.SetViolenceCheck(c.vc) // Add monitor for this configuration file, // any changes of this file will refresh its cache in Config object. if filePath != "" && !gres.Contains(filePath) { diff --git a/os/gcfg/gcfg_instance.go b/os/gcfg/gcfg_instance.go index 8484e5d12..9ef0fc951 100644 --- a/os/gcfg/gcfg_instance.go +++ b/os/gcfg/gcfg_instance.go @@ -7,6 +7,7 @@ package gcfg import ( + "fmt" "github.com/gogf/gf/container/gmap" ) @@ -16,18 +17,25 @@ const ( ) var ( - // Instances map. + // Instances map containing configuration instances. instances = gmap.NewStrAnyMap(true) ) // Instance returns an instance of Config with default settings. -// The parameter is the name for the instance. +// The parameter is the name for the instance. But very note that, if the file "name.toml" +// exists in the configuration directory, it then sets it as the default configuration file. The +// toml file type is the default configuration file type. func Instance(name ...string) *Config { key := DEFAULT_NAME if len(name) > 0 && name[0] != "" { key = name[0] } return instances.GetOrSetFuncLock(key, func() interface{} { - return New() + c := New() + file := fmt.Sprintf(`%s.toml`, key) + if c.Available(file) { + c.SetFileName(file) + } + return c }).(*Config) } diff --git a/os/gcfg/gcfg_z_unit_test.go b/os/gcfg/gcfg_z_unit_test.go index 5df2a0066..924e09959 100644 --- a/os/gcfg/gcfg_z_unit_test.go +++ b/os/gcfg/gcfg_z_unit_test.go @@ -9,6 +9,7 @@ package gcfg_test import ( + "github.com/gogf/gf/debug/gdebug" "io/ioutil" "os" "testing" @@ -420,6 +421,14 @@ func TestCfg_Instance(t *testing.T) { gtest.Case(t, func() { gtest.Assert(gcfg.Instance("gf") != nil, true) }) + gtest.Case(t, func() { + pwd := gfile.Pwd() + gfile.Chdir(gfile.Join(gdebug.CallerDirectory(), "testdata")) + defer gfile.Chdir(pwd) + gtest.Assert(gcfg.Instance("c1") != nil, true) + gtest.Assert(gcfg.Instance("c1").Get("my-config"), "1") + gtest.Assert(gcfg.Instance("folder1/c1").Get("my-config"), "2") + }) } func TestCfg_Config(t *testing.T) { diff --git a/os/gcfg/testdata/c1.toml b/os/gcfg/testdata/c1.toml new file mode 100644 index 000000000..0ebee44aa --- /dev/null +++ b/os/gcfg/testdata/c1.toml @@ -0,0 +1,2 @@ + +my-config = "1" \ No newline at end of file diff --git a/os/gcfg/testdata/folder1/c1.toml b/os/gcfg/testdata/folder1/c1.toml new file mode 100644 index 000000000..eff89f9f5 --- /dev/null +++ b/os/gcfg/testdata/folder1/c1.toml @@ -0,0 +1,2 @@ + +my-config = "2" \ No newline at end of file From 8230c72ec68d1a928a9240802e8f752cc0ab3f6e Mon Sep 17 00:00:00 2001 From: John Date: Sun, 15 Mar 2020 20:11:38 +0800 Subject: [PATCH 21/32] improve file uploading feature for ghttp.Request --- net/ghttp/ghttp_request_param_file.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/net/ghttp/ghttp_request_param_file.go b/net/ghttp/ghttp_request_param_file.go index 65f3e8f81..b808d6cf1 100644 --- a/net/ghttp/ghttp_request_param_file.go +++ b/net/ghttp/ghttp_request_param_file.go @@ -36,7 +36,7 @@ type UploadFiles []*UploadFile // Note that it will overwrite the target file if there's already a same name file exist. func (f *UploadFile) Save(dirPath string, randomlyRename ...bool) (filename string, err error) { if f == nil { - return + return "", errors.New("file is empty, maybe you retrieve it from invalid field name or form enctype") } if !gfile.Exists(dirPath) { if err = gfile.Mkdir(dirPath); err != nil { @@ -77,7 +77,7 @@ func (f *UploadFile) Save(dirPath string, randomlyRename ...bool) (filename stri // The parameter specifies whether randomly renames all the file names. func (fs UploadFiles) Save(dirPath string, randomlyRename ...bool) (filenames []string, err error) { if len(fs) == 0 { - return nil, nil + return nil, errors.New("file array is empty, maybe you retrieve it from invalid field name or form enctype") } for _, f := range fs { if filename, err := f.Save(dirPath, randomlyRename...); err != nil { From 2438f565e9495a4f7c04e5d9892b53feabee197b Mon Sep 17 00:00:00 2001 From: sunmoon Date: Mon, 16 Mar 2020 11:18:06 +0800 Subject: [PATCH 22/32] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=AD=97=E7=AC=A6?= =?UTF-8?q?=E4=B8=B2ip=E5=9C=B0=E5=9D=80=E4=B8=8ELong=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=E4=BA=92=E8=BD=AC=E7=9A=84=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- util/gconv/gconv.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/util/gconv/gconv.go b/util/gconv/gconv.go index c34d0a569..045176cdc 100644 --- a/util/gconv/gconv.go +++ b/util/gconv/gconv.go @@ -8,9 +8,11 @@ package gconv import ( + "encoding/binary" "encoding/json" "fmt" "github.com/gogf/gf/os/gtime" + "net" "reflect" "strconv" "strings" @@ -165,6 +167,23 @@ func Runes(i interface{}) []rune { } return []rune(String(i)) } +// convert ip to long, ex ip 106.8.36.164 to 1778918564 +func Ip2long(ipStr string) uint32 { + ip := net.ParseIP(ipStr) + if ip == nil { + return 0 + } + ip = ip.To4() + return binary.BigEndian.Uint32(ip) +} +// convert long to ip string, ex int 1778918564 to ip 106.8.36.164 +func Long2ip(ipLong uint32) string { + ipByte := make([]byte, 4) + binary.BigEndian.PutUint32(ipByte, ipLong) + ip := net.IP(ipByte) + return ip.String() +} + // String converts to string. // It's most common used converting function. From 8ed3cf9c9779f55c3aae0d5a4d54e3cdc1fdff8e Mon Sep 17 00:00:00 2001 From: John Date: Mon, 16 Mar 2020 20:46:03 +0800 Subject: [PATCH 23/32] fix issue in unit testing case of package gcdg --- os/gcfg/gcfg_z_unit_test.go | 51 ++++++++++++++++++++++++------------- os/gspath/gspath.go | 7 +++-- 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/os/gcfg/gcfg_z_unit_test.go b/os/gcfg/gcfg_z_unit_test.go index 924e09959..08a3582a5 100644 --- a/os/gcfg/gcfg_z_unit_test.go +++ b/os/gcfg/gcfg_z_unit_test.go @@ -10,7 +10,7 @@ package gcfg_test import ( "github.com/gogf/gf/debug/gdebug" - "io/ioutil" + "github.com/gogf/gf/os/gtime" "os" "testing" @@ -40,9 +40,7 @@ array = [1,2,3] path := gcfg.DEFAULT_CONFIG_FILE err := gfile.PutContents(path, config) gtest.Assert(err, nil) - defer func() { - _ = gfile.Remove(path) - }() + defer gfile.Remove(path) c := gcfg.New() gtest.Assert(c.Get("v1"), 1) @@ -310,9 +308,8 @@ func TestCfg_New(t *testing.T) { configPath := gfile.Pwd() + gfile.Separator + "config" _ = gfile.Mkdir(configPath) - defer func() { - _ = gfile.Remove(configPath) - }() + defer gfile.Remove(configPath) + c = gcfg.New("config.yml") gtest.Assert(c.Get("name"), nil) @@ -364,12 +361,21 @@ func TestCfg_FilePath(t *testing.T) { func TestCfg_Get(t *testing.T) { gtest.Case(t, func() { - configPath := gfile.Pwd() + gfile.Separator + "config" - _ = gfile.Mkdir(configPath) - defer func() { - _ = gfile.Remove(configPath) - }() - _ = ioutil.WriteFile(configPath+gfile.Separator+"config.yml", []byte("wrong config"), 0644) + var err error + configPath := gfile.Join(gfile.TempDir(), gtime.TimestampNanoStr()) + err = gfile.Mkdir(configPath) + gtest.Assert(err, nil) + defer gfile.Remove(configPath) + + defer gfile.Chdir(gfile.Pwd()) + err = gfile.Chdir(configPath) + gtest.Assert(err, nil) + + err = gfile.PutContents( + gfile.Join(configPath, "config.yml"), + "wrong config", + ) + gtest.Assert(err, nil) c := gcfg.New("config.yml") gtest.Assert(c.Get("name"), nil) gtest.Assert(c.GetVar("name").Val(), nil) @@ -404,13 +410,24 @@ func TestCfg_Get(t *testing.T) { c.Clear() - arr, _ := gjson.Encode(g.Map{"name": "gf", "time": "2019-06-12", "person": g.Map{"name": "gf"}, "floats": g.Slice{1, 2, 3}}) - _ = ioutil.WriteFile(configPath+gfile.Separator+"config.yml", arr, 0644) + arr, _ := gjson.Encode( + g.Map{ + "name": "gf", + "time": "2019-06-12", + "person": g.Map{"name": "gf"}, + "floats": g.Slice{1, 2, 3}, + }, + ) + err = gfile.PutBytes( + gfile.Join(configPath, "config.yml"), + arr, + ) + gtest.Assert(err, nil) gtest.Assert(c.GetTime("time").Format("2006-01-02"), "2019-06-12") gtest.Assert(c.GetGTime("time").Format("Y-m-d"), "2019-06-12") gtest.Assert(c.GetDuration("time").String(), "0s") - //t.Log(c.GetString("person")) - err := c.GetStruct("person", &name) + + err = c.GetStruct("person", &name) gtest.Assert(err, nil) gtest.Assert(name.Name, "gf") gtest.Assert(c.GetFloats("floats") == nil, false) diff --git a/os/gspath/gspath.go b/os/gspath/gspath.go index 89d2d06d5..0ab5b2634 100644 --- a/os/gspath/gspath.go +++ b/os/gspath/gspath.go @@ -7,12 +7,14 @@ // Package gspath implements file index and search for folders. // // It searches file internally with high performance in order by the directory adding sequence. -// Note that: If caching feature enabled, there would be a searching delay after adding/deleting files. +// Note that: +// If caching feature enabled, there would be a searching delay after adding/deleting files. package gspath import ( "errors" "fmt" + "github.com/gogf/gf/internal/intlog" "os" "sort" "strings" @@ -50,7 +52,7 @@ func New(path string, cache bool) *SPath { } if len(path) > 0 { if _, err := sp.Add(path); err != nil { - //fmt.Errorf(err.Error()) + intlog.Print(err) } } return sp @@ -111,6 +113,7 @@ func (sp *SPath) Set(path string) (realPath string, err error) { sp.removeMonitorByPath(v) } } + intlog.Print("paths clear:", sp.paths) sp.paths.Clear() if sp.cache != nil { sp.cache.Clear() From c578df06a3a0a3bbbd3e8563460168c5eeda1928 Mon Sep 17 00:00:00 2001 From: John Date: Mon, 16 Mar 2020 20:51:19 +0800 Subject: [PATCH 24/32] fix issue in unit testing case of package gfile --- os/gfile/gfile_z_time_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/os/gfile/gfile_z_time_test.go b/os/gfile/gfile_z_time_test.go index f70b814df..8e5e7a152 100644 --- a/os/gfile/gfile_z_time_test.go +++ b/os/gfile/gfile_z_time_test.go @@ -9,6 +9,7 @@ package gfile_test import ( "os" "testing" + "time" "github.com/gogf/gf/os/gfile" "github.com/gogf/gf/test/gtest" @@ -18,7 +19,7 @@ func Test_MTime(t *testing.T) { gtest.Case(t, func() { var ( - file1 string = "/testfile_t1.txt" + file1 = "/testfile_t1.txt" err error fileobj os.FileInfo ) @@ -36,7 +37,7 @@ func Test_MTime(t *testing.T) { func Test_MTimeMillisecond(t *testing.T) { gtest.Case(t, func() { var ( - file1 string = "/testfile_t1.txt" + file1 = "/testfile_t1.txt" err error fileobj os.FileInfo ) @@ -46,7 +47,8 @@ func Test_MTimeMillisecond(t *testing.T) { fileobj, err = os.Stat(testpath() + file1) gtest.Assert(err, nil) - gtest.AssertGE(gfile.MTimeMillisecond(testpath()+file1), fileobj.ModTime().Nanosecond()/1000000) + time.Sleep(time.Millisecond * 100) + gtest.AssertGE(gfile.MTimeMillisecond(testpath()+file1), fileobj.ModTime().UnixNano()/1000000) gtest.Assert(gfile.MTimeMillisecond(""), 0) }) } From 9f0548c03d43ea37e91604c53eddabeb88382502 Mon Sep 17 00:00:00 2001 From: John Date: Mon, 16 Mar 2020 22:47:39 +0800 Subject: [PATCH 25/32] improve testdata directory retrieving for unit testing cases --- database/gdb/gdb.go | 28 +- debug/gdebug/gdebug.go | 269 ---------------- debug/gdebug/gdebug_caller.go | 174 +++++++++++ debug/gdebug/gdebug_stack.go | 87 ++++++ debug/gdebug/gdebug_testdata.go | 17 + debug/gdebug/gdebug_version.go | 36 +++ ...g_bench_test.go => gdebug_z_bench_test.go} | 0 encoding/gbase64/gbase64_test.go | 2 +- .../gcompress/gcompress_z_unit_gzip_test.go | 2 +- .../gcompress/gcompress_z_unit_zip_test.go | 22 +- frame/gins/gins_z_unit_basic_test.go | 29 +- frame/gins/gins_z_unit_config_test.go | 295 ++++++++++-------- frame/gins/gins_z_unit_database_test.go | 59 ++-- frame/gins/gins_z_unit_redis_test.go | 64 ++-- frame/gins/gins_z_unit_view_test.go | 8 +- frame/gins/testdata/config/config.toml | 29 ++ frame/gins/testdata/database/config.toml | 29 ++ frame/gins/testdata/redis/config.toml | 32 ++ net/ghttp/ghttp_unit_middleware_basic_test.go | 4 +- net/ghttp/ghttp_unit_param_file_test.go | 14 +- net/ghttp/ghttp_unit_static_test.go | 2 +- net/ghttp/ghttp_unit_template_test.go | 4 +- os/gcfg/gcfg_z_unit_test.go | 2 +- os/gfile/gfile_z_time_test.go | 5 +- os/gview/gview_unit_config_test.go | 4 +- test/gtest/gtest.go | 10 +- 26 files changed, 683 insertions(+), 544 deletions(-) create mode 100644 debug/gdebug/gdebug_caller.go create mode 100644 debug/gdebug/gdebug_stack.go create mode 100644 debug/gdebug/gdebug_testdata.go create mode 100644 debug/gdebug/gdebug_version.go rename debug/gdebug/{gdebug_bench_test.go => gdebug_z_bench_test.go} (100%) create mode 100644 frame/gins/testdata/config/config.toml create mode 100644 frame/gins/testdata/database/config.toml create mode 100644 frame/gins/testdata/redis/config.toml diff --git a/database/gdb/gdb.go b/database/gdb/gdb.go index 40280f893..1c9936c96 100644 --- a/database/gdb/gdb.go +++ b/database/gdb/gdb.go @@ -182,12 +182,13 @@ type Map = map[string]interface{} type List = []Map const ( - gINSERT_OPTION_DEFAULT = 0 - gINSERT_OPTION_REPLACE = 1 - gINSERT_OPTION_SAVE = 2 - gINSERT_OPTION_IGNORE = 3 - gDEFAULT_BATCH_NUM = 10 // Per count for batch insert/replace/save - gDEFAULT_CONN_MAX_LIFE_TIME = 30 // Max life time for per connection in pool in seconds. + gINSERT_OPTION_DEFAULT = 0 + gINSERT_OPTION_REPLACE = 1 + gINSERT_OPTION_SAVE = 2 + gINSERT_OPTION_IGNORE = 3 + gDEFAULT_BATCH_NUM = 10 // Per count for batch insert/replace/save + gDEFAULT_CONN_MAX_IDLE_COUNT = 10 // Max idle connection count in pool. + gDEFAULT_CONN_MAX_LIFE_TIME = 30 // Max life time for per connection in pool in seconds. ) var ( @@ -226,13 +227,14 @@ func New(name ...string) (db DB, err error) { if _, ok := configs.config[group]; ok { if node, err := getConfigNodeByGroup(group, true); err == nil { c := &Core{ - group: group, - debug: gtype.NewBool(), - cache: gcache.New(), - schema: gtype.NewString(), - logger: glog.New(), - prefix: node.Prefix, - maxConnLifetime: gDEFAULT_CONN_MAX_LIFE_TIME, // Default max connection life time if user does not configure. + group: group, + debug: gtype.NewBool(), + cache: gcache.New(), + schema: gtype.NewString(), + logger: glog.New(), + prefix: node.Prefix, + maxIdleConnCount: gDEFAULT_CONN_MAX_IDLE_COUNT, + maxConnLifetime: gDEFAULT_CONN_MAX_LIFE_TIME, // Default max connection life time if user does not configure. } if v, ok := driverMap[node.Type]; ok { c.DB, err = v.New(c, node) diff --git a/debug/gdebug/gdebug.go b/debug/gdebug/gdebug.go index e4d897a7f..4771ee648 100644 --- a/debug/gdebug/gdebug.go +++ b/debug/gdebug/gdebug.go @@ -6,272 +6,3 @@ // Package gdebug contains facilities for programs to debug themselves while they are running. package gdebug - -import ( - "bytes" - "fmt" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "reflect" - "runtime" - "strconv" - "strings" - - "github.com/gogf/gf/encoding/ghash" - - "github.com/gogf/gf/crypto/gmd5" -) - -const ( - gMAX_DEPTH = 1000 - gFILTER_KEY = "/debug/gdebug/gdebug" -) - -var ( - goRootForFilter = runtime.GOROOT() // goRootForFilter is used for stack filtering purpose. - binaryVersion = "" // The version of current running binary(uint64 hex). - binaryVersionMd5 = "" // The version of current running binary(MD5). - selfPath = "" // Current running binary absolute path. -) - -func init() { - if goRootForFilter != "" { - goRootForFilter = strings.Replace(goRootForFilter, "\\", "/", -1) - } - // Initialize internal package variable: selfPath. - selfPath, _ := exec.LookPath(os.Args[0]) - if selfPath != "" { - selfPath, _ = filepath.Abs(selfPath) - } - if selfPath == "" { - selfPath, _ = filepath.Abs(os.Args[0]) - } -} - -// BinVersion returns the version of current running binary. -// It uses ghash.BKDRHash+BASE36 algorithm to calculate the unique version of the binary. -func BinVersion() string { - if binaryVersion == "" { - binaryContent, _ := ioutil.ReadFile(selfPath) - binaryVersion = strconv.FormatInt( - int64(ghash.BKDRHash(binaryContent)), - 36, - ) - } - return binaryVersion -} - -// BinVersionMd5 returns the version of current running binary. -// It uses MD5 algorithm to calculate the unique version of the binary. -func BinVersionMd5() string { - if binaryVersionMd5 == "" { - binaryVersionMd5, _ = gmd5.EncryptFile(selfPath) - } - return binaryVersionMd5 -} - -// PrintStack prints to standard error the stack trace returned by runtime.Stack. -func PrintStack(skip ...int) { - fmt.Print(Stack(skip...)) -} - -// Stack returns a formatted stack trace of the goroutine that calls it. -// It calls runtime.Stack with a large enough buffer to capture the entire trace. -func Stack(skip ...int) string { - return StackWithFilter("", skip...) -} - -// StackWithFilter returns a formatted stack trace of the goroutine that calls it. -// It calls runtime.Stack with a large enough buffer to capture the entire trace. -// -// The parameter is used to filter the path of the caller. -func StackWithFilter(filter string, skip ...int) string { - return StackWithFilters([]string{filter}, skip...) -} - -// StackWithFilters returns a formatted stack trace of the goroutine that calls it. -// It calls runtime.Stack with a large enough buffer to capture the entire trace. -// -// The parameter is a slice of strings, which are used to filter the path of the caller. -func StackWithFilters(filters []string, skip ...int) string { - number := 0 - if len(skip) > 0 { - number = skip[0] - } - name := "" - space := " " - index := 1 - buffer := bytes.NewBuffer(nil) - filtered := false - ok := true - pc, file, line, start := callerFromIndex(filters) - for i := start + number; i < gMAX_DEPTH; i++ { - if i != start { - pc, file, line, ok = runtime.Caller(i) - } - if ok { - if goRootForFilter != "" && len(file) >= len(goRootForFilter) && file[0:len(goRootForFilter)] == goRootForFilter { - continue - } - filtered = false - for _, filter := range filters { - if filter != "" && strings.Contains(file, filter) { - filtered = true - break - } - } - if filtered { - continue - } - if strings.Contains(file, gFILTER_KEY) { - continue - } - if fn := runtime.FuncForPC(pc); fn == nil { - name = "unknown" - } else { - name = fn.Name() - } - if index > 9 { - space = " " - } - buffer.WriteString(fmt.Sprintf("%d.%s%s\n %s:%d\n", index, space, name, file, line)) - index++ - } else { - break - } - } - return buffer.String() -} - -// CallerPath returns the function name and the absolute file path along with its line number of the caller. -func Caller(skip ...int) (function string, path string, line int) { - return CallerWithFilter("", skip...) -} - -// CallerPathWithFilter returns the function name and the absolute file path along with its line number of the caller. -// -// The parameter is used to filter the path of the caller. -func CallerWithFilter(filter string, skip ...int) (function string, path string, line int) { - number := 0 - if len(skip) > 0 { - number = skip[0] - } - ok := true - pc, file, line, start := callerFromIndex([]string{filter}) - if start != -1 { - for i := start + number; i < gMAX_DEPTH; i++ { - if i != start { - pc, file, line, ok = runtime.Caller(i) - } - if ok { - if filter != "" && strings.Contains(file, filter) { - continue - } - if strings.Contains(file, gFILTER_KEY) { - continue - } - function := "" - if fn := runtime.FuncForPC(pc); fn == nil { - function = "unknown" - } else { - function = fn.Name() - } - return function, file, line - } else { - break - } - } - } - return "", "", -1 -} - -// callerFromIndex returns the caller position and according information exclusive of the debug package. -func callerFromIndex(filters []string) (pc uintptr, file string, line int, index int) { - var filtered, ok bool - for index = 0; index < gMAX_DEPTH; index++ { - if pc, file, line, ok = runtime.Caller(index); ok { - filtered = false - for _, filter := range filters { - if filter != "" && strings.Contains(file, filter) { - filtered = true - break - } - } - if filtered { - continue - } - if strings.Contains(file, gFILTER_KEY) { - continue - } - return - } - } - return 0, "", -1, -1 -} - -// CallerPackage returns the package name of the caller. -func CallerPackage() string { - function, _, _ := Caller() - indexSplit := strings.LastIndexByte(function, '/') - if indexSplit == -1 { - return function[:strings.IndexByte(function, '.')] - } else { - leftPart := function[:indexSplit+1] - rightPart := function[indexSplit+1:] - indexDot := strings.IndexByte(function, '.') - rightPart = rightPart[:indexDot-1] - return leftPart + rightPart - } -} - -// CallerFunction returns the function name of the caller. -func CallerFunction() string { - function, _, _ := Caller() - function = function[strings.LastIndexByte(function, '/')+1:] - function = function[strings.IndexByte(function, '.')+1:] - return function -} - -// CallerFilePath returns the file path of the caller. -func CallerFilePath() string { - _, path, _ := Caller() - return path -} - -// CallerDirectory returns the directory of the caller. -func CallerDirectory() string { - _, path, _ := Caller() - return filepath.Dir(path) -} - -// CallerFileLine returns the file path along with the line number of the caller. -func CallerFileLine() string { - _, path, line := Caller() - return fmt.Sprintf(`%s:%d`, path, line) -} - -// CallerFileLineShort returns the file name along with the line number of the caller. -func CallerFileLineShort() string { - _, path, line := Caller() - return fmt.Sprintf(`%s:%d`, filepath.Base(path), line) -} - -// FuncPath returns the complete function path of given . -func FuncPath(f interface{}) string { - return runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name() -} - -// FuncName returns the function name of given . -func FuncName(f interface{}) string { - path := FuncPath(f) - if path == "" { - return "" - } - index := strings.LastIndexByte(path, '/') - if index < 0 { - index = strings.LastIndexByte(path, '\\') - } - return path[index+1:] -} diff --git a/debug/gdebug/gdebug_caller.go b/debug/gdebug/gdebug_caller.go new file mode 100644 index 000000000..13f7391cf --- /dev/null +++ b/debug/gdebug/gdebug_caller.go @@ -0,0 +1,174 @@ +// Copyright 2019-2020 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gdebug + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "runtime" + "strings" +) + +const ( + gMAX_DEPTH = 1000 + gFILTER_KEY = "/debug/gdebug/gdebug" +) + +var ( + goRootForFilter = runtime.GOROOT() // goRootForFilter is used for stack filtering purpose. + binaryVersion = "" // The version of current running binary(uint64 hex). + binaryVersionMd5 = "" // The version of current running binary(MD5). + selfPath = "" // Current running binary absolute path. +) + +func init() { + if goRootForFilter != "" { + goRootForFilter = strings.Replace(goRootForFilter, "\\", "/", -1) + } + // Initialize internal package variable: selfPath. + selfPath, _ := exec.LookPath(os.Args[0]) + if selfPath != "" { + selfPath, _ = filepath.Abs(selfPath) + } + if selfPath == "" { + selfPath, _ = filepath.Abs(os.Args[0]) + } +} + +// CallerPath returns the function name and the absolute file path along with its line number of the caller. +func Caller(skip ...int) (function string, path string, line int) { + return CallerWithFilter("", skip...) +} + +// CallerPathWithFilter returns the function name and the absolute file path along with its line number of the caller. +// +// The parameter is used to filter the path of the caller. +func CallerWithFilter(filter string, skip ...int) (function string, path string, line int) { + number := 0 + if len(skip) > 0 { + number = skip[0] + } + ok := true + pc, file, line, start := callerFromIndex([]string{filter}) + if start != -1 { + for i := start + number; i < gMAX_DEPTH; i++ { + if i != start { + pc, file, line, ok = runtime.Caller(i) + } + if ok { + if filter != "" && strings.Contains(file, filter) { + continue + } + if strings.Contains(file, gFILTER_KEY) { + continue + } + function := "" + if fn := runtime.FuncForPC(pc); fn == nil { + function = "unknown" + } else { + function = fn.Name() + } + return function, file, line + } else { + break + } + } + } + return "", "", -1 +} + +// callerFromIndex returns the caller position and according information exclusive of the debug package. +func callerFromIndex(filters []string) (pc uintptr, file string, line int, index int) { + var filtered, ok bool + for index = 0; index < gMAX_DEPTH; index++ { + if pc, file, line, ok = runtime.Caller(index); ok { + filtered = false + for _, filter := range filters { + if filter != "" && strings.Contains(file, filter) { + filtered = true + break + } + } + if filtered { + continue + } + if strings.Contains(file, gFILTER_KEY) { + continue + } + return + } + } + return 0, "", -1, -1 +} + +// CallerPackage returns the package name of the caller. +func CallerPackage() string { + function, _, _ := Caller() + indexSplit := strings.LastIndexByte(function, '/') + if indexSplit == -1 { + return function[:strings.IndexByte(function, '.')] + } else { + leftPart := function[:indexSplit+1] + rightPart := function[indexSplit+1:] + indexDot := strings.IndexByte(function, '.') + rightPart = rightPart[:indexDot-1] + return leftPart + rightPart + } +} + +// CallerFunction returns the function name of the caller. +func CallerFunction() string { + function, _, _ := Caller() + function = function[strings.LastIndexByte(function, '/')+1:] + function = function[strings.IndexByte(function, '.')+1:] + return function +} + +// CallerFilePath returns the file path of the caller. +func CallerFilePath() string { + _, path, _ := Caller() + return path +} + +// CallerDirectory returns the directory of the caller. +func CallerDirectory() string { + _, path, _ := Caller() + return filepath.Dir(path) +} + +// CallerFileLine returns the file path along with the line number of the caller. +func CallerFileLine() string { + _, path, line := Caller() + return fmt.Sprintf(`%s:%d`, path, line) +} + +// CallerFileLineShort returns the file name along with the line number of the caller. +func CallerFileLineShort() string { + _, path, line := Caller() + return fmt.Sprintf(`%s:%d`, filepath.Base(path), line) +} + +// FuncPath returns the complete function path of given . +func FuncPath(f interface{}) string { + return runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name() +} + +// FuncName returns the function name of given . +func FuncName(f interface{}) string { + path := FuncPath(f) + if path == "" { + return "" + } + index := strings.LastIndexByte(path, '/') + if index < 0 { + index = strings.LastIndexByte(path, '\\') + } + return path[index+1:] +} diff --git a/debug/gdebug/gdebug_stack.go b/debug/gdebug/gdebug_stack.go new file mode 100644 index 000000000..498f03bf2 --- /dev/null +++ b/debug/gdebug/gdebug_stack.go @@ -0,0 +1,87 @@ +// Copyright 2019-2020 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gdebug + +import ( + "bytes" + "fmt" + "runtime" + "strings" +) + +// PrintStack prints to standard error the stack trace returned by runtime.Stack. +func PrintStack(skip ...int) { + fmt.Print(Stack(skip...)) +} + +// Stack returns a formatted stack trace of the goroutine that calls it. +// It calls runtime.Stack with a large enough buffer to capture the entire trace. +func Stack(skip ...int) string { + return StackWithFilter("", skip...) +} + +// StackWithFilter returns a formatted stack trace of the goroutine that calls it. +// It calls runtime.Stack with a large enough buffer to capture the entire trace. +// +// The parameter is used to filter the path of the caller. +func StackWithFilter(filter string, skip ...int) string { + return StackWithFilters([]string{filter}, skip...) +} + +// StackWithFilters returns a formatted stack trace of the goroutine that calls it. +// It calls runtime.Stack with a large enough buffer to capture the entire trace. +// +// The parameter is a slice of strings, which are used to filter the path of the caller. +func StackWithFilters(filters []string, skip ...int) string { + number := 0 + if len(skip) > 0 { + number = skip[0] + } + name := "" + space := " " + index := 1 + buffer := bytes.NewBuffer(nil) + filtered := false + ok := true + pc, file, line, start := callerFromIndex(filters) + for i := start + number; i < gMAX_DEPTH; i++ { + if i != start { + pc, file, line, ok = runtime.Caller(i) + } + if ok { + if goRootForFilter != "" && len(file) >= len(goRootForFilter) && file[0:len(goRootForFilter)] == goRootForFilter { + continue + } + filtered = false + for _, filter := range filters { + if filter != "" && strings.Contains(file, filter) { + filtered = true + break + } + } + if filtered { + continue + } + if strings.Contains(file, gFILTER_KEY) { + continue + } + if fn := runtime.FuncForPC(pc); fn == nil { + name = "unknown" + } else { + name = fn.Name() + } + if index > 9 { + space = " " + } + buffer.WriteString(fmt.Sprintf("%d.%s%s\n %s:%d\n", index, space, name, file, line)) + index++ + } else { + break + } + } + return buffer.String() +} diff --git a/debug/gdebug/gdebug_testdata.go b/debug/gdebug/gdebug_testdata.go new file mode 100644 index 000000000..e7f209970 --- /dev/null +++ b/debug/gdebug/gdebug_testdata.go @@ -0,0 +1,17 @@ +// Copyright 2019-2020 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gdebug + +import ( + "path/filepath" +) + +// TestDataPath retrieves and returns the testdata path of current package. +// It is used for unit testing cases only. +func TestDataPath() string { + return CallerDirectory() + string(filepath.Separator) + "testdata" +} diff --git a/debug/gdebug/gdebug_version.go b/debug/gdebug/gdebug_version.go new file mode 100644 index 000000000..69a5fc97e --- /dev/null +++ b/debug/gdebug/gdebug_version.go @@ -0,0 +1,36 @@ +// Copyright 2019-2020 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gdebug + +import ( + "github.com/gogf/gf/crypto/gmd5" + "github.com/gogf/gf/encoding/ghash" + "io/ioutil" + "strconv" +) + +// BinVersion returns the version of current running binary. +// It uses ghash.BKDRHash+BASE36 algorithm to calculate the unique version of the binary. +func BinVersion() string { + if binaryVersion == "" { + binaryContent, _ := ioutil.ReadFile(selfPath) + binaryVersion = strconv.FormatInt( + int64(ghash.BKDRHash(binaryContent)), + 36, + ) + } + return binaryVersion +} + +// BinVersionMd5 returns the version of current running binary. +// It uses MD5 algorithm to calculate the unique version of the binary. +func BinVersionMd5() string { + if binaryVersionMd5 == "" { + binaryVersionMd5, _ = gmd5.EncryptFile(selfPath) + } + return binaryVersionMd5 +} diff --git a/debug/gdebug/gdebug_bench_test.go b/debug/gdebug/gdebug_z_bench_test.go similarity index 100% rename from debug/gdebug/gdebug_bench_test.go rename to debug/gdebug/gdebug_z_bench_test.go diff --git a/encoding/gbase64/gbase64_test.go b/encoding/gbase64/gbase64_test.go index 378166193..f13393587 100644 --- a/encoding/gbase64/gbase64_test.go +++ b/encoding/gbase64/gbase64_test.go @@ -67,7 +67,7 @@ func Test_Basic(t *testing.T) { } func Test_File(t *testing.T) { - path := gfile.Join(gdebug.CallerDirectory(), "testdata", "test") + path := gfile.Join(gdebug.TestDataPath(), "test") expect := "dGVzdA==" gtest.Case(t, func() { b, err := gbase64.EncodeFile(path) diff --git a/encoding/gcompress/gcompress_z_unit_gzip_test.go b/encoding/gcompress/gcompress_z_unit_gzip_test.go index 14ffab44a..694105344 100644 --- a/encoding/gcompress/gcompress_z_unit_gzip_test.go +++ b/encoding/gcompress/gcompress_z_unit_gzip_test.go @@ -43,7 +43,7 @@ func Test_Gzip_UnGzip(t *testing.T) { } func Test_Gzip_UnGzip_File(t *testing.T) { - srcPath := gfile.Join(gdebug.CallerDirectory(), "testdata", "gzip", "file.txt") + srcPath := gfile.Join(gdebug.TestDataPath(), "gzip", "file.txt") dstPath1 := gfile.Join(gfile.TempDir(), gtime.TimestampNanoStr(), "gzip.zip") dstPath2 := gfile.Join(gfile.TempDir(), gtime.TimestampNanoStr(), "file.txt") diff --git a/encoding/gcompress/gcompress_z_unit_zip_test.go b/encoding/gcompress/gcompress_z_unit_zip_test.go index 9fb0f3b1f..399bc2783 100644 --- a/encoding/gcompress/gcompress_z_unit_zip_test.go +++ b/encoding/gcompress/gcompress_z_unit_zip_test.go @@ -20,8 +20,8 @@ import ( func Test_ZipPath(t *testing.T) { // file gtest.Case(t, func() { - srcPath := gfile.Join(gdebug.CallerDirectory(), "testdata", "zip", "path1", "1.txt") - dstPath := gfile.Join(gdebug.CallerDirectory(), "testdata", "zip", "zip.zip") + srcPath := gfile.Join(gdebug.TestDataPath(), "zip", "path1", "1.txt") + dstPath := gfile.Join(gdebug.TestDataPath(), "zip", "zip.zip") gtest.Assert(gfile.Exists(dstPath), false) err := gcompress.ZipPath(srcPath, dstPath) @@ -44,8 +44,8 @@ func Test_ZipPath(t *testing.T) { }) // directory gtest.Case(t, func() { - srcPath := gfile.Join(gdebug.CallerDirectory(), "testdata", "zip") - dstPath := gfile.Join(gdebug.CallerDirectory(), "testdata", "zip", "zip.zip") + srcPath := gfile.Join(gdebug.TestDataPath(), "zip") + dstPath := gfile.Join(gdebug.TestDataPath(), "zip", "zip.zip") pwd := gfile.Pwd() err := gfile.Chdir(srcPath) @@ -77,10 +77,10 @@ func Test_ZipPath(t *testing.T) { }) // multiple paths joined using char ',' gtest.Case(t, func() { - srcPath := gfile.Join(gdebug.CallerDirectory(), "testdata", "zip") - srcPath1 := gfile.Join(gdebug.CallerDirectory(), "testdata", "zip", "path1") - srcPath2 := gfile.Join(gdebug.CallerDirectory(), "testdata", "zip", "path2") - dstPath := gfile.Join(gdebug.CallerDirectory(), "testdata", "zip", "zip.zip") + srcPath := gfile.Join(gdebug.TestDataPath(), "zip") + srcPath1 := gfile.Join(gdebug.TestDataPath(), "zip", "path1") + srcPath2 := gfile.Join(gdebug.TestDataPath(), "zip", "path2") + dstPath := gfile.Join(gdebug.TestDataPath(), "zip", "zip.zip") pwd := gfile.Pwd() err := gfile.Chdir(srcPath) @@ -116,9 +116,9 @@ func Test_ZipPath(t *testing.T) { func Test_ZipPathWriter(t *testing.T) { gtest.Case(t, func() { - srcPath := gfile.Join(gdebug.CallerDirectory(), "testdata", "zip") - srcPath1 := gfile.Join(gdebug.CallerDirectory(), "testdata", "zip", "path1") - srcPath2 := gfile.Join(gdebug.CallerDirectory(), "testdata", "zip", "path2") + srcPath := gfile.Join(gdebug.TestDataPath(), "zip") + srcPath1 := gfile.Join(gdebug.TestDataPath(), "zip", "path1") + srcPath2 := gfile.Join(gdebug.TestDataPath(), "zip", "path2") pwd := gfile.Pwd() err := gfile.Chdir(srcPath) diff --git a/frame/gins/gins_z_unit_basic_test.go b/frame/gins/gins_z_unit_basic_test.go index 8322dacb7..5820720e8 100644 --- a/frame/gins/gins_z_unit_basic_test.go +++ b/frame/gins/gins_z_unit_basic_test.go @@ -4,9 +4,10 @@ // If a copy of the MIT was not distributed with this file, // You can obtain one at https://github.com/gogf/gf. -package gins +package gins_test import ( + "github.com/gogf/gf/frame/gins" "testing" "github.com/gogf/gf/test/gtest" @@ -14,30 +15,30 @@ import ( func Test_SetGet(t *testing.T) { gtest.Case(t, func() { - Set("test-user", 1) - gtest.Assert(Get("test-user"), 1) - gtest.Assert(Get("none-exists"), nil) + gins.Set("test-user", 1) + gtest.Assert(gins.Get("test-user"), 1) + gtest.Assert(gins.Get("none-exists"), nil) }) gtest.Case(t, func() { - gtest.Assert(GetOrSet("test-1", 1), 1) - gtest.Assert(Get("test-1"), 1) + gtest.Assert(gins.GetOrSet("test-1", 1), 1) + gtest.Assert(gins.Get("test-1"), 1) }) gtest.Case(t, func() { - gtest.Assert(GetOrSetFunc("test-2", func() interface{} { + gtest.Assert(gins.GetOrSetFunc("test-2", func() interface{} { return 2 }), 2) - gtest.Assert(Get("test-2"), 2) + gtest.Assert(gins.Get("test-2"), 2) }) gtest.Case(t, func() { - gtest.Assert(GetOrSetFuncLock("test-3", func() interface{} { + gtest.Assert(gins.GetOrSetFuncLock("test-3", func() interface{} { return 3 }), 3) - gtest.Assert(Get("test-3"), 3) + gtest.Assert(gins.Get("test-3"), 3) }) gtest.Case(t, func() { - gtest.Assert(SetIfNotExist("test-4", 4), true) - gtest.Assert(Get("test-4"), 4) - gtest.Assert(SetIfNotExist("test-4", 5), false) - gtest.Assert(Get("test-4"), 4) + gtest.Assert(gins.SetIfNotExist("test-4", 4), true) + gtest.Assert(gins.Get("test-4"), 4) + gtest.Assert(gins.SetIfNotExist("test-4", 5), false) + gtest.Assert(gins.Get("test-4"), 4) }) } diff --git a/frame/gins/gins_z_unit_config_test.go b/frame/gins/gins_z_unit_config_test.go index 412d4957f..74c03d401 100644 --- a/frame/gins/gins_z_unit_config_test.go +++ b/frame/gins/gins_z_unit_config_test.go @@ -4,10 +4,12 @@ // If a copy of the MIT was not distributed with this file, // You can obtain one at https://github.com/gogf/gf. -package gins +package gins_test import ( "fmt" + "github.com/gogf/gf/debug/gdebug" + "github.com/gogf/gf/frame/gins" "testing" "time" @@ -18,157 +20,186 @@ import ( "github.com/gogf/gf/test/gtest" ) -func Test_Config(t *testing.T) { - config := ` -# 模板引擎目录 -viewpath = "/home/www/templates/" -test = "v=1" -# MySQL数据库配置 -[database] - [[database.default]] - host = "127.0.0.1" - port = "3306" - user = "root" - pass = "" - name = "test" - type = "mysql" - role = "master" - charset = "utf8" - priority = "1" - [[database.default]] - host = "127.0.0.1" - port = "3306" - user = "root" - pass = "8692651" - name = "test" - type = "mysql" - role = "master" - charset = "utf8" - priority = "1" -# Redis数据库配置 -[redis] - disk = "127.0.0.1:6379,0" - cache = "127.0.0.1:6379,1" -` - gtest.Case(t, func() { - gtest.AssertNE(Config(), nil) - }) +var ( + configContent = gfile.GetContents( + gfile.Join(gdebug.TestDataPath(), "config", "config.toml"), + ) +) +func Test_Config1(t *testing.T) { + gtest.Case(t, func() { + gtest.AssertNE(configContent, "") + }) + gtest.Case(t, func() { + gtest.AssertNE(gins.Config(), nil) + }) +} + +func Test_Config2(t *testing.T) { // relative path gtest.Case(t, func() { - path := "config.toml" - err := gfile.PutContents(path, config) + var err error + dirPath := gfile.Join(gfile.TempDir(), gtime.TimestampNanoStr()) + err = gfile.Mkdir(dirPath) gtest.Assert(err, nil) - defer gfile.Remove(path) - defer Config().Clear() - gtest.Assert(Config().Get("test"), "v=1") - gtest.Assert(Config().Get("database.default.1.host"), "127.0.0.1") - gtest.Assert(Config().Get("redis.disk"), "127.0.0.1:6379,0") + defer gfile.Remove(dirPath) + + name := "config.toml" + err = gfile.PutContents(gfile.Join(dirPath, name), configContent) + gtest.Assert(err, nil) + + err = gins.Config().AddPath(dirPath) + gtest.Assert(err, nil) + + defer gins.Config().Clear() + + gtest.Assert(gins.Config().Get("test"), "v=1") + gtest.Assert(gins.Config().Get("database.default.1.host"), "127.0.0.1") + gtest.Assert(gins.Config().Get("redis.disk"), "127.0.0.1:6379,0") }) // for gfsnotify callbacks to refresh cache of config file time.Sleep(500 * time.Millisecond) // relative path, config folder gtest.Case(t, func() { - path := "config/config.toml" - err := gfile.PutContents(path, config) + var err error + dirPath := gfile.Join(gfile.TempDir(), gtime.TimestampNanoStr()) + err = gfile.Mkdir(dirPath) gtest.Assert(err, nil) - defer gfile.Remove(path) - defer Config().Clear() - gtest.Assert(Config().Get("test"), "v=1") - gtest.Assert(Config().Get("database.default.1.host"), "127.0.0.1") - gtest.Assert(Config().Get("redis.disk"), "127.0.0.1:6379,0") - }) - // for gfsnotify callbacks to refresh cache of config file - time.Sleep(500 * time.Millisecond) + defer gfile.Remove(dirPath) - gtest.Case(t, func() { - path := "test.toml" - err := gfile.PutContents(path, config) + name := "config/config.toml" + err = gfile.PutContents(gfile.Join(dirPath, name), configContent) gtest.Assert(err, nil) - defer gfile.Remove(path) - defer Config("test").Clear() - Config("test").SetFileName("test.toml") - gtest.Assert(Config("test").Get("test"), "v=1") - gtest.Assert(Config("test").Get("database.default.1.host"), "127.0.0.1") - gtest.Assert(Config("test").Get("redis.disk"), "127.0.0.1:6379,0") - }) - // for gfsnotify callbacks to refresh cache of config file - time.Sleep(500 * time.Millisecond) - gtest.Case(t, func() { - path := "config/test.toml" - err := gfile.PutContents(path, config) + err = gins.Config().AddPath(dirPath) gtest.Assert(err, nil) - defer gfile.Remove(path) - defer Config("test").Clear() - Config("test").SetFileName("test.toml") - gtest.Assert(Config("test").Get("test"), "v=1") - gtest.Assert(Config("test").Get("database.default.1.host"), "127.0.0.1") - gtest.Assert(Config("test").Get("redis.disk"), "127.0.0.1:6379,0") - }) - // for gfsnotify callbacks to refresh cache of config file - time.Sleep(500 * time.Millisecond) - // absolute path - gtest.Case(t, func() { - path := fmt.Sprintf(`%s/%d`, gfile.TempDir(), gtime.TimestampNano()) - file := fmt.Sprintf(`%s/%s`, path, "config.toml") - err := gfile.PutContents(file, config) - gtest.Assert(err, nil) - defer gfile.Remove(file) - defer Config().Clear() - gtest.Assert(Config().AddPath(path), nil) - gtest.Assert(Config().Get("test"), "v=1") - gtest.Assert(Config().Get("database.default.1.host"), "127.0.0.1") - gtest.Assert(Config().Get("redis.disk"), "127.0.0.1:6379,0") - }) - time.Sleep(500 * time.Millisecond) + defer gins.Config().Clear() - gtest.Case(t, func() { - path := fmt.Sprintf(`%s/%d/config`, gfile.TempDir(), gtime.TimestampNano()) - file := fmt.Sprintf(`%s/%s`, path, "config.toml") - err := gfile.PutContents(file, config) - gtest.Assert(err, nil) - defer gfile.Remove(file) - defer Config().Clear() - gtest.Assert(Config().AddPath(path), nil) - gtest.Assert(Config().Get("test"), "v=1") - gtest.Assert(Config().Get("database.default.1.host"), "127.0.0.1") - gtest.Assert(Config().Get("redis.disk"), "127.0.0.1:6379,0") - }) - time.Sleep(500 * time.Millisecond) + gtest.Assert(gins.Config().Get("test"), "v=1") + gtest.Assert(gins.Config().Get("database.default.1.host"), "127.0.0.1") + gtest.Assert(gins.Config().Get("redis.disk"), "127.0.0.1:6379,0") - gtest.Case(t, func() { - path := fmt.Sprintf(`%s/%d`, gfile.TempDir(), gtime.TimestampNano()) - file := fmt.Sprintf(`%s/%s`, path, "test.toml") - err := gfile.PutContents(file, config) - gtest.Assert(err, nil) - defer gfile.Remove(file) - defer Config("test").Clear() - Config("test").SetFileName("test.toml") - gtest.Assert(Config("test").AddPath(path), nil) - gtest.Assert(Config("test").Get("test"), "v=1") - gtest.Assert(Config("test").Get("database.default.1.host"), "127.0.0.1") - gtest.Assert(Config("test").Get("redis.disk"), "127.0.0.1:6379,0") - }) - time.Sleep(500 * time.Millisecond) - - gtest.Case(t, func() { - path := fmt.Sprintf(`%s/%d/config`, gfile.TempDir(), gtime.TimestampNano()) - file := fmt.Sprintf(`%s/%s`, path, "test.toml") - err := gfile.PutContents(file, config) - gtest.Assert(err, nil) - defer gfile.Remove(file) - defer Config().Clear() - Config("test").SetFileName("test.toml") - gtest.Assert(Config("test").AddPath(path), nil) - gtest.Assert(Config("test").Get("test"), "v=1") - gtest.Assert(Config("test").Get("database.default.1.host"), "127.0.0.1") - gtest.Assert(Config("test").Get("redis.disk"), "127.0.0.1:6379,0") + // for gfsnotify callbacks to refresh cache of config file + time.Sleep(500 * time.Millisecond) }) } +func Test_Config3(t *testing.T) { + gtest.Case(t, func() { + gtest.Case(t, func() { + var err error + dirPath := gfile.Join(gfile.TempDir(), gtime.TimestampNanoStr()) + err = gfile.Mkdir(dirPath) + gtest.Assert(err, nil) + defer gfile.Remove(dirPath) + + name := "test.toml" + err = gfile.PutContents(gfile.Join(dirPath, name), configContent) + gtest.Assert(err, nil) + + err = gins.Config("test").AddPath(dirPath) + gtest.Assert(err, nil) + + defer gins.Config("test").Clear() + gins.Config("test").SetFileName("test.toml") + + gtest.Assert(gins.Config("test").Get("test"), "v=1") + gtest.Assert(gins.Config("test").Get("database.default.1.host"), "127.0.0.1") + gtest.Assert(gins.Config("test").Get("redis.disk"), "127.0.0.1:6379,0") + }) + // for gfsnotify callbacks to refresh cache of config file + time.Sleep(500 * time.Millisecond) + + gtest.Case(t, func() { + var err error + dirPath := gfile.Join(gfile.TempDir(), gtime.TimestampNanoStr()) + err = gfile.Mkdir(dirPath) + gtest.Assert(err, nil) + defer gfile.Remove(dirPath) + + name := "config/test.toml" + err = gfile.PutContents(gfile.Join(dirPath, name), configContent) + gtest.Assert(err, nil) + + err = gins.Config("test").AddPath(dirPath) + gtest.Assert(err, nil) + + defer gins.Config("test").Clear() + gins.Config("test").SetFileName("test.toml") + + gtest.Assert(gins.Config("test").Get("test"), "v=1") + gtest.Assert(gins.Config("test").Get("database.default.1.host"), "127.0.0.1") + gtest.Assert(gins.Config("test").Get("redis.disk"), "127.0.0.1:6379,0") + }) + // for gfsnotify callbacks to refresh cache of config file for next unit testing case. + time.Sleep(500 * time.Millisecond) + }) +} + +func Test_Config4(t *testing.T) { + gtest.Case(t, func() { + // absolute path + gtest.Case(t, func() { + path := fmt.Sprintf(`%s/%d`, gfile.TempDir(), gtime.TimestampNano()) + file := fmt.Sprintf(`%s/%s`, path, "config.toml") + err := gfile.PutContents(file, configContent) + gtest.Assert(err, nil) + defer gfile.Remove(file) + defer gins.Config().Clear() + + gtest.Assert(gins.Config().AddPath(path), nil) + gtest.Assert(gins.Config().Get("test"), "v=1") + gtest.Assert(gins.Config().Get("database.default.1.host"), "127.0.0.1") + gtest.Assert(gins.Config().Get("redis.disk"), "127.0.0.1:6379,0") + }) + time.Sleep(500 * time.Millisecond) + + gtest.Case(t, func() { + path := fmt.Sprintf(`%s/%d/config`, gfile.TempDir(), gtime.TimestampNano()) + file := fmt.Sprintf(`%s/%s`, path, "config.toml") + err := gfile.PutContents(file, configContent) + gtest.Assert(err, nil) + defer gfile.Remove(file) + defer gins.Config().Clear() + gtest.Assert(gins.Config().AddPath(path), nil) + gtest.Assert(gins.Config().Get("test"), "v=1") + gtest.Assert(gins.Config().Get("database.default.1.host"), "127.0.0.1") + gtest.Assert(gins.Config().Get("redis.disk"), "127.0.0.1:6379,0") + }) + time.Sleep(500 * time.Millisecond) + + gtest.Case(t, func() { + path := fmt.Sprintf(`%s/%d`, gfile.TempDir(), gtime.TimestampNano()) + file := fmt.Sprintf(`%s/%s`, path, "test.toml") + err := gfile.PutContents(file, configContent) + gtest.Assert(err, nil) + defer gfile.Remove(file) + defer gins.Config("test").Clear() + gins.Config("test").SetFileName("test.toml") + gtest.Assert(gins.Config("test").AddPath(path), nil) + gtest.Assert(gins.Config("test").Get("test"), "v=1") + gtest.Assert(gins.Config("test").Get("database.default.1.host"), "127.0.0.1") + gtest.Assert(gins.Config("test").Get("redis.disk"), "127.0.0.1:6379,0") + }) + time.Sleep(500 * time.Millisecond) + + gtest.Case(t, func() { + path := fmt.Sprintf(`%s/%d/config`, gfile.TempDir(), gtime.TimestampNano()) + file := fmt.Sprintf(`%s/%s`, path, "test.toml") + err := gfile.PutContents(file, configContent) + gtest.Assert(err, nil) + defer gfile.Remove(file) + defer gins.Config().Clear() + gins.Config("test").SetFileName("test.toml") + gtest.Assert(gins.Config("test").AddPath(path), nil) + gtest.Assert(gins.Config("test").Get("test"), "v=1") + gtest.Assert(gins.Config("test").Get("database.default.1.host"), "127.0.0.1") + gtest.Assert(gins.Config("test").Get("redis.disk"), "127.0.0.1:6379,0") + }) + }) +} func Test_Basic2(t *testing.T) { config := `log-path = "logs"` gtest.Case(t, func() { @@ -179,6 +210,6 @@ func Test_Basic2(t *testing.T) { _ = gfile.Remove(path) }() - gtest.Assert(Config().Get("log-path"), "logs") + gtest.Assert(gins.Config().Get("log-path"), "logs") }) } diff --git a/frame/gins/gins_z_unit_database_test.go b/frame/gins/gins_z_unit_database_test.go index 9e652c881..16d2b3a03 100644 --- a/frame/gins/gins_z_unit_database_test.go +++ b/frame/gins/gins_z_unit_database_test.go @@ -4,9 +4,12 @@ // If a copy of the MIT was not distributed with this file, // You can obtain one at https://github.com/gogf/gf. -package gins +package gins_test import ( + "github.com/gogf/gf/debug/gdebug" + "github.com/gogf/gf/frame/gins" + "github.com/gogf/gf/os/gtime" "testing" "time" @@ -15,42 +18,22 @@ import ( ) func Test_Database(t *testing.T) { - config := ` -# 模板引擎目录 -viewpath = "/home/www/templates/" -test = "v=2" -# MySQL数据库配置 -[database] - [[database.default]] - host = "127.0.0.1" - port = "3306" - user = "root" - pass = "12345678" - name = "test" - type = "mysql" - role = "master" - weight = "1" - charset = "utf8" - [[database.test]] - host = "127.0.0.1" - port = "3306" - user = "root" - pass = "12345678" - name = "test" - type = "mysql" - role = "master" - weight = "1" - charset = "utf8" -# Redis数据库配置 -[redis] - default = "127.0.0.1:6379,0" - cache = "127.0.0.1:6379,1" -` - path := "config.toml" - err := gfile.PutContents(path, config) + databaseContent := gfile.GetContents(gfile.Join(gdebug.TestDataPath(), "database", "config.toml")) + + var err error + dirPath := gfile.Join(gfile.TempDir(), gtime.TimestampNanoStr()) + err = gfile.Mkdir(dirPath) gtest.Assert(err, nil) - defer gfile.Remove(path) - defer Config().Clear() + defer gfile.Remove(dirPath) + + name := "config.toml" + err = gfile.PutContents(gfile.Join(dirPath, name), databaseContent) + gtest.Assert(err, nil) + + err = gins.Config().AddPath(dirPath) + gtest.Assert(err, nil) + + defer gins.Config().Clear() // for gfsnotify callbacks to refresh cache of config file time.Sleep(500 * time.Millisecond) @@ -58,8 +41,8 @@ test = "v=2" gtest.Case(t, func() { //fmt.Println("gins Test_Database", Config().Get("test")) - dbDefault := Database() - dbTest := Database("test") + dbDefault := gins.Database() + dbTest := gins.Database("test") gtest.AssertNE(dbDefault, nil) gtest.AssertNE(dbTest, nil) diff --git a/frame/gins/gins_z_unit_redis_test.go b/frame/gins/gins_z_unit_redis_test.go index 933203b49..1101a81a8 100644 --- a/frame/gins/gins_z_unit_redis_test.go +++ b/frame/gins/gins_z_unit_redis_test.go @@ -4,9 +4,12 @@ // If a copy of the MIT was not distributed with this file, // You can obtain one at https://github.com/gogf/gf. -package gins +package gins_test import ( + "github.com/gogf/gf/debug/gdebug" + "github.com/gogf/gf/frame/gins" + "github.com/gogf/gf/os/gtime" "testing" "time" @@ -15,45 +18,22 @@ import ( ) func Test_Redis(t *testing.T) { - config := ` -# 模板引擎目录 -viewpath = "/home/www/templates/" -test = "v=3" -# MySQL数据库配置 -[database] - [[database.default]] - host = "127.0.0.1" - port = "3306" - user = "root" - pass = "" - # pass = "12345678" - name = "test" - type = "mysql" - role = "master" - charset = "utf8" - priority = "1" - [[database.test]] - host = "127.0.0.1" - port = "3306" - user = "root" - pass = "" - # pass = "12345678" - name = "test" - type = "mysql" - role = "master" - charset = "utf8" - priority = "1" -# Redis数据库配置 -[redis] - default = "127.0.0.1:6379,7" - cache = "127.0.0.1:6379,8" - disk = "127.0.0.1:6379,9,?maxIdle=1&maxActive=10&idleTimeout=10&maxConnLifetime=10" -` - path := "config.toml" - err := gfile.PutContents(path, config) + redisContent := gfile.GetContents(gfile.Join(gdebug.TestDataPath(), "redis", "config.toml")) + + var err error + dirPath := gfile.Join(gfile.TempDir(), gtime.TimestampNanoStr()) + err = gfile.Mkdir(dirPath) gtest.Assert(err, nil) - defer gfile.Remove(path) - defer Config().Clear() + defer gfile.Remove(dirPath) + + name := "config.toml" + err = gfile.PutContents(gfile.Join(dirPath, name), redisContent) + gtest.Assert(err, nil) + + err = gins.Config().AddPath(dirPath) + gtest.Assert(err, nil) + + defer gins.Config().Clear() // for gfsnotify callbacks to refresh cache of config file time.Sleep(500 * time.Millisecond) @@ -61,9 +41,9 @@ test = "v=3" gtest.Case(t, func() { //fmt.Println("gins Test_Redis", Config().Get("test")) - redisDefault := Redis() - redisCache := Redis("cache") - redisDisk := Redis("disk") + redisDefault := gins.Redis() + redisCache := gins.Redis("cache") + redisDisk := gins.Redis("disk") gtest.AssertNE(redisDefault, nil) gtest.AssertNE(redisCache, nil) gtest.AssertNE(redisDisk, nil) diff --git a/frame/gins/gins_z_unit_view_test.go b/frame/gins/gins_z_unit_view_test.go index d156bb398..a3ea55bb6 100644 --- a/frame/gins/gins_z_unit_view_test.go +++ b/frame/gins/gins_z_unit_view_test.go @@ -52,7 +52,7 @@ func Test_View(t *testing.T) { func Test_View_Config(t *testing.T) { // view1 test1 gtest.Case(t, func() { - dirPath := gfile.Join(gdebug.CallerDirectory(), "testdata", "view1") + dirPath := gfile.Join(gdebug.TestDataPath(), "view1") gcfg.SetContent(gfile.GetContents(gfile.Join(dirPath, "config.toml"))) defer gcfg.ClearContent() defer instances.Clear() @@ -74,7 +74,7 @@ func Test_View_Config(t *testing.T) { }) // view1 test2 gtest.Case(t, func() { - dirPath := gfile.Join(gdebug.CallerDirectory(), "testdata", "view1") + dirPath := gfile.Join(gdebug.TestDataPath(), "view1") gcfg.SetContent(gfile.GetContents(gfile.Join(dirPath, "config.toml"))) defer gcfg.ClearContent() defer instances.Clear() @@ -96,7 +96,7 @@ func Test_View_Config(t *testing.T) { }) // view2 gtest.Case(t, func() { - dirPath := gfile.Join(gdebug.CallerDirectory(), "testdata", "view2") + dirPath := gfile.Join(gdebug.TestDataPath(), "view2") gcfg.SetContent(gfile.GetContents(gfile.Join(dirPath, "config.toml"))) defer gcfg.ClearContent() defer instances.Clear() @@ -118,7 +118,7 @@ func Test_View_Config(t *testing.T) { }) // view2 gtest.Case(t, func() { - dirPath := gfile.Join(gdebug.CallerDirectory(), "testdata", "view2") + dirPath := gfile.Join(gdebug.TestDataPath(), "view2") gcfg.SetContent(gfile.GetContents(gfile.Join(dirPath, "config.toml"))) defer gcfg.ClearContent() defer instances.Clear() diff --git a/frame/gins/testdata/config/config.toml b/frame/gins/testdata/config/config.toml new file mode 100644 index 000000000..7d6ad4b03 --- /dev/null +++ b/frame/gins/testdata/config/config.toml @@ -0,0 +1,29 @@ +# 模板引擎目录 +viewpath = "/home/www/templates/" +test = "v=1" +# MySQL数据库配置 +[database] + [[database.default]] + host = "127.0.0.1" + port = "3306" + user = "root" + pass = "" + name = "test" + type = "mysql" + role = "master" + charset = "utf8" + priority = "1" + [[database.default]] + host = "127.0.0.1" + port = "3306" + user = "root" + pass = "8692651" + name = "test" + type = "mysql" + role = "master" + charset = "utf8" + priority = "1" +# Redis数据库配置 +[redis] + disk = "127.0.0.1:6379,0" + cache = "127.0.0.1:6379,1" \ No newline at end of file diff --git a/frame/gins/testdata/database/config.toml b/frame/gins/testdata/database/config.toml new file mode 100644 index 000000000..d2b72adfb --- /dev/null +++ b/frame/gins/testdata/database/config.toml @@ -0,0 +1,29 @@ +# 模板引擎目录 +viewpath = "/home/www/templates/" +test = "v=2" +# MySQL数据库配置 +[database] + [[database.default]] + host = "127.0.0.1" + port = "3306" + user = "root" + pass = "12345678" + name = "test" + type = "mysql" + role = "master" + weight = "1" + charset = "utf8" + [[database.test]] + host = "127.0.0.1" + port = "3306" + user = "root" + pass = "12345678" + name = "test" + type = "mysql" + role = "master" + weight = "1" + charset = "utf8" +# Redis数据库配置 +[redis] + default = "127.0.0.1:6379,0" + cache = "127.0.0.1:6379,1" \ No newline at end of file diff --git a/frame/gins/testdata/redis/config.toml b/frame/gins/testdata/redis/config.toml new file mode 100644 index 000000000..624ba1abb --- /dev/null +++ b/frame/gins/testdata/redis/config.toml @@ -0,0 +1,32 @@ +# 模板引擎目录 +viewpath = "/home/www/templates/" +test = "v=3" +# MySQL数据库配置 +[database] + [[database.default]] + host = "127.0.0.1" + port = "3306" + user = "root" + pass = "" + # pass = "12345678" + name = "test" + type = "mysql" + role = "master" + charset = "utf8" + priority = "1" + [[database.test]] + host = "127.0.0.1" + port = "3306" + user = "root" + pass = "" + # pass = "12345678" + name = "test" + type = "mysql" + role = "master" + charset = "utf8" + priority = "1" +# Redis数据库配置 +[redis] + default = "127.0.0.1:6379,7" + cache = "127.0.0.1:6379,8" + disk = "127.0.0.1:6379,9,?maxIdle=1&maxActive=10&idleTimeout=10&maxConnLifetime=10" \ No newline at end of file diff --git a/net/ghttp/ghttp_unit_middleware_basic_test.go b/net/ghttp/ghttp_unit_middleware_basic_test.go index 0ff2c4beb..aa4eec40e 100644 --- a/net/ghttp/ghttp_unit_middleware_basic_test.go +++ b/net/ghttp/ghttp_unit_middleware_basic_test.go @@ -178,7 +178,7 @@ func Test_Middleware_With_Static(t *testing.T) { }) s.SetPort(p) s.SetDumpRouterMap(false) - s.SetServerRoot(gfile.Join(gdebug.CallerDirectory(), "testdata", "static1")) + s.SetServerRoot(gfile.Join(gdebug.TestDataPath(), "static1")) s.Start() defer s.Shutdown() time.Sleep(100 * time.Millisecond) @@ -250,7 +250,7 @@ func Test_Middleware_Hook_With_Static(t *testing.T) { }) s.SetPort(p) //s.SetDumpRouterMap(false) - s.SetServerRoot(gfile.Join(gdebug.CallerDirectory(), "testdata", "static1")) + s.SetServerRoot(gfile.Join(gdebug.TestDataPath(), "static1")) s.Start() defer s.Shutdown() time.Sleep(100 * time.Millisecond) diff --git a/net/ghttp/ghttp_unit_param_file_test.go b/net/ghttp/ghttp_unit_param_file_test.go index 357eb3e4c..de0dd51ba 100644 --- a/net/ghttp/ghttp_unit_param_file_test.go +++ b/net/ghttp/ghttp_unit_param_file_test.go @@ -45,7 +45,7 @@ func Test_Params_File_Single(t *testing.T) { client := ghttp.NewClient() client.SetPrefix(fmt.Sprintf("http://127.0.0.1:%d", p)) - srcPath := gfile.Join(gdebug.CallerDirectory(), "testdata", "upload", "file1.txt") + srcPath := gfile.Join(gdebug.TestDataPath(), "upload", "file1.txt") dstPath := gfile.Join(dstDirPath, "file1.txt") content := client.PostContent("/upload/single", g.Map{ "file": "@file:" + srcPath, @@ -61,7 +61,7 @@ func Test_Params_File_Single(t *testing.T) { client := ghttp.NewClient() client.SetPrefix(fmt.Sprintf("http://127.0.0.1:%d", p)) - srcPath := gfile.Join(gdebug.CallerDirectory(), "testdata", "upload", "file2.txt") + srcPath := gfile.Join(gdebug.TestDataPath(), "upload", "file2.txt") content := client.PostContent("/upload/single", g.Map{ "file": "@file:" + srcPath, "randomlyRename": true, @@ -98,7 +98,7 @@ func Test_Params_File_CustomName(t *testing.T) { client := ghttp.NewClient() client.SetPrefix(fmt.Sprintf("http://127.0.0.1:%d", p)) - srcPath := gfile.Join(gdebug.CallerDirectory(), "testdata", "upload", "file1.txt") + srcPath := gfile.Join(gdebug.TestDataPath(), "upload", "file1.txt") dstPath := gfile.Join(dstDirPath, "my.txt") content := client.PostContent("/upload/single", g.Map{ "file": "@file:" + srcPath, @@ -135,8 +135,8 @@ func Test_Params_File_Batch(t *testing.T) { client := ghttp.NewClient() client.SetPrefix(fmt.Sprintf("http://127.0.0.1:%d", p)) - srcPath1 := gfile.Join(gdebug.CallerDirectory(), "testdata", "upload", "file1.txt") - srcPath2 := gfile.Join(gdebug.CallerDirectory(), "testdata", "upload", "file2.txt") + srcPath1 := gfile.Join(gdebug.TestDataPath(), "upload", "file1.txt") + srcPath2 := gfile.Join(gdebug.TestDataPath(), "upload", "file2.txt") dstPath1 := gfile.Join(dstDirPath, "file1.txt") dstPath2 := gfile.Join(dstDirPath, "file2.txt") content := client.PostContent("/upload/batch", g.Map{ @@ -155,8 +155,8 @@ func Test_Params_File_Batch(t *testing.T) { client := ghttp.NewClient() client.SetPrefix(fmt.Sprintf("http://127.0.0.1:%d", p)) - srcPath1 := gfile.Join(gdebug.CallerDirectory(), "testdata", "upload", "file1.txt") - srcPath2 := gfile.Join(gdebug.CallerDirectory(), "testdata", "upload", "file2.txt") + srcPath1 := gfile.Join(gdebug.TestDataPath(), "upload", "file1.txt") + srcPath2 := gfile.Join(gdebug.TestDataPath(), "upload", "file2.txt") content := client.PostContent("/upload/batch", g.Map{ "file[0]": "@file:" + srcPath1, "file[1]": "@file:" + srcPath2, diff --git a/net/ghttp/ghttp_unit_static_test.go b/net/ghttp/ghttp_unit_static_test.go index 1ca72d502..361c82959 100644 --- a/net/ghttp/ghttp_unit_static_test.go +++ b/net/ghttp/ghttp_unit_static_test.go @@ -66,7 +66,7 @@ func Test_Static_ServerRoot_Security(t *testing.T) { gtest.Case(t, func() { p := ports.PopRand() s := g.Server(p) - s.SetServerRoot(gfile.Join(gdebug.CallerDirectory(), "testdata", "static1")) + s.SetServerRoot(gfile.Join(gdebug.TestDataPath(), "static1")) s.SetPort(p) s.Start() defer s.Shutdown() diff --git a/net/ghttp/ghttp_unit_template_test.go b/net/ghttp/ghttp_unit_template_test.go index bf2d1e956..c42609a94 100644 --- a/net/ghttp/ghttp_unit_template_test.go +++ b/net/ghttp/ghttp_unit_template_test.go @@ -24,7 +24,7 @@ import ( func Test_Template_Layout1(t *testing.T) { gtest.Case(t, func() { - v := gview.New(gfile.Join(gdebug.CallerDirectory(), "testdata", "template", "layout1")) + v := gview.New(gfile.Join(gdebug.TestDataPath(), "template", "layout1")) p := ports.PopRand() s := g.Server(p) s.SetView(v) @@ -54,7 +54,7 @@ func Test_Template_Layout1(t *testing.T) { func Test_Template_Layout2(t *testing.T) { gtest.Case(t, func() { - v := gview.New(gfile.Join(gdebug.CallerDirectory(), "testdata", "template", "layout2")) + v := gview.New(gfile.Join(gdebug.TestDataPath(), "template", "layout2")) p := ports.PopRand() s := g.Server(p) s.SetView(v) diff --git a/os/gcfg/gcfg_z_unit_test.go b/os/gcfg/gcfg_z_unit_test.go index 08a3582a5..e9b16a645 100644 --- a/os/gcfg/gcfg_z_unit_test.go +++ b/os/gcfg/gcfg_z_unit_test.go @@ -440,7 +440,7 @@ func TestCfg_Instance(t *testing.T) { }) gtest.Case(t, func() { pwd := gfile.Pwd() - gfile.Chdir(gfile.Join(gdebug.CallerDirectory(), "testdata")) + gfile.Chdir(gfile.Join(gdebug.TestDataPath())) defer gfile.Chdir(pwd) gtest.Assert(gcfg.Instance("c1") != nil, true) gtest.Assert(gcfg.Instance("c1").Get("my-config"), "1") diff --git a/os/gfile/gfile_z_time_test.go b/os/gfile/gfile_z_time_test.go index 8e5e7a152..843c1ad1e 100644 --- a/os/gfile/gfile_z_time_test.go +++ b/os/gfile/gfile_z_time_test.go @@ -48,7 +48,10 @@ func Test_MTimeMillisecond(t *testing.T) { gtest.Assert(err, nil) time.Sleep(time.Millisecond * 100) - gtest.AssertGE(gfile.MTimeMillisecond(testpath()+file1), fileobj.ModTime().UnixNano()/1000000) + gtest.AssertGE( + gfile.MTimeMillisecond(testpath()+file1), + fileobj.ModTime().UnixNano()/1000000, + ) gtest.Assert(gfile.MTimeMillisecond(""), 0) }) } diff --git a/os/gview/gview_unit_config_test.go b/os/gview/gview_unit_config_test.go index aac9bf88a..162873454 100644 --- a/os/gview/gview_unit_config_test.go +++ b/os/gview/gview_unit_config_test.go @@ -18,7 +18,7 @@ import ( func Test_Config(t *testing.T) { gtest.Case(t, func() { config := gview.Config{ - Paths: []string{gfile.Join(gdebug.CallerDirectory(), "testdata", "config")}, + Paths: []string{gfile.Join(gdebug.TestDataPath(), "config")}, Data: g.Map{ "name": "gf", }, @@ -45,7 +45,7 @@ func Test_ConfigWithMap(t *testing.T) { gtest.Case(t, func() { view := gview.New() err := view.SetConfigWithMap(g.Map{ - "Paths": []string{gfile.Join(gdebug.CallerDirectory(), "testdata", "config")}, + "Paths": []string{gfile.Join(gdebug.TestDataPath(), "config")}, "DefaultFile": "test.html", "Delimiters": []string{"${", "}"}, "Data": g.Map{ diff --git a/test/gtest/gtest.go b/test/gtest/gtest.go index 489e62240..2ffe21ed2 100644 --- a/test/gtest/gtest.go +++ b/test/gtest/gtest.go @@ -132,16 +132,20 @@ func AssertGE(value, expect interface{}) { passed = gconv.String(value) >= gconv.String(expect) case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - passed = gconv.Int(value) >= gconv.Int(expect) + passed = gconv.Int64(value) >= gconv.Int64(expect) case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - passed = gconv.Uint(value) >= gconv.Uint(expect) + passed = gconv.Uint64(value) >= gconv.Uint64(expect) case reflect.Float32, reflect.Float64: passed = gconv.Float64(value) >= gconv.Float64(expect) } if !passed { - panic(fmt.Sprintf(`[ASSERT] EXPECT %v >= %v`, value, expect)) + panic(fmt.Sprintf( + `[ASSERT] EXPECT %v(%v) >= %v(%v)`, + value, reflect.ValueOf(value).Kind(), + expect, reflect.ValueOf(expect).Kind(), + )) } } From c7f911cae298b77a506dd4b2c2ea5670c7c28af0 Mon Sep 17 00:00:00 2001 From: John Date: Mon, 16 Mar 2020 23:03:35 +0800 Subject: [PATCH 26/32] change function time.Duration.Milliseconds to time.Duration.Nanoseconds for compatibility of Golang version < 1.13 --- container/gpool/gpool.go | 4 +++- os/glog/glog_logger_rotate.go | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/container/gpool/gpool.go b/container/gpool/gpool.go index 93df5b3ba..efb84cf8e 100644 --- a/container/gpool/gpool.go +++ b/container/gpool/gpool.go @@ -82,7 +82,9 @@ func (p *Pool) Put(value interface{}) error { if p.TTL == 0 { item.expire = 0 } else { - item.expire = gtime.TimestampMilli() + p.TTL.Milliseconds() + // As for Golang version <= 1.13, there's no method Milliseconds for time.Duration. + // So we need calculate the milliseconds using its nanoseconds value. + item.expire = gtime.TimestampMilli() + p.TTL.Nanoseconds()/1000000 } p.list.PushBack(item) return nil diff --git a/os/glog/glog_logger_rotate.go b/os/glog/glog_logger_rotate.go index e61d8a952..0dc853640 100644 --- a/os/glog/glog_logger_rotate.go +++ b/os/glog/glog_logger_rotate.go @@ -141,7 +141,9 @@ func (l *Logger) rotateChecks() { // Expiration checks. if l.config.RotateExpire > 0 { nowTimestampMilli := gtime.TimestampMilli() - expireMillisecond := l.config.RotateExpire.Milliseconds() + // As for Golang version <= 1.13, there's no method Milliseconds for time.Duration. + // So we need calculate the milliseconds using its nanoseconds value. + expireMillisecond := l.config.RotateExpire.Nanoseconds() / 1000000 for _, array := range backupFilesMap { array.Iterator(func(_ int, v interface{}) bool { path := v.(string) From 2b46e765c42b1ae378b11a3e21e223d3c4d92dac Mon Sep 17 00:00:00 2001 From: John Date: Mon, 16 Mar 2020 23:03:37 +0800 Subject: [PATCH 27/32] change function time.Duration.Milliseconds to time.Duration.Nanoseconds for compatibility of Golang version < 1.13 --- container/gpool/gpool.go | 2 +- os/glog/glog_logger_rotate.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/container/gpool/gpool.go b/container/gpool/gpool.go index efb84cf8e..4e4d5d816 100644 --- a/container/gpool/gpool.go +++ b/container/gpool/gpool.go @@ -82,7 +82,7 @@ func (p *Pool) Put(value interface{}) error { if p.TTL == 0 { item.expire = 0 } else { - // As for Golang version <= 1.13, there's no method Milliseconds for time.Duration. + // As for Golang version < 1.13, there's no method Milliseconds for time.Duration. // So we need calculate the milliseconds using its nanoseconds value. item.expire = gtime.TimestampMilli() + p.TTL.Nanoseconds()/1000000 } diff --git a/os/glog/glog_logger_rotate.go b/os/glog/glog_logger_rotate.go index 0dc853640..843e77f45 100644 --- a/os/glog/glog_logger_rotate.go +++ b/os/glog/glog_logger_rotate.go @@ -141,7 +141,7 @@ func (l *Logger) rotateChecks() { // Expiration checks. if l.config.RotateExpire > 0 { nowTimestampMilli := gtime.TimestampMilli() - // As for Golang version <= 1.13, there's no method Milliseconds for time.Duration. + // As for Golang version < 1.13, there's no method Milliseconds for time.Duration. // So we need calculate the milliseconds using its nanoseconds value. expireMillisecond := l.config.RotateExpire.Nanoseconds() / 1000000 for _, array := range backupFilesMap { From 2812a247aa4f5c894c259a1e9ed25aa281e338c3 Mon Sep 17 00:00:00 2001 From: sunmoon Date: Tue, 17 Mar 2020 11:19:43 +0800 Subject: [PATCH 28/32] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E9=AA=8C=E8=AF=81?= =?UTF-8?q?=E4=B8=AD=E8=8B=B1=E6=96=87=E6=B7=B7=E5=90=88=E5=AD=97=E7=AC=A6?= =?UTF-8?q?=E4=B8=B2=E9=95=BF=E5=BA=A6=E7=9A=84=E5=88=A4=E6=96=AD=E8=A7=84?= =?UTF-8?q?=E5=88=99,=E6=8C=89utf8=E8=AE=A1=E7=AE=97,=E4=B8=80=E4=B8=AA?= =?UTF-8?q?=E4=B8=AD=E6=96=87=E7=AE=97=E4=B8=80=E4=B8=AA=E5=AD=97=E7=AC=A6?= =?UTF-8?q?,cn-length,cn-min-length,cn-max-length?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- util/gvalid/gvalid_check.go | 77 +++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/util/gvalid/gvalid_check.go b/util/gvalid/gvalid_check.go index 59e3eadcc..ef020d44d 100644 --- a/util/gvalid/gvalid_check.go +++ b/util/gvalid/gvalid_check.go @@ -10,6 +10,7 @@ import ( "regexp" "strconv" "strings" + "unicode/utf8" "github.com/gogf/gf/container/gmap" "github.com/gogf/gf/encoding/gjson" @@ -78,6 +79,9 @@ var ( "length": {}, "min-length": {}, "max-length": {}, + "cn-length": {}, + "cn-min-length": {}, + "cn-max-length": {}, "between": {}, "min": {}, "max": {}, @@ -192,6 +196,17 @@ func Check(value interface{}, rules string, msgs interface{}, params ...interfac } else { match = true } + // 中英文混合长度判断 + case "cn-length": + fallthrough + case "cn-min-length": + fallthrough + case "cn-max-length": + if msg := checkCnLength(val, ruleKey, ruleVal, customMsgMap); msg != "" { + errorMsgs[ruleKey] = msg + } else { + match = true + } // 大小范围 case "min": @@ -522,7 +537,69 @@ func checkRequired(value, ruleKey, ruleVal string, params map[string]string) boo return true } } +// 对中英文混合字段值长度进行检测 +func checkCnLength(value, ruleKey, ruleVal string, customMsgMap map[string]string) string { + msg := "" + strLen := utf8.RuneCountInString(value) + switch ruleKey { + // 长度范围 + case "cn-length": + array := strings.Split(ruleVal, ",") + min := 0 + max := 0 + if len(array) > 0 { + if v, err := strconv.Atoi(strings.TrimSpace(array[0])); err == nil { + min = v + } + } + if len(array) > 1 { + if v, err := strconv.Atoi(strings.TrimSpace(array[1])); err == nil { + max = v + } + } + if strLen < min || strLen > max { + if v, ok := customMsgMap[ruleKey]; !ok { + msg = errorMsgMap.Get(ruleKey) + } else { + msg = v + } + msg = strings.Replace(msg, ":min", strconv.Itoa(min), -1) + msg = strings.Replace(msg, ":max", strconv.Itoa(max), -1) + return msg + } + // 最小长度 + case "cn-min-length": + if min, err := strconv.Atoi(ruleVal); err == nil { + if strLen < min { + if v, ok := customMsgMap[ruleKey]; !ok { + msg = errorMsgMap.Get(ruleKey) + } else { + msg = v + } + msg = strings.Replace(msg, ":min", strconv.Itoa(min), -1) + } + } else { + msg = "校验参数[" + ruleVal + "]应当为整数类型" + } + + // 最大长度 + case "cn-max-length": + if max, err := strconv.Atoi(ruleVal); err == nil { + if strLen > max { + if v, ok := customMsgMap[ruleKey]; !ok { + msg = errorMsgMap.Get(ruleKey) + } else { + msg = v + } + msg = strings.Replace(msg, ":max", strconv.Itoa(max), -1) + } + } else { + msg = "校验参数[" + ruleVal + "]应当为整数类型" + } + } + return msg +} // 对字段值长度进行检测 func checkLength(value, ruleKey, ruleVal string, customMsgMap map[string]string) string { msg := "" From 4fb01e68f7f0e781a25b4a1eb7168376094a67a2 Mon Sep 17 00:00:00 2001 From: sunmoon Date: Tue, 17 Mar 2020 14:26:36 +0800 Subject: [PATCH 29/32] =?UTF-8?q?=E7=A7=BB=E9=99=A4=E5=AD=97=E7=AC=A6?= =?UTF-8?q?=E4=B8=B2ip=E5=9C=B0=E5=9D=80=E4=B8=8ELong=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=E4=BA=92=E8=BD=AC=E7=9A=84=E6=96=B9=E6=B3=95,=E4=BF=AE?= =?UTF-8?q?=E6=94=B9=E4=B8=AD=E6=96=87=E9=95=BF=E5=BA=A6=E6=A0=A1=E9=AA=8C?= =?UTF-8?q?=E7=9A=84=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- util/gconv/gconv.go | 19 -------- util/gvalid/gvalid_check.go | 94 ++++--------------------------------- 2 files changed, 8 insertions(+), 105 deletions(-) diff --git a/util/gconv/gconv.go b/util/gconv/gconv.go index 045176cdc..c34d0a569 100644 --- a/util/gconv/gconv.go +++ b/util/gconv/gconv.go @@ -8,11 +8,9 @@ package gconv import ( - "encoding/binary" "encoding/json" "fmt" "github.com/gogf/gf/os/gtime" - "net" "reflect" "strconv" "strings" @@ -167,23 +165,6 @@ func Runes(i interface{}) []rune { } return []rune(String(i)) } -// convert ip to long, ex ip 106.8.36.164 to 1778918564 -func Ip2long(ipStr string) uint32 { - ip := net.ParseIP(ipStr) - if ip == nil { - return 0 - } - ip = ip.To4() - return binary.BigEndian.Uint32(ip) -} -// convert long to ip string, ex int 1778918564 to ip 106.8.36.164 -func Long2ip(ipLong uint32) string { - ipByte := make([]byte, 4) - binary.BigEndian.PutUint32(ipByte, ipLong) - ip := net.IP(ipByte) - return ip.String() -} - // String converts to string. // It's most common used converting function. diff --git a/util/gvalid/gvalid_check.go b/util/gvalid/gvalid_check.go index ef020d44d..0992ce8fc 100644 --- a/util/gvalid/gvalid_check.go +++ b/util/gvalid/gvalid_check.go @@ -7,11 +7,6 @@ package gvalid import ( - "regexp" - "strconv" - "strings" - "unicode/utf8" - "github.com/gogf/gf/container/gmap" "github.com/gogf/gf/encoding/gjson" "github.com/gogf/gf/net/gipv4" @@ -19,6 +14,9 @@ import ( "github.com/gogf/gf/os/gtime" "github.com/gogf/gf/text/gregex" "github.com/gogf/gf/util/gconv" + "regexp" + "strconv" + "strings" ) const ( @@ -79,9 +77,6 @@ var ( "length": {}, "min-length": {}, "max-length": {}, - "cn-length": {}, - "cn-min-length": {}, - "cn-max-length": {}, "between": {}, "min": {}, "max": {}, @@ -196,18 +191,6 @@ func Check(value interface{}, rules string, msgs interface{}, params ...interfac } else { match = true } - // 中英文混合长度判断 - case "cn-length": - fallthrough - case "cn-min-length": - fallthrough - case "cn-max-length": - if msg := checkCnLength(val, ruleKey, ruleVal, customMsgMap); msg != "" { - errorMsgs[ruleKey] = msg - } else { - match = true - } - // 大小范围 case "min": fallthrough @@ -537,72 +520,11 @@ func checkRequired(value, ruleKey, ruleVal string, params map[string]string) boo return true } } -// 对中英文混合字段值长度进行检测 -func checkCnLength(value, ruleKey, ruleVal string, customMsgMap map[string]string) string { - msg := "" - strLen := utf8.RuneCountInString(value) - switch ruleKey { - // 长度范围 - case "cn-length": - array := strings.Split(ruleVal, ",") - min := 0 - max := 0 - if len(array) > 0 { - if v, err := strconv.Atoi(strings.TrimSpace(array[0])); err == nil { - min = v - } - } - if len(array) > 1 { - if v, err := strconv.Atoi(strings.TrimSpace(array[1])); err == nil { - max = v - } - } - if strLen < min || strLen > max { - if v, ok := customMsgMap[ruleKey]; !ok { - msg = errorMsgMap.Get(ruleKey) - } else { - msg = v - } - msg = strings.Replace(msg, ":min", strconv.Itoa(min), -1) - msg = strings.Replace(msg, ":max", strconv.Itoa(max), -1) - return msg - } - - // 最小长度 - case "cn-min-length": - if min, err := strconv.Atoi(ruleVal); err == nil { - if strLen < min { - if v, ok := customMsgMap[ruleKey]; !ok { - msg = errorMsgMap.Get(ruleKey) - } else { - msg = v - } - msg = strings.Replace(msg, ":min", strconv.Itoa(min), -1) - } - } else { - msg = "校验参数[" + ruleVal + "]应当为整数类型" - } - - // 最大长度 - case "cn-max-length": - if max, err := strconv.Atoi(ruleVal); err == nil { - if strLen > max { - if v, ok := customMsgMap[ruleKey]; !ok { - msg = errorMsgMap.Get(ruleKey) - } else { - msg = v - } - msg = strings.Replace(msg, ":max", strconv.Itoa(max), -1) - } - } else { - msg = "校验参数[" + ruleVal + "]应当为整数类型" - } - } - return msg -} // 对字段值长度进行检测 func checkLength(value, ruleKey, ruleVal string, customMsgMap map[string]string) string { msg := "" + runeArray := gconv.Runes(value) + valueLen := len(runeArray) switch ruleKey { // 长度范围 case "length": @@ -619,7 +541,7 @@ func checkLength(value, ruleKey, ruleVal string, customMsgMap map[string]string) max = v } } - if len(value) < min || len(value) > max { + if valueLen < min || valueLen > max { if v, ok := customMsgMap[ruleKey]; !ok { msg = errorMsgMap.Get(ruleKey) } else { @@ -633,7 +555,7 @@ func checkLength(value, ruleKey, ruleVal string, customMsgMap map[string]string) // 最小长度 case "min-length": if min, err := strconv.Atoi(ruleVal); err == nil { - if len(value) < min { + if valueLen < min { if v, ok := customMsgMap[ruleKey]; !ok { msg = errorMsgMap.Get(ruleKey) } else { @@ -648,7 +570,7 @@ func checkLength(value, ruleKey, ruleVal string, customMsgMap map[string]string) // 最大长度 case "max-length": if max, err := strconv.Atoi(ruleVal); err == nil { - if len(value) > max { + if valueLen > max { if v, ok := customMsgMap[ruleKey]; !ok { msg = errorMsgMap.Get(ruleKey) } else { From f3d859159da8b6efbdc054cb344885d6383a076c Mon Sep 17 00:00:00 2001 From: John Date: Tue, 17 Mar 2020 14:48:52 +0800 Subject: [PATCH 30/32] improve router group for duplicated router registering for package ghttp.Server --- encoding/gjson/gjson_z_unit_basic_test.go | 36 ----- encoding/gjson/gjson_z_unit_struct_test.go | 132 +++++++++++++++++++ frame/gmvc/model.go | 10 +- net/ghttp/ghttp_server.go | 3 +- net/ghttp/ghttp_server_domain.go | 127 +++++++++++------- net/ghttp/ghttp_server_router.go | 13 +- net/ghttp/ghttp_server_router_group.go | 126 +++++++++++------- net/ghttp/ghttp_server_router_hook.go | 5 + net/ghttp/ghttp_server_service_controller.go | 40 ++++-- net/ghttp/ghttp_server_service_handler.go | 8 +- net/ghttp/ghttp_server_service_object.go | 25 +++- net/ghttp/ghttp_unit_middleware_cors_test.go | 3 + 12 files changed, 370 insertions(+), 158 deletions(-) create mode 100644 encoding/gjson/gjson_z_unit_struct_test.go diff --git a/encoding/gjson/gjson_z_unit_basic_test.go b/encoding/gjson/gjson_z_unit_basic_test.go index ed62b7c39..7d89c2407 100644 --- a/encoding/gjson/gjson_z_unit_basic_test.go +++ b/encoding/gjson/gjson_z_unit_basic_test.go @@ -462,39 +462,3 @@ func Test_IsNil(t *testing.T) { gtest.Assert(j.IsNil(), true) }) } - -func Test_ToStructDeep(t *testing.T) { - gtest.Case(t, func() { - type Item struct { - Title string `json:"title"` - Key string `json:"key"` - } - - type M struct { - Id string `json:"id"` - Me map[string]interface{} `json:"me"` - Txt string `json:"txt"` - Items []*Item `json:"items"` - } - - txt := `{ - "id":"88888", - "me":{"name":"mikey","day":"20009"}, - "txt":"hello", - "items":null - }` - - j, err := gjson.LoadContent(txt) - gtest.Assert(err, nil) - gtest.Assert(j.GetString("me.name"), "mikey") - gtest.Assert(j.GetString("items"), "") - gtest.Assert(j.GetBool("items"), false) - gtest.Assert(j.GetArray("items"), nil) - m := new(M) - err = j.ToStructDeep(m) - gtest.Assert(err, nil) - gtest.AssertNE(m.Me, nil) - gtest.Assert(m.Me["day"], "20009") - gtest.Assert(m.Items, nil) - }) -} diff --git a/encoding/gjson/gjson_z_unit_struct_test.go b/encoding/gjson/gjson_z_unit_struct_test.go new file mode 100644 index 000000000..decc310ee --- /dev/null +++ b/encoding/gjson/gjson_z_unit_struct_test.go @@ -0,0 +1,132 @@ +// Copyright 2017 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gjson_test + +import ( + "github.com/gogf/gf/encoding/gjson" + "github.com/gogf/gf/frame/g" + "github.com/gogf/gf/test/gtest" + "testing" +) + +func Test_ToStruct1(t *testing.T) { + gtest.Case(t, func() { + type BaseInfoItem struct { + IdCardNumber string `db:"id_card_number" json:"idCardNumber" field:"id_card_number"` + IsHouseholder bool `db:"is_householder" json:"isHouseholder" field:"is_householder"` + HouseholderRelation string `db:"householder_relation" json:"householderRelation" field:"householder_relation"` + UserName string `db:"user_name" json:"userName" field:"user_name"` + UserSex string `db:"user_sex" json:"userSex" field:"user_sex"` + UserAge int `db:"user_age" json:"userAge" field:"user_age"` + UserNation string `db:"user_nation" json:"userNation" field:"user_nation"` + } + + type UserCollectionAddReq struct { + BaseInfo []BaseInfoItem `db:"_" json:"baseInfo" field:"_"` + } + jsonContent := `{ + "baseInfo": [{ + "idCardNumber": "520101199412141111", + "isHouseholder": true, + "householderRelation": "户主", + "userName": "李四", + "userSex": "男", + "userAge": 32, + "userNation": "苗族", + "userPhone": "13084183323", + "liveAddress": {}, + "occupationInfo": [{ + "occupationType": "经商", + "occupationBusinessInfo": [{ + "occupationClass": "制造业", + "businessLicenseNumber": "32020000012300", + "businessName": "土灶柴火鸡", + "spouseName": "", + "spouseIdCardNumber": "", + "businessLicensePhotoId": 125, + "businessPlace": "租赁房产", + "hasGoodsInsurance": true, + "businessScopeStr": "柴火鸡;烧烤", + "businessAddress": {}, + "businessPerformAbility": { + "businessType": "服务业", + "businessLife": 5, + "salesRevenue": 8000, + "familyEquity": 6000 + } + }], + "occupationWorkInfo": { + "occupationClass": "", + "companyName": "", + "companyType": "", + "workYearNum": 0, + "spouseName": "", + "spouseIdCardNumber": "", + "spousePhone": "", + "spouseEducation": "", + "spouseCompanyName": "", + "workLevel": "", + "workAddress": {}, + "workPerformAbility": { + "familyAnnualIncome": 0, + "familyEquity": 0, + "workCooperationState": "", + "workMoneyCooperationState": "" + } + }, + "occupationAgricultureInfo": [] + }], + "assetsInfo": [], + "expenditureInfo": [], + "incomeInfo": [], + "liabilityInfo": [] + }] +}` + data := new(UserCollectionAddReq) + j, err := gjson.LoadJson(jsonContent) + gtest.Assert(err, nil) + err = j.ToStruct(data) + gtest.Assert(err, nil) + g.Dump(data) + }) +} + +func Test_ToStructDeep(t *testing.T) { + gtest.Case(t, func() { + type Item struct { + Title string `json:"title"` + Key string `json:"key"` + } + + type M struct { + Id string `json:"id"` + Me map[string]interface{} `json:"me"` + Txt string `json:"txt"` + Items []*Item `json:"items"` + } + + txt := `{ + "id":"88888", + "me":{"name":"mikey","day":"20009"}, + "txt":"hello", + "items":null + }` + + j, err := gjson.LoadContent(txt) + gtest.Assert(err, nil) + gtest.Assert(j.GetString("me.name"), "mikey") + gtest.Assert(j.GetString("items"), "") + gtest.Assert(j.GetBool("items"), false) + gtest.Assert(j.GetArray("items"), nil) + m := new(M) + err = j.ToStructDeep(m) + gtest.Assert(err, nil) + gtest.AssertNE(m.Me, nil) + gtest.Assert(m.Me["day"], "20009") + gtest.Assert(m.Items, nil) + }) +} diff --git a/frame/gmvc/model.go b/frame/gmvc/model.go index 81a0ea2a5..5eb656d0d 100644 --- a/frame/gmvc/model.go +++ b/frame/gmvc/model.go @@ -8,9 +8,7 @@ package gmvc import "github.com/gogf/gf/database/gdb" -// M is alias for Model, -// just for short write purpose. -type M = Model - -// Model is alias for *gdb.Model. -type Model = *gdb.Model +type ( + M = Model // M is alias for Model, just for short write purpose. + Model = *gdb.Model // Model is alias for *gdb.Model. +) diff --git a/net/ghttp/ghttp_server.go b/net/ghttp/ghttp_server.go index b446a17e8..a9e219b09 100644 --- a/net/ghttp/ghttp_server.go +++ b/net/ghttp/ghttp_server.go @@ -89,6 +89,7 @@ type ( ctrlInfo *handlerController // Controller information for reflect usage. hookName string // Hook type name. router *Router // Router object. + source string // Source file path:line when registering. } // handlerParsedItem is the item parsed from URL.Path. @@ -105,7 +106,7 @@ type ( // registeredRouteItem stores the information of the router and is used for route map. registeredRouteItem struct { - file string // Source file path and its line number. + source string // Source file path and its line number. handler *handlerItem // Handler object. } diff --git a/net/ghttp/ghttp_server_domain.go b/net/ghttp/ghttp_server_domain.go index a7a8c8564..25faf9ea0 100644 --- a/net/ghttp/ghttp_server_domain.go +++ b/net/ghttp/ghttp_server_domain.go @@ -12,121 +12,148 @@ import ( // 域名管理器对象 type Domain struct { - s *Server // 所属Server - m map[string]bool // 多域名 + server *Server // 所属Server + domains map[string]struct{} // 多域名 } // 生成一个域名对象, 参数 domains 支持给定多个域名。 func (s *Server) Domain(domains string) *Domain { d := &Domain{ - s: s, - m: make(map[string]bool), + server: s, + domains: make(map[string]struct{}), } for _, v := range strings.Split(domains, ",") { - d.m[strings.TrimSpace(v)] = true + d.domains[strings.TrimSpace(v)] = struct{}{} } return d } func (d *Domain) BindHandler(pattern string, handler HandlerFunc) { - for domain, _ := range d.m { - d.s.BindHandler(pattern+"@"+domain, handler) + for domain, _ := range d.domains { + d.server.BindHandler(pattern+"@"+domain, handler) } } -func (d *Domain) doBindHandler(pattern string, handler HandlerFunc, middleware []HandlerFunc) { - for domain, _ := range d.m { - d.s.doBindHandler(pattern+"@"+domain, handler, middleware) +func (d *Domain) doBindHandler( + pattern string, handler HandlerFunc, + middleware []HandlerFunc, source string, +) { + for domain, _ := range d.domains { + d.server.doBindHandler(pattern+"@"+domain, handler, middleware, source) } } func (d *Domain) BindObject(pattern string, obj interface{}, methods ...string) { - for domain, _ := range d.m { - d.s.BindObject(pattern+"@"+domain, obj, methods...) + for domain, _ := range d.domains { + d.server.BindObject(pattern+"@"+domain, obj, methods...) } } -func (d *Domain) doBindObject(pattern string, obj interface{}, methods string, middleware []HandlerFunc) { - for domain, _ := range d.m { - d.s.doBindObject(pattern+"@"+domain, obj, methods, middleware) +func (d *Domain) doBindObject( + pattern string, obj interface{}, methods string, + middleware []HandlerFunc, source string, +) { + for domain, _ := range d.domains { + d.server.doBindObject(pattern+"@"+domain, obj, methods, middleware, source) } } func (d *Domain) BindObjectMethod(pattern string, obj interface{}, method string) { - for domain, _ := range d.m { - d.s.BindObjectMethod(pattern+"@"+domain, obj, method) + for domain, _ := range d.domains { + d.server.BindObjectMethod(pattern+"@"+domain, obj, method) } } -func (d *Domain) doBindObjectMethod(pattern string, obj interface{}, method string, middleware []HandlerFunc) { - for domain, _ := range d.m { - d.s.doBindObjectMethod(pattern+"@"+domain, obj, method, middleware) +func (d *Domain) doBindObjectMethod( + pattern string, obj interface{}, method string, + middleware []HandlerFunc, source string, +) { + for domain, _ := range d.domains { + d.server.doBindObjectMethod(pattern+"@"+domain, obj, method, middleware, source) } } func (d *Domain) BindObjectRest(pattern string, obj interface{}) { - for domain, _ := range d.m { - d.s.BindObjectRest(pattern+"@"+domain, obj) + for domain, _ := range d.domains { + d.server.BindObjectRest(pattern+"@"+domain, obj) } } -func (d *Domain) doBindObjectRest(pattern string, obj interface{}, middleware []HandlerFunc) { - for domain, _ := range d.m { - d.s.doBindObjectRest(pattern+"@"+domain, obj, middleware) +func (d *Domain) doBindObjectRest( + pattern string, obj interface{}, + middleware []HandlerFunc, source string, +) { + for domain, _ := range d.domains { + d.server.doBindObjectRest(pattern+"@"+domain, obj, middleware, source) } } func (d *Domain) BindController(pattern string, c Controller, methods ...string) { - for domain, _ := range d.m { - d.s.BindController(pattern+"@"+domain, c, methods...) + for domain, _ := range d.domains { + d.server.BindController(pattern+"@"+domain, c, methods...) } } -func (d *Domain) doBindController(pattern string, c Controller, methods string, middleware []HandlerFunc) { - for domain, _ := range d.m { - d.s.doBindController(pattern+"@"+domain, c, methods, middleware) +func (d *Domain) doBindController( + pattern string, c Controller, methods string, + middleware []HandlerFunc, source string, +) { + for domain, _ := range d.domains { + d.server.doBindController(pattern+"@"+domain, c, methods, middleware, source) } } func (d *Domain) BindControllerMethod(pattern string, c Controller, method string) { - for domain, _ := range d.m { - d.s.BindControllerMethod(pattern+"@"+domain, c, method) + for domain, _ := range d.domains { + d.server.BindControllerMethod(pattern+"@"+domain, c, method) } } -func (d *Domain) doBindControllerMethod(pattern string, c Controller, method string, middleware []HandlerFunc) { - for domain, _ := range d.m { - d.s.doBindControllerMethod(pattern+"@"+domain, c, method, middleware) +func (d *Domain) doBindControllerMethod( + pattern string, c Controller, method string, + middleware []HandlerFunc, source string, +) { + for domain, _ := range d.domains { + d.server.doBindControllerMethod(pattern+"@"+domain, c, method, middleware, source) } } func (d *Domain) BindControllerRest(pattern string, c Controller) { - for domain, _ := range d.m { - d.s.BindControllerRest(pattern+"@"+domain, c) + for domain, _ := range d.domains { + d.server.BindControllerRest(pattern+"@"+domain, c) } } -func (d *Domain) doBindControllerRest(pattern string, c Controller, middleware []HandlerFunc) { - for domain, _ := range d.m { - d.s.doBindControllerRest(pattern+"@"+domain, c, middleware) +func (d *Domain) doBindControllerRest( + pattern string, c Controller, + middleware []HandlerFunc, source string, +) { + for domain, _ := range d.domains { + d.server.doBindControllerRest(pattern+"@"+domain, c, middleware, source) } } func (d *Domain) BindHookHandler(pattern string, hook string, handler HandlerFunc) { - for domain, _ := range d.m { - d.s.BindHookHandler(pattern+"@"+domain, hook, handler) + for domain, _ := range d.domains { + d.server.BindHookHandler(pattern+"@"+domain, hook, handler) + } +} + +func (d *Domain) doBindHookHandler(pattern string, hook string, handler HandlerFunc, source string) { + for domain, _ := range d.domains { + d.server.doBindHookHandler(pattern+"@"+domain, hook, handler, source) } } func (d *Domain) BindHookHandlerByMap(pattern string, hookmap map[string]HandlerFunc) { - for domain, _ := range d.m { - d.s.BindHookHandlerByMap(pattern+"@"+domain, hookmap) + for domain, _ := range d.domains { + d.server.BindHookHandlerByMap(pattern+"@"+domain, hookmap) } } func (d *Domain) BindStatusHandler(status int, handler HandlerFunc) { - for domain, _ := range d.m { - d.s.setStatusHandler(d.s.statusHandlerKey(status, domain), handler) + for domain, _ := range d.domains { + d.server.setStatusHandler(d.server.statusHandlerKey(status, domain), handler) } } @@ -137,14 +164,14 @@ func (d *Domain) BindStatusHandlerByMap(handlerMap map[int]HandlerFunc) { } func (d *Domain) BindMiddleware(pattern string, handlers ...HandlerFunc) { - for domain, _ := range d.m { - d.s.BindMiddleware(pattern+"@"+domain, handlers...) + for domain, _ := range d.domains { + d.server.BindMiddleware(pattern+"@"+domain, handlers...) } } func (d *Domain) BindMiddlewareDefault(handlers ...HandlerFunc) { - for domain, _ := range d.m { - d.s.BindMiddleware(gDEFAULT_MIDDLEWARE_PATTERN+"@"+domain, handlers...) + for domain, _ := range d.domains { + d.server.BindMiddleware(gDEFAULT_MIDDLEWARE_PATTERN+"@"+domain, handlers...) } } diff --git a/net/ghttp/ghttp_server_router.go b/net/ghttp/ghttp_server_router.go index e27767893..00ff298f9 100644 --- a/net/ghttp/ghttp_server_router.go +++ b/net/ghttp/ghttp_server_router.go @@ -67,6 +67,10 @@ func (s *Server) parsePattern(pattern string) (domain, method, path string, err // is the well designed router storage structure for router searching when the request is under serving. func (s *Server) setHandler(pattern string, handler *handlerItem) { handler.itemId = handlerIdGenerator.Add(1) + if handler.source == "" { + _, file, line := gdebug.CallerWithFilter(gFILTER_KEY) + handler.source = fmt.Sprintf(`%s:%d`, file, line) + } domain, method, uri, err := s.parsePattern(pattern) if err != nil { s.Logger().Fatal("invalid pattern:", pattern, err) @@ -83,7 +87,10 @@ func (s *Server) setHandler(pattern string, handler *handlerItem) { switch handler.itemType { case gHANDLER_TYPE_HANDLER, gHANDLER_TYPE_OBJECT, gHANDLER_TYPE_CONTROLLER: if item, ok := s.routesMap[routerKey]; ok { - s.Logger().Fatalf(`duplicated route registry "%s", already registered at %s`, pattern, item[0].file) + s.Logger().Fatalf( + `duplicated route registry "%s" at %s , already registered at %s`, + pattern, handler.source, item[0].source, + ) return } } @@ -184,9 +191,9 @@ func (s *Server) setHandler(pattern string, handler *handlerItem) { if _, ok := s.routesMap[routerKey]; !ok { s.routesMap[routerKey] = make([]registeredRouteItem, 0) } - _, file, line := gdebug.CallerWithFilter(gFILTER_KEY) + routeItem := registeredRouteItem{ - file: fmt.Sprintf(`%s:%d`, file, line), + source: handler.source, handler: handler, } switch handler.itemType { diff --git a/net/ghttp/ghttp_server_router_group.go b/net/ghttp/ghttp_server_router_group.go index 64915eedd..53480e5e6 100644 --- a/net/ghttp/ghttp_server_router_group.go +++ b/net/ghttp/ghttp_server_router_group.go @@ -7,6 +7,8 @@ package ghttp import ( + "fmt" + "github.com/gogf/gf/debug/gdebug" "reflect" "strings" @@ -36,6 +38,7 @@ type ( pattern string object interface{} // Can be handler, controller or object. params []interface{} // Extra parameters for route registering depending on the type. + source string // // Handler is register at certain source file path:line. } ) @@ -53,10 +56,10 @@ func (s *Server) handlePreBindItems() { if item.group.server != nil && item.group.server != s { continue } - if item.group.domain != nil && item.group.domain.s != s { + if item.group.domain != nil && item.group.domain.server != s { continue } - item.group.doBind(item.bindType, item.pattern, item.object, item.params...) + item.group.doBindRoutersToServer(item) } preBindItems = preBindItems[:0] } @@ -147,9 +150,9 @@ func (g *RouterGroup) Bind(items []GroupItem) *RouterGroup { bindType := gstr.ToUpper(gconv.String(item[0])) switch bindType { case "REST": - group.preBind("REST", gconv.String(item[0])+":"+gconv.String(item[1]), item[2]) + group.preBindToLocalArray("REST", gconv.String(item[0])+":"+gconv.String(item[1]), item[2]) case "MIDDLEWARE": - group.preBind("MIDDLEWARE", gconv.String(item[0])+":"+gconv.String(item[1]), item[2]) + group.preBindToLocalArray("MIDDLEWARE", gconv.String(item[0])+":"+gconv.String(item[1]), item[2]) default: if strings.EqualFold(bindType, "ALL") { bindType = "" @@ -157,9 +160,9 @@ func (g *RouterGroup) Bind(items []GroupItem) *RouterGroup { bindType += ":" } if len(item) > 3 { - group.preBind("HANDLER", bindType+gconv.String(item[1]), item[2], item[3]) + group.preBindToLocalArray("HANDLER", bindType+gconv.String(item[1]), item[2], item[3]) } else { - group.preBind("HANDLER", bindType+gconv.String(item[1]), item[2]) + group.preBindToLocalArray("HANDLER", bindType+gconv.String(item[1]), item[2]) } } } @@ -168,62 +171,62 @@ func (g *RouterGroup) Bind(items []GroupItem) *RouterGroup { // ALL registers a http handler to given route pattern and all http methods. func (g *RouterGroup) ALL(pattern string, object interface{}, params ...interface{}) *RouterGroup { - return g.Clone().preBind("HANDLER", gDEFAULT_METHOD+":"+pattern, object, params...) + return g.Clone().preBindToLocalArray("HANDLER", gDEFAULT_METHOD+":"+pattern, object, params...) } // GET registers a http handler to given route pattern and http method: GET. func (g *RouterGroup) GET(pattern string, object interface{}, params ...interface{}) *RouterGroup { - return g.Clone().preBind("HANDLER", "GET:"+pattern, object, params...) + return g.Clone().preBindToLocalArray("HANDLER", "GET:"+pattern, object, params...) } // PUT registers a http handler to given route pattern and http method: PUT. func (g *RouterGroup) PUT(pattern string, object interface{}, params ...interface{}) *RouterGroup { - return g.Clone().preBind("HANDLER", "PUT:"+pattern, object, params...) + return g.Clone().preBindToLocalArray("HANDLER", "PUT:"+pattern, object, params...) } // POST registers a http handler to given route pattern and http method: POST. func (g *RouterGroup) POST(pattern string, object interface{}, params ...interface{}) *RouterGroup { - return g.Clone().preBind("HANDLER", "POST:"+pattern, object, params...) + return g.Clone().preBindToLocalArray("HANDLER", "POST:"+pattern, object, params...) } // DELETE registers a http handler to given route pattern and http method: DELETE. func (g *RouterGroup) DELETE(pattern string, object interface{}, params ...interface{}) *RouterGroup { - return g.Clone().preBind("HANDLER", "DELETE:"+pattern, object, params...) + return g.Clone().preBindToLocalArray("HANDLER", "DELETE:"+pattern, object, params...) } // PATCH registers a http handler to given route pattern and http method: PATCH. func (g *RouterGroup) PATCH(pattern string, object interface{}, params ...interface{}) *RouterGroup { - return g.Clone().preBind("HANDLER", "PATCH:"+pattern, object, params...) + return g.Clone().preBindToLocalArray("HANDLER", "PATCH:"+pattern, object, params...) } // HEAD registers a http handler to given route pattern and http method: HEAD. func (g *RouterGroup) HEAD(pattern string, object interface{}, params ...interface{}) *RouterGroup { - return g.Clone().preBind("HANDLER", "HEAD:"+pattern, object, params...) + return g.Clone().preBindToLocalArray("HANDLER", "HEAD:"+pattern, object, params...) } // CONNECT registers a http handler to given route pattern and http method: CONNECT. func (g *RouterGroup) CONNECT(pattern string, object interface{}, params ...interface{}) *RouterGroup { - return g.Clone().preBind("HANDLER", "CONNECT:"+pattern, object, params...) + return g.Clone().preBindToLocalArray("HANDLER", "CONNECT:"+pattern, object, params...) } // OPTIONS registers a http handler to given route pattern and http method: OPTIONS. func (g *RouterGroup) OPTIONS(pattern string, object interface{}, params ...interface{}) *RouterGroup { - return g.Clone().preBind("HANDLER", "OPTIONS:"+pattern, object, params...) + return g.Clone().preBindToLocalArray("HANDLER", "OPTIONS:"+pattern, object, params...) } // TRACE registers a http handler to given route pattern and http method: TRACE. func (g *RouterGroup) TRACE(pattern string, object interface{}, params ...interface{}) *RouterGroup { - return g.Clone().preBind("HANDLER", "TRACE:"+pattern, object, params...) + return g.Clone().preBindToLocalArray("HANDLER", "TRACE:"+pattern, object, params...) } // REST registers a http handler to given route pattern according to REST rule. func (g *RouterGroup) REST(pattern string, object interface{}) *RouterGroup { - return g.Clone().preBind("REST", pattern, object) + return g.Clone().preBindToLocalArray("REST", pattern, object) } // Hook registers a hook to given route pattern. func (g *RouterGroup) Hook(pattern string, hook string, handler HandlerFunc) *RouterGroup { - return g.Clone().preBind("HANDLER", pattern, handler, hook) + return g.Clone().preBindToLocalArray("HANDLER", pattern, handler, hook) } // Middleware binds one or more middleware to the router group. @@ -232,14 +235,16 @@ func (g *RouterGroup) Middleware(handlers ...HandlerFunc) *RouterGroup { return g } -// preBind adds the route registering parameters to internal variable array for lazily registering feature. -func (g *RouterGroup) preBind(bindType string, pattern string, object interface{}, params ...interface{}) *RouterGroup { +// preBindToLocalArray adds the route registering parameters to internal variable array for lazily registering feature. +func (g *RouterGroup) preBindToLocalArray(bindType string, pattern string, object interface{}, params ...interface{}) *RouterGroup { + _, file, line := gdebug.CallerWithFilter(gFILTER_KEY) preBindItems = append(preBindItems, preBindItem{ group: g, bindType: bindType, pattern: pattern, object: object, params: params, + source: fmt.Sprintf(`%s:%d`, file, line), }) return g } @@ -255,8 +260,15 @@ func (g *RouterGroup) getPrefix() string { return prefix } -// doBind does really registering for the group. -func (g *RouterGroup) doBind(bindType string, pattern string, object interface{}, params ...interface{}) *RouterGroup { +// doBindRoutersToServer does really registering for the group. +func (g *RouterGroup) doBindRoutersToServer(item preBindItem) *RouterGroup { + var ( + bindType = item.bindType + pattern = item.pattern + object = item.object + params = item.params + source = item.source + ) prefix := g.getPrefix() // Route check. if len(prefix) > 0 { @@ -271,7 +283,9 @@ func (g *RouterGroup) doBind(bindType string, pattern string, object interface{} if bindType == "REST" { pattern = prefix + "/" + strings.TrimLeft(path, "/") } else { - pattern = g.server.serveHandlerKey(method, prefix+"/"+strings.TrimLeft(path, "/"), domain) + pattern = g.server.serveHandlerKey( + method, prefix+"/"+strings.TrimLeft(path, "/"), domain, + ) } } // Filter repeated char '/'. @@ -286,78 +300,102 @@ func (g *RouterGroup) doBind(bindType string, pattern string, object interface{} case "HANDLER": if h, ok := object.(HandlerFunc); ok { if g.server != nil { - g.server.doBindHandler(pattern, h, g.middleware) + g.server.doBindHandler(pattern, h, g.middleware, source) } else { - g.domain.doBindHandler(pattern, h, g.middleware) + g.domain.doBindHandler(pattern, h, g.middleware, source) } } else if g.isController(object) { if len(extras) > 0 { if g.server != nil { if gstr.Contains(extras[0], ",") { - g.server.doBindController(pattern, object.(Controller), extras[0], g.middleware) + g.server.doBindController( + pattern, object.(Controller), extras[0], g.middleware, source, + ) } else { - g.server.doBindControllerMethod(pattern, object.(Controller), extras[0], g.middleware) + g.server.doBindControllerMethod( + pattern, object.(Controller), extras[0], g.middleware, source, + ) } } else { if gstr.Contains(extras[0], ",") { - g.domain.doBindController(pattern, object.(Controller), extras[0], g.middleware) + g.domain.doBindController( + pattern, object.(Controller), extras[0], g.middleware, source, + ) } else { - g.domain.doBindControllerMethod(pattern, object.(Controller), extras[0], g.middleware) + g.domain.doBindControllerMethod( + pattern, object.(Controller), extras[0], g.middleware, source, + ) } } } else { if g.server != nil { - g.server.doBindController(pattern, object.(Controller), "", g.middleware) + g.server.doBindController( + pattern, object.(Controller), "", g.middleware, source, + ) } else { - g.domain.doBindController(pattern, object.(Controller), "", g.middleware) + g.domain.doBindController( + pattern, object.(Controller), "", g.middleware, source, + ) } } } else { if len(extras) > 0 { if g.server != nil { if gstr.Contains(extras[0], ",") { - g.server.doBindObject(pattern, object, extras[0], g.middleware) + g.server.doBindObject( + pattern, object, extras[0], g.middleware, source, + ) } else { - g.server.doBindObjectMethod(pattern, object, extras[0], g.middleware) + g.server.doBindObjectMethod( + pattern, object, extras[0], g.middleware, source, + ) } } else { if gstr.Contains(extras[0], ",") { - g.domain.doBindObject(pattern, object, extras[0], g.middleware) + g.domain.doBindObject( + pattern, object, extras[0], g.middleware, source, + ) } else { - g.domain.doBindObjectMethod(pattern, object, extras[0], g.middleware) + g.domain.doBindObjectMethod( + pattern, object, extras[0], g.middleware, source, + ) } } } else { if g.server != nil { - g.server.doBindObject(pattern, object, "", g.middleware) + g.server.doBindObject(pattern, object, "", g.middleware, source) } else { - g.domain.doBindObject(pattern, object, "", g.middleware) + g.domain.doBindObject(pattern, object, "", g.middleware, source) } } } case "REST": if g.isController(object) { if g.server != nil { - g.server.doBindControllerRest(pattern, object.(Controller), g.middleware) + g.server.doBindControllerRest( + pattern, object.(Controller), g.middleware, source, + ) } else { - g.domain.doBindControllerRest(pattern, object.(Controller), g.middleware) + g.domain.doBindControllerRest( + pattern, object.(Controller), g.middleware, source, + ) } } else { if g.server != nil { - g.server.doBindObjectRest(pattern, object, g.middleware) + g.server.doBindObjectRest(pattern, object, g.middleware, source) } else { - g.domain.doBindObjectRest(pattern, object, g.middleware) + g.domain.doBindObjectRest(pattern, object, g.middleware, source) } } case "HOOK": if h, ok := object.(HandlerFunc); ok { if g.server != nil { - g.server.BindHookHandler(pattern, extras[0], h) + g.server.doBindHookHandler(pattern, extras[0], h, source) } else { - g.domain.BindHookHandler(pattern, extras[0], h) + g.domain.doBindHookHandler(pattern, extras[0], h, source) } } else { - g.server.Logger().Fatalf("invalid hook handler for pattern:%s", pattern) + g.server.Logger().Fatalf("invalid hook handler for pattern: %s", pattern) } } return g diff --git a/net/ghttp/ghttp_server_router_hook.go b/net/ghttp/ghttp_server_router_hook.go index 63d170dba..e918c6e30 100644 --- a/net/ghttp/ghttp_server_router_hook.go +++ b/net/ghttp/ghttp_server_router_hook.go @@ -13,11 +13,16 @@ import ( // 绑定指定的hook回调函数, pattern参数同BindHandler，支持命名路由；hook参数的值由ghttp server设定，参数不区分大小写 func (s *Server) BindHookHandler(pattern string, hook string, handler HandlerFunc) { + s.doBindHookHandler(pattern, hook, handler, "") +} + +func (s *Server) doBindHookHandler(pattern string, hook string, handler HandlerFunc, source string) { s.setHandler(pattern, &handlerItem{ itemType: gHANDLER_TYPE_HOOK, itemName: gdebug.FuncPath(handler), itemFunc: handler, hookName: hook, + source: source, }) } diff --git a/net/ghttp/ghttp_server_service_controller.go b/net/ghttp/ghttp_server_service_controller.go index ef6f4c8c6..e86f14cad 100644 --- a/net/ghttp/ghttp_server_service_controller.go +++ b/net/ghttp/ghttp_server_service_controller.go @@ -24,12 +24,12 @@ func (s *Server) BindController(pattern string, controller Controller, method .. if len(method) > 0 { bindMethod = method[0] } - s.doBindController(pattern, controller, bindMethod, nil) + s.doBindController(pattern, controller, bindMethod, nil, "") } // 绑定路由到指定的方法执行, 第三个参数method仅支持一个方法注册，不支持多个，并且区分大小写。 func (s *Server) BindControllerMethod(pattern string, controller Controller, method string) { - s.doBindControllerMethod(pattern, controller, method, nil) + s.doBindControllerMethod(pattern, controller, method, nil, "") } // 绑定控制器(RESTFul)，控制器需要实现gmvc.Controller接口 @@ -37,10 +37,13 @@ func (s *Server) BindControllerMethod(pattern string, controller Controller, met // 因此只会绑定HTTP Method对应的方法，其他方法不会自动注册绑定 // 这种方式绑定的控制器每一次请求都会初始化一个新的控制器对象进行处理，对应不同的请求会话 func (s *Server) BindControllerRest(pattern string, controller Controller) { - s.doBindControllerRest(pattern, controller, nil) + s.doBindControllerRest(pattern, controller, nil, "") } -func (s *Server) doBindController(pattern string, controller Controller, method string, middleware []HandlerFunc) { +func (s *Server) doBindController( + pattern string, controller Controller, method string, + middleware []HandlerFunc, source string, +) { // Convert input method to map for convenience and high performance searching. var methodMap map[string]bool if len(method) > 0 { @@ -98,6 +101,7 @@ func (s *Server) doBindController(pattern string, controller Controller, method reflect: v.Elem().Type(), }, middleware: middleware, + source: source, } // 如果方法中带有Index方法，那么额外自动增加一个路由规则匹配主URI， // 例如: pattern为/user, 那么会同时注册/user及/user/index， @@ -117,13 +121,20 @@ func (s *Server) doBindController(pattern string, controller Controller, method reflect: v.Elem().Type(), }, middleware: middleware, + source: source, } } } s.bindHandlerByMap(m) } -func (s *Server) doBindControllerMethod(pattern string, controller Controller, method string, middleware []HandlerFunc) { +func (s *Server) doBindControllerMethod( + pattern string, + controller Controller, + method string, + middleware []HandlerFunc, + source string, +) { m := make(map[string]*handlerItem) v := reflect.ValueOf(controller) t := v.Type() @@ -141,8 +152,10 @@ func (s *Server) doBindControllerMethod(pattern string, controller Controller, m ctlName = fmt.Sprintf(`(%s)`, ctlName) } if _, ok := methodValue.Interface().(func()); !ok { - s.Logger().Errorf(`invalid route method: %s.%s.%s defined as "%s", but "func()" is required for controller registry`, - pkgPath, ctlName, methodName, methodValue.Type().String()) + s.Logger().Errorf( + `invalid route method: %s.%s.%s defined as "%s", but "func()" is required for controller registry`, + pkgPath, ctlName, methodName, methodValue.Type().String(), + ) return } key := s.mergeBuildInNameToPattern(pattern, structName, methodName, false) @@ -154,11 +167,15 @@ func (s *Server) doBindControllerMethod(pattern string, controller Controller, m reflect: v.Elem().Type(), }, middleware: middleware, + source: source, } s.bindHandlerByMap(m) } -func (s *Server) doBindControllerRest(pattern string, controller Controller, middleware []HandlerFunc) { +func (s *Server) doBindControllerRest( + pattern string, controller Controller, + middleware []HandlerFunc, source string, +) { // 遍历控制器，获取方法列表，并构造成uri m := make(map[string]*handlerItem) v := reflect.ValueOf(controller) @@ -177,8 +194,10 @@ func (s *Server) doBindControllerRest(pattern string, controller Controller, mid ctlName = fmt.Sprintf(`(%s)`, ctlName) } if _, ok := v.Method(i).Interface().(func()); !ok { - s.Logger().Errorf(`invalid route method: %s.%s.%s defined as "%s", but "func()" is required for controller registry`, - pkgPath, ctlName, methodName, v.Method(i).Type().String()) + s.Logger().Errorf( + `invalid route method: %s.%s.%s defined as "%s", but "func()" is required for controller registry`, + pkgPath, ctlName, methodName, v.Method(i).Type().String(), + ) return } key := s.mergeBuildInNameToPattern(methodName+":"+pattern, structName, methodName, false) @@ -190,6 +209,7 @@ func (s *Server) doBindControllerRest(pattern string, controller Controller, mid reflect: v.Elem().Type(), }, middleware: middleware, + source: source, } } s.bindHandlerByMap(m) diff --git a/net/ghttp/ghttp_server_service_handler.go b/net/ghttp/ghttp_server_service_handler.go index a53807bce..7a1c8750d 100644 --- a/net/ghttp/ghttp_server_service_handler.go +++ b/net/ghttp/ghttp_server_service_handler.go @@ -16,18 +16,22 @@ import ( // 注意该方法是直接绑定函数的内存地址，执行的时候直接执行该方法，不会存在初始化新的控制器逻辑 func (s *Server) BindHandler(pattern string, handler HandlerFunc) { - s.doBindHandler(pattern, handler, nil) + s.doBindHandler(pattern, handler, nil, "") } // 绑定URI到操作函数/方法 // pattern的格式形如：/user/list, put:/user, delete:/user, post:/user@johng.cn // 支持RESTful的请求格式，具体业务逻辑由绑定的处理方法来执行 -func (s *Server) doBindHandler(pattern string, handler HandlerFunc, middleware []HandlerFunc) { +func (s *Server) doBindHandler( + pattern string, handler HandlerFunc, + middleware []HandlerFunc, source string, +) { s.setHandler(pattern, &handlerItem{ itemName: gdebug.FuncPath(handler), itemType: gHANDLER_TYPE_HANDLER, itemFunc: handler, middleware: middleware, + source: source, }) } diff --git a/net/ghttp/ghttp_server_service_object.go b/net/ghttp/ghttp_server_service_object.go index 4fd135822..d6a167cdb 100644 --- a/net/ghttp/ghttp_server_service_object.go +++ b/net/ghttp/ghttp_server_service_object.go @@ -23,22 +23,25 @@ func (s *Server) BindObject(pattern string, object interface{}, method ...string if len(method) > 0 { bindMethod = method[0] } - s.doBindObject(pattern, object, bindMethod, nil) + s.doBindObject(pattern, object, bindMethod, nil, "") } // 绑定对象到URI请求处理中，会自动识别方法名称，并附加到对应的URI地址后面， // 第三个参数method仅支持一个方法注册，不支持多个，并且区分大小写。 func (s *Server) BindObjectMethod(pattern string, object interface{}, method string) { - s.doBindObjectMethod(pattern, object, method, nil) + s.doBindObjectMethod(pattern, object, method, nil, "") } // 绑定对象到URI请求处理中，会自动识别方法名称，并附加到对应的URI地址后面, // 需要注意对象方法的定义必须按照 ghttp.HandlerFunc 来定义 func (s *Server) BindObjectRest(pattern string, object interface{}) { - s.doBindObjectRest(pattern, object, nil) + s.doBindObjectRest(pattern, object, nil, "") } -func (s *Server) doBindObject(pattern string, object interface{}, method string, middleware []HandlerFunc) { +func (s *Server) doBindObject( + pattern string, object interface{}, method string, + middleware []HandlerFunc, source string, +) { // Convert input method to map for convenience and high performance searching purpose. var methodMap map[string]bool if len(method) > 0 { @@ -107,6 +110,7 @@ func (s *Server) doBindObject(pattern string, object interface{}, method string, initFunc: initFunc, shutFunc: shutFunc, middleware: middleware, + source: source, } // 如果方法中带有Index方法，那么额外自动增加一个路由规则匹配主URI。 // 注意，当pattern带有内置变量时，不会自动加该路由。 @@ -123,6 +127,7 @@ func (s *Server) doBindObject(pattern string, object interface{}, method string, initFunc: initFunc, shutFunc: shutFunc, middleware: middleware, + source: source, } } } @@ -131,7 +136,10 @@ func (s *Server) doBindObject(pattern string, object interface{}, method string, // 绑定对象到URI请求处理中，会自动识别方法名称，并附加到对应的URI地址后面， // 第三个参数method仅支持一个方法注册，不支持多个，并且区分大小写。 -func (s *Server) doBindObjectMethod(pattern string, object interface{}, method string, middleware []HandlerFunc) { +func (s *Server) doBindObjectMethod( + pattern string, object interface{}, method string, + middleware []HandlerFunc, source string, +) { m := make(map[string]*handlerItem) v := reflect.ValueOf(object) t := v.Type() @@ -170,12 +178,16 @@ func (s *Server) doBindObjectMethod(pattern string, object interface{}, method s initFunc: initFunc, shutFunc: shutFunc, middleware: middleware, + source: source, } s.bindHandlerByMap(m) } -func (s *Server) doBindObjectRest(pattern string, object interface{}, middleware []HandlerFunc) { +func (s *Server) doBindObjectRest( + pattern string, object interface{}, + middleware []HandlerFunc, source string, +) { m := make(map[string]*handlerItem) v := reflect.ValueOf(object) t := v.Type() @@ -213,6 +225,7 @@ func (s *Server) doBindObjectRest(pattern string, object interface{}, middleware initFunc: initFunc, shutFunc: shutFunc, middleware: middleware, + source: source, } } s.bindHandlerByMap(m) diff --git a/net/ghttp/ghttp_unit_middleware_cors_test.go b/net/ghttp/ghttp_unit_middleware_cors_test.go index c99e16c97..4044d59e6 100644 --- a/net/ghttp/ghttp_unit_middleware_cors_test.go +++ b/net/ghttp/ghttp_unit_middleware_cors_test.go @@ -40,6 +40,7 @@ func Test_Middleware_CORS(t *testing.T) { resp, err := client.Get("/api.v2/user/list") gtest.Assert(err, nil) gtest.Assert(len(resp.Header["Access-Control-Allow-Headers"]), 0) + gtest.Assert(resp.StatusCode, 404) resp.Close() // POST request matches the route and CORS middleware. @@ -61,6 +62,7 @@ func Test_Middleware_CORS(t *testing.T) { gtest.Assert(err, nil) gtest.Assert(len(resp.Header["Access-Control-Allow-Headers"]), 0) gtest.Assert(resp.ReadAllString(), "Not Found") + gtest.Assert(resp.StatusCode, 404) resp.Close() }) // OPTIONS POST @@ -71,6 +73,7 @@ func Test_Middleware_CORS(t *testing.T) { resp, err := client.Options("/api.v2/user/list") gtest.Assert(err, nil) gtest.Assert(len(resp.Header["Access-Control-Allow-Headers"]), 1) + gtest.Assert(resp.StatusCode, 200) resp.Close() }) } From 33ae93e050cc5535bdbd2fafb03ddb6885975348 Mon Sep 17 00:00:00 2001 From: John Date: Tue, 17 Mar 2020 17:46:43 +0800 Subject: [PATCH 31/32] improve CORS feature for ghttp.Server --- net/ghttp/ghttp_response_cors.go | 23 ++----- net/ghttp/ghttp_unit_middleware_cors_test.go | 68 +++++++++++++++++++- 2 files changed, 71 insertions(+), 20 deletions(-) diff --git a/net/ghttp/ghttp_response_cors.go b/net/ghttp/ghttp_response_cors.go index 824ed5500..df2bc5f6f 100644 --- a/net/ghttp/ghttp_response_cors.go +++ b/net/ghttp/ghttp_response_cors.go @@ -94,28 +94,13 @@ func (r *Response) CORS(options CORSOptions) { r.Header().Set("Access-Control-Allow-Headers", options.AllowHeaders) } // No continue service handling if it's OPTIONS request. + // Note that there's special checks in previous router searching, + // so if it goes to here it means there's already serving handler exist. if gstr.Equal(r.Request.Method, "OPTIONS") { - // Request method handler searching. - // It here simply uses Server.routesMap attribute enhancing the searching performance. - if method := r.Request.Header.Get("Access-Control-Request-Method"); method != "" { - routerKey := "" - for _, domain := range []string{gDEFAULT_DOMAIN, r.Request.GetHost()} { - for _, v := range []string{gDEFAULT_METHOD, method} { - routerKey = r.Server.routerMapKey("", v, r.Request.URL.Path, domain) - if r.Server.routesMap[routerKey] != nil { - if r.Status == 0 { - r.Status = http.StatusOK - } - // No continue serving. - r.Request.ExitAll() - } - } - } - } - // Cannot find the request serving handler, it then responses 404. if r.Status == 0 { - r.Status = http.StatusNotFound + r.Status = http.StatusOK } + // No continue serving. r.Request.ExitAll() } } diff --git a/net/ghttp/ghttp_unit_middleware_cors_test.go b/net/ghttp/ghttp_unit_middleware_cors_test.go index 4044d59e6..c5d24529e 100644 --- a/net/ghttp/ghttp_unit_middleware_cors_test.go +++ b/net/ghttp/ghttp_unit_middleware_cors_test.go @@ -15,7 +15,7 @@ import ( "time" ) -func Test_Middleware_CORS(t *testing.T) { +func Test_Middleware_CORS1(t *testing.T) { p := ports.PopRand() s := g.Server(p) s.Group("/api.v2", func(group *ghttp.RouterGroup) { @@ -77,3 +77,69 @@ func Test_Middleware_CORS(t *testing.T) { resp.Close() }) } + +func Test_Middleware_CORS2(t *testing.T) { + p := ports.PopRand() + s := g.Server(p) + s.Group("/api.v2", func(group *ghttp.RouterGroup) { + group.Middleware(MiddlewareCORS) + group.GET("/user/list/{type}", func(r *ghttp.Request) { + r.Response.Write(r.Get("type")) + }) + }) + s.SetPort(p) + s.SetDumpRouterMap(false) + s.Start() + defer s.Shutdown() + time.Sleep(100 * time.Millisecond) + gtest.Case(t, func() { + client := ghttp.NewClient() + client.SetPrefix(fmt.Sprintf("http://127.0.0.1:%d", p)) + // Common Checks. + gtest.Assert(client.GetContent("/"), "Not Found") + gtest.Assert(client.GetContent("/api.v2"), "Not Found") + // Get request. + resp, err := client.Get("/api.v2/user/list/1") + gtest.Assert(err, nil) + gtest.Assert(len(resp.Header["Access-Control-Allow-Headers"]), 1) + gtest.Assert(resp.Header["Access-Control-Allow-Headers"][0], "Origin,Content-Type,Accept,User-Agent,Cookie,Authorization,X-Auth-Token,X-Requested-With") + gtest.Assert(resp.Header["Access-Control-Allow-Methods"][0], "GET,PUT,POST,DELETE,PATCH,HEAD,CONNECT,OPTIONS,TRACE") + gtest.Assert(resp.Header["Access-Control-Allow-Origin"][0], "*") + gtest.Assert(resp.Header["Access-Control-Max-Age"][0], "3628800") + gtest.Assert(resp.ReadAllString(), "1") + resp.Close() + }) + // OPTIONS GET None. + gtest.Case(t, func() { + client := ghttp.NewClient() + client.SetPrefix(fmt.Sprintf("http://127.0.0.1:%d", p)) + client.SetHeader("Access-Control-Request-Method", "GET") + resp, err := client.Options("/api.v2/user") + gtest.Assert(err, nil) + gtest.Assert(len(resp.Header["Access-Control-Allow-Headers"]), 0) + gtest.Assert(resp.StatusCode, 404) + resp.Close() + }) + // OPTIONS GET + gtest.Case(t, func() { + client := ghttp.NewClient() + client.SetPrefix(fmt.Sprintf("http://127.0.0.1:%d", p)) + client.SetHeader("Access-Control-Request-Method", "GET") + resp, err := client.Options("/api.v2/user/list/1") + gtest.Assert(err, nil) + gtest.Assert(len(resp.Header["Access-Control-Allow-Headers"]), 1) + gtest.Assert(resp.StatusCode, 200) + resp.Close() + }) + // OPTIONS POST + gtest.Case(t, func() { + client := ghttp.NewClient() + client.SetPrefix(fmt.Sprintf("http://127.0.0.1:%d", p)) + client.SetHeader("Access-Control-Request-Method", "POST") + resp, err := client.Options("/api.v2/user/list/1") + gtest.Assert(err, nil) + gtest.Assert(len(resp.Header["Access-Control-Allow-Headers"]), 0) + gtest.Assert(resp.StatusCode, 404) + resp.Close() + }) +} From 5a646179adac075612543c55daf5da193265f44c Mon Sep 17 00:00:00 2001 From: John Date: Tue, 17 Mar 2020 21:38:49 +0800 Subject: [PATCH 32/32] fix issue in limit...offset statement of postgresql for packagegdb --- database/gdb/gdb_driver_pgsql.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/gdb/gdb_driver_pgsql.go b/database/gdb/gdb_driver_pgsql.go index 76db87476..1e0bc5a88 100644 --- a/database/gdb/gdb_driver_pgsql.go +++ b/database/gdb/gdb_driver_pgsql.go @@ -66,7 +66,7 @@ func (d *DriverPgsql) HandleSqlBeforeCommit(link Link, sql string, args []interf index++ return fmt.Sprintf("$%d", index) }) - sql, _ = gregex.ReplaceString(` LIMIT (\d+),\s*(\d+)`, ` LIMIT $1 OFFSET $2`, sql) + sql, _ = gregex.ReplaceString(` LIMIT (\d+),\s*(\d+)`, ` LIMIT $2 OFFSET $1`, sql) return sql, args }