From 69684f50239daa75a0186fb455b6eaf8f782d59a Mon Sep 17 00:00:00 2001 From: John Date: Tue, 10 Sep 2019 20:38:23 +0800 Subject: [PATCH] improve gredis instance initialization for gins --- .../database/gredis/migration/migration.go | 60 +++++++ .example/other/test.go | 37 ++++- database/gredis/gredis.go | 10 ++ database/gredis/gredis_config.go | 77 ++++++++- frame/g/g_object.go | 6 - frame/gins/gins.go | 148 ++++++------------ frame/gins/gins_kvdb_test.go | 44 ------ 7 files changed, 221 insertions(+), 161 deletions(-) create mode 100644 .example/database/gredis/migration/migration.go delete mode 100644 frame/gins/gins_kvdb_test.go diff --git a/.example/database/gredis/migration/migration.go b/.example/database/gredis/migration/migration.go new file mode 100644 index 000000000..ac9c0ab17 --- /dev/null +++ b/.example/database/gredis/migration/migration.go @@ -0,0 +1,60 @@ +package main + +import ( + "fmt" + + "github.com/gogf/gf/util/gconv" + + "github.com/gogf/gf/os/gcmd" + + "github.com/gogf/gf/container/garray" + "github.com/gogf/gf/database/gredis" + "github.com/gogf/gf/frame/g" +) + +func main() { + batchNumber := 1000 + redis1Config, err := gredis.ConfigFromStr("im-redis-slave:6379,9") + if err != nil { + panic(err) + } + redis2Config, err := gredis.ConfigFromStr("r-bp1f0a5d4efd8744.redis.rds.aliyuncs.com:6379,9") + if err != nil { + panic(err) + } + gredis.SetConfig(redis1Config) + + v, err := g.Redis().DoVar("keys", "*") + if err != nil { + panic(err) + } + array := garray.NewStrArrayFrom(v.Strings()) + for { + slice := array.PopLefts(batchNumber) + if len(slice) > 0 { + // `migrate %s %d "" 0 2000 copy replace auth %s keys %s`, + params := g.Slice{ + redis2Config.Host, + redis2Config.Port, + "", + redis2Config.Db, + 2000, + "copy", + "replace", + "keys", + } + params = append(params, gconv.Interfaces(slice)...) + fmt.Println(params) + if gcmd.GetOpt("dryrun") == "0" { + if v, err := g.Redis().DoVar("migrate", params...); err != nil { + panic(err) + } else { + fmt.Println(v.String()) + } + } + } else { + break + } + } + fmt.Println("done") +} diff --git a/.example/other/test.go b/.example/other/test.go index 463496435..26624be03 100644 --- a/.example/other/test.go +++ b/.example/other/test.go @@ -1,13 +1,38 @@ package main import ( - "fmt" - "net/url" + "encoding/json" + "strings" + + "github.com/gogf/gf/util/gconv" + + "github.com/gogf/gf/frame/g" + + "github.com/gogf/gf/text/gregex" + + "github.com/gogf/gf/container/gset" + + "github.com/gogf/gf/os/gfile" ) func main() { - parse1, _ := url.Parse("https://gf.cdn.johng.cn") - parse2, _ := url.Parse("https://gf.cdn.johng.cn/cli/") - fmt.Println(parse1.Host) - fmt.Println(parse2.Host) + path1 := "/Users/john/Temp/downloaded_data_parsed.txt" + path2 := "/Users/john/Temp/downloaded_data_parsed_mapping.txt" + array := strings.Split(gfile.GetContents(path1), "\n") + mapping := make(map[string]*gset.Set) + for _, line := range array { + array, _ := gregex.MatchString(`add group success \[\[(\d+),\[*(\d+)\]*`, line) + if len(array) != 3 { + g.Dump(line) + g.Dump(array) + continue + } + if _, ok := mapping[array[1]]; !ok { + mapping[array[1]] = gset.New() + } + mapping[array[1]].Add(gconv.Interfaces(strings.Split(array[2], ","))...) + } + g.Dump(mapping) + b, _ := json.Marshal(mapping) + gfile.PutBytes(path2, b) } diff --git a/database/gredis/gredis.go b/database/gredis/gredis.go index eb679fb42..fb4ac3803 100644 --- a/database/gredis/gredis.go +++ b/database/gredis/gredis.go @@ -104,6 +104,16 @@ func New(config Config) *Redis { } } +// NewFromStr creates a redis client object with given configuration string. +// Redis client maintains a connection pool automatically. +func NewFromStr(str string) (*Redis, error) { + config, err := ConfigFromStr(str) + if err != nil { + return nil, err + } + return New(config), nil +} + // Close closes the redis connection pool, // it will release all connections reserved by this pool. // It is not necessary to call Close manually. diff --git a/database/gredis/gredis_config.go b/database/gredis/gredis_config.go index 79a56706b..65a799ee5 100644 --- a/database/gredis/gredis_config.go +++ b/database/gredis/gredis_config.go @@ -6,11 +6,20 @@ package gredis -import "github.com/gogf/gf/container/gmap" +import ( + "time" + + "github.com/gogf/gf/errors/gerror" + + "github.com/gogf/gf/container/gmap" + "github.com/gogf/gf/text/gregex" + "github.com/gogf/gf/text/gstr" + "github.com/gogf/gf/util/gconv" +) const ( - // Default configuration group name. - DEFAULT_GROUP_NAME = "default" + DEFAULT_GROUP_NAME = "default" // Default configuration group name. + DEFAULT_REDIS_PORT = 6379 // Default redis port configuration if not passed. ) var ( @@ -29,6 +38,22 @@ func SetConfig(config Config, name ...string) { instances.Remove(group) } +// SetConfigByStr sets the global configuration for specified group with string. +// If is not passed, it sets configuration for the default group name. +func SetConfigByStr(str string, name ...string) error { + group := DEFAULT_GROUP_NAME + if len(name) > 0 { + group = name[0] + } + config, err := ConfigFromStr(str) + if err != nil { + return err + } + configs.Set(group, config) + instances.Remove(group) + return nil +} + // GetConfig returns the global configuration with specified group name. // If is not passed, it returns configuration of the default group name. func GetConfig(name ...string) (config Config, ok bool) { @@ -53,6 +78,52 @@ func RemoveConfig(name ...string) { instances.Remove(group) } +// ConfigFromStr parses and returns config from given str. +// Eg: host:port[,db,pass?maxIdle=x&maxActive=x&idleTimeout=x&maxConnLifetime=x] +func ConfigFromStr(str string) (config Config, err error) { + array, _ := gregex.MatchString(`([^:]+):*(\d*),{0,1}(\d*),{0,1}(.*)\?(.+)`, str) + if len(array) == 6 { + parse, _ := gstr.Parse(array[5]) + config = Config{ + Host: array[1], + Port: gconv.Int(array[2]), + Db: gconv.Int(array[3]), + Pass: array[4], + } + if config.Port == 0 { + config.Port = DEFAULT_REDIS_PORT + } + if v, ok := parse["maxIdle"]; ok { + config.MaxIdle = gconv.Int(v) + } + if v, ok := parse["maxActive"]; ok { + config.MaxActive = gconv.Int(v) + } + if v, ok := parse["idleTimeout"]; ok { + config.IdleTimeout = gconv.Duration(v) * time.Second + } + if v, ok := parse["maxConnLifetime"]; ok { + config.MaxConnLifetime = gconv.Duration(v) * time.Second + } + return + } + array, _ = gregex.MatchString(`([^:]+):*(\d*),{0,1}(\d*),{0,1}(.*)`, str) + if len(array) == 5 { + config = Config{ + Host: array[1], + Port: gconv.Int(array[2]), + Db: gconv.Int(array[3]), + Pass: array[4], + } + if config.Port == 0 { + config.Port = DEFAULT_REDIS_PORT + } + } else { + err = gerror.Newf(`invalid redis configuration: "%s"`, str) + } + return +} + // ClearConfig removes all configurations and instances of redis. func ClearConfig() { configs.Clear() diff --git a/frame/g/g_object.go b/frame/g/g_object.go index 4a3d00035..74e9417de 100644 --- a/frame/g/g_object.go +++ b/frame/g/g_object.go @@ -8,7 +8,6 @@ package g import ( "github.com/gogf/gf/database/gdb" - "github.com/gogf/gf/database/gkvdb" "github.com/gogf/gf/database/gredis" "github.com/gogf/gf/frame/gins" "github.com/gogf/gf/i18n/gi18n" @@ -80,11 +79,6 @@ func DB(name ...string) gdb.DB { return gins.Database(name...) } -// KV returns an instance of gkvdb with specified configuration group name. -func KV(name ...string) *gkvdb.DB { - return gins.KV(name...) -} - // Redis returns an instance of redis client with specified configuration group name. func Redis(name ...string) *gredis.Redis { return gins.Redis(name...) diff --git a/frame/gins/gins.go b/frame/gins/gins.go index d043f0c49..46730411b 100644 --- a/frame/gins/gins.go +++ b/frame/gins/gins.go @@ -9,12 +9,9 @@ package gins import ( "fmt" - "time" "github.com/gogf/gf/os/gfile" - "github.com/gogf/gf/database/gkvdb" - "github.com/gogf/gf/container/gmap" "github.com/gogf/gf/database/gdb" "github.com/gogf/gf/database/gredis" @@ -25,7 +22,6 @@ import ( "github.com/gogf/gf/os/gres" "github.com/gogf/gf/os/gview" "github.com/gogf/gf/text/gregex" - "github.com/gogf/gf/text/gstr" "github.com/gogf/gf/util/gconv" ) @@ -102,43 +98,49 @@ func Database(name ...string) gdb.DB { } instanceKey := fmt.Sprintf("%s.%s", gFRAME_CORE_COMPONENT_NAME_DATABASE, group) db := instances.GetOrSetFuncLock(instanceKey, func() interface{} { - if gdb.GetConfig(group) == nil { - m := config.GetMap("database") - if m == nil { - glog.Error(`database init failed: "database" node not found, is config file or configuration missing?`) - return nil + if gdb.GetConfig(group) != nil { + db, err := gdb.Instance(group) + if err != nil { + glog.Error(err) } - // Parse as map-slice. - for group, groupConfig := range m { - cg := gdb.ConfigGroup{} - switch value := groupConfig.(type) { - case []interface{}: - for _, v := range value { - if node := parseDBConfigNode(v); node != nil { - cg = append(cg, *node) - } - } - case map[string]interface{}: - if node := parseDBConfigNode(value); node != nil { + return db + } + m := config.GetMap("database") + if m == nil { + glog.Error(`database init failed: "database" node not found, is config file or configuration missing?`) + return nil + } + // Parse as map-slice. + for group, groupConfig := range m { + cg := gdb.ConfigGroup{} + switch value := groupConfig.(type) { + case []interface{}: + for _, v := range value { + if node := parseDBConfigNode(v); node != nil { cg = append(cg, *node) } } - if len(cg) > 0 { - gdb.SetConfigGroup(group, cg) - } - } - // Parse as a single node configuration. - if node := parseDBConfigNode(m); node != nil { - cg := gdb.ConfigGroup{} - if node.LinkInfo != "" || node.Host != "" { + case map[string]interface{}: + if node := parseDBConfigNode(value); node != nil { cg = append(cg, *node) } - if len(cg) > 0 { - gdb.SetConfigGroup(group, cg) - } } - addConfigMonitor(instanceKey, config) + if len(cg) > 0 { + gdb.SetConfigGroup(group, cg) + } } + // Parse as a single node configuration. + if node := parseDBConfigNode(m); node != nil { + cg := gdb.ConfigGroup{} + if node.LinkInfo != "" || node.Host != "" { + cg = append(cg, *node) + } + if len(cg) > 0 { + gdb.SetConfigGroup(group, cg) + } + } + addConfigMonitor(instanceKey, config) + if db, err := gdb.New(name...); err == nil { return db } else { @@ -238,46 +240,20 @@ func Redis(name ...string) *gredis.Redis { } instanceKey := fmt.Sprintf("%s.%s", gFRAME_CORE_COMPONENT_NAME_REDIS, group) result := instances.GetOrSetFuncLock(instanceKey, func() interface{} { + // If already configured, it returns the redis instance. + if _, ok := gredis.GetConfig(group); ok { + return gredis.Instance(group) + } + // Or else, it parses the default configuration file and returns a new redis instance. if m := config.GetMap("redis"); m != nil { - // host:port[,db,pass?maxIdle=x&maxActive=x&idleTimeout=x&maxConnLifetime=x] if v, ok := m[group]; ok { - line := gconv.String(v) - array, _ := gregex.MatchString(`(.+):(\d+),{0,1}(\d*),{0,1}(.*)\?(.+)`, line) - if len(array) == 6 { - parse, _ := gstr.Parse(array[5]) - redisConfig := gredis.Config{ - Host: array[1], - Port: gconv.Int(array[2]), - Db: gconv.Int(array[3]), - Pass: array[4], - } - if v, ok := parse["maxIdle"]; ok { - redisConfig.MaxIdle = gconv.Int(v) - } - if v, ok := parse["maxActive"]; ok { - redisConfig.MaxActive = gconv.Int(v) - } - if v, ok := parse["idleTimeout"]; ok { - redisConfig.IdleTimeout = gconv.Duration(v) * time.Second - } - if v, ok := parse["maxConnLifetime"]; ok { - redisConfig.MaxConnLifetime = gconv.Duration(v) * time.Second - } - addConfigMonitor(instanceKey, config) - return gredis.New(redisConfig) - } - array, _ = gregex.MatchString(`(.+):(\d+),{0,1}(\d*),{0,1}(.*)`, line) - if len(array) == 5 { - addConfigMonitor(instanceKey, config) - return gredis.New(gredis.Config{ - Host: array[1], - Port: gconv.Int(array[2]), - Db: gconv.Int(array[3]), - Pass: array[4], - }) - } else { - glog.Errorf(`invalid redis node configuration: "%s"`, line) + redisConfig, err := gredis.ConfigFromStr(gconv.String(v)) + if err != nil { + glog.Error(err) + return nil } + addConfigMonitor(instanceKey, config) + return gredis.New(redisConfig) } else { glog.Errorf(`configuration for redis not found for group "%s"`, group) } @@ -292,38 +268,6 @@ func Redis(name ...string) *gredis.Redis { return nil } -// KV returns an instance of gkvdb with specified configuration group name. -func KV(name ...string) *gkvdb.DB { - config := Config() - group := "default" - if len(name) > 0 && name[0] != "" { - group = name[0] - } - instanceKey := fmt.Sprintf("%s.%s", gFRAME_CORE_COMPONENT_NAME_GKVDB, group) - result := instances.GetOrSetFuncLock(instanceKey, func() interface{} { - key := fmt.Sprintf("kvdb.%s", group) - if s := config.GetString(key); s != "" { - db := gkvdb.Instance(group) - parse, _ := gstr.Parse(s) - if value, ok := parse["path"]; ok { - db.SetPath(gconv.String(value)) - } - if value, ok := parse["sync"]; ok { - db.Options().SyncWrites = gconv.Bool(value) - } - addConfigMonitor(instanceKey, config) - return db - } else { - glog.Errorf(`incomplete configuration for gkvdb: "%s" node not found in config file "%s"`, key, config.FilePath()) - } - return nil - }) - if result != nil { - return result.(*gkvdb.DB) - } - return nil -} - func addConfigMonitor(key string, config *gcfg.Config) { if path := config.FilePath(); path != "" && gfile.Exists(path) { gfsnotify.Add(path, func(event *gfsnotify.Event) { diff --git a/frame/gins/gins_kvdb_test.go b/frame/gins/gins_kvdb_test.go deleted file mode 100644 index 3a562ad47..000000000 --- a/frame/gins/gins_kvdb_test.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2017 gf Author(https://github.com/gogf/gf). All Rights Reserved. -// -// This Source Code Form is subject to the terms of the MIT License. -// If a copy of the MIT was not distributed with this file, -// You can obtain one at https://github.com/gogf/gf. - -package gins_test - -import ( - "testing" - "time" - - "github.com/gogf/gf/frame/gins" - "github.com/gogf/gf/os/gfile" - "github.com/gogf/gf/test/gtest" -) - -func Test_KV(t *testing.T) { - config := ` -[kvdb] - default = "path=/tmp/gkvdb&sync=false" - cache = "path=/tmp/gkvdb-cache&sync=true" -` - path := "config.toml" - err := gfile.PutContents(path, config) - gtest.Assert(err, nil) - defer gfile.Remove(path) - defer gins.Config().Clear() - - // for gfsnotify callbacks to refresh cache of config file - time.Sleep(500 * time.Millisecond) - - gtest.Case(t, func() { - kvDefault := gins.KV() - kvCache := gins.KV("cache") - key := []byte("key") - value := []byte("value") - err := kvDefault.Set(key, value) - gtest.Assert(err, nil) - - gtest.Assert(kvDefault.Get(key), value) - gtest.Assert(kvCache.Get(key), nil) - }) -}