bingyi/gf
1
0
Fork 0
mirror of https://gitee.com/johng/gf synced 2026-06-06 16:21:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

improve gredis instance initialization for gins

Browse Source
This commit is contained in:
John
2019-09-10 20:38:23 +08:00
parent 2ed1854f0c
commit 69684f5023
7 changed files with 221 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
.example/database/gredis/migration/migration.go Normal file
View File

@ -0,0 +1,60 @@
package main
import (
"fmt"
"github.com/gogf/gf/util/gconv"
"github.com/gogf/gf/os/gcmd"
"github.com/gogf/gf/container/garray"
"github.com/gogf/gf/database/gredis"
"github.com/gogf/gf/frame/g"
)
func main() {
batchNumber := 1000
redis1Config, err := gredis.ConfigFromStr("im-redis-slave:6379,9")
if err != nil {
panic(err)
}
redis2Config, err := gredis.ConfigFromStr("r-bp1f0a5d4efd8744.redis.rds.aliyuncs.com:6379,9")
if err != nil {
panic(err)
}
gredis.SetConfig(redis1Config)
v, err := g.Redis().DoVar("keys", "*")
if err != nil {
panic(err)
}
array := garray.NewStrArrayFrom(v.Strings())
for {
slice := array.PopLefts(batchNumber)
if len(slice) > 0 {
// `migrate %s %d "" 0 2000 copy replace auth %s keys %s`,
params := g.Slice{
redis2Config.Host,
redis2Config.Port,
"",
redis2Config.Db,
2000,
"copy",
"replace",
"keys",
}
params = append(params, gconv.Interfaces(slice)...)
fmt.Println(params)
if gcmd.GetOpt("dryrun") == "0" {
if v, err := g.Redis().DoVar("migrate", params...); err != nil {
panic(err)
} else {
fmt.Println(v.String())
}
}
} else {
break
}
}
fmt.Println("done")
}

37
.example/other/test.go
View File

@ -1,13 +1,38 @@
package main
import (
"fmt"
"net/url"
"encoding/json"
"strings"
"github.com/gogf/gf/util/gconv"
"github.com/gogf/gf/frame/g"
"github.com/gogf/gf/text/gregex"
"github.com/gogf/gf/container/gset"
"github.com/gogf/gf/os/gfile"
)
func main() {
parse1, _ := url.Parse("https://gf.cdn.johng.cn")
parse2, _ := url.Parse("https://gf.cdn.johng.cn/cli/")
fmt.Println(parse1.Host)
fmt.Println(parse2.Host)
path1 := "/Users/john/Temp/downloaded_data_parsed.txt"
path2 := "/Users/john/Temp/downloaded_data_parsed_mapping.txt"
array := strings.Split(gfile.GetContents(path1), "\n")
mapping := make(map[string]*gset.Set)
for _, line := range array {
array, _ := gregex.MatchString(`add group success \[\[(\d+),\[*(\d+)\]*`, line)
if len(array) != 3 {
g.Dump(line)
g.Dump(array)
continue
}
if _, ok := mapping[array[1]]; !ok {
mapping[array[1]] = gset.New()
}
mapping[array[1]].Add(gconv.Interfaces(strings.Split(array[2], ","))...)
}
g.Dump(mapping)
b, _ := json.Marshal(mapping)
gfile.PutBytes(path2, b)
}

10
database/gredis/gredis.go
View File

@ -104,6 +104,16 @@ func New(config Config) *Redis {
}
}
// NewFromStr creates a redis client object with given configuration string.
// Redis client maintains a connection pool automatically.
func NewFromStr(str string) (*Redis, error) {
config, err := ConfigFromStr(str)
if err != nil {
return nil, err
}
return New(config), nil
}
// Close closes the redis connection pool,
// it will release all connections reserved by this pool.
// It is not necessary to call Close manually.

77
database/gredis/gredis_config.go
View File

@ -6,11 +6,20 @@
package gredis
import "github.com/gogf/gf/container/gmap"
import (
"time"
"github.com/gogf/gf/errors/gerror"
"github.com/gogf/gf/container/gmap"
"github.com/gogf/gf/text/gregex"
"github.com/gogf/gf/text/gstr"
"github.com/gogf/gf/util/gconv"
)
const (
// Default configuration group name.
DEFAULT_GROUP_NAME = "default"
DEFAULT_GROUP_NAME = "default" // Default configuration group name.
DEFAULT_REDIS_PORT = 6379 // Default redis port configuration if not passed.
)
var (
@ -29,6 +38,22 @@ func SetConfig(config Config, name ...string) {
instances.Remove(group)
}
// SetConfigByStr sets the global configuration for specified group with string.
// If <name> is not passed, it sets configuration for the default group name.
func SetConfigByStr(str string, name ...string) error {
group := DEFAULT_GROUP_NAME
if len(name) > 0 {
group = name[0]
}
config, err := ConfigFromStr(str)
if err != nil {
return err
}
configs.Set(group, config)
instances.Remove(group)
return nil
}
// GetConfig returns the global configuration with specified group name.
// If <name> is not passed, it returns configuration of the default group name.
func GetConfig(name ...string) (config Config, ok bool) {
@ -53,6 +78,52 @@ func RemoveConfig(name ...string) {
instances.Remove(group)
}
// ConfigFromStr parses and returns config from given str.
// Eg: host:port[,db,pass?maxIdle=x&maxActive=x&idleTimeout=x&maxConnLifetime=x]
func ConfigFromStr(str string) (config Config, err error) {
array, _ := gregex.MatchString(`([^:]+):*(\d*),{0,1}(\d*),{0,1}(.*)\?(.+)`, str)
if len(array) == 6 {
parse, _ := gstr.Parse(array[5])
config = Config{
Host: array[1],
Port: gconv.Int(array[2]),
Db: gconv.Int(array[3]),
Pass: array[4],
}
if config.Port == 0 {
config.Port = DEFAULT_REDIS_PORT
}
if v, ok := parse["maxIdle"]; ok {
config.MaxIdle = gconv.Int(v)
}
if v, ok := parse["maxActive"]; ok {
config.MaxActive = gconv.Int(v)
}
if v, ok := parse["idleTimeout"]; ok {
config.IdleTimeout = gconv.Duration(v) * time.Second
}
if v, ok := parse["maxConnLifetime"]; ok {
config.MaxConnLifetime = gconv.Duration(v) * time.Second
}
return
}
array, _ = gregex.MatchString(`([^:]+):*(\d*),{0,1}(\d*),{0,1}(.*)`, str)
if len(array) == 5 {
config = Config{
Host: array[1],
Port: gconv.Int(array[2]),
Db: gconv.Int(array[3]),
Pass: array[4],
}
if config.Port == 0 {
config.Port = DEFAULT_REDIS_PORT
}
} else {
err = gerror.Newf(`invalid redis configuration: "%s"`, str)
}
return
}
// ClearConfig removes all configurations and instances of redis.
func ClearConfig() {
configs.Clear()

6
frame/g/g_object.go
View File

@ -8,7 +8,6 @@ package g
import (
"github.com/gogf/gf/database/gdb"
"github.com/gogf/gf/database/gkvdb"
"github.com/gogf/gf/database/gredis"
"github.com/gogf/gf/frame/gins"
"github.com/gogf/gf/i18n/gi18n"
@ -80,11 +79,6 @@ func DB(name ...string) gdb.DB {
return gins.Database(name...)
}
// KV returns an instance of gkvdb with specified configuration group name.
func KV(name ...string) *gkvdb.DB {
return gins.KV(name...)
}
// Redis returns an instance of redis client with specified configuration group name.
func Redis(name ...string) *gredis.Redis {
return gins.Redis(name...)

148
frame/gins/gins.go
View File

@ -9,12 +9,9 @@ package gins
import (
"fmt"
"time"
"github.com/gogf/gf/os/gfile"
"github.com/gogf/gf/database/gkvdb"
"github.com/gogf/gf/container/gmap"
"github.com/gogf/gf/database/gdb"
"github.com/gogf/gf/database/gredis"
@ -25,7 +22,6 @@ import (
"github.com/gogf/gf/os/gres"
"github.com/gogf/gf/os/gview"
"github.com/gogf/gf/text/gregex"
"github.com/gogf/gf/text/gstr"
"github.com/gogf/gf/util/gconv"
)
@ -102,43 +98,49 @@ func Database(name ...string) gdb.DB {
}
instanceKey := fmt.Sprintf("%s.%s", gFRAME_CORE_COMPONENT_NAME_DATABASE, group)
db := instances.GetOrSetFuncLock(instanceKey, func() interface{} {
if gdb.GetConfig(group) == nil {
m := config.GetMap("database")
if m == nil {
glog.Error(`database init failed: "database" node not found, is config file or configuration missing?`)
return nil
if gdb.GetConfig(group) != nil {
db, err := gdb.Instance(group)
if err != nil {
glog.Error(err)
}
// Parse <m> as map-slice.
for group, groupConfig := range m {
cg := gdb.ConfigGroup{}
switch value := groupConfig.(type) {
case []interface{}:
for _, v := range value {
if node := parseDBConfigNode(v); node != nil {
cg = append(cg, *node)
}
}
case map[string]interface{}:
if node := parseDBConfigNode(value); node != nil {
return db
}
m := config.GetMap("database")
if m == nil {
glog.Error(`database init failed: "database" node not found, is config file or configuration missing?`)
return nil
}
// Parse <m> as map-slice.
for group, groupConfig := range m {
cg := gdb.ConfigGroup{}
switch value := groupConfig.(type) {
case []interface{}:
for _, v := range value {
if node := parseDBConfigNode(v); node != nil {
cg = append(cg, *node)
}
}
if len(cg) > 0 {
gdb.SetConfigGroup(group, cg)
}
}
// Parse <m> as a single node configuration.
if node := parseDBConfigNode(m); node != nil {
cg := gdb.ConfigGroup{}
if node.LinkInfo != "" || node.Host != "" {
case map[string]interface{}:
if node := parseDBConfigNode(value); node != nil {
cg = append(cg, *node)
}
if len(cg) > 0 {
gdb.SetConfigGroup(group, cg)
}
}
addConfigMonitor(instanceKey, config)
if len(cg) > 0 {
gdb.SetConfigGroup(group, cg)
}
}
// Parse <m> as a single node configuration.
if node := parseDBConfigNode(m); node != nil {
cg := gdb.ConfigGroup{}
if node.LinkInfo != "" || node.Host != "" {
cg = append(cg, *node)
}
if len(cg) > 0 {
gdb.SetConfigGroup(group, cg)
}
}
addConfigMonitor(instanceKey, config)
if db, err := gdb.New(name...); err == nil {
return db
} else {
@ -238,46 +240,20 @@ func Redis(name ...string) *gredis.Redis {
}
instanceKey := fmt.Sprintf("%s.%s", gFRAME_CORE_COMPONENT_NAME_REDIS, group)
result := instances.GetOrSetFuncLock(instanceKey, func() interface{} {
// If already configured, it returns the redis instance.
if _, ok := gredis.GetConfig(group); ok {
return gredis.Instance(group)
}
// Or else, it parses the default configuration file and returns a new redis instance.
if m := config.GetMap("redis"); m != nil {
// host:port[,db,pass?maxIdle=x&maxActive=x&idleTimeout=x&maxConnLifetime=x]
if v, ok := m[group]; ok {
line := gconv.String(v)
array, _ := gregex.MatchString(`(.+):(\d+),{0,1}(\d*),{0,1}(.*)\?(.+)`, line)
if len(array) == 6 {
parse, _ := gstr.Parse(array[5])
redisConfig := gredis.Config{
Host: array[1],
Port: gconv.Int(array[2]),
Db: gconv.Int(array[3]),
Pass: array[4],
}
if v, ok := parse["maxIdle"]; ok {
redisConfig.MaxIdle = gconv.Int(v)
}
if v, ok := parse["maxActive"]; ok {
redisConfig.MaxActive = gconv.Int(v)
}
if v, ok := parse["idleTimeout"]; ok {
redisConfig.IdleTimeout = gconv.Duration(v) * time.Second
}
if v, ok := parse["maxConnLifetime"]; ok {
redisConfig.MaxConnLifetime = gconv.Duration(v) * time.Second
}
addConfigMonitor(instanceKey, config)
return gredis.New(redisConfig)
}
array, _ = gregex.MatchString(`(.+):(\d+),{0,1}(\d*),{0,1}(.*)`, line)
if len(array) == 5 {
addConfigMonitor(instanceKey, config)
return gredis.New(gredis.Config{
Host: array[1],
Port: gconv.Int(array[2]),
Db: gconv.Int(array[3]),
Pass: array[4],
})
} else {
glog.Errorf(`invalid redis node configuration: "%s"`, line)
redisConfig, err := gredis.ConfigFromStr(gconv.String(v))
if err != nil {
glog.Error(err)
return nil
}
addConfigMonitor(instanceKey, config)
return gredis.New(redisConfig)
} else {
glog.Errorf(`configuration for redis not found for group "%s"`, group)
}
@ -292,38 +268,6 @@ func Redis(name ...string) *gredis.Redis {
return nil
}
// KV returns an instance of gkvdb with specified configuration group name.
func KV(name ...string) *gkvdb.DB {
config := Config()
group := "default"
if len(name) > 0 && name[0] != "" {
group = name[0]
}
instanceKey := fmt.Sprintf("%s.%s", gFRAME_CORE_COMPONENT_NAME_GKVDB, group)
result := instances.GetOrSetFuncLock(instanceKey, func() interface{} {
key := fmt.Sprintf("kvdb.%s", group)
if s := config.GetString(key); s != "" {
db := gkvdb.Instance(group)
parse, _ := gstr.Parse(s)
if value, ok := parse["path"]; ok {
db.SetPath(gconv.String(value))
}
if value, ok := parse["sync"]; ok {
db.Options().SyncWrites = gconv.Bool(value)
}
addConfigMonitor(instanceKey, config)
return db
} else {
glog.Errorf(`incomplete configuration for gkvdb: "%s" node not found in config file "%s"`, key, config.FilePath())
}
return nil
})
if result != nil {
return result.(*gkvdb.DB)
}
return nil
}
func addConfigMonitor(key string, config *gcfg.Config) {
if path := config.FilePath(); path != "" && gfile.Exists(path) {
gfsnotify.Add(path, func(event *gfsnotify.Event) {

44
frame/gins/gins_kvdb_test.go
View File

@ -1,44 +0,0 @@
// Copyright 2017 gf Author(https://github.com/gogf/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.
package gins_test
import (
"testing"
"time"
"github.com/gogf/gf/frame/gins"
"github.com/gogf/gf/os/gfile"
"github.com/gogf/gf/test/gtest"
)
func Test_KV(t *testing.T) {
config := `
[kvdb]
default = "path=/tmp/gkvdb&sync=false"
cache = "path=/tmp/gkvdb-cache&sync=true"
`
path := "config.toml"
err := gfile.PutContents(path, config)
gtest.Assert(err, nil)
defer gfile.Remove(path)
defer gins.Config().Clear()
// for gfsnotify callbacks to refresh cache of config file
time.Sleep(500 * time.Millisecond)
gtest.Case(t, func() {
kvDefault := gins.KV()
kvCache := gins.KV("cache")
key := []byte("key")
value := []byte("value")
err := kvDefault.Set(key, value)
gtest.Assert(err, nil)
gtest.Assert(kvDefault.Get(key), value)
gtest.Assert(kvCache.Get(key), nil)
})
}