bingyi/gf
1
0
Fork 0
mirror of https://gitee.com/johng/gf synced 2026-06-07 02:12:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

improve package gpool/gfpool; donator updates

Browse Source
This commit is contained in:
John
2020-03-14 17:12:44 +08:00
parent e513cd10ed
commit 0a92df691b
13 changed files with 178 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.example/container/gpool/gpool.go
View File

@ -9,7 +9,7 @@ import (
func main() {
// 创建一个对象池，过期时间为1000毫秒
p := gpool.New(1000, nil)
p := gpool.New(1000*time.Millisecond, nil)
// 从池中取一个对象，返回nil及错误信息
fmt.Println(p.Get())

2
.example/container/gpool/gpool_expirefunc.go
View File

@ -11,7 +11,7 @@ import (
func main() {
// 创建对象复用池，对象过期时间为3000毫秒，并给定创建及销毁方法
p := gpool.New(3000, func() (interface{}, error) {
p := gpool.New(3000*time.Millisecond, func() (interface{}, error) {
return gtcp.NewConn("www.baidu.com:80")
}, func(i interface{}) {
glog.Println("expired")

2
.example/os/gfpool/gfpool.go
View File

@ -11,7 +11,7 @@ import (
func main() {
for {
time.Sleep(time.Second)
if f, err := gfpool.Open("/home/john/temp/log.log", os.O_RDONLY, 0666, 60000000*1000); err == nil {
if f, err := gfpool.Open("/home/john/temp/log.log", os.O_RDONLY, 0666, time.Hour); err == nil {
fmt.Println(f.Name())
f.Close()
} else {

17
DONATOR.MD
View File

@ -40,7 +40,7 @@ We currently accept donation by Alipay/WechatPay, please note your github/gitee
|R*s|wechat|￥18.88| 谢谢GF！辛苦了！
|粟*e|wechat|￥50.00|
|[李超](https://github.com/effortlee)|wechat|￥124.00|
|张炳贤|wechat+qq|￥600.00|
|soidea666|wechat+qq|￥800.00|
|[王哈哈](https://gitee.com/develop1024)|wechat|￥6.66| 希望gf越来越好
|夕景|alipay+qq|￥9.96+3.57|
|struggler|alipay|￥18.80|
@ -51,6 +51,21 @@ We currently accept donation by Alipay/WechatPay, please note your github/gitee
|[Zeroing-ZY](https://gitee.com/yunjieg)|gitee|￥20.00| 感谢您的开源项目！
|[katydid酱](https://gitee.com/katydid2005)|gitee|￥50.00| 感谢您的开源项目！框架给予了很大的帮助！谢谢大佬！
|[李海峰](https://gitee.com/dlhf)|gitee|￥10.00| 希望GF越来越好，框架很牛逼！
|陆昱天|alipay|￥100.00|
|[Dockercore](https://github.com/dockercore)|wechat|￥200.00| 非常喜欢！简洁好用！文档超级全！
|🚶|wechat|￥6.88| 喝杯冰阔落
|a*l|wechat|￥10.00| gf
|[wxkj](https://gitee.com/wxkj)|wechat|￥10.00|
|*包|wechat|￥9.99|
|重庆宝尔威科技|wechat|￥6.66|
|琦玉-QPT|wechat|￥6.66|
|sailsea|wechat|￥11.00|
|[seny0929](https://gitee.com/seny0929)|wechat|￥99.90|
|*华|wechat|￥6.66| 感谢郭强的热心
|[Playhi](https://github.com/Playhi)|alipay|￥10.00|
|北京京纬互动科技有限公司|alipay|￥200.00|
|米司特包|wechat|￥99.99|
|金毛|alipay|￥100.00|
<img src="https://goframe.org/images/donate.png"/>

94
container/gpool/gpool.go
View File

@ -17,22 +17,31 @@ import (
"github.com/gogf/gf/os/gtimer"
)
// Object-Reusable Pool.
// Pool is an Object-Reusable Pool.
type Pool struct {
list *glist.List // Available/idle list.
closed *gtype.Bool // Whether the pool is closed.
Expire int64 // Max idle time(ms), after which it is recycled.
NewFunc func() (interface{}, error) // Callback function to create item.
ExpireFunc func(interface{}) // Expired destruction function for objects.
// This function needs to be defined when the pool object
// needs to perform additional destruction operations.
// Available/idle items list.
list *glist.List
// Whether the pool is closed.
closed *gtype.Bool
// Time To Live for pool items.
TTL time.Duration
// Callback function to create pool item.
NewFunc func() (interface{}, error)
// ExpireFunc is the for expired items destruction.
// This function needs to be defined when the pool items
// need to perform additional destruction operations.
// Eg: net.Conn, os.File, etc.
ExpireFunc func(interface{})
}
// Pool item.
type poolItem struct {
expire int64 // Expire time(millisecond).
value interface{} // Value.
expire int64 // Expire timestamp in milliseconds.
value interface{} // Item value.
}
// Creation function for object.
@ -41,47 +50,62 @@ type NewFunc func() (interface{}, error)
// Destruction function for object.
type ExpireFunc func(interface{})
// New returns a new object pool.
// New creates and returns a new object pool.
// To ensure execution efficiency, the expiration time cannot be modified once it is set.
//
// Expiration logic:
// expire = 0 : not expired;
// expire < 0 : immediate expired after use;
// expire > 0 : timeout expired;
// Note that the expiration time unit is ** milliseconds **.
func New(expire int, newFunc NewFunc, expireFunc ...ExpireFunc) *Pool {
// Note the expiration logic:
// ttl = 0 : not expired;
// ttl < 0 : immediate expired after use;
// ttl > 0 : timeout expired;
func New(ttl time.Duration, newFunc NewFunc, expireFunc ...ExpireFunc) *Pool {
r := &Pool{
list: glist.New(true),
closed: gtype.NewBool(),
Expire: int64(expire),
TTL: ttl,
NewFunc: newFunc,
}
if len(expireFunc) > 0 {
r.ExpireFunc = expireFunc[0]
}
gtimer.AddSingleton(time.Second, r.checkExpire)
gtimer.AddSingleton(time.Second, r.checkExpireItems)
return r
}
// Put puts an item to pool.
func (p *Pool) Put(value interface{}) {
func (p *Pool) Put(value interface{}) error {
if p.closed.Val() {
return errors.New("pool is closed")
}
item := &poolItem{
value: value,
}
if p.Expire == 0 {
if p.TTL == 0 {
item.expire = 0
} else {
item.expire = gtime.TimestampMilli() + p.Expire
item.expire = gtime.TimestampMilli() + p.TTL.Milliseconds()
}
p.list.PushBack(item)
return nil
}
// Clear clears pool, which means it will remove all items from pool.
func (p *Pool) Clear() {
p.list.RemoveAll()
if p.ExpireFunc != nil {
for {
if r := p.list.PopFront(); r != nil {
p.ExpireFunc(r.(*poolItem).value)
} else {
break
}
}
} else {
p.list.RemoveAll()
}
}
// Get picks an item from pool.
// Get picks and returns an item from pool. If the pool is empty and NewFunc is defined,
// it creates and returns one from NewFunc.
func (p *Pool) Get() (interface{}, error) {
for !p.closed.Val() {
if r := p.list.PopFront(); r != nil {
@ -106,12 +130,13 @@ func (p *Pool) Size() int {
// Close closes the pool. If <p> has ExpireFunc,
// then it automatically closes all items using this function before it's closed.
// Commonly you do not need call this function manually.
func (p *Pool) Close() {
p.closed.Set(true)
}
// checkExpire removes expired items from pool every second.
func (p *Pool) checkExpire() {
// checkExpire removes expired items from pool in every second.
func (p *Pool) checkExpireItems() {
if p.closed.Val() {
// If p has ExpireFunc,
// then it must close all items using this function.
@ -126,11 +151,24 @@ func (p *Pool) checkExpire() {
}
gtimer.Exit()
}
// All items do not expire.
if p.TTL == 0 {
return
}
// The latest item expire timestamp in milliseconds.
var latestExpire int64 = -1
// Retrieve the current timestamp in milliseconds, it expires the items
// by comparing with this timestamp. It is not accurate comparison for
// every items expired, but high performance.
var timestampMilli = gtime.TimestampMilli()
for {
// TODO Do not use Pop and Push mechanism, which is not graceful.
if latestExpire > timestampMilli {
break
}
if r := p.list.PopFront(); r != nil {
item := r.(*poolItem)
if item.expire == 0 || item.expire > gtime.TimestampMilli() {
latestExpire = item.expire
if item.expire > timestampMilli {
p.list.PushFront(item)
break
}

3
container/gpool/gpool_bench_test.go
View File

@ -11,11 +11,12 @@ package gpool_test
import (
"sync"
"testing"
"time"
"github.com/gogf/gf/container/gpool"
)
var pool = gpool.New(99999999, nil)
var pool = gpool.New(time.Hour, nil)
var syncp = sync.Pool{}
func BenchmarkGPoolPut(b *testing.B) {

2
container/gpool/gpool_z_unit_test.go
View File

@ -62,7 +62,7 @@ func Test_Gpool(t *testing.T) {
gtest.Case(t, func() {
//
//expire > 0
p2 := gpool.New(2000, nil, ef)
p2 := gpool.New(2*time.Second, nil, ef)
for index := 0; index < 10; index++ {
p2.Put(index)
}

6
container/gtype/z_bench_basic_test.go
View File

@ -157,6 +157,12 @@ func BenchmarkBool_Val(b *testing.B) {
}
}
func BenchmarkBool_Cas(b *testing.B) {
for i := 0; i < b.N; i++ {
bl.Cas(false, true)
}
}
func BenchmarkString_Set(b *testing.B) {
for i := 0; i < b.N; i++ {
str.Set(strconv.Itoa(i))

14
net/ghttp/ghttp_request.go
View File

@ -23,23 +23,23 @@ import (
// Request is the context object for a request.
type Request struct {
*http.Request
Server *Server // Parent server.
Server *Server // Server.
Cookie *Cookie // Cookie.
Session *gsession.Session // Session.
Response *Response // Corresponding Response of this request.
Router *Router // Matched Router for this request. Note that it's only available in HTTP handler, not in HOOK or MiddleWare.
Router *Router // Matched Router for this request. Note that it's not available in HOOK handler.
EnterTime int64 // Request starting time in microseconds.
LeaveTime int64 // Request ending time in microseconds.
Middleware *Middleware // The middleware manager.
StaticFile *StaticFile // Static file object when static file serving.
Context context.Context // Custom context map for internal usage purpose.
handlers []*handlerParsedItem // All matched handlers containing handler, hook and middleware for this request .
Middleware *Middleware // Middleware manager.
StaticFile *StaticFile // Static file object for static file serving.
Context context.Context // Custom context for internal usage purpose.
handlers []*handlerParsedItem // All matched handlers containing handler, hook and middleware for this request.
hasHookHandler bool // A bool marking whether there's hook handler in the handlers for performance purpose.
hasServeHandler bool // A bool marking whether there's serving handler in the handlers for performance purpose.
parsedQuery bool // A bool marking whether the GET parameters parsed.
parsedBody bool // A bool marking whether the request body parsed.
parsedForm bool // A bool marking whether request Form parsed for HTTP method PUT, POST, PATCH.
paramsMap map[string]interface{} // Custom parameters.
paramsMap map[string]interface{} // Custom parameters map.
routerMap map[string]string // Router parameters map, which might be nil if there're no router parameters.
queryMap map[string]interface{} // Query parameters map, which is nil if there's no query string.
formMap map[string]interface{} // Form parameters map, which is nil if there's no form data from client.

8
net/gtcp/gtcp_pool.go
View File

@ -23,10 +23,10 @@ type PoolConn struct {
}
const (
gDEFAULT_POOL_EXPIRE = 10000 // (Millisecond) Default TTL for connection in the pool.
gCONN_STATUS_UNKNOWN = 0 // Means it is unknown it's connective or not.
gCONN_STATUS_ACTIVE = 1 // Means it is now connective.
gCONN_STATUS_ERROR = 2 // Means it should be closed and removed from pool.
gDEFAULT_POOL_EXPIRE = 10 * time.Second // Default TTL for connection in the pool.
gCONN_STATUS_UNKNOWN = 0 // Means it is unknown it's connective or not.
gCONN_STATUS_ACTIVE = 1 // Means it is now connective.
gCONN_STATUS_ERROR = 2 // Means it should be closed and removed from pool.
)
var (

124
os/gfpool/gfpool.go
View File

@ -9,8 +9,9 @@ package gfpool
import (
"fmt"
"github.com/gogf/gf/os/gfile"
"os"
"sync"
"time"
"github.com/gogf/gf/container/gmap"
"github.com/gogf/gf/container/gpool"
@ -20,71 +21,84 @@ import (
// File pointer pool.
type Pool struct {
id *gtype.Int // 指针池ID，用以识别指针池是否需要重建
pool *gpool.Pool // 底层对象池
inited *gtype.Bool // 是否初始化(在执行第一次执行File方法后初始化，主要用于文件监听的添加，但是只能添加一次)
expire int // 过期时间
id *gtype.Int // Pool id, which is used to mark this pool whether recreated.
pool *gpool.Pool // Underlying pool.
init *gtype.Bool // Whether initialized, used for marking this file added to fsnotify, and it can only be added just once.
ttl time.Duration // Time to live for file pointer items.
}
// 文件指针池指针
// File is an item in the pool.
type File struct {
*os.File // 底层文件指针
mu sync.RWMutex // 互斥锁
pool *Pool // 所属池
poolid int // 所属池ID，如果池ID不同表示池已经重建，那么该文件指针也应当销毁，不能重新丢到原有的池中
flag int // 打开标志
perm os.FileMode // 打开权限
path string // 绝对路径
*os.File // Underlying file pointer.
pid int // Belonging pool id, which is set when file pointer created. It's used to check whether the pool is recreated.
pool *Pool // Belonging ool.
flag int // Flash for opening file.
perm os.FileMode // Permission for opening file.
path string // Absolute path of the file.
}
var (
// 全局文件指针池Map, 不过期
// Global file pointer pool.
pools = gmap.NewStrAnyMap(true)
)
// 获得文件对象，并自动创建指针池(过期时间单位：毫秒)
func Open(path string, flag int, perm os.FileMode, expire ...int) (file *File, err error) {
fpExpire := 0
if len(expire) > 0 {
fpExpire = expire[0]
// Open creates and returns a file item with given file path, flag and opening permission.
// It automatically creates an associated file pointer pool internally when it's called first time.
// It retrieves a file item from the file pointer pool after then.
func Open(path string, flag int, perm os.FileMode, ttl ...time.Duration) (file *File, err error) {
var fpTTL time.Duration
if len(ttl) > 0 {
fpTTL = ttl[0]
}
pool := pools.GetOrSetFuncLock(fmt.Sprintf("%s&%d&%d&%d", path, flag, expire, perm), func() interface{} {
return New(path, flag, perm, fpExpire)
}).(*Pool)
path, err = gfile.Search(path)
if err != nil {
return nil, err
}
pool := pools.GetOrSetFuncLock(
fmt.Sprintf("%s&%d&%d&%d", path, flag, fpTTL, perm),
func() interface{} {
return New(path, flag, perm, fpTTL)
},
).(*Pool)
return pool.File()
}
// 创建一个文件指针池，expire = 0表示不过期，expire < 0表示使用完立即回收，expire > 0表示超时回收，默认值为0表示不过期。
// 注意过期时间单位为：毫秒。
func New(path string, flag int, perm os.FileMode, expire ...int) *Pool {
fpExpire := 0
if len(expire) > 0 {
fpExpire = expire[0]
// New creates and returns a file pointer pool with given file path, flag and opening permission.
//
// Note the expiration logic:
// ttl = 0 : not expired;
// ttl < 0 : immediate expired after use;
// ttl > 0 : timeout expired;
// It is not expired in default.
func New(path string, flag int, perm os.FileMode, ttl ...time.Duration) *Pool {
var fpTTL time.Duration
if len(ttl) > 0 {
fpTTL = ttl[0]
}
p := &Pool{
id: gtype.NewInt(),
expire: fpExpire,
inited: gtype.NewBool(),
id: gtype.NewInt(),
ttl: fpTTL,
init: gtype.NewBool(),
}
p.pool = newFilePool(p, path, flag, perm, fpExpire)
p.pool = newFilePool(p, path, flag, perm, fpTTL)
return p
}
// 创建文件指针池
func newFilePool(p *Pool, path string, flag int, perm os.FileMode, expire int) *gpool.Pool {
pool := gpool.New(expire, func() (interface{}, error) {
// newFilePool creates and returns a file pointer pool with given file path, flag and opening permission.
func newFilePool(p *Pool, path string, flag int, perm os.FileMode, ttl time.Duration) *gpool.Pool {
pool := gpool.New(ttl, func() (interface{}, error) {
file, err := os.OpenFile(path, flag, perm)
if err != nil {
return nil, err
}
return &File{
File: file,
pool: p,
poolid: p.id.Val(),
flag: flag,
perm: perm,
path: path,
File: file,
pool: p,
pid: p.id.Val(),
flag: flag,
perm: perm,
path: path,
}, nil
}, func(i interface{}) {
_ = i.(*File).File.Close()
@ -92,7 +106,10 @@ func newFilePool(p *Pool, path string, flag int, perm os.FileMode, expire int) *
return pool
}
// 获得一个文件打开指针
// File retrieves file item from the file pointer pool and returns it. It creates one if
// the file pointer pool is empty.
// Note that it should be closed when it will never be used. When it's closed, it is not
// really closed the underlying file pointer but put back to the file pinter pool.
func (p *Pool) File() (*File, error) {
if v, err := p.pool.Get(); err != nil {
return nil, err
@ -127,18 +144,17 @@ func (p *Pool) File() (*File, error) {
return nil, err
}
}
// 优先使用 !p.inited.Val() 原子读取操作判断，保证判断操作的效率；
// p.inited.Set(true) == false 使用原子写入操作，保证该操作的原子性；
if !p.inited.Val() && p.inited.Set(true) == false {
// It firstly checks using !p.init.Val() for performance purpose.
if !p.init.Val() && p.init.Cas(false, true) {
_, _ = gfsnotify.Add(f.path, func(event *gfsnotify.Event) {
// 如果文件被删除或者重命名，立即重建指针池
// If teh file is removed or renamed, recreates the pool by increasing the pool id.
if event.IsRemove() || event.IsRename() {
// 原有的指针都不要了
// It drops the old pool.
p.id.Add(1)
// Clear相当于重建指针池
// Clears the pool items staying in the pool.
p.pool.Clear()
// 为保证原子操作，但又不想加锁，
// 这里再执行一次原子Add，将在两次Add中间可能分配出去的文件指针丢弃掉
// It uses another adding to drop the file items between the two adding.
// Whenever the pool id changes, the pool will be recreated.
p.id.Add(1)
}
}, false)
@ -147,15 +163,15 @@ func (p *Pool) File() (*File, error) {
}
}
// 关闭指针池
// Close closes current file pointer pool.
func (p *Pool) Close() {
p.pool.Close()
}
// 获得底层文件指针(返回error是标准库io.ReadWriteCloser接口实现)
// Close puts the file pointer back to the file pointer pool.
func (f *File) Close() error {
if f.poolid == f.pool.id.Val() {
f.pool.pool.Put(f)
if f.pid == f.pool.id.Val() {
return f.pool.pool.Put(f)
}
return nil
}

4
os/gfpool/gfpool_z_unit_test.go
View File

@ -79,12 +79,12 @@ func TestOpenExpire(t *testing.T) {
testFile := start("TestOpenExpire.txt")
gtest.Case(t, func() {
f, err := gfpool.Open(testFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_APPEND, 0666, 100)
f, err := gfpool.Open(testFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_APPEND, 0666, 100*time.Millisecond)
gtest.AssertEQ(err, nil)
f.Close()
time.Sleep(150 * time.Millisecond)
f2, err1 := gfpool.Open(testFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_APPEND, 0666, 100)
f2, err1 := gfpool.Open(testFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_APPEND, 0666, 100*time.Millisecond)
gtest.AssertEQ(err1, nil)
//gtest.AssertNE(f, f2)
f2.Close()

2
os/glog/glog_logger.go
View File

@ -34,7 +34,7 @@ const (
gDEFAULT_FILE_FORMAT = `{Y-m-d}.log`
gDEFAULT_FILE_POOL_FLAGS = os.O_CREATE | os.O_WRONLY | os.O_APPEND
gDEFAULT_FPOOL_PERM = os.FileMode(0666)
gDEFAULT_FPOOL_EXPIRE = 60000
gDEFAULT_FPOOL_EXPIRE = time.Minute
gPATH_FILTER_KEY = "/os/glog/glog"
)