From adbf6bc57939fc10431980543dbf8cc29b056337 Mon Sep 17 00:00:00 2001 From: john Date: Thu, 26 Jul 2018 13:00:04 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0gmap.Map=E5=AF=B9=E8=B1=A1?= =?UTF-8?q?=EF=BC=9B=E6=94=B9=E8=BF=9Bgfilepool=E6=96=87=E4=BB=B6=E6=8C=87?= =?UTF-8?q?=E9=92=88=E6=B1=A0=E8=AE=BE=E8=AE=A1=EF=BC=9B=E6=94=B9=E8=BF=9B?= =?UTF-8?q?gfile=E6=96=87=E6=9C=AC=E5=86=85=E5=AE=B9=E5=86=99=E5=85=A5?= =?UTF-8?q?=EF=BC=8C=E5=A2=9E=E5=8A=A0=E6=8C=87=E9=92=88=E6=B1=A0=E4=BD=BF?= =?UTF-8?q?=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- g/container/gmap/gmap.go | 11 +++ g/container/gmap/gmap_bench_test.go | 84 +++++++++++++++++--- g/container/gpool/gpool.go | 33 +++++--- g/os/gcache/gcache.go | 4 +- g/os/gfile/gfile.go | 8 +- g/os/gfilepool/gfilepool.go | 116 ++++++++-------------------- g/os/glog/glog_logger.go | 23 +++--- geg/other/test.go | 20 +---- 8 files changed, 164 insertions(+), 135 deletions(-) diff --git a/g/container/gmap/gmap.go b/g/container/gmap/gmap.go index afd26bd24..9e8e6d732 100644 --- a/g/container/gmap/gmap.go +++ b/g/container/gmap/gmap.go @@ -6,3 +6,14 @@ // 并发安全的哈希MAP. package gmap + +// 默认的Map对象其实就是InterfaceInterfaceMap的别名。 +// 注意 +// 1、这个Map是所有并发安全Map中效率最低的，如果对效率要求比较高的场合，请合理选择对应数据类型的Map； +// 2、这个Map的优点是使用简便，由于键值都是interface{}类型，因此对键值的数据类型要求不高； +// 3、底层实现比较类似于sync.Map； +type Map = InterfaceInterfaceMap + +func NewMap() *Map { + return NewInterfaceInterfaceMap() +} \ No newline at end of file diff --git a/g/container/gmap/gmap_bench_test.go b/g/container/gmap/gmap_bench_test.go index ccba0b6b0..24930012c 100644 --- a/g/container/gmap/gmap_bench_test.go +++ b/g/container/gmap/gmap_bench_test.go @@ -25,63 +25,127 @@ var sifm = NewStringInterfaceMap() var ssm = NewStringStringMap() var uifm = NewUintInterfaceMap() -func BenchmarkIntBoolMap_Set(b *testing.B) { +// 写入性能测试 + +func Benchmark_IntBoolMap_Set(b *testing.B) { for i := 0; i < b.N; i++ { ibm.Set(i, true) } } -func BenchmarkIntIntMap_Set(b *testing.B) { +func Benchmark_IntIntMap_Set(b *testing.B) { for i := 0; i < b.N; i++ { iim.Set(i, i) } } -func BenchmarkIntInterfaceMap_Set(b *testing.B) { +func Benchmark_IntInterfaceMap_Set(b *testing.B) { for i := 0; i < b.N; i++ { iifm.Set(i, i) } } -func BenchmarkIntStringMap_Set(b *testing.B) { +func Benchmark_IntStringMap_Set(b *testing.B) { for i := 0; i < b.N; i++ { ism.Set(i, strconv.Itoa(i)) } } -func BenchmarkInterfaceInterfaceMap_Set(b *testing.B) { +func Benchmark_InterfaceInterfaceMap_Set(b *testing.B) { for i := 0; i < b.N; i++ { ififm.Set(i, i) } } -func BenchmarkStringBoolMap_Set(b *testing.B) { +func Benchmark_StringBoolMap_Set(b *testing.B) { for i := 0; i < b.N; i++ { sbm.Set(strconv.Itoa(i), true) } } -func BenchmarkStringIntMap_Set(b *testing.B) { +func Benchmark_StringIntMap_Set(b *testing.B) { for i := 0; i < b.N; i++ { sim.Set(strconv.Itoa(i), i) } } -func BenchmarkStringInterfaceMap_Set(b *testing.B) { +func Benchmark_StringInterfaceMap_Set(b *testing.B) { for i := 0; i < b.N; i++ { sifm.Set(strconv.Itoa(i), i) } } -func BenchmarkStringStringMap_Set(b *testing.B) { +func Benchmark_StringStringMap_Set(b *testing.B) { for i := 0; i < b.N; i++ { ssm.Set(strconv.Itoa(i), strconv.Itoa(i)) } } -func BenchmarkUintInterfaceMap_Set(b *testing.B) { +func Benchmark_UintInterfaceMap_Set(b *testing.B) { for i := 0; i < b.N; i++ { uifm.Set(uint(i), i) } } +// 读取性能测试 + +func Benchmark_IntBoolMap_Get(b *testing.B) { + for i := 0; i < b.N; i++ { + ibm.Get(i) + } +} + +func Benchmark_IntIntMap_Get(b *testing.B) { + for i := 0; i < b.N; i++ { + iim.Get(i) + } +} + +func Benchmark_IntInterfaceMap_Get(b *testing.B) { + for i := 0; i < b.N; i++ { + iifm.Get(i) + } +} + +func Benchmark_IntStringMap_Get(b *testing.B) { + for i := 0; i < b.N; i++ { + ism.Get(i) + } +} + +func Benchmark_InterfaceInterfaceMap_Get(b *testing.B) { + for i := 0; i < b.N; i++ { + ififm.Get(i) + } +} + +func Benchmark_StringBoolMap_Get(b *testing.B) { + for i := 0; i < b.N; i++ { + sbm.Get(strconv.Itoa(i)) + } +} + +func Benchmark_StringIntMap_Get(b *testing.B) { + for i := 0; i < b.N; i++ { + sim.Get(strconv.Itoa(i)) + } +} + +func Benchmark_StringInterfaceMap_Get(b *testing.B) { + for i := 0; i < b.N; i++ { + sifm.Get(strconv.Itoa(i)) + } +} + +func Benchmark_StringStringMap_Get(b *testing.B) { + for i := 0; i < b.N; i++ { + ssm.Get(strconv.Itoa(i)) + } +} + +func Benchmark_UintInterfaceMap_Get(b *testing.B) { + for i := 0; i < b.N; i++ { + uifm.Get(uint(i)) + } +} + diff --git a/g/container/gpool/gpool.go b/g/container/gpool/gpool.go index bbde2db0b..a8332b5da 100644 --- a/g/container/gpool/gpool.go +++ b/g/container/gpool/gpool.go @@ -9,18 +9,21 @@ package gpool import ( "time" + "errors" "gitee.com/johng/gf/g/os/gtime" "gitee.com/johng/gf/g/container/glist" "gitee.com/johng/gf/g/container/gtype" - "errors" + "math" ) // 对象池 type Pool struct { - list *glist.List // 可用/闲置的文件指针链表 - closed *gtype.Bool // 连接池是否已关闭 - Expire int64 // (毫秒)闲置最大时间，超过该时间则被系统回收 - NewFunc func()(interface{}, error) // 创建对象的方法定义 + list *glist.List // 可用/闲置的文件指针链表 + closed *gtype.Bool // 连接池是否已关闭 + Expire int64 // (毫秒)闲置最大时间，超过该时间则被系统回收 + NewFunc func()(interface{}, error) // 创建对象的方法定义 + ExpireFunc func(interface{}) // 对象的过期销毁方法(当池对象销毁需要执行额外的销毁操作时，需要定义该方法) + // 例如: net.Conn, os.File等对象都需要执行额外关闭操作 } // 对象池数据项 @@ -30,8 +33,12 @@ type poolItem struct { } // 创建一个对象池，为保证执行效率，过期时间一旦设定之后无法修改 +// expire = 0表示不过期，expire < 0表示使用完立即回收，expire > 0表示超时回收 // 注意过期时间单位为**毫秒** func New(expire int, newFunc...func() (interface{}, error)) *Pool { + if expire == 0 { + expire = math.MaxInt64 + } r := &Pool { list : glist.New(), closed : gtype.NewBool(), @@ -44,9 +51,14 @@ func New(expire int, newFunc...func() (interface{}, error)) *Pool { return r } +// 设置对象过期销毁时的关闭方法 +func (p *Pool) SetExpireFunc(expireFunc func(interface{})) { + p.ExpireFunc = expireFunc +} + // 放一个临时对象到池中 func (p *Pool) Put(item interface{}) { - p.list.PushBack(&poolItem{ + p.list.PushBack(&poolItem { expire : gtime.Millisecond() + p.Expire, value : item, }) @@ -85,11 +97,14 @@ func (p *Pool) expireCheckingLoop() { for !p.closed.Val() { for { if r := p.list.PopFront(); r != nil { - f := r.(*poolItem) - if f.expire > gtime.Millisecond() { - p.list.PushFront(f) + item := r.(*poolItem) + if item.expire > gtime.Millisecond() { + p.list.PushFront(item) break } + if p.ExpireFunc != nil { + p.ExpireFunc(item.value) + } } } time.Sleep(time.Second) diff --git a/g/os/gcache/gcache.go b/g/os/gcache/gcache.go index 21eb27aa7..28b74ec17 100644 --- a/g/os/gcache/gcache.go +++ b/g/os/gcache/gcache.go @@ -71,12 +71,12 @@ func SetCap(cap int) { cache.cap.Set(cap) } -// (使用全局KV缓存对象)设置kv缓存键值对，过期时间单位为毫秒 +// (使用全局KV缓存对象)设置kv缓存键值对，过期时间单位为**毫秒** func Set(key string, value interface{}, expire int) { cache.Set(key, value, expire) } -// (使用全局KV缓存对象)批量设置kv缓存键值对，过期时间单位为毫秒 +// (使用全局KV缓存对象)批量设置kv缓存键值对，过期时间单位为**毫秒** func BatchSet(data map[string]interface{}, expire int) { cache.BatchSet(data, expire) } diff --git a/g/os/gfile/gfile.go b/g/os/gfile/gfile.go index 757b23a6b..3167308e0 100644 --- a/g/os/gfile/gfile.go +++ b/g/os/gfile/gfile.go @@ -23,6 +23,7 @@ import ( "path/filepath" "gitee.com/johng/gf/g/util/gregex" "gitee.com/johng/gf/g/container/gtype" + "gitee.com/johng/gf/g/os/gfilepool" ) // 封装了常用的文件操作方法，如需更详细的文件控制，请查看官方os包 @@ -328,14 +329,13 @@ func putContents(path string, data []byte, flag int, perm os.FileMode) error { return err } } - // 创建/打开文件 - f, err := os.OpenFile(path, flag, perm) + // 创建/打开文件，使用文件指针池，默认为60秒 + f, err := gfilepool.OpenWithPool(path, flag, perm, 60000) if err != nil { return err } defer f.Close() - n, err := f.Write(data) - if err != nil { + if n, err := f.Write(data); err != nil { return err } else if n < len(data) { return io.ErrShortWrite diff --git a/g/os/gfilepool/gfilepool.go b/g/os/gfilepool/gfilepool.go index 7cb0fe313..70801be5b 100644 --- a/g/os/gfilepool/gfilepool.go +++ b/g/os/gfilepool/gfilepool.go @@ -9,123 +9,73 @@ package gfilepool import ( "os" - "time" "sync" - "strconv" - "gitee.com/johng/gf/g/os/gtime" "gitee.com/johng/gf/g/container/gmap" - "gitee.com/johng/gf/g/container/glist" - "gitee.com/johng/gf/g/container/gtype" + "gitee.com/johng/gf/g/container/gpool" + "fmt" ) // 文件指针池 type Pool struct { - path string // 文件绝对路径 - flag int // 文件打开标识 - list *glist.List // 可用/闲置的文件指针链表 - idle int // 闲置最大时间，超过该时间则被系统回收(秒) - closed *gtype.Bool // 连接池是否已关闭 + pool *gpool.Pool // 底层对象池 } // 文件指针池指针 -type PoolItem struct { - mu sync.RWMutex +type File struct { + *os.File // 底层文件指针 + mu sync.RWMutex // 互斥锁 pool *Pool // 所属池 - file *os.File // 指针对象 - expire *gtype.Int64 // 过期时间(秒) } // 全局指针池，expire < 0表示不过期，expire = 0表示使用完立即回收，expire > 0表示超时回收 var pools = gmap.NewStringInterfaceMap() // 获得文件对象，并自动创建指针池 -func OpenWithPool(path string, flag int, expire int) (*PoolItem, error) { - key := path + strconv.Itoa(flag) + strconv.Itoa(expire) +func OpenWithPool(path string, flag int, perm os.FileMode, expire int) (*File, error) { + key := fmt.Sprintf("%s&%d&%d&%d", path, flag, expire, perm) result := pools.Get(key) if result != nil { return result.(*Pool).File() } - pool := New(path, flag, expire) + pool := New(path, flag, perm, expire) pools.Set(key, pool) return pool.File() } -// 创建一个文件指针池，expire < 0表示不过期，expire = 0表示使用完立即回收，expire > 0表示超时回收 -func New(path string, flag int, expire int) *Pool { - r := &Pool { - path : path, - flag : flag, - list : glist.New(), - idle : expire, - closed : gtype.NewBool(), - } - // 独立的线程执行过期清理工作 - if expire != -1 { - go func(p *Pool) { - for !p.closed.Val() { - if r := p.list.PopFront(); r != nil { - f := r.(*PoolItem) - // 必须小于，中间有1秒的缓存时间，防止同时获取和判断过期时冲突 - if f.expire.Val() < gtime.Second() { - f.destroy() - } else { - // 重新推回去 - p.list.PushFront(f) - break - } - } - time.Sleep(3 * time.Second) - } - }(r) - } - return r +// 创建一个文件指针池，expire = 0表示不过期，expire < 0表示使用完立即回收，expire > 0表示超时回收 +func New(path string, flag int, perm os.FileMode, expire int) *Pool { + p := &Pool {} + p.pool = gpool.New(expire, func() (interface{}, error) { + file, err := os.OpenFile(path, flag, perm) + if err != nil { + return nil, err + } + return &File{ + File : file, + pool : p, + }, nil + }) + p.pool.SetExpireFunc(func(i interface{}) { + i.(*File).File.Close() + }) + return p } // 获得一个文件打开指针 -func (p *Pool) File() (*PoolItem, error) { - if p.list.Len() > 0 { - for { - // 从队列头依次查找，返回一个未过期的指针 - if r := p.list.PopFront(); r != nil { - f := r.(*PoolItem) - if f.expire.Val() > gtime.Second() { - return f, nil - } else if f.file != nil { - f.destroy() - } - } else { - break - } - } - } - file, err := os.OpenFile(p.path, p.flag, 0666) - if err != nil { +func (p *Pool) File() (*File, error) { + if v, err := p.pool.Get(); err != nil { return nil, err + } else { + return v.(*File), nil } - return &PoolItem { - pool : p, - file : file, - expire : gtype.NewInt64(), - }, nil } // 关闭指针池 func (p *Pool) Close() { - p.closed.Set(true) + p.pool.Close() } // 获得底层文件指针 -func (f *PoolItem) File() *os.File { - return f.file +func (f *File) Close() { + f.pool.pool.Put(f) } - -// 关闭指针链接(软关闭)，放回池中重复使用 -func (f *PoolItem) Close() { - f.expire.Set(gtime.Second() + int64(f.pool.idle)) - f.pool.list.PushBack(f) -} - -// 销毁指针 -func (f *PoolItem) destroy() { - f.file.Close() -} \ No newline at end of file diff --git a/g/os/glog/glog_logger.go b/g/os/glog/glog_logger.go index 33a8487ef..8916ff2bb 100644 --- a/g/os/glog/glog_logger.go +++ b/g/os/glog/glog_logger.go @@ -57,10 +57,10 @@ func (l *Logger) GetIO() io.Writer { } // 获取默认的文件IO -func (l *Logger) getFileByPool() *gfilepool.PoolItem { +func (l *Logger) getFileByPool() *gfilepool.File { if path := l.path.Val(); path != "" { fpath := path + gfile.Separator + time.Now().Format("2006-01-02.log") - if fp, err := gfilepool.OpenWithPool(fpath, gDEFAULT_FILE_POOL_FLAGS, 86400); err == nil { + if fp, err := gfilepool.OpenWithPool(fpath, gDEFAULT_FILE_POOL_FLAGS, 0666, 86400000); err == nil { return fp } else { fmt.Fprintln(os.Stderr, err) @@ -104,21 +104,22 @@ func (l *Logger) SetStdPrint(open bool) { // 这里的写锁保证统一时刻只会写入一行日志，防止串日志的情况 func (l *Logger) print(defaultIO io.Writer, s string) { - w := l.GetIO() - if w == nil { - if v := l.getFileByPool(); v != nil { - w = v.File() - // 同时输出到文件和终端 @author zseeker + writer := l.GetIO() + if writer == nil { + if f := l.getFileByPool(); f != nil { + writer = f + // 同时输出到文件和终端 + // @author zseeker if l.stdprint.Val() { - w = io.MultiWriter(w, defaultIO) + writer = io.MultiWriter(writer, defaultIO) } - defer v.Close() + defer f.Close() } else { - w = defaultIO + writer = defaultIO } } l.mu.Lock() - fmt.Fprint(w, l.format(s)) + fmt.Fprint(writer, l.format(s)) l.mu.Unlock() } diff --git a/geg/other/test.go b/geg/other/test.go index d7479fe51..b599cbbee 100644 --- a/geg/other/test.go +++ b/geg/other/test.go @@ -1,23 +1,11 @@ package main import ( - "fmt" - "time" + "gitee.com/johng/gf/g/os/gfile" ) -var c = make(chan int, 10) - -func Chan() chan int { - fmt.Println("yes chan") - return c -} - func main() { - for { - select { - case <- Chan(): - default: - time.Sleep(time.Second) - } - } + gfile.PutContentsAppend("/tmp/test", "1") + gfile.PutContentsAppend("/tmp/test", "2") + gfile.PutContentsAppend("/tmp/test", "3") } \ No newline at end of file