From 92926db8dc0c80c8582c06cb34e12e2f3b11b9d0 Mon Sep 17 00:00:00 2001 From: john Date: Mon, 17 Sep 2018 18:43:59 +0800 Subject: [PATCH] =?UTF-8?q?=E6=94=B9=E8=BF=9Bgfpool=EF=BC=8C=E6=94=B9?= =?UTF-8?q?=E8=BF=9Bgmap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- g/container/gmap/string_interface_map.go | 17 ++++ g/os/gcache/gcache_mem_cache.go | 17 ++-- g/os/gfile/gfile.go | 5 +- g/os/gfpool/gfpool.go | 124 +++++++++++++++++++++++ g/os/glog/glog_logger.go | 5 +- 5 files changed, 158 insertions(+), 10 deletions(-) create mode 100644 g/os/gfpool/gfpool.go diff --git a/g/container/gmap/string_interface_map.go b/g/container/gmap/string_interface_map.go index 796cf3a30..144152b4d 100644 --- a/g/container/gmap/string_interface_map.go +++ b/g/container/gmap/string_interface_map.go @@ -81,6 +81,23 @@ func (this *StringInterfaceMap) GetWithDefault(key string, value interface{}) in return val } +func (this *StringInterfaceMap) GetOrSetFunc(key string, f func() interface{}) interface{} { + if v := this.Get(key); v == nil { + this.mu.Lock() + defer this.mu.Unlock() + // 写锁二次检索确认 + if v, ok := this.m[key]; !ok { + v = f() + this.m[key] = v + return v + } else { + return v + } + } else { + return v + } +} + // 删除键值对 func (this *StringInterfaceMap) Remove(key string) { this.mu.Lock() diff --git a/g/os/gcache/gcache_mem_cache.go b/g/os/gcache/gcache_mem_cache.go index a760e26e9..9769983db 100644 --- a/g/os/gcache/gcache_mem_cache.go +++ b/g/os/gcache/gcache_mem_cache.go @@ -46,7 +46,7 @@ type memCacheEvent struct { const ( // 这个数值不能太大，否则初始化会占用太多无意义的内存 // 60W，这个数值是创始人的机器上支持基准测试的参考结果 - gEVENT_QUEUE_SIZE = 600000 + gEVENT_QUEUE_SIZE = 10000000 ) // 创建底层的缓存对象 @@ -91,7 +91,12 @@ func (c *memCache) getOrNewExpireSet(expire int64) *gset.Set { if ekset := c.getExpireSet(expire); ekset == nil { set := gset.New() c.smu.Lock() - c.eksets[expire] = set + // 二次检索确认 + if ekset, ok := c.eksets[expire]; !ok { + c.eksets[expire] = set + } else { + set = ekset + } c.smu.Unlock() return set } else { @@ -145,6 +150,10 @@ func (c *memCache) Get(key interface{}) interface{} { item, ok := c.data[key] c.dmu.RUnlock() if ok && !item.IsExpired() { + // LRU(Least Recently Used)操作记录 + if c.cap.Val() > 0 { + c.lru.Push(key) + } return item.v } return nil @@ -263,10 +272,6 @@ func (c *memCache) autoSyncLoop() { c.emu.Lock() c.ekmap[item.k] = newe c.emu.Unlock() - // LRU操作记录(只有新增和修改操作才会记录到LRU管理对象中，删除不会) - if c.cap.Val() > 0 { - c.lru.Push(item.k) - } } } diff --git a/g/os/gfile/gfile.go b/g/os/gfile/gfile.go index c606dafdb..c842706fa 100644 --- a/g/os/gfile/gfile.go +++ b/g/os/gfile/gfile.go @@ -24,6 +24,7 @@ import ( "gitee.com/johng/gf/g/container/gtype" "sort" "gitee.com/johng/gf/g/util/gconv" + "gitee.com/johng/gf/g/os/gfpool" ) // 封装了常用的文件操作方法，如需更详细的文件控制，请查看官方os包 @@ -382,8 +383,8 @@ func putContents(path string, data []byte, flag int, perm os.FileMode) error { return err } } - // 创建/打开文件 - f, err := os.OpenFile(path, flag, perm) + // 创建/打开文件，使用文件指针池，默认60秒 + f, err := gfpool.OpenFile(path, flag, perm, 60000) if err != nil { return err } diff --git a/g/os/gfpool/gfpool.go b/g/os/gfpool/gfpool.go new file mode 100644 index 000000000..8cfc63f73 --- /dev/null +++ b/g/os/gfpool/gfpool.go @@ -0,0 +1,124 @@ +// Copyright 2017 gf Author(https://gitee.com/johng/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://gitee.com/johng/gf. + +// 文件指针池 +package gfpool + +import ( + "os" + "sync" + "gitee.com/johng/gf/g/container/gmap" + "gitee.com/johng/gf/g/container/gpool" + "fmt" +) + +// 文件指针池 +type Pool struct { + pool *gpool.Pool // 底层对象池 +} + +// 文件指针池指针 +type File struct { + os.File // 底层文件指针 + mu sync.RWMutex // 互斥锁 + pool *Pool // 所属池 + flag int // 打开标志 + perm os.FileMode // 打开权限 + path string // 绝对路径 +} + +// 全局指针池，expire < 0表示不过期，expire = 0表示使用完立即回收，expire > 0表示超时回收 +var pools = gmap.NewStringInterfaceMap() + +// 获得文件对象，并自动创建指针池 +func Open(path string, flag int, perm os.FileMode, expire...int) (*File, error) { + fpExpire := 0 + if len(expire) > 0 { + fpExpire = expire[0] + } + key := fmt.Sprintf("%s&%d&%d&%d", path, flag, expire, perm) + result := pools.Get(key) + if result != nil { + return result.(*Pool).File() + } + pool := New(path, flag, perm, fpExpire) + pools.Set(key, pool) + return pool.File() +} + +func OpenFile(path string, flag int, perm os.FileMode, expire...int) (*File, error) { + return Open(path, flag, perm, expire...) +} + +// 创建一个文件指针池，expire = 0表示不过期，expire < 0表示使用完立即回收，expire > 0表示超时回收，默认值为0不过期 +func New(path string, flag int, perm os.FileMode, expire...int) *Pool { + fpExpire := 0 + if len(expire) > 0 { + fpExpire = expire[0] + } + p := &Pool {} + p.pool = gpool.New(fpExpire, func() (interface{}, error) { + file, err := os.OpenFile(path, flag, perm) + if err != nil { + return nil, err + } + return &File{ + File : *file, + pool : p, + flag : flag, + perm : perm, + path : path, + }, nil + }) + p.pool.SetExpireFunc(func(i interface{}) { + i.(*File).File.Close() + }) + return p +} + +// 获得一个文件打开指针 +func (p *Pool) File() (*File, error) { + if v, err := p.pool.Get(); err != nil { + return nil, err + } else { + f := v.(*File) + if f.flag & os.O_APPEND > 0 { + if _, err := f.Seek(0, 2); err != nil { + return nil, err + } + } + if f.flag & os.O_CREATE > 0 { + _, err := f.Stat() + if err != nil { + if !os.IsExist(err) { + if file, err := os.OpenFile(f.path, f.flag, f.perm); err != nil { + return nil, err + } else { + f.File = *file + } + } + } + } + if f.flag & os.O_TRUNC > 0 { + if err := f.Truncate(0); err != nil { + return nil, err + } + } + return f, nil + } +} + +// 关闭指针池(返回error是标准库io.ReadWriteCloser接口实现) +func (p *Pool) Close() error { + p.pool.Close() + return nil +} + +// 获得底层文件指针(返回error是标准库io.ReadWriteCloser接口实现) +func (f *File) Close() error { + f.pool.pool.Put(f) + return nil +} diff --git a/g/os/glog/glog_logger.go b/g/os/glog/glog_logger.go index 2d764f8eb..27bffada7 100644 --- a/g/os/glog/glog_logger.go +++ b/g/os/glog/glog_logger.go @@ -19,6 +19,7 @@ import ( "gitee.com/johng/gf/g/util/gregex" "gitee.com/johng/gf/g/container/gtype" "gitee.com/johng/gf/g/os/gmlock" + "gitee.com/johng/gf/g/os/gfpool" ) const ( @@ -100,10 +101,10 @@ func (l *Logger) GetIO() io.Writer { } // 获取默认的文件IO -func (l *Logger) getFilePointer() *os.File { +func (l *Logger) getFilePointer() *gfpool.File { if path := l.path.Val(); path != "" { fpath := path + gfile.Separator + time.Now().Format("2006-01-02.log") - if fp, err := os.OpenFile(fpath, gDEFAULT_FILE_POOL_FLAGS, 0666); err == nil { + if fp, err := gfpool.Open(fpath, gDEFAULT_FILE_POOL_FLAGS, 0666); err == nil { return fp } else { fmt.Fprintln(os.Stderr, err)