diff --git a/g/container/glist/glist.go b/g/container/glist/glist.go index 6361b566b..2a43e699e 100644 --- a/g/container/glist/glist.go +++ b/g/container/glist/glist.go @@ -22,7 +22,7 @@ type List struct { // 获得一个变长链表指针 func New(safe...bool) *List { - return &List{ + return &List { mu : rwmutex.New(safe...), list : list.New(), } diff --git a/g/container/glist/glist_test.go b/g/container/glist/glist_test.go index e8d41d438..24b74b60c 100644 --- a/g/container/glist/glist_test.go +++ b/g/container/glist/glist_test.go @@ -14,31 +14,31 @@ import ( var l = New() -func BenchmarkPushBack(b *testing.B) { +func Benchmark_PushBack(b *testing.B) { for i := 0; i < b.N; i++ { l.PushBack(i) } } -func BenchmarkPopFront(b *testing.B) { +func Benchmark_PopFront(b *testing.B) { for i := 0; i < b.N; i++ { l.PopFront() } } -func BenchmarkPushFront(b *testing.B) { +func Benchmark_PushFront(b *testing.B) { for i := 0; i < b.N; i++ { l.PushFront(i) } } -func BenchmarkPopBack(b *testing.B) { +func Benchmark_PopBack(b *testing.B) { for i := 0; i < b.N; i++ { l.PopBack() } } -func BenchmarkLen(b *testing.B) { +func Benchmark_Len(b *testing.B) { for i := 0; i < b.N; i++ { l.Len() } diff --git a/g/container/gset/gset_test.go b/g/container/gset/gset_test.go index f032ad968..f127748e9 100644 --- a/g/container/gset/gset_test.go +++ b/g/container/gset/gset_test.go @@ -18,55 +18,55 @@ var ints = gset.NewIntSet() var itfs = gset.NewInterfaceSet() var strs = gset.NewStringSet() -func BenchmarkIntSet_Add(b *testing.B) { +func Benchmark_IntSet_Add(b *testing.B) { for i := 0; i < b.N; i++ { ints.Add(i) } } -func BenchmarkIntSet_Contains(b *testing.B) { +func Benchmark_IntSet_Contains(b *testing.B) { for i := 0; i < b.N; i++ { ints.Contains(i) } } -func BenchmarkIntSet_Remove(b *testing.B) { +func Benchmark_IntSet_Remove(b *testing.B) { for i := 0; i < b.N; i++ { ints.Remove(i) } } -func BenchmarkInterfaceSet_Add(b *testing.B) { +func Benchmark_InterfaceSet_Add(b *testing.B) { for i := 0; i < b.N; i++ { itfs.Add(i) } } -func BenchmarkInterfaceSet_Contains(b *testing.B) { +func Benchmark_InterfaceSet_Contains(b *testing.B) { for i := 0; i < b.N; i++ { itfs.Contains(i) } } -func BenchmarkInterfaceSet_Remove(b *testing.B) { +func Benchmark_InterfaceSet_Remove(b *testing.B) { for i := 0; i < b.N; i++ { itfs.Remove(i) } } -func BenchmarkStringSet_Add(b *testing.B) { +func Benchmark_StringSet_Add(b *testing.B) { for i := 0; i < b.N; i++ { strs.Add(strconv.Itoa(i)) } } -func BenchmarkStringSet_Contains(b *testing.B) { +func Benchmark_StringSet_Contains(b *testing.B) { for i := 0; i < b.N; i++ { strs.Contains(strconv.Itoa(i)) } } -func BenchmarkStringSet_Remove(b *testing.B) { +func Benchmark_StringSet_Remove(b *testing.B) { for i := 0; i < b.N; i++ { strs.Remove(strconv.Itoa(i)) } diff --git a/g/container/gset/gset_unsafe_test.go b/g/container/gset/gset_unsafe_test.go new file mode 100644 index 000000000..94b379738 --- /dev/null +++ b/g/container/gset/gset_unsafe_test.go @@ -0,0 +1,73 @@ +// Copyright 2017 gf Author(https://gitee.com/johng/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://gitee.com/johng/gf. + +// go test *.go -bench=".*" + +package gset_test + +import ( + "testing" + "strconv" + "gitee.com/johng/gf/g/container/gset" +) + +var intsUnsafe = gset.NewIntSet(false) +var itfsUnsafe = gset.NewInterfaceSet(false) +var strsUnsafe = gset.NewStringSet(false) + +func Benchmark_Unsafe_IntSet_Add(b *testing.B) { + for i := 0; i < b.N; i++ { + intsUnsafe.Add(i) + } +} + +func Benchmark_Unsafe_IntSet_Contains(b *testing.B) { + for i := 0; i < b.N; i++ { + intsUnsafe.Contains(i) + } +} + +func Benchmark_Unsafe_IntSet_Remove(b *testing.B) { + for i := 0; i < b.N; i++ { + intsUnsafe.Remove(i) + } +} + +func Benchmark_Unsafe_InterfaceSet_Add(b *testing.B) { + for i := 0; i < b.N; i++ { + itfsUnsafe.Add(i) + } +} + +func Benchmark_Unsafe_InterfaceSet_Contains(b *testing.B) { + for i := 0; i < b.N; i++ { + itfsUnsafe.Contains(i) + } +} + +func Benchmark_Unsafe_InterfaceSet_Remove(b *testing.B) { + for i := 0; i < b.N; i++ { + itfsUnsafe.Remove(i) + } +} + +func Benchmark_Unsafe_StringSet_Add(b *testing.B) { + for i := 0; i < b.N; i++ { + strsUnsafe.Add(strconv.Itoa(i)) + } +} + +func Benchmark_Unsafe_StringSet_Contains(b *testing.B) { + for i := 0; i < b.N; i++ { + strsUnsafe.Contains(strconv.Itoa(i)) + } +} + +func Benchmark_Unsafe_StringSet_Remove(b *testing.B) { + for i := 0; i < b.N; i++ { + strsUnsafe.Remove(strconv.Itoa(i)) + } +} \ No newline at end of file diff --git a/g/encoding/ghash/ghash_test.go b/g/encoding/ghash/ghash_test.go index 78214bf6f..7720f4f31 100644 --- a/g/encoding/ghash/ghash_test.go +++ b/g/encoding/ghash/ghash_test.go @@ -9,103 +9,106 @@ package ghash_test import ( - "testing" "gitee.com/johng/gf/g/encoding/ghash" - "gitee.com/johng/gf/g/encoding/gbinary" + "testing" +) + +var ( + str = []byte("This is the test string for hash.") ) func BenchmarkBKDRHash(b *testing.B) { for i := 0; i < b.N; i++ { - ghash.BKDRHash(gbinary.EncodeInt(i)) + ghash.BKDRHash(str) } } func BenchmarkBKDRHash64(b *testing.B) { for i := 0; i < b.N; i++ { - ghash.BKDRHash64(gbinary.EncodeInt(i)) + ghash.BKDRHash64(str) } } func BenchmarkSDBMHash(b *testing.B) { for i := 0; i < b.N; i++ { - ghash.SDBMHash(gbinary.EncodeInt(i)) + ghash.SDBMHash(str) } } func BenchmarkSDBMHash64(b *testing.B) { for i := 0; i < b.N; i++ { - ghash.SDBMHash64(gbinary.EncodeInt(i)) + ghash.SDBMHash64(str) } } func BenchmarkRSHash(b *testing.B) { for i := 0; i < b.N; i++ { - ghash.RSHash(gbinary.EncodeInt(i)) + ghash.RSHash(str) } } func BenchmarkSRSHash64(b *testing.B) { for i := 0; i < b.N; i++ { - ghash.RSHash64(gbinary.EncodeInt(i)) + ghash.RSHash64(str) } } func BenchmarkJSHash(b *testing.B) { for i := 0; i < b.N; i++ { - ghash.JSHash(gbinary.EncodeInt(i)) + ghash.JSHash(str) } } func BenchmarkJSHash64(b *testing.B) { for i := 0; i < b.N; i++ { - ghash.JSHash64(gbinary.EncodeInt(i)) + ghash.JSHash64(str) } } func BenchmarkPJWHash(b *testing.B) { for i := 0; i < b.N; i++ { - ghash.PJWHash(gbinary.EncodeInt(i)) + ghash.PJWHash(str) } } func BenchmarkPJWHash64(b *testing.B) { for i := 0; i < b.N; i++ { - ghash.PJWHash64(gbinary.EncodeInt(i)) + ghash.PJWHash64(str) } } func BenchmarkELFHash(b *testing.B) { for i := 0; i < b.N; i++ { - ghash.ELFHash(gbinary.EncodeInt(i)) + ghash.ELFHash(str) } } func BenchmarkELFHash64(b *testing.B) { for i := 0; i < b.N; i++ { - ghash.ELFHash64(gbinary.EncodeInt(i)) + ghash.ELFHash64(str) } } func BenchmarkDJBHash(b *testing.B) { for i := 0; i < b.N; i++ { - ghash.DJBHash(gbinary.EncodeInt(i)) + ghash.DJBHash(str) } } func BenchmarkDJBHash64(b *testing.B) { for i := 0; i < b.N; i++ { - ghash.DJBHash64(gbinary.EncodeInt(i)) + ghash.DJBHash64(str) } } func BenchmarkAPHash(b *testing.B) { for i := 0; i < b.N; i++ { - ghash.APHash(gbinary.EncodeInt(i)) + ghash.APHash(str) } } func BenchmarkAPHash64(b *testing.B) { for i := 0; i < b.N; i++ { - ghash.APHash64(gbinary.EncodeInt(i)) + ghash.APHash64(str) } } diff --git a/g/os/gcache/gcache.go b/g/os/gcache/gcache.go index 379779a7b..e6478f564 100644 --- a/g/os/gcache/gcache.go +++ b/g/os/gcache/gcache.go @@ -1,17 +1,12 @@ -// Copyright 2017 gf Author(https://gitee.com/johng/gf). All Rights Reserved. +// Copyright 2017-2018 gf Author(https://gitee.com/johng/gf). All Rights Reserved. // // This Source Code Form is subject to the terms of the MIT License. // If a copy of the MIT was not distributed with this file, // You can obtain one at https://gitee.com/johng/gf. -// 单进程高速缓存. +// 并发安全的单进程高速缓存. package gcache -const ( - // 当数据不过期时,默认设置的过期属性值,相当于:math.MaxInt64/1000000 - gDEFAULT_MAX_EXPIRE = 9223372036854 -) - // 全局缓存管理对象 var cache = New() diff --git a/g/os/gcache/gcache_cache.go b/g/os/gcache/gcache_cache.go index 13f2c491b..ad65e3332 100644 --- a/g/os/gcache/gcache_cache.go +++ b/g/os/gcache/gcache_cache.go @@ -22,8 +22,7 @@ func New(lruCap...int) *Cache { c := &Cache { memCache : newMemCache(lruCap...), } - go c.autoSyncLoop() - go c.autoClearLoop() + go c.autoLoop() return c } diff --git a/g/os/gcache/gcache_mem_cache.go b/g/os/gcache/gcache_mem_cache.go index 265693a32..5529cf7ff 100644 --- a/g/os/gcache/gcache_mem_cache.go +++ b/g/os/gcache/gcache_mem_cache.go @@ -7,33 +7,37 @@ package gcache import ( - "math" - "gitee.com/johng/gf/g/container/gset" - "gitee.com/johng/gf/g/os/gtime" - "sync" - "gitee.com/johng/gf/g/util/gconv" "gitee.com/johng/gf/g/container/glist" + "gitee.com/johng/gf/g/container/gset" + "gitee.com/johng/gf/g/container/gtype" + "gitee.com/johng/gf/g/os/gtime" + "gitee.com/johng/gf/g/util/gconv" + "math" + "sync" "time" ) + // 缓存对象 type memCache struct { - dmu sync.RWMutex // data锁(自定义锁的目的是除去键值的断言转换造成的性能损耗) - emu sync.RWMutex // ekmap锁(expire key map) - smu sync.RWMutex // eksets锁(expire key sets) - lru *memCacheLru // LRU缓存限制(只有限定池大小时才启用) - cap int // 控制缓存池大小,超过大小则按照LRU算法进行缓存过期处理(默认为0表示不进行限制) - data map[interface{}]memCacheItem // 缓存数据(所有的缓存数据存放哈希表) - ekmap map[interface{}]int64 // 键名对应的分组过期时间(用于相同键名过期时间快速更新),键值为10秒级时间戳 - eksets map[int64]*gset.Set // 分组过期时间对应的键名列表(用于自动过期快速删除),键值为10秒级时间戳 - eventList *glist.List // 异步处理队列 - lruGetList *glist.List // 获取方法的LRU列表 - stopChan chan struct{} // 关闭时间通知 + dataMu sync.RWMutex + expireTimeMu sync.RWMutex + expireSetMu sync.RWMutex + + cap int // 控制缓存池大小,超过大小则按照LRU算法进行缓存过期处理(默认为0表示不进行限制) + data map[interface{}]memCacheItem // 缓存数据(所有的缓存数据存放哈希表) + expireTimes map[interface{}]int64 // 键名对应的分组过期时间(用于相同键名过期时间快速更新),键值为10秒级时间戳 + expireSets map[int64]*gset.Set // 分组过期时间对应的键名列表(用于自动过期快速删除),键值为10秒级时间戳 + + lru *memCacheLru // LRU缓存限制(只有限定cap池大小时才启用) + lruGetList *glist.List // Get操作的LRU记录 + eventList *glist.List // 异步处理队列 + closed *gtype.Bool // 关闭事件通知 } // 缓存数据项 type memCacheItem struct { - v interface{} // 缓存键值 + v interface{} // 键值 e int64 // 过期时间 } @@ -43,81 +47,82 @@ type memCacheEvent struct { e int64 // 过期时间 } +const ( + // 当数据不过期时,默认设置的过期属性值,相当于:math.MaxInt64/1000000 + gDEFAULT_MAX_EXPIRE = 9223372036854 +) + + // 创建底层的缓存对象 func newMemCache(lruCap...int) *memCache { c := &memCache { - lru : newMemCacheLru(), - data : make(map[interface{}]memCacheItem), - ekmap : make(map[interface{}]int64), - eksets : make(map[int64]*gset.Set), - stopChan : make(chan struct{}), - eventList : glist.New(), - lruGetList : glist.New(), + lruGetList : glist.New(), + data : make(map[interface{}]memCacheItem), + expireTimes : make(map[interface{}]int64), + expireSets : make(map[int64]*gset.Set), + eventList : glist.New(), + closed : gtype.NewBool(), } if len(lruCap) > 0 { - c.cap = lruCap[0] + c.lru = newMemCacheLru(c) + c.cap = lruCap[0] } return c } -// 计算过期缓存的键名(将毫秒换算成秒的整数毫秒) +// 计算过期缓存的键名(将毫秒换算成秒的整数毫秒,按照10秒进行分组) func (c *memCache) makeExpireKey(expire int64) int64 { return int64(math.Ceil(float64(expire/10000) + 1)*10000) } // 获取一个过期键名存放Set,如果没有则返回nil -func (c *memCache) getExpireSet(expire int64) *gset.Set { - c.smu.RLock() - if ekset, ok := c.eksets[expire]; ok { - c.smu.RUnlock() - return ekset - } - c.smu.RUnlock() +func (c *memCache) getExpireSet(expire int64) (expireSet *gset.Set) { + c.expireSetMu.RLock() + expireSet, _ = c.expireSets[expire] + c.expireSetMu.RUnlock() return nil } // 获取或者创建一个过期键名存放Set(由于是异步单线程执行,因此不会出现创建set时的覆盖问题) -func (c *memCache) getOrNewExpireSet(expire int64) *gset.Set { - if ekset := c.getExpireSet(expire); ekset == nil { - set := gset.New() - c.smu.Lock() - // 二次检索确认 - if ekset, ok := c.eksets[expire]; !ok { - c.eksets[expire] = set +func (c *memCache) getOrNewExpireSet(expire int64) (expireSet *gset.Set) { + if expireSet = c.getExpireSet(expire); expireSet == nil { + expireSet = gset.New() + c.expireSetMu.Lock() + // 写锁二次检索确认 + if es, ok := c.expireSets[expire]; ok { + expireSet = es } else { - set = ekset + c.expireSets[expire] = expireSet } - c.smu.Unlock() - return set - } else { - return ekset + c.expireSetMu.Unlock() } + return } // 设置kv缓存键值对,过期时间单位为毫秒,expire<=0表示不过期 func (c *memCache) Set(key interface{}, value interface{}, expire int) { - expireTimestamp := c.getInternalExpire(expire) - c.dmu.Lock() - c.data[key] = memCacheItem{v : value, e : expireTimestamp} - c.dmu.Unlock() - c.eventList.PushBack(memCacheEvent{k : key, e : expireTimestamp}) + expireTime := c.getInternalExpire(expire) + c.dataMu.Lock() + c.data[key] = memCacheItem{v : value, e : expireTime} + c.dataMu.Unlock() + c.eventList.PushBack(&memCacheEvent{k : key, e : expireTime}) } // 设置kv缓存键值对,内部会对键名的存在性使用写锁进行二次检索确认,如果存在则不再写入;返回键名对应的键值。 // 在高并发下有用,防止数据写入的并发逻辑错误。 func (c *memCache) doSetWithLockCheck(key interface{}, value interface{}, expire int) interface{} { expireTimestamp := c.getInternalExpire(expire) - c.dmu.Lock() + c.dataMu.Lock() if v, ok := c.data[key]; ok && !v.IsExpired() { - c.dmu.Unlock() + c.dataMu.Unlock() return v } if f, ok := value.(func() interface {}); ok { value = f() } c.data[key] = memCacheItem{v : value, e : expireTimestamp} - c.dmu.Unlock() - c.eventList.PushBack(memCacheEvent{k : key, e : expireTimestamp}) + c.dataMu.Unlock() + c.eventList.PushBack(&memCacheEvent{k : key, e : expireTimestamp}) return value } @@ -140,23 +145,23 @@ func (c *memCache) SetIfNotExist(key interface{}, value interface{}, expire int) } // 批量设置 -func (c *memCache) BatchSet(data map[interface{}]interface{}, expire int) { - expireTimestamp := c.getInternalExpire(expire) +func (c *memCache) BatchSet(data map[interface{}]interface{}, expire int) { + expireTime := c.getInternalExpire(expire) for k, v := range data { - c.dmu.Lock() - c.data[k] = memCacheItem{v: v, e: expireTimestamp} - c.dmu.Unlock() - c.eventList.PushBack(memCacheEvent{k: k, e: expireTimestamp}) + c.dataMu.Lock() + c.data[k] = memCacheItem{v: v, e: expireTime} + c.dataMu.Unlock() + c.eventList.PushBack(&memCacheEvent{k: k, e: expireTime}) } } // 获取指定键名的值 func (c *memCache) Get(key interface{}) interface{} { - c.dmu.RLock() + c.dataMu.RLock() item, ok := c.data[key] - c.dmu.RUnlock() + c.dataMu.RUnlock() if ok && !item.IsExpired() { - // LRU(Least Recently Used)操作记录 + // 增加LRU(Least Recently Used)操作记录 if c.cap > 0 { c.lruGetList.PushBack(key) } @@ -199,35 +204,37 @@ func (c *memCache) Contains(key interface{}) bool { } // 删除指定键值对,并返回被删除的键值 -func (c *memCache) Remove(key interface{}) interface{} { - c.dmu.Lock() +func (c *memCache) Remove(key interface{}) (value interface{}) { + c.dataMu.RLock() item, ok := c.data[key] + c.dataMu.RUnlock() if ok { + value = item.v + c.dataMu.Lock() delete(c.data, key) + c.dataMu.Unlock() + c.eventList.PushBack(&memCacheEvent{k: key, e: gtime.Millisecond() - 1000}) } - c.dmu.Unlock() - return item.v + return } // 批量删除键值对,并返回被删除的键值对数据 func (c *memCache) BatchRemove(keys []interface{}) { - c.dmu.Lock() for _, key := range keys { - delete(c.data, key) + c.Remove(key) } - c.dmu.Unlock() } // 获得所有的键名,组成数组返回 func (c *memCache) Keys() []interface{} { keys := make([]interface{}, 0) - c.dmu.RLock() + c.dataMu.RLock() for k, v := range c.data { if !v.IsExpired() { keys = append(keys, k) } } - c.dmu.RUnlock() + c.dataMu.RUnlock() return keys } @@ -239,129 +246,118 @@ func (c *memCache) KeyStrings() []string { // 获得所有的值,组成数组返回 func (c *memCache) Values() []interface{} { values := make([]interface{}, 0) - c.dmu.RLock() + c.dataMu.RLock() for _, v := range c.data { if !v.IsExpired() { values = append(values, v.v) } } - c.dmu.RUnlock() + c.dataMu.RUnlock() return values } // 获得缓存对象的键值对数量 -func (c *memCache) Size() int { - c.dmu.RLock() - length := len(c.data) - c.dmu.RUnlock() - return length +func (c *memCache) Size() (size int) { + c.dataMu.RLock() + size = len(c.data) + c.dataMu.RUnlock() + return } // 删除缓存对象 func (c *memCache) Close() { - close(c.stopChan) + c.closed.Set(true) c.lru.Close() } -// 数据自动同步循环 -func (c *memCache) autoSyncLoop() { - newe := int64(0) +// 数据异步任务循环: +// 1、将事件列表中的数据异步处理,并同步结果到expireTimes和expireSets属性中; +// 2、清理过期键值对数据; +func (c *memCache) autoLoop() { + event := (*memCacheEvent)(nil) + oldExpireTime := int64(0) + newExpireTime := int64(0) for { - select { - case <-c.stopChan: - return - default: - for { - v := c.eventList.PopFront() - if v == nil { - break - } - item := v.(memCacheEvent) - nowm := gtime.Millisecond() - // 如果用户设置的时间比当前时间还小,那么表示要自动清除了, - // 这里赋值一个当前时间-10秒的时间,在自动清理的goroutine中会自动检测删除该key - if item.e < nowm { - newe = c.makeExpireKey(nowm) - 10000 - } else { - newe = c.makeExpireKey(item.e) - } - // 添加该key到对应的过期集合中 - // 注意:这里不需要检查存在性, - // 因为在key过期的时候,会和原始的键值对中的过期时间做核对。 - c.getOrNewExpireSet(newe).Add(item.k) - // 重新设置对应键名的过期时间 - c.emu.Lock() - c.ekmap[item.k] = newe - c.emu.Unlock() - // LRU(Least Recently Used)操作记录 - if c.cap > 0 { - c.lru.Push(item.k) - } - } - if c.cap > 0 { - // 优先级高的lru key放后面,读取列表 - for { - if v := c.lruGetList.PopFront(); v != nil { - c.lru.Push(v) - } else { - break - } - } - } - time.Sleep(10 * time.Second) + if c.closed.Val() { + return + } + // ======================== + // 数据同步处理 + // ======================== + for { + v := c.eventList.PopFront() + if v == nil { + break + } + event = v.(*memCacheEvent) + // 获得旧的过期时间分组 + c.expireTimeMu.RLock() + oldExpireTime = c.expireTimes[event.k] + c.expireTimeMu.RUnlock() + // 计算新的过期时间分组 + newExpireTime = c.makeExpireKey(event.e) + if newExpireTime != oldExpireTime { + c.getOrNewExpireSet(newExpireTime).Add(event.k) + if oldExpireTime != 0 { + c.getOrNewExpireSet(oldExpireTime).Remove(event.k) + } + // 重新设置对应键名的过期时间 + c.expireTimeMu.Lock() + c.expireTimes[event.k] = newExpireTime + c.expireTimeMu.Unlock() + } + // 写入操作也会增加到LRU(Least Recently Used)操作记录 + if c.cap > 0 { + c.lru.Push(event.k) + } + } + // 异步处理读取操作的LRU列表 + if c.cap > 0 && c.lruGetList.Len() > 0 { + for { + if v := c.lruGetList.PopFront(); v != nil { + c.lru.Push(v) + } else { + break + } + } + } + // ======================== + // 缓存过期处理 + // ======================== + ek := c.makeExpireKey(gtime.Millisecond()) + eks := []int64{ek - 10000, ek - 20000, ek - 30000, ek - 40000, ek - 50000} + for _, expireTime := range eks { + if expireSet := c.getExpireSet(expireTime); expireSet != nil { + // 遍历Set,执行数据过期删除 + expireSet.Iterator(func(key interface{}) bool { + return c.clearByKey(key) + }) + // Set数据处理完之后删除该Set + c.expireSetMu.Lock() + delete(c.expireSets, expireTime) + c.expireSetMu.Unlock() + } } - } -} -// LRU缓存淘汰处理+自动清理过期键值对 -// 每隔10秒清除过去60秒的键值对数据 -func (c *memCache) autoClearLoop() { - for { - select { - case <- c.stopChan: - return - default: - // 缓存过期处理 - ek := c.makeExpireKey(gtime.Millisecond()) - eks := []int64{ek - 10000, ek - 20000, ek - 30000, ek - 40000, ek - 50000, ek - 60000} - for _, v := range eks { - if ekset := c.getExpireSet(v); ekset != nil { - ekset.Iterator(func(v interface{}) bool { - return c.clearByKey(v) - }) - } - // 数据处理完之后从集合中删除该时间段 - c.smu.Lock() - delete(c.eksets, v) - c.smu.Unlock() - } - // LRU缓存淘汰清理 - if c.cap > 0 { - for i := c.Size() - c.cap; i > 0; i-- { - if s := c.lru.Pop(); s != nil { - c.clearByKey(s, true) - } - } - } - time.Sleep(10*time.Second) - } - } + // 每间隔1秒批量处理一次 + time.Sleep(time.Second) + } } // 删除对应键名的缓存数据 func (c *memCache) clearByKey(key interface{}, force...bool) bool { // 删除缓存数据 - c.dmu.Lock() + c.dataMu.Lock() // 删除核对,真正的过期才删除 if item, ok := c.data[key]; (ok && item.IsExpired()) || (len(force) > 0 && force[0]) { delete(c.data, key) } - c.dmu.Unlock() + c.dataMu.Unlock() // 删除异步处理数据项 - c.emu.Lock() - delete(c.ekmap, key) - c.emu.Unlock() + c.expireTimeMu.Lock() + delete(c.expireTimes, key) + c.expireTimeMu.Unlock() // 删除LRU管理对象中指定键名 c.lru.Remove(key) diff --git a/g/os/gcache/gcache_mem_cache_lru.go b/g/os/gcache/gcache_mem_cache_lru.go index 9073a53dd..a7fb5e9c5 100644 --- a/g/os/gcache/gcache_mem_cache_lru.go +++ b/g/os/gcache/gcache_mem_cache_lru.go @@ -7,27 +7,31 @@ package gcache import ( - "fmt" "container/list" + "fmt" "gitee.com/johng/gf/g/container/glist" - "gitee.com/johng/gf/g/container/gqueue" "gitee.com/johng/gf/g/container/gmap" - + "gitee.com/johng/gf/g/container/gtype" + "time" ) // LRU算法实现对象,底层双向链表使用了标准库的list.List type memCacheLru struct { - data *gmap.Map // 记录键名与链表中的位置项指针 - list *glist.List // 键名历史记录链表 - queue *gqueue.Queue // 事件队列 + cache *memCache // 所属Cache对象 + data *gmap.Map // 记录键名与链表中的位置项指针 + list *glist.List // 键名历史记录链表 + rawList *glist.List // 事件列表 + closed *gtype.Bool // 是否关闭 } // 创建LRU管理对象 -func newMemCacheLru() *memCacheLru { +func newMemCacheLru(cache *memCache) *memCacheLru { lru := &memCacheLru { - list : glist.New(), - data : gmap.New(), - queue : gqueue.New(), + cache : cache, + data : gmap.New(), + list : glist.New(), + rawList : glist.New(), + closed : gtype.NewBool(), } go lru.StartAutoLoop() return lru @@ -35,7 +39,7 @@ func newMemCacheLru() *memCacheLru { // 关闭LRU对象 func (lru *memCacheLru) Close() { - lru.queue.Close() + lru.closed.Set(true) } // 删除指定数据项 @@ -53,7 +57,7 @@ func (lru *memCacheLru) Size() int { // 添加LRU数据项 func (lru *memCacheLru) Push(key interface{}) { - lru.queue.Push(key) + lru.rawList.PushBack(key) } // 从链表尾删除LRU数据项,并返回对应数据 @@ -70,20 +74,34 @@ func (lru *memCacheLru) Print() { for _, v := range lru.list.FrontAll() { fmt.Printf("%v ", v) } + fmt.Println() } // 异步执行协程,将queue中的数据同步到list中 func (lru *memCacheLru) StartAutoLoop() { for { - if v := lru.queue.Pop(); v != nil { - // 删除对应链表项 - if v := lru.data.Get(v); v != nil { - lru.list.Remove(v.(*list.Element)) - } - // 将数据插入到链表头,并记录对应的链表项到哈希表中,便于检索 - lru.data.Set(v, lru.list.PushFront(v)) - } else { - break + if lru.closed.Val() { + return } + // 数据同步 + for { + if v := lru.rawList.PopFront(); v != nil { + // 删除对应链表项 + if v := lru.data.Get(v); v != nil { + lru.list.Remove(v.(*list.Element)) + } + // 将数据插入到链表头,并记录对应的链表项到哈希表中,便于检索 + lru.data.Set(v, lru.list.PushFront(v)) + } else { + break + } + } + // 数据清理 + for i := lru.Size() - lru.cache.cap; i > 0; i-- { + if s := lru.Pop(); s != nil { + lru.cache.clearByKey(s, true) + } + } + time.Sleep(time.Second) } } \ No newline at end of file diff --git a/g/util/gconv/gconv_map.go b/g/util/gconv/gconv_map.go index 6d5e31fab..2bb3f573c 100644 --- a/g/util/gconv/gconv_map.go +++ b/g/util/gconv/gconv_map.go @@ -12,7 +12,8 @@ import ( // 任意类型转换为 map[string]interface{} 类型, // 如果给定的输入参数i不是map类型,那么转换会失败,返回nil. -func Map(i interface{}) map[string]interface{} { +// 当i为struct对象时,第二个参数noTagCheck表示不检测json标签,否则将会使用json tag作为map的键名。 +func Map(i interface{}, noTagCheck...bool) map[string]interface{} { if i == nil { return nil } @@ -95,9 +96,13 @@ func Map(i interface{}) map[string]interface{} { m[String(k.Interface())] = rv.MapIndex(k).Interface() } } else if kind == reflect.Struct { - rt := rv.Type() + rt := rv.Type() + name := "" for i := 0; i < rv.NumField(); i++ { - m[rt.Field(i).Name] = rv.Field(i).Interface() + if name = rt.Field(i).Tag.Get("json"); name == "" { + name = rt.Field(i).Name + } + m[name] = rv.Field(i).Interface() } } else { return nil diff --git a/g/util/gstr/gstr_test.go b/g/util/gstr/gstr_test.go new file mode 100644 index 000000000..9e6b03701 --- /dev/null +++ b/g/util/gstr/gstr_test.go @@ -0,0 +1,34 @@ +// Copyright 2018 gf Author(https://gitee.com/johng/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://gitee.com/johng/gf. + +// go test *.go -bench=".*" + +package gstr_test + +import ( + "testing" +) + +var ( + str = "This is the test string for gstr." + bytes = []byte(str) +) + +func Benchmark_StringToBytes(b *testing.B) { + for i := 0; i < b.N; i++ { + if []byte(str) != nil { + + } + } +} + +func Benchmark_BytesToString(b *testing.B) { + for i := 0; i < b.N; i++ { + if string(bytes) != "" { + + } + } +} \ No newline at end of file diff --git a/geg/container/gqueue/gqueue2.go b/geg/container/gqueue/gqueue2.go new file mode 100644 index 000000000..44bda1b2e --- /dev/null +++ b/geg/container/gqueue/gqueue2.go @@ -0,0 +1,28 @@ +package main + +import ( + "fmt" + "time" + "gitee.com/johng/gf/g/os/gtime" + "gitee.com/johng/gf/g/container/gqueue" +) + +func main() { + q := gqueue.New() + // 数据生产者,每隔1秒往队列写数据 + gtime.SetInterval(time.Second, func() bool { + for i := 0; i < 10; i++ { + q.Push(i) + } + return true + }) + + // 消费者,不停读取队列数据并输出到终端 + for { + if v := q.Pop(); v != nil { + fmt.Println(" Pop:", v) + } else { + break + } + } +} diff --git a/geg/os/gcache/usage_lru.go b/geg/os/gcache/usage_lru.go index 438dea2fb..7d4864565 100644 --- a/geg/os/gcache/usage_lru.go +++ b/geg/os/gcache/usage_lru.go @@ -20,8 +20,8 @@ func main() { // 读取键名1,保证该键名是优先保留 fmt.Println(c.Get(1)) - // 等待一定时间后(默认10秒检查一次),元素会被按照从旧到新的顺序进行淘汰 - time.Sleep(10*time.Second) + // 等待一定时间后(默认1秒检查一次),元素会被按照从旧到新的顺序进行淘汰 + time.Sleep(2*time.Second) fmt.Println(c.Size()) fmt.Println(c.Keys()) } \ No newline at end of file diff --git a/geg/other/test.go b/geg/other/test.go index 7a79d5cf6..356b18d0a 100644 --- a/geg/other/test.go +++ b/geg/other/test.go @@ -1,39 +1,17 @@ package main import ( - "fmt" - "gitee.com/johng/gf/g/encoding/gparser" + "fmt" + "gitee.com/johng/gf/g/encoding/gparser" ) func main() { - - - type DemoInfo struct { - Name string - Age string + type User struct { + Uid int `json:"uid"` + Name string `json:"name"` } - - - //r.Response.Write("Hello World") - l := map[interface{}][]DemoInfo{} - - el := [...]DemoInfo{ - {Name:"Bala", Age:"15"}, - {Name:"CeCe", Age:"18"}, - {Name:"ChenLo", Age:"28"}, - {Name:"Bii", Age:"22"}, - {Name:"Ann", Age:"23"}, - {Name:"Bmx", Age:"88"}, - } - - for _,v := range el{ - l[string(v.Name[0])] = append(l[string(v.Name[0])],v) - } - //fmt.Println(l) - - - - b, err := gparser.VarToJson(l) - fmt.Println(err) - fmt.Println(string(b)) + user := User{1, "john"} + b, err := gparser.VarToJson(user) + fmt.Println(err) + fmt.Println(string(b)) } diff --git a/geg/other/test2.go b/geg/other/test2.go index b3e410bf2..c7635e6ed 100644 --- a/geg/other/test2.go +++ b/geg/other/test2.go @@ -2,10 +2,22 @@ package main import ( "fmt" - "gitee.com/johng/gf/g/util/gconv" + "reflect" ) +type Home struct { + i int `nljb:"100"` +} + func main() { - fmt.Println(int(gconv.Float64("2.99s"))) - //fmt.Println(strconv.Atoi(strings.TrimSpace("1.99"))) + home := new(Home) + home.i = 5 + rcvr := reflect.ValueOf(home) + typ := reflect.Indirect(rcvr).Type() + fmt.Println(typ.Kind().String()) + x := typ.NumField() + for i := 0; i < x; i++ { + nljb := typ.Field(0).Tag.Get("nljb") + fmt.Println(nljb) + } } \ No newline at end of file