diff --git a/g/container/gset/gset_test.go b/g/container/gset/gset_test.go index 7ac049e00..0a4ecbc6f 100644 --- a/g/container/gset/gset_test.go +++ b/g/container/gset/gset_test.go @@ -14,9 +14,10 @@ import ( "gitee.com/johng/gf/g/container/gset" ) -var ints = gset.NewIntSet() -var itfs = gset.NewInterfaceSet() -var strs = gset.NewStringSet() +var ints = gset.NewIntSet() +var uints = gset.NewUintSet() +var itfs = gset.NewInterfaceSet() +var strs = gset.NewStringSet() func BenchmarkIntSet_Add(b *testing.B) { for i := 0; i < b.N; i++ { @@ -36,6 +37,24 @@ func BenchmarkIntSet_Remove(b *testing.B) { } } +func BenchmarkUintSet_Add(b *testing.B) { + for i := 0; i < b.N; i++ { + uints.Add(uint(i)) + } +} + +func BenchmarkUintSet_Contains(b *testing.B) { + for i := 0; i < b.N; i++ { + uints.Contains(uint(i)) + } +} + +func BenchmarkUintSet_Remove(b *testing.B) { + for i := 0; i < b.N; i++ { + uints.Remove(uint(i)) + } +} + func BenchmarkInterfaceSet_Add(b *testing.B) { for i := 0; i < b.N; i++ { itfs.Add(i) diff --git a/g/container/gset/uint_set.go b/g/container/gset/uint_set.go new file mode 100644 index 000000000..d72021542 --- /dev/null +++ b/g/container/gset/uint_set.go @@ -0,0 +1,97 @@ +// Copyright 2017 gf Author(https://gitee.com/johng/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://gitee.com/johng/gf. +// + +package gset + +import ( + "fmt" + "sync" +) + +type UintSet struct { + mu sync.RWMutex + m map[uint]struct{} +} + +func NewUintSet() *UintSet { + return &UintSet{m: make(map[uint]struct{})} +} + +// 给定回调函数对原始内容进行遍历 +func (this *UintSet) Iterator(f func (v uint)) { + this.mu.RLock() + for k, _ := range this.m { + f(k) + } + this.mu.RUnlock() +} + +// 添加 +func (this *UintSet) Add(item uint) *UintSet { + this.mu.Lock() + this.m[item] = struct{}{} + this.mu.Unlock() + return this +} + +// 批量添加 +func (this *UintSet) BatchAdd(items []uint) *UintSet { + this.mu.Lock() + for _, item := range items { + this.m[item] = struct{}{} + } + this.mu.Unlock() + return this +} + +// 键是否存在 +func (this *UintSet) Contains(item uint) bool { + this.mu.RLock() + _, exists := this.m[item] + this.mu.RUnlock() + return exists +} + +// 删除键值对 +func (this *UintSet) Remove(key uint) { + this.mu.Lock() + delete(this.m, key) + this.mu.Unlock() +} + +// 大小 +func (this *UintSet) Size() int { + this.mu.RLock() + l := len(this.m) + this.mu.RUnlock() + return l +} + +// 清空set +func (this *UintSet) Clear() { + this.mu.Lock() + this.m = make(map[uint]struct{}) + this.mu.Unlock() +} + +// 转换为数组 +func (this *UintSet) Slice() []uint { + this.mu.RLock() + i := 0 + ret := make([]uint, len(this.m)) + for item := range this.m { + ret[i] = item + i++ + } + this.mu.RUnlock() + return ret +} + +// 转换为字符串 +func (this *UintSet) String() string { + return fmt.Sprint(this.Slice()) +} diff --git a/g/os/gcache/gcache.go b/g/os/gcache/gcache.go index 7c3c12e40..49fb5b841 100644 --- a/g/os/gcache/gcache.go +++ b/g/os/gcache/gcache.go @@ -4,33 +4,28 @@ // If a copy of the MIT was not distributed with this file, // You can obtain one at https://gitee.com/johng/gf. -// 单进程缓存. -// @todo 需要新增一个MAP,用于时间与过期键值对的快速处理。 +// 单进程高速缓存. package gcache import ( "sync" "time" + "math" "gitee.com/johng/gf/g/os/gtime" "gitee.com/johng/gf/g/encoding/ghash" -) - -const ( - gDEFAULT_CACHE_GROUP_SIZE = 4 // 默认缓存分区大小,不能超过uint8的最大值 + "gitee.com/johng/gf/g/container/gset" + "gitee.com/johng/gf/g/container/gqueue" ) // 缓存对象 type Cache struct { - sync.RWMutex - g uint8 // 分区大小 - m map[uint8]*CacheMap // 以分区大小数字作为索引 -} - -// 缓存分区对象 -type CacheMap struct { - sync.RWMutex - closed bool // 对象是否已删除,以便判断停止goroutine - m map[string]CacheItem // 键值对 + dmu sync.RWMutex // data锁 + emu sync.RWMutex // ekmap锁 + data map[uint64]CacheItem // 缓存数据(使用键名哈希作为键) + ekmap map[uint64]int64 // 键名哈希对应的分组过期时间 + eksets map[int64]*gset.UintSet // 分组过期时间对应的键名哈希列表 + eventQueue *gqueue.Queue // 异步处理队列 + stopEvents chan struct{} // 关闭时间通知 } // 缓存数据项 @@ -39,209 +34,208 @@ type CacheItem struct { e int64 // 过期时间 } +// 异步队列数据项 +type EventItem struct { + k uint64 // 键名uint64哈希值 + e int64 // 过期时间 +} + // 全局缓存管理对象 -var cache *Cache = New(gDEFAULT_CACHE_GROUP_SIZE) +var cache *Cache = New() // Cache对象按照缓存键名首字母做了分组 -func New(group uint8) *Cache { +func New() *Cache { c := &Cache { - g : group, - m : make(map[uint8]*CacheMap), - } - // 初始化分区对象 - var i uint8 = 0 - for ; i < group; i++ { - m := &CacheMap { - m : make(map[string]CacheItem), - } - c.m[i] = m - go m.autoClearLoop() + data : make(map[uint64]CacheItem), + ekmap : make(map[uint64]int64), + eksets : make(map[int64]*gset.UintSet), + eventQueue : gqueue.New(), + stopEvents : make(chan struct{}, 2), } + go c.autoSyncLoop() + go c.autoClearLoop() return c } // (使用全局KV缓存对象)设置kv缓存键值对,过期时间单位为毫秒 -func Set(k string, v interface{}, expired int64) { - cache.Set(k, v, expired) +func Set(key string, value interface{}, expire int64) { + cache.Set(key, value, expire) } // (使用全局KV缓存对象)批量设置kv缓存键值对,过期时间单位为毫秒 -func BatchSet(m map[string]interface{}, expired int64) { - cache.BatchSet(m, expired) +func BatchSet(data map[string]interface{}, expire int64) { + cache.BatchSet(data, expire) } // (使用全局KV缓存对象)获取指定键名的值 -func Get(k string) interface{} { - return cache.Get(k) +func Get(key string) interface{} { + return cache.Get(key) } // (使用全局KV缓存对象)删除指定键值对 -func Remove(k string) { - cache.Remove(k) +func Remove(key string) { + cache.Remove(key) } // (使用全局KV缓存对象)批量删除指定键值对 -func BatchRemove(l []string) { - cache.BatchRemove(l) +func BatchRemove(keys []string) { + cache.BatchRemove(keys) } -// 设置kv缓存键值对,过期时间单位为毫秒 -func (c *Cache) Set(k string, v interface{}, expired int64) { - c.RLock() - c.m[c.getIndex(k)].Set(k, v, expired) - c.RUnlock() +// 计算过期缓存的键名 +func (c *Cache) makeExpireKey(expire int64) int64 { + return int64(math.Ceil(float64(expire/1000) + 1)*1000) +} + +// 获取一个过期键名存放Set,如果没有则返回nil +func (c *Cache) getExpireSet(expire int64) *gset.UintSet { + c.emu.RLock() + if ekset, ok := c.eksets[expire]; ok { + c.emu.RUnlock() + return ekset + } + c.emu.RUnlock() + return nil +} + +// 获取或者创建一个过期键名存放Set(由于是异步单线程执行,因此不会出现创建set时的覆盖问题) +func (c *Cache) getOrNewExpireSet(expire int64) *gset.UintSet { + if ekset := c.getExpireSet(expire); ekset == nil { + set := gset.NewUintSet() + c.emu.Lock() + c.eksets[expire] = set + c.emu.Unlock() + return set + } else { + return ekset + } +} + +// 设置kv缓存键值对,过期时间单位为毫秒,expire<=0表示不过期 +func (c *Cache) Set(key string, value interface{}, expire int64) { + e := int64(math.MaxInt64) + if expire > 0 { + e = gtime.Millisecond() + int64(expire) + } + h := ghash.BKDRHash64([]byte(key)) + c.dmu.Lock() + c.data[h] = CacheItem{v: value, e: e} + c.dmu.Unlock() + c.eventQueue.PushBack(EventItem{k:h, e:e}) } // 批量设置 -func (c *Cache) BatchSet(m map[string]interface{}, expired int64) { - c.RLock() - for k, v := range m { - c.m[c.getIndex(k)].Set(k, v, expired) +func (c *Cache) BatchSet(data map[string]interface{}, expire int64) { + e := int64(math.MaxInt64) + if expire > 0 { + e = gtime.Millisecond() + int64(expire) + } + for k, v := range data { + h := ghash.BKDRHash64([]byte(k)) + c.dmu.Lock() + c.data[h] = CacheItem{v: v, e: e} + c.dmu.Unlock() + c.eventQueue.PushBack(EventItem{k:h, e:e}) } - c.RUnlock() } // 获取指定键名的值 -func (c *Cache) Get(k string) interface{} { - c.RLock() - r := c.m[c.getIndex(k)].Get(k) - c.RUnlock() - return r +func (c *Cache) Get(key string) interface{} { + c.dmu.RLock() + item, ok := c.data[ghash.BKDRHash64([]byte(key))] + c.dmu.RUnlock() + if ok { + if item.e > gtime.Millisecond() { + return item.v + } + } + return nil } // 删除指定键值对 -func (c *Cache) Remove(k string) { - c.RLock() - c.m[c.getIndex(k)].Remove(k) - c.RUnlock() +func (c *Cache) Remove(key string) { + c.Set(key, nil, -1) } // 批量删除键值对 -func (c *Cache) BatchRemove(l []string) { - c.RLock() - for _, k := range l { - c.m[c.getIndex(k)].Remove(k) +func (c *Cache) BatchRemove(keys []string) { + for _, key := range keys { + h := ghash.BKDRHash64([]byte(key)) + c.dmu.Lock() + c.data[h] = CacheItem{v: nil, e: -1} + c.dmu.Unlock() + c.eventQueue.PushBack(EventItem{k:h, e: -1}) } - c.RUnlock() } // 获得所有的键名,组成字符串数组返回 -func (c *Cache) Keys() []string { - l := make([]string, 0) - c.RLock() - for _, cm := range c.m { - cm.RLock() - for k2, _ := range cm.m { - l = append(l, k2) - } - cm.RUnlock() - } - c.RUnlock() - return l -} - -// 获得所有的值,组成数组返回 -func (c *Cache) Values() []interface{} { - l := make([]interface{}, 0) - c.RLock() - for _, cm := range c.m { - cm.RLock() - for _, v2 := range cm.m { - l = append(l, v2.v) - } - cm.RUnlock() - } - c.RUnlock() - return l -} +//func (c *Cache) Keys() []string { +// return append(c.temp.Keys(), c.data.Keys()...) +//} +// +//// 获得所有的值,组成数组返回 +//func (c *Cache) Values() []interface{} { +// return append(c.temp.Values(), c.data.Values()...) +//} // 获得缓存对象的键值对数量 func (c *Cache) Size() int { - var size int - c.RLock() - for _, cm := range c.m { - cm.RLock() - size += len(cm.m) - cm.RUnlock() - } - c.RUnlock() - return size + c.dmu.RLock() + length := len(c.data) + c.dmu.RUnlock() + return length } // 删除缓存对象 func (c *Cache) Close() { - c.RLock() - for _, cm := range c.m { - cm.Close() - } - c.RUnlock() + c.stopEvents <- struct{}{} + c.eventQueue.Close() } -// 计算缓存的索引 -func (c *Cache) getIndex(k string) uint8 { - return uint8(ghash.BKDRHash([]byte(k)) % uint32(c.g)) -} - -// 设置kv缓存键值对,过期时间单位为毫秒 -func (cm *CacheMap) Set(k string, v interface{}, expired int64) { - var e int64 - if expired > 0 { - e = gtime.Millisecond() + int64(expired) - } - cm.Lock() - cm.m[k] = CacheItem{v: v, e: e} - cm.Unlock() -} - -// 获取指定键名的值 -func (cm *CacheMap) Get(k string) interface{} { - var v interface{} - cm.RLock() - if r, ok := cm.m[k]; ok { - if r.e > 0 && r.e < gtime.Millisecond() { - v = nil - } else { - v = r.v - } - } - cm.RUnlock() - return v -} - -// 删除指定键值对 -func (cm *CacheMap) Remove(k string) { - cm.Lock() - delete(cm.m, k) - cm.Unlock() -} - -// 关闭缓存分区 -func (cm *CacheMap) Close() { - cm.Lock() - cm.closed = true - cm.Unlock() -} - -// 是否删除 -func (cm *CacheMap) isClosed() bool { - cm.RLock() - r := cm.closed - cm.RUnlock() - return r -} - -// 自动清理过期键值对(每间隔3秒执行) -func (cm *CacheMap) autoClearLoop() { - for !cm.isClosed() { - cm.Lock() - for k, v := range cm.m { - if v.e > 0 && v.e < gtime.Millisecond() { - delete(cm.m, k) +// 数据自动同步循环 +func (c *Cache) autoSyncLoop() { + for { + if r := c.eventQueue.PopFront(); r != nil { + item := r.(EventItem) + newe := c.makeExpireKey(item.e) + if olde, ok := c.ekmap[item.k]; ok { + if newe != olde { + if ekset := c.getExpireSet(olde); ekset != nil { + ekset.Remove(uint(item.k)) + } + } } + c.getOrNewExpireSet(newe).Add(uint(item.k)) + c.ekmap[item.k] = newe + } else { + break } - cm.Unlock() - time.Sleep(3 * time.Second) } } - +// 自动清理过期键值对 +// 每隔1秒清除过去3秒的键值对数据 +func (c *Cache) autoClearLoop() { + for { + select { + case <- c.stopEvents: + return + default: + ek := c.makeExpireKey(gtime.Millisecond()) + eks := []int64{ek - 2000, ek - 3000, ek - 4000} + for _, v := range eks { + if ekset := c.getExpireSet(v); ekset != nil { + c.dmu.Lock() + ekset.Iterator(func(v uint) { + delete(c.data, uint64(v)) + }) + c.dmu.Unlock() + } + c.emu.Lock() + delete(c.eksets, v) + c.emu.Unlock() + } + time.Sleep(time.Second) + } + } +}