1

pkg/utility/cmap/concurrentmap.go

@@ -0,0 +1,197 @@
+package cmap
+
+import (
+	"sync"
+)
+
+var SHARD_COUNT = 32
+
+// 一个分片的map存储器 可并发
+
+const ShardCount = 31 // 分区数量
+
+// ConcurrentMap A "thread" safe map of type string:Anything.
+// To avoid lock bottlenecks this map is dived to several (ShardCount) map shards.
+type ConcurrentMap []*ConcurrentMapShared // 分片存储map 可并发
+
+// ConcurrentMapShared A "thread" safe string to anything map.
+type ConcurrentMapShared struct {
+	items        map[string]interface{}
+	sync.RWMutex // Read Write mutex, guards access to internal map.
+}
+
+// New Creates a new concurrent map.
+func New() ConcurrentMap {
+	m := make(ConcurrentMap, SHARD_COUNT)
+	for i := 0; i < SHARD_COUNT; i++ {
+		m[i] = &ConcurrentMapShared{items: make(map[string]interface{})}
+	}
+	return m
+}
+
+// GetNoLock retrieves an element from map under given key.
+func (m ConcurrentMap) GetNoLock(shard *ConcurrentMapShared, key string) (interface{}, bool) {
+	// Get item from shard.
+	val, ok := shard.items[key]
+	return val, ok
+}
+
+// SetNoLock retrieves an element from map under given key.
+func (m ConcurrentMap) SetNoLock(shard *ConcurrentMapShared, key string, value interface{}) {
+	shard.items[key] = value
+}
+
+// NewConcurrentMap 创建
+func NewConcurrentMap() ConcurrentMap {
+	m := make(ConcurrentMap, ShardCount)
+	for i := 0; i < ShardCount; i++ {
+		m[i] = &ConcurrentMapShared{items: make(map[string]interface{})}
+	}
+	return m
+}
+
+// GetShard 返回给定键下的分片
+func (m ConcurrentMap) GetShard(key string) *ConcurrentMapShared {
+	return m[fnv32(key)&ShardCount]
+}
+
+// MSet 存储一组map
+func (m ConcurrentMap) MSet(data map[string]interface{}) {
+	for key, value := range data {
+		shard := m.GetShard(key)
+		shard.Lock()
+		shard.items[key] = value
+		shard.Unlock()
+	}
+}
+
+// Set the given value under the specified key.
+// 在指定的键下设置给定的值。
+func (m ConcurrentMap) Set(key string, value interface{}) {
+	// Get map shard.
+	shard := m.GetShard(key)
+	shard.Lock()
+	shard.items[key] = value
+	shard.Unlock()
+}
+
+// UpsertCb Callback to return new element to be inserted into the map
+// It is called while lock is held, therefore it MUST NOT
+// try to access other keys in same map, as it can lead to deadlock since
+// Go sync.RWLock is not reentrant
+// 回调函数返回新元素插入到映射中。它在锁被持有时被调用，因此它一定不要试图访问同一映射中的其他键，因为它可能导致死锁。 RWLock是不可重入的
+type UpsertCb func(exist bool, valueInMap interface{}, newValue interface{}) interface{}
+
+// Upsert Insert or Update - updates existing element or inserts a new one using UpsertCb
+// 插入或更新——使用UpsertCb更新现有元素或插入新元素
+func (m ConcurrentMap) Upsert(key string, value interface{}, cb UpsertCb) (res interface{}) {
+	shard := m.GetShard(key)
+	shard.Lock()
+	v, ok := shard.items[key]
+	res = cb(ok, v, value)
+	shard.items[key] = res
+	shard.Unlock()
+	return res
+}
+
+// SetIfAbsent Sets the given value under the specified key if no value was associated with it.
+// 如果没有值与指定键关联，则在指定键下设置给定值。
+func (m ConcurrentMap) SetIfAbsent(key string, value interface{}) bool {
+	// Get map shard.
+	shard := m.GetShard(key)
+	shard.Lock()
+	_, ok := shard.items[key]
+	if !ok {
+		shard.items[key] = value
+	}
+	shard.Unlock()
+	return !ok
+}
+
+// Get retrieves an element from map under given key.
+func (m ConcurrentMap) Get(key string) (interface{}, bool) {
+	shard := m.GetShard(key)
+	shard.RLock()
+	val, ok := shard.items[key]
+	shard.RUnlock()
+	return val, ok
+}
+
+// Count returns the number of elements within the map.
+func (m ConcurrentMap) Count() int {
+	count := 0
+	for i := 0; i < ShardCount; i++ {
+		shard := m[i]
+		shard.RLock()
+		count += len(shard.items)
+		shard.RUnlock()
+	}
+	return count
+}
+
+// Has Looks up an item under specified key 存在性
+func (m ConcurrentMap) Has(key string) bool {
+	shard := m.GetShard(key)
+	shard.RLock()
+	_, ok := shard.items[key]
+	shard.RUnlock()
+	return ok
+}
+
+// Remove removes an element from the map. 移除
+func (m ConcurrentMap) Remove(key string) {
+	// Try to get shard.
+	shard := m.GetShard(key)
+	shard.Lock()
+	delete(shard.items, key)
+	shard.Unlock()
+}
+
+// RemoveCb is a callback executed in a map.RemoveCb() call, while Lock is held
+// If returns true, the element will be removed from the map
+// 是一个在map.RemoveCb()调用中执行的回调函数，当Lock被持有时，如果返回true，该元素将从map中移除
+type RemoveCb func(key string, v interface{}, exists bool) bool
+
+// RemoveCb locks the shard containing the key, retrieves its current value and calls the callback with those params
+// If callback returns true and element exists, it will remove it from the map
+// Returns the value returned by the callback (even if element was not present in the map)
+// 如果callback返回true且element存在，则将其从map中移除。返回callback返回的值(即使element不存在于map中)
+func (m ConcurrentMap) RemoveCb(key string, cb RemoveCb) bool {
+	shard := m.GetShard(key)
+	shard.Lock()
+	v, ok := shard.items[key]
+	remove := cb(key, v, ok)
+	if remove && ok {
+		delete(shard.items, key)
+	}
+	shard.Unlock()
+	return remove
+}
+
+// Pop removes an element from the map and returns it
+// 从映射中移除一个元素并返回它
+func (m ConcurrentMap) Pop(key string) (v interface{}, exists bool) {
+	// Try to get shard.
+	shard := m.GetShard(key)
+	shard.Lock()
+	v, exists = shard.items[key]
+	delete(shard.items, key)
+	shard.Unlock()
+	return v, exists
+}
+
+// IsEmpty checks if map is empty. 是否是空的
+func (m ConcurrentMap) IsEmpty() bool {
+	return m.Count() == 0
+}
+
+// 将键值映射为数字uint32
+func fnv32(key string) uint32 {
+	const prime32 = uint32(16777619)
+	hash := uint32(2166136261)
+	for i := 0; i < len(key); i++ {
+		hash *= prime32
+		hash ^= uint32(key[i])
+	}
+	return hash
+}

pkg/utility/cmap/concurrentmap_test.go

@@ -0,0 +1,60 @@
+package cmap
+
+import (
+	"fmt"
+	"go-admin/pkg/utility"
+	"hash/crc32"
+	"testing"
+)
+
+// 离散性测试
+func Test_fnv32(t *testing.T) {
+	st := make(map[uint32]int)
+	for i := 0; i < 1000000; i++ {
+		fnv := crc32.ChecksumIEEE([]byte(utility.GenerateRandString(8)))
+		k := fnv & 15
+		count, ok := st[k]
+		if !ok {
+			st[k] = 1
+		}
+		st[k] = count + 1
+	}
+	for k, v := range st {
+		fmt.Println(k, "\t", float64(v)/1000000)
+	}
+}
+
+// go test -bench=_QE_ -benchmem -run=^$
+// -benchtime 默认为1秒  -benchmem 获得内存分配的统计数据
+// Benchmark_QE_1-6          146641              8192 ns/op              32 B/op          3 allocs/op
+// Benchmark_QE_2-6          143118              8246 ns/op              40 B/op          4 allocs/op
+//
+// Benchmark_QE_1-6          146289              8212 ns/op              32 B/op          3 allocs/op
+// Benchmark_QE_2-6          144918              8239 ns/op              40 B/op          4 allocs/op
+func Benchmark_QE_1(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		fnv32(utility.GenerateRandString(8))
+	}
+}
+
+func Benchmark_QE_2(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		crc32.ChecksumIEEE([]byte(utility.GenerateRandString(8)))
+	}
+}
+
+// go test -bench=_QE2_ -benchmem -benchtime=5s -run=^$
+// -benchtime 默认为1秒  -benchmem 获得内存分配的统计数据
+// Benchmark_QE2_1-6       1000000000               0.2623 ns/op          0 B/op          0 allocs/op
+// Benchmark_QE2_2-6       1000000000               0.2631 ns/op          0 B/op          0 allocs/op
+func Benchmark_QE2_1(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		_ = i & 31
+	}
+}
+
+func Benchmark_QE2_2(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		_ = i % 31
+	}
+}

pkg/utility/cmap/iterator.go

@@ -0,0 +1,160 @@
+package cmap
+
+import (
+	"sync"
+
+	"github.com/bytedance/sonic"
+)
+
+// 迭代器部分
+
+// Tuple Used by the Iter & IterBuffered functions to wrap two variables together over a channel
+// 由Iter & IterBuffered函数使用，在一个通道上封装两个变量，
+type Tuple struct {
+	Key string
+	Val interface{}
+}
+
+// Iter returns an iterator which could be used in a for range loop.
+// 返回一个可用于for范围循环的迭代器。
+// Deprecated: using IterBuffered() will get a better performence
+// 使用IterBuffered()将获得更好的性能
+func (m ConcurrentMap) Iter() <-chan Tuple {
+	chans := snapshot(m)
+	ch := make(chan Tuple) // 不带缓冲
+	go fanIn(chans, ch)
+	return ch
+}
+
+// IterBuffered returns a buffered iterator which could be used in a for range loop.
+// 返回一个可用于for范围循环的缓冲迭代器。
+func (m ConcurrentMap) IterBuffered() <-chan Tuple {
+	chans := snapshot(m)
+	total := 0
+	for _, c := range chans {
+		total += cap(c)
+	}
+	ch := make(chan Tuple, total) // 一次性写完到缓冲中
+	go fanIn(chans, ch)
+	return ch
+}
+
+// Returns a array of channels that contains elements in each shard,
+// which likely takes a snapshot of `m`.
+// It returns once the size of each buffered channel is determined,
+// before all the channels are populated using goroutines.
+// 返回一个通道数组，其中包含每个shard中的元素，它可能会获取' m '的快照。
+// 一旦确定了每个缓冲通道的大小，在使用goroutines填充所有通道之前，它将返回。
+func snapshot(m ConcurrentMap) (chans []chan Tuple) {
+	chans = make([]chan Tuple, ShardCount)
+	wg := sync.WaitGroup{}
+	wg.Add(ShardCount)
+	// Foreach shard.
+	for index, shard := range m {
+		go func(index int, shard *ConcurrentMapShared) {
+			shard.RLock()
+			chans[index] = make(chan Tuple, len(shard.items))
+			wg.Done() // 只要创建了通道就不用再阻塞了
+			for key, val := range shard.items {
+				chans[index] <- Tuple{key, val}
+			}
+			shard.RUnlock()
+			close(chans[index])
+		}(index, shard)
+	}
+	wg.Wait()
+	return chans
+}
+
+// fanIn reads elements from channels `chans` into channel `out`
+// 从通道' chans '读取元素到通道' out '
+func fanIn(chans []chan Tuple, out chan Tuple) {
+	wg := sync.WaitGroup{}
+	wg.Add(len(chans))
+	for _, ch := range chans {
+		go func(ch chan Tuple) {
+			for t := range ch {
+				out <- t
+			}
+			wg.Done()
+		}(ch)
+	}
+	wg.Wait()
+	close(out)
+}
+
+// Items returns all items as map[string]interface{}
+// 返回所有条目作为map[string]interface{}
+func (m ConcurrentMap) Items() map[string]interface{} {
+	tmp := make(map[string]interface{})
+
+	// Insert items to temporary map. 向临时映射中插入项目。
+	for item := range m.IterBuffered() {
+		tmp[item.Key] = item.Val
+	}
+
+	return tmp
+}
+
+// IterCb Iterator callback,called for every key,value found in
+// maps. RLock is held for all calls for a given shard
+// therefore callback sess consistent view of a shard,
+// but not across the shards
+// 迭代器回调函数，在map中找到的每个键和值都会被调用。
+// RLock对给定分片的所有调用都保持，因此回调获得一个分片的一致视图，但不跨分片
+type IterCb func(key string, v interface{})
+
+// IterCb Callback based iterator, cheapest way to read all elements in a map.
+// 基于回调的迭代器，读取映射中所有元素的最便宜方法。
+func (m ConcurrentMap) IterCb(fn IterCb) {
+	for idx := range m {
+		shard := (m)[idx]
+		shard.RLock()
+		for key, value := range shard.items {
+			fn(key, value)
+		}
+		shard.RUnlock()
+	}
+}
+
+// Keys returns all keys as []string
+// 返回所有键为[]字符串
+func (m ConcurrentMap) Keys() []string {
+	count := m.Count()
+	ch := make(chan string, count)
+	go func() {
+		// Foreach shard.
+		wg := sync.WaitGroup{}
+		wg.Add(ShardCount)
+		for _, shard := range m {
+			go func(shard *ConcurrentMapShared) {
+				// Foreach key, value pair.
+				shard.RLock()
+				for key := range shard.items {
+					ch <- key
+				}
+				shard.RUnlock()
+				wg.Done()
+			}(shard)
+		}
+		wg.Wait()
+		close(ch)
+	}()
+
+	// Generate keys
+	keys := make([]string, 0, count)
+	for k := range ch {
+		keys = append(keys, k)
+	}
+	return keys
+}
+
+// MarshalJSON Reviles ConcurrentMap "private" variables to json marshal.
+// 将存储的所有数据json序列化输出
+func (m ConcurrentMap) MarshalJSON() ([]byte, error) {
+	tmp := make(map[string]interface{})
+	for item := range m.IterBuffered() {
+		tmp[item.Key] = item.Val
+	}
+	return sonic.Marshal(tmp)
+}