From 0dc9a14751b0ee822fc2b2b06ac5ca62c0b14143 Mon Sep 17 00:00:00 2001 From: hlccd <56643462+hlccd@users.noreply.github.com> Date: Fri, 24 Dec 2021 14:06:37 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=BA=86LRU=E9=93=BE?= =?UTF-8?q?=E5=92=8C=E4=B8=80=E8=87=B4=E6=80=A7hash=E7=9A=84=E5=AE=9E?= =?UTF-8?q?=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../consistentHash/consistentHash.go | 149 ++++++++++++++++++ goSTL/data_structure/lru/lru.go | 91 +++++++++++ 2 files changed, 240 insertions(+) create mode 100644 goSTL/data_structure/consistentHash/consistentHash.go create mode 100644 goSTL/data_structure/lru/lru.go diff --git a/goSTL/data_structure/consistentHash/consistentHash.go b/goSTL/data_structure/consistentHash/consistentHash.go new file mode 100644 index 0000000..7323e90 --- /dev/null +++ b/goSTL/data_structure/consistentHash/consistentHash.go @@ -0,0 +1,149 @@ +package consistentHash + +import ( + "fmt" + "sync" +) + +const ( + maxReplicas = 32 +) + +var primes = []byte{3, 5, 7, 11, 13, 17, 19, 23, 29, 31, + 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, + 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, + 131, 137, 139, 149, 151, 157, 163, 167, 173, 179, + 121, 191, 193, 197, 199, 211, 223, 227, 229, 233, + 239, 241, 251, +} + +type ConsistentHash struct { + minReplicas int + keys []uint32 //存储的结点和虚拟结点的集合 + hashMap map[uint32]interface{} //hash与结点之间的映射 + nodeMap map[interface{}][]uint32 //结点所对应的虚拟节点的hash值 + mutex sync.Mutex +} + +func hash(id int, v interface{}) (h uint32) { + prime := primes[(id*id+len(primes))%len(primes)] + h = uint32(0) + s := fmt.Sprintf("%d-%v-%d", id*int(prime), v, prime) + bs := []byte(s) + for i := range bs { + h += uint32(bs[i] * prime) + } + return h +} +func New(minReplicas int) *ConsistentHash { + if minReplicas > maxReplicas { + minReplicas = maxReplicas + } + if minReplicas < 0 { + minReplicas = 1 + } + ch := &ConsistentHash{ + minReplicas: minReplicas, + keys: make([]uint32, 0, 0), + hashMap: make(map[uint32]interface{}), + nodeMap: make(map[interface{}][]uint32), + mutex: sync.Mutex{}, + } + return ch +} +func (ch *ConsistentHash) Insert(keys ...interface{}) (nums []int) { + nums = make([]int, 0, len(keys)) + ch.mutex.Lock() + for _, key := range keys { + num := 0 + _, exist := ch.nodeMap[key] + if !exist { + for i := 0; i < maxReplicas || num < ch.minReplicas; i++ { + h := uint32(hash(i, key)) + _, ok := ch.hashMap[h] + if !ok { + num++ + ch.keys = append(ch.keys, h) + ch.hashMap[h] = key + ch.nodeMap[key] = append(ch.nodeMap[key], h) + } + } + } + nums = append(nums, num) + } + //对keys进行排序,以方便后续查找 + ch.sort(0, len(ch.keys)-1) + ch.mutex.Unlock() + return nums +} +func (ch *ConsistentHash) sort(L, R int) { + if L >= R { + return + } + l, r, m := L-1, R+1, ch.keys[(L+R)/2] + for l < r { + l++ + for ch.keys[l] < m { + l++ + } + r-- + for ch.keys[r] > m { + r-- + } + if l < r { + tmp := ch.keys[l] + ch.keys[l] = ch.keys[r] + ch.keys[r] = tmp + } + } + ch.sort(L, l-1) + ch.sort(r+1, R) +} +func (ch *ConsistentHash) Get(key interface{}) (ans interface{}) { + if len(ch.keys) == 0 { + return nil + } + ch.mutex.Lock() + h := hash(0, key) + idx := ch.search(h) + ans = ch.hashMap[ch.keys[idx]] + ch.mutex.Unlock() + return ans +} +func (ch *ConsistentHash) search(h uint32) (idx uint32) { + l, m, r := uint32(0), uint32(len(ch.keys)/2), uint32(len(ch.keys)) + for l < r { + m = (l + r) / 2 + if ch.keys[m] >= h { + r = m + } else { + l = m + 1 + } + } + return l % uint32(len(ch.keys)) +} +func (ch *ConsistentHash) Erase(key interface{}) { + ch.mutex.Lock() + hs, ok := ch.nodeMap[key] + if ok { + delete(ch.nodeMap, key) + for i := range hs { + delete(ch.hashMap, hs[i]) + } + arr := make([]uint32, 0, len(ch.keys)-len(hs)) + for i := range ch.keys { + h := ch.keys[i] + flag := true + for j := range hs { + if hs[j] == h { + flag = false + } + } + if flag { + arr = append(arr, h) + } + } + ch.keys = arr + } + ch.mutex.Unlock() +} diff --git a/goSTL/data_structure/lru/lru.go b/goSTL/data_structure/lru/lru.go new file mode 100644 index 0000000..dad2e2f --- /dev/null +++ b/goSTL/data_structure/lru/lru.go @@ -0,0 +1,91 @@ +package lru + +import ( + "container/list" + "sync" +) + +type LRU struct { + maxBytes int64 + nowBytes int64 + ll *list.List + cache map[string]*list.Element + onRemove func(key string, value Value) + mutex sync.Mutex //并发控制锁 +} +type indexes struct { + key string + value Value +} + +type Value interface { + Len() int +} + +func New(maxBytes int64, onRemove func(string, Value)) *LRU { + return &LRU{ + maxBytes: maxBytes, + nowBytes: 0, + ll: list.New(), + cache: make(map[string]*list.Element), + onRemove: onRemove, + } +} +func (l *LRU) Add(key string, value Value) { + l.mutex.Lock() + if ele, ok := l.cache[key]; ok { + l.ll.MoveToFront(ele) + kv := ele.Value.(*indexes) + //此处是一个替换,即将cache中的value替换为新的value,同时根据实际存储量修改其当前存储的实际大小 + l.nowBytes += int64(value.Len()) - int64(kv.value.Len()) + kv.value = value + } else { + ele := l.ll.PushFront(&indexes{key, value}) + l.cache[key] = ele + //此处是一个增加操作,即原本不存在,所以直接插入即可,同时在当前数值范围内增加对应的占用空间 + l.nowBytes += int64(len(key)) + int64(value.Len()) + } + //添加完成后根据删除掉尾部一部分数据以保证实际使用空间小于设定的空间上限 + for l.maxBytes != 0 && l.maxBytes < l.nowBytes { + ele := l.ll.Back() + if ele != nil { + l.ll.Remove(ele) + kv := ele.Value.(*indexes) + //删除最末尾的同时将其占用空间减去 + delete(l.cache, kv.key) + l.nowBytes -= int64(len(kv.key)) + int64(kv.value.Len()) + if l.onRemove != nil { + //删除后的回调函数,可用于持久化该部分数据 + l.onRemove(kv.key, kv.value) + } + } + } + l.mutex.Unlock() +} +func (l *LRU) RemoveOldest() { + l.mutex.Lock() + ele := l.ll.Back() + if ele != nil { + l.ll.Remove(ele) + kv := ele.Value.(*indexes) + //删除最末尾的同时将其占用空间减去 + delete(l.cache, kv.key) + l.nowBytes -= int64(len(kv.key)) + int64(kv.value.Len()) + if l.onRemove != nil { + //删除后的回调函数,可用于持久化该部分数据 + l.onRemove(kv.key, kv.value) + } + } + l.mutex.Unlock() +} +func (l *LRU) Get(key string) (value Value, ok bool) { + l.mutex.Lock() + if ele, ok := l.cache[key]; ok { + l.ll.MoveToFront(ele) + kv := ele.Value.(*indexes) + l.mutex.Unlock() + return kv.value, true + } + l.mutex.Unlock() + return nil, false +}