preface
- When we use redis to combine golang, we see a lot of problems about cache penetration and cache avalanche on the Internet, but there are few really combined codes. In fact, we can refer to other people's codes to write our own templates
- However, for a complete set of systems like this, it's best to use ready-made ones (laughter)
What are cache penetration, cache avalanche, cache breakdown
1. Cache avalanche
A large number of caches expire at the same time
Solution: set the cache time to a certain range of random numbers
2. Cache penetration
Neither the cache nor the database exists (the requested data is not intercepted by the cache. I have been looking for the database, but the database does not exist, so I have been looking for it)
Solution: set the cache value to DISABLE when hitting for the first time, and then only hit the cache every time
3. Buffer breakdown
After the cache expires, some key s are accessed with high concurrency
- Solution: use mutex lock. When there is a lock, wait for it to be obtained
Solution of golang cache penetration + cache avalanche + cache breakdown
1. Third party package
- Database mapping framework ORM: Gorm
- Cache connection tool: go redis
- Error encapsulation tool: pkg/errors
2. Configure structure + write cache key + cache setting time
// First, select a table in the database
// The structure selected for the cache is the serialized string---
// (hash is not selected because it cannot set the expiration time at the same time to prevent the cache from becoming a permanent cache -- ha ha, I won't say it's because of laziness)
type CatUser struct {
ID uint `gorm:"column:id" json:"id"`
CreatedAt time.Time `gorm:"column:created_at" json:"createdAt"`
UpdatedAt time.Time `gorm:"column:updated_at" json:"updatedAt"`
UserId int `gorm:"column:user_id" json:"user_id"`
Name string `gorm:"column:name" json:"name"`
Password string `gorm:"cloumn:password" json:"password"`
}
// RedisKey cache key
func (o *CatUser) RedisKey() string {
// Recommended here: 1 Separated by: 1 If there is an id that can identify the unique id, use it -- (id is also OK)
// user_id can uniquely identify the data -- similar to id
return fmt.Sprintf("cat_user:%d", o.UserId)
}
func (o *CatUser) ArrayRedisKey() string {
return fmt.Sprintf("cat_user")
}
// Cache time
func (o *CatUser) RedisDuration() time.Duration {
// At this time, random time can be used to solve the cache avalanche problem
// Set 30 ~ 60 minutes -- remember not to set 0 ~ n time here, because if it is 0, it is equivalent to no setting
return time.Duration((rand.Intn(60-30) + 30)) * time.Minute
}
3. Write cache operation
1. Synchronous cache
// SyncToRedis add cache
// Use the serialized string type to store the cache
func (o *CatUser) SyncToRedis(conn *redis.Conn) error {
if o.RedisKey() == "" {
return errors.New("not set redis key")
}
buf, err := json.Marshal(o)
if err != nil {
return errors.WithStack(err)
}
if err = conn.SetEX(context.Background(), o.RedisKey(), string(buf), o.RedisDuration()).Err(); err != nil {
return errors.WithStack(err)
}
return nil
}2. Get cache
// Get cache
// 1. Judge whether there is a key
// 2. Get whether it is empty
// 3. Judge whether cache penetration
// 3. Deserialization after obtaining
// GetFromRedis get cache
func (o *CatUser) GetFromRedis(conn *redis.Conn) error {
if o.RedisKey() == "" {
return errors.New("not set redis key")
}
buf, err := conn.Get(context.Background(), o.RedisKey()).Bytes()
if err != nil {
if err == redis.Nil {
return redis.Nil
}
return errors.WithStack(err)
}
// Has cache penetration occurred
if string(buf) == "DISABLE" {
return errors.New("not found data in redis nor db")
}
if err = json.Unmarshal(buf, o); err != nil {
return errors.WithStack(err)
}
return nil
}3. Delete cache
func (o *CatUser) DeleteFromRedis(conn *redis.Conn) error {
if o.RedisKey() != "" {
if err := conn.Del(context.Background(), o.RedisKey()).Err(); err != nil {
return errors.WithStack(err)
}
}
// Delete array cache at the same time
if o.ArrayRedisKey() != "" {
if err := conn.Del(context.Background(), o.ArrayRedisKey()).Err(); err != nil {
return errors.WithStack(err)
}
}
return nil
}4. Focus - get data
// MustGet get data
// 1. Get from cache first
// 2. If not found -- find the database (also not found -- set DISABLE to prevent cache penetration)
func (o *CatUser) MustGet(engine *gorm.DB, conn *redis.Conn) error {
err := o.GetFromRedis(conn)
// If it is empty and the certificate is found, return in advance, regardless of subsequent operations
if err == nil {
return nil
}
if err != nil && err != redis.Nil {
return errors.WithStack(err)
}
// If this data is not found in the cache, find it from the database
var count int64
if err = engine.Count(&count).Error; err != nil {
return errors.WithStack(err)
}
// If count =0 is set to DISABLE, cache penetration is prevented
if count == 0 {
if err = conn.SetNX(context.Background(), o.RedisKey(), "DISABLE", o.RedisDuration()).Err(); err != nil {
return errors.WithStack(err)
}
return errors.New("not found data in redis nor db")
}
// At this time, it is found -- and there is data in the database -- lock to prevent cache breakdown
// Set the mutex time of 5 seconds
var mutex = o.RedisKey() + "_MUTEX"
if err = conn.Get(context.Background(), mutex).Err(); err != nil {
// The non cache is empty, which is an abnormal error, and an error is reported in advance
if err != redis.Nil {
return errors.WithStack(err)
}
// err == redis.Nil
// Set the mutex time of 5 s
if err = conn.SetNX(context.Background(), mutex, 1, 5*time.Second).Err(); err != nil {
return errors.WithStack(err)
}
// Find from database
if err = engine.First(&o).Error; err != nil {
return errors.WithStack(err)
}
// Synchronous cache
if err = o.SyncToRedis(conn); err != nil {
return errors.WithStack(err)
}
// Delete lock
if err = conn.Del(context.Background(), mutex).Err(); err != nil {
return errors.WithStack(err)
}
} else {
// At this time, it is not empty. It is locked -- wait for a cycle
time.Sleep(50 * time.Millisecond)
if err = o.MustGet(engine, conn); err != nil {
return errors.WithStack(err)
}
}
return nil
}5. The array operation is the same
func (o *CatUser) ArraySyncToRedis(list []CatUser, conn *redis.Conn) error {
if o.ArrayRedisKey() == "" {
return errors.New("not set redis key")
}
buf, err := json.Marshal(list)
if err != nil {
return errors.WithStack(err)
}
if err = conn.SetEX(context.Background(), o.ArrayRedisKey(), string(buf), o.RedisDuration()).Err(); err != nil {
return errors.WithStack(err)
}
return nil
}
func (o *CatUser) ArrayGetFromRedis(conn *redis.Conn) ([]CatUser, error) {
if o.RedisKey() == "" {
return nil, errors.New("not set redis key")
}
buf, err := conn.Get(context.Background(), o.ArrayRedisKey()).Bytes()
if err != nil {
if err == redis.Nil {
return nil, redis.Nil
}
return nil, errors.WithStack(err)
}
if string(buf) == "DISABLE" {
return nil, errors.New("not found data in redis nor db")
}
var list []CatUser
if err = json.Unmarshal(buf, &list); err != nil {
return nil, errors.WithStack(err)
}
return list, nil
}
func (o *CatUser) ArrayDeleteFromRedis(conn *redis.Conn) error {
return o.DeleteFromRedis(conn)
}
// ArrayMustGet
func (o *CatUser) ArrayMustGet(engine *gorm.DB, conn *redis.Conn) ([]CatUser, error) {
list, err := o.ArrayGetFromRedis(conn)
if err == nil {
return list, nil
}
if err != nil && err != redis.Nil {
return nil, errors.WithStack(err)
}
// not found in redis
var count int64
if err = engine.Count(&count).Error; err != nil {
return nil, errors.WithStack(err)
}
if count == 0 {
if err = conn.SetNX(context.Background(), o.ArrayRedisKey(), "DISABLE", o.RedisDuration()).Err(); err != nil {
return nil, errors.WithStack(err)
}
return nil, errors.New("not found data in redis nor db")
}
var mutex = o.ArrayRedisKey() + "_MUTEX"
if err = conn.Get(context.Background(), mutex).Err(); err != nil {
if err != redis.Nil {
return nil, errors.WithStack(err)
}
// err = redis.Nil
if err = conn.SetNX(context.Background(), mutex, 1, 5*time.Second).Err(); err != nil {
return nil, errors.WithStack(err)
}
if err = engine.Find(&list).Error; err != nil {
return nil, errors.WithStack(err)
}
if err = o.ArraySyncToRedis(list, conn); err != nil {
return nil, errors.WithStack(err)
}
if err = conn.Del(context.Background(), mutex).Err(); err != nil {
return nil, errors.WithStack(err)
}
} else {
time.Sleep(50 * time.Millisecond)
list, err = o.ArrayMustGet(engine, conn)
if err != nil {
return nil, errors.WithStack(err)
}
}
return list, nil
}
6. Unit test
func TestCatUser(t *testing.T) {
db := InitDb()
conn := InitRedis()
t.Run("single", func(t *testing.T) {
var cu CatUser
engine := db.Model(&CatUser{}).Where("user_id=?", 1)
cu.UserId = 1
cu.DeleteFromRedis(conn)
if err := cu.MustGet(engine, conn); err != nil {
fmt.Printf("%+v", err)
panic(err)
}
fmt.Println(cu)
})
t.Run("list", func(t *testing.T) {
var cu CatUser
engine := db.Model(&CatUser{})
list, err := cu.ArrayMustGet(engine, conn)
if err != nil {
panic(err)
}
fmt.Println(list)
})
}7. Code summary
package main
import (
"context"
"encoding/json"
"fmt"
"github.com/catbugdemo/errors"
"github.com/go-redis/redis/v8"
"gorm.io/gorm"
"math/rand"
"time"
)
// First, select a table in the database
// The structure selected for the cache is the serialized string---
// (hash is not selected because it cannot set the expiration time at the same time to prevent the cache from becoming a permanent cache -- ha ha, I won't say it's because of laziness)
// Identify a structure
type CatUser struct {
ID uint `gorm:"column:id" json:"id"`
CreatedAt time.Time `gorm:"column:created_at" json:"createdAt"`
UpdatedAt time.Time `gorm:"column:updated_at" json:"updatedAt"`
UserId int `gorm:"column:user_id" json:"user_id"`
Name string `gorm:"column:name" json:"name"`
Password string `gorm:"cloumn:password" json:"password"`
}
// Then write and set the cache -- (because this is the beginning -- and the simplest)
// RedisKey cache key
func (o *CatUser) RedisKey() string {
// Recommended here: 1 Separated by: 1 If there is an id that can identify the unique id, use it -- (id is also OK)
// user_id can uniquely identify the data -- similar to id
return fmt.Sprintf("cat_user:%d", o.UserId)
}
func (o *CatUser) ArrayRedisKey() string {
return fmt.Sprintf("cat_user")
}
// time
func (o *CatUser) RedisDuration() time.Duration {
// Identify by seconds
// At this time, random time can be used to solve the cache avalanche problem
// Set 30 ~ 60 minutes -- remember not to set 0 ~ n time here, because if it is 0, it is equivalent to no setting
// If it is - 1, it is set to permanent
return time.Duration((rand.Intn(60-30) + 30)) * time.Minute
}
// SyncToRedis add cache
// Use the serialized string type to store the cache
func (o *CatUser) SyncToRedis(conn *redis.Conn) error {
if o.RedisKey() == "" {
return errors.New("not set redis key")
}
buf, err := json.Marshal(o)
if err != nil {
return errors.WithStack(err)
}
if err = conn.SetEX(context.Background(), o.RedisKey(), string(buf), o.RedisDuration()).Err(); err != nil {
return errors.WithStack(err)
}
return nil
}
// Get cache
// 1. Judge whether there is a key
// 2. Get whether it is empty
// 3. Judge whether cache penetration
// 3. Deserialization after obtaining
// GetFromRedis get cache
func (o *CatUser) GetFromRedis(conn *redis.Conn) error {
if o.RedisKey() == "" {
return errors.New("not set redis key")
}
buf, err := conn.Get(context.Background(), o.RedisKey()).Bytes()
if err != nil {
if err == redis.Nil {
return redis.Nil
}
return errors.WithStack(err)
}
if string(buf) == "DISABLE" {
return errors.New("not found data in redis nor db")
}
if err = json.Unmarshal(buf, o); err != nil {
return errors.WithStack(err)
}
return nil
}
func (o *CatUser) DeleteFromRedis(conn *redis.Conn) error {
if o.RedisKey() != "" {
if err := conn.Del(context.Background(), o.RedisKey()).Err(); err != nil {
return errors.WithStack(err)
}
}
// set up
if o.ArrayRedisKey() != "" {
if err := conn.Del(context.Background(), o.ArrayRedisKey()).Err(); err != nil {
return errors.WithStack(err)
}
}
return nil
}
// MustGet get data
// 1. Get from cache first
// 2. If not found -- find the database (also not found -- set DISABLE to prevent cache penetration)
func (o *CatUser) MustGet(engine *gorm.DB, conn *redis.Conn) error {
err := o.GetFromRedis(conn)
if err == nil {
return nil
}
if err != nil && err != redis.Nil {
return errors.WithStack(err)
}
// If this data is not found in the cache, find it from the database
var count int64
if err = engine.Count(&count).Error; err != nil {
return errors.WithStack(err)
}
// If count =0 is set to DISABLE, cache penetration is prevented
if count == 0 {
if err = conn.SetNX(context.Background(), o.RedisKey(), "DISABLE", o.RedisDuration()).Err(); err != nil {
return errors.WithStack(err)
}
return errors.New("not found data in redis nor db")
}
// At this time, it is found -- and there is data in the database -- lock to prevent cache breakdown
// Set the mutex time of 5 seconds
var mutex = o.RedisKey() + "_MUTEX"
if err = conn.Get(context.Background(), mutex).Err(); err != nil {
if err != redis.Nil {
return errors.WithStack(err)
}
// err == redis.Nil
if err = conn.SetNX(context.Background(), mutex, 1, 5*time.Second).Err(); err != nil {
return errors.WithStack(err)
}
// Find from database
if err = engine.First(&o).Error; err != nil {
return errors.WithStack(err)
}
// Synchronous cache
if err = o.SyncToRedis(conn); err != nil {
return errors.WithStack(err)
}
// Delete lock
if err = conn.Del(context.Background(), mutex).Err(); err != nil {
return errors.WithStack(err)
}
} else {
// At this time, it is not empty. It is locked -- wait for a cycle
time.Sleep(50 * time.Millisecond)
if err = o.MustGet(engine, conn); err != nil {
return errors.WithStack(err)
}
}
return nil
}
// Also operate on arrays
// ArraySyncToRedis add cache
// Use the serialized string type to store the cache
func (o *CatUser) ArraySyncToRedis(list []CatUser, conn *redis.Conn) error {
if o.ArrayRedisKey() == "" {
return errors.New("not set redis key")
}
buf, err := json.Marshal(list)
if err != nil {
return errors.WithStack(err)
}
if err = conn.SetEX(context.Background(), o.ArrayRedisKey(), string(buf), o.RedisDuration()).Err(); err != nil {
return errors.WithStack(err)
}
return nil
}
// Get cache
// 1. Judge whether there is a key
// 2. Get whether it is empty
// 3. Judge whether cache penetration
// 3. Deserialization after obtaining
// ArrayGetFromRedis get cache
func (o *CatUser) ArrayGetFromRedis(conn *redis.Conn) ([]CatUser, error) {
if o.RedisKey() == "" {
return nil, errors.New("not set redis key")
}
buf, err := conn.Get(context.Background(), o.ArrayRedisKey()).Bytes()
if err != nil {
if err == redis.Nil {
return nil, redis.Nil
}
return nil, errors.WithStack(err)
}
if string(buf) == "DISABLE" {
return nil, errors.New("not found data in redis nor db")
}
var list []CatUser
if err = json.Unmarshal(buf, &list); err != nil {
return nil, errors.WithStack(err)
}
return list, nil
}
// ArrayDeleteFromRedis delete cache
func (o *CatUser) ArrayDeleteFromRedis(conn *redis.Conn) error {
return o.DeleteFromRedis(conn)
}
// ArrayMustGet
func (o *CatUser) ArrayMustGet(engine *gorm.DB, conn *redis.Conn) ([]CatUser, error) {
list, err := o.ArrayGetFromRedis(conn)
if err == nil {
return list, nil
}
if err != nil && err != redis.Nil {
return nil, errors.WithStack(err)
}
// not found in redis
var count int64
if err = engine.Count(&count).Error; err != nil {
return nil, errors.WithStack(err)
}
if count == 0 {
if err = conn.SetNX(context.Background(), o.ArrayRedisKey(), "DISABLE", o.RedisDuration()).Err(); err != nil {
return nil, errors.WithStack(err)
}
return nil, errors.New("not found data in redis nor db")
}
var mutex = o.ArrayRedisKey() + "_MUTEX"
if err = conn.Get(context.Background(), mutex).Err(); err != nil {
if err != redis.Nil {
return nil, errors.WithStack(err)
}
// err = redis.Nil
if err = conn.SetNX(context.Background(), mutex, 1, 5*time.Second).Err(); err != nil {
return nil, errors.WithStack(err)
}
if err = engine.Find(&list).Error; err != nil {
return nil, errors.WithStack(err)
}
if err = o.ArraySyncToRedis(list, conn); err != nil {
return nil, errors.WithStack(err)
}
if err = conn.Del(context.Background(), mutex).Err(); err != nil {
return nil, errors.WithStack(err)
}
} else {
time.Sleep(50 * time.Millisecond)
list, err = o.ArrayMustGet(engine, conn)
if err != nil {
return nil, errors.WithStack(err)
}
}
return list, nil
}
summary
- According to the above method, we can collect solutions including cache penetration + cache avalanche + cache breakdown
- But I have to write so much every time. A single one is OK. I can't afford more
- It doesn't matter. I wrote an automatically generated template and just use it directly
Automatically generate template address
epilogue
- Thank you for your great reading