common
/
machinery-plugins
Private
1
Fork 0
Code Pull Requests Releases Activity

add backend

Browse Source
Operations
Operations
Create branch
Create tag
Revert
Cherry-pick
main
git 2024-05-25 01:03:13 +08:00
parent 0749384c84
commit 46efefa31d
Signed by: git
GPG Key ID: 3F65EFFA44207ADD
8 changed files with 404 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

190
backends/etcd/etcd.go Normal file
View File

@ -0,0 +1,190 @@
package etcd
import (
"context"
"encoding/json"
"fmt"
"time"
"github.com/RichardKnop/machinery/v2/backends/iface"
"github.com/RichardKnop/machinery/v2/common"
"github.com/RichardKnop/machinery/v2/config"
"github.com/RichardKnop/machinery/v2/tasks"
clientv3 "go.etcd.io/etcd/client/v3"
)
type etcdBackend struct {
common.Backend
ctx context.Context
conf *config.Config
client *clientv3.Client
}
func New(ctx context.Context, cnf *config.Config) (iface.Backend, error) {
etcdConf := clientv3.Config{
Endpoints: []string{cnf.ResultBackend},
Context: ctx,
DialTimeout: time.Second * 5,
}
client, err := clientv3.New(etcdConf)
if err != nil {
return nil, err
}
backend := etcdBackend{
Backend: common.NewBackend(cnf),
ctx: ctx,
client: client,
conf: cnf,
}
return &backend, nil
}
// Group related functions
func (b *etcdBackend) InitGroup(groupUUID string, taskUUIDs []string) error {
groupMeta := &tasks.GroupMeta{
GroupUUID: groupUUID,
TaskUUIDs: taskUUIDs,
CreatedAt: time.Now().UTC(),
}
encoded, err := json.Marshal(groupMeta)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(b.ctx, time.Second*2)
defer cancel()
key := fmt.Sprintf("/machinery/v2/backend/%s", groupUUID)
_, err = b.client.KV.Put(ctx, key, string(encoded))
return err
}
func (b *etcdBackend) GroupCompleted(groupUUID string, groupTaskCount int) (bool, error) {
groupMeta, err := b.getGroupMeta(groupUUID)
if err != nil {
return false, err
}
taskStates, err := b.getStates(groupMeta.TaskUUIDs...)
if err != nil {
return false, err
}
var countSuccessTasks = 0
for _, taskState := range taskStates {
if taskState.IsCompleted() {
countSuccessTasks++
}
}
return countSuccessTasks == groupTaskCount, nil
}
func (b *etcdBackend) getGroupMeta(groupUUID string) (*tasks.GroupMeta, error) {
return nil, nil
}
func (b *etcdBackend) GroupTaskStates(groupUUID string, groupTaskCount int) ([]*tasks.TaskState, error) {
groupMeta, err := b.getGroupMeta(groupUUID)
if err != nil {
return []*tasks.TaskState{}, err
}
return b.getStates(groupMeta.TaskUUIDs...)
}
func (b *etcdBackend) TriggerChord(groupUUID string) (bool, error) {
return false, nil
}
// Setting / getting task state
// SetStatePending updates task state to PENDING
func (b *etcdBackend) SetStatePending(signature *tasks.Signature) error {
taskState := tasks.NewPendingTaskState(signature)
return b.updateState(taskState)
}
// SetStateReceived updates task state to RECEIVED
func (b *etcdBackend) SetStateReceived(signature *tasks.Signature) error {
taskState := tasks.NewReceivedTaskState(signature)
b.mergeNewTaskState(taskState)
return b.updateState(taskState)
}
// SetStateStarted updates task state to STARTED
func (b *etcdBackend) SetStateStarted(signature *tasks.Signature) error {
taskState := tasks.NewStartedTaskState(signature)
b.mergeNewTaskState(taskState)
return b.updateState(taskState)
}
// SetStateRetry updates task state to RETRY
func (b *etcdBackend) SetStateRetry(signature *tasks.Signature) error {
taskState := tasks.NewRetryTaskState(signature)
b.mergeNewTaskState(taskState)
return b.updateState(taskState)
}
// SetStateSuccess updates task state to SUCCESS
func (b *etcdBackend) SetStateSuccess(signature *tasks.Signature, results []*tasks.TaskResult) error {
taskState := tasks.NewSuccessTaskState(signature, results)
b.mergeNewTaskState(taskState)
return b.updateState(taskState)
}
// SetStateFailure updates task state to FAILURE
func (b *etcdBackend) SetStateFailure(signature *tasks.Signature, err string) error {
taskState := tasks.NewFailureTaskState(signature, err)
b.mergeNewTaskState(taskState)
return b.updateState(taskState)
}
func (b *etcdBackend) GetState(taskUUID string) (*tasks.TaskState, error) {
return nil, nil
}
// Purging stored stored tasks states and group meta data
func (b *etcdBackend) IsAMQP() bool {
return false
}
func (b *etcdBackend) mergeNewTaskState(newState *tasks.TaskState) {
state, err := b.GetState(newState.TaskUUID)
if err == nil {
newState.CreatedAt = state.CreatedAt
newState.TaskName = state.TaskName
}
}
func (b *etcdBackend) PurgeState(taskUUID string) error {
return nil
}
func (b *etcdBackend) PurgeGroupMeta(groupUUID string) error {
return nil
}
// getStates returns multiple task states
func (b *etcdBackend) getStates(taskUUIDs ...string) ([]*tasks.TaskState, error) {
taskStates := make([]*tasks.TaskState, len(taskUUIDs))
return taskStates, nil
}
// updateState saves current task state
func (b *etcdBackend) updateState(taskState *tasks.TaskState) error {
return nil
}
// getExpiration returns expiration for a stored task state
func (b *etcdBackend) getExpiration() time.Duration {
expiresIn := b.GetConfig().ResultsExpireIn
if expiresIn == 0 {
// expire results after 1 hour by default
expiresIn = config.DefaultResultsExpireIn
}
return time.Duration(expiresIn) * time.Second
}

161
backends/mysql/mysql.go Normal file
View File

@ -0,0 +1,161 @@
package mysql
import (
"encoding/json"
"errors"
"fmt"
"log"
"time"
"github.com/RichardKnop/machinery/v2/backends/iface"
"github.com/RichardKnop/machinery/v2/config"
"github.com/RichardKnop/machinery/v2/tasks"
"github.com/jmoiron/sqlx"
)
var (
ErrRedisLockFailed = errors.New("redis lock: failed to acquire lock")
)
type mysqlBackend struct {
globalConf *config.Config
db *sqlx.DB
retries int
interval time.Duration
}
func New(cnf *config.Config) (iface.Backend, error) {
db, err := sqlx.Connect("mysql", "xx")
if err != nil {
log.Fatalln(err)
}
backend := mysqlBackend{
globalConf: cnf,
db: db,
}
return &backend, nil
}
// Group related functions
func (b mysqlBackend) InitGroup(groupUUID string, taskUUIDs []string) error {
groupMeta := &tasks.GroupMeta{
GroupUUID: groupUUID,
TaskUUIDs: taskUUIDs,
CreatedAt: time.Now().UTC(),
}
encoded, err := json.Marshal(groupMeta)
if err != nil {
return err
}
sql := "insert into %s (uuid, encoded) values($1, $2)"
_, err = b.db.Exec(sql, groupUUID, encoded)
return err
}
func (b mysqlBackend) GroupCompleted(groupUUID string, groupTaskCount int) (bool, error) {
return false, nil
}
func (b mysqlBackend) getGroupMeta(groupUUID string) (*tasks.GroupMeta, error) {
sql := "select * from %s where group_uuid = $1"
metas := []*tasks.GroupMeta{}
if err := b.db.Select(&metas, sql); err != nil {
return nil, err
}
if len(metas) != 1 {
return nil, fmt.Errorf("not valid groupUUID")
}
return metas[0], nil
}
func (b mysqlBackend) GroupTaskStates(groupUUID string, groupTaskCount int) ([]*tasks.TaskState, error) {
return nil, nil
}
func (b mysqlBackend) TriggerChord(groupUUID string) (bool, error) {
return false, nil
}
// Setting / getting task state
// SetStatePending updates task state to PENDING
func (b *mysqlBackend) SetStatePending(signature *tasks.Signature) error {
taskState := tasks.NewPendingTaskState(signature)
return b.updateState(taskState)
}
// SetStateReceived updates task state to RECEIVED
func (b *mysqlBackend) SetStateReceived(signature *tasks.Signature) error {
taskState := tasks.NewReceivedTaskState(signature)
b.mergeNewTaskState(taskState)
return b.updateState(taskState)
}
// SetStateStarted updates task state to STARTED
func (b *mysqlBackend) SetStateStarted(signature *tasks.Signature) error {
taskState := tasks.NewStartedTaskState(signature)
b.mergeNewTaskState(taskState)
return b.updateState(taskState)
}
// SetStateRetry updates task state to RETRY
func (b *mysqlBackend) SetStateRetry(signature *tasks.Signature) error {
taskState := tasks.NewRetryTaskState(signature)
b.mergeNewTaskState(taskState)
return b.updateState(taskState)
}
// SetStateSuccess updates task state to SUCCESS
func (b *mysqlBackend) SetStateSuccess(signature *tasks.Signature, results []*tasks.TaskResult) error {
taskState := tasks.NewSuccessTaskState(signature, results)
b.mergeNewTaskState(taskState)
return b.updateState(taskState)
}
// SetStateFailure updates task state to FAILURE
func (b *mysqlBackend) SetStateFailure(signature *tasks.Signature, err string) error {
taskState := tasks.NewFailureTaskState(signature, err)
b.mergeNewTaskState(taskState)
return b.updateState(taskState)
}
func (b mysqlBackend) GetState(taskUUID string) (*tasks.TaskState, error) {
return nil, nil
}
// Purging stored stored tasks states and group meta data
func (b mysqlBackend) IsAMQP() bool {
return false
}
func (b *mysqlBackend) mergeNewTaskState(newState *tasks.TaskState) {
state, err := b.GetState(newState.TaskUUID)
if err == nil {
newState.CreatedAt = state.CreatedAt
newState.TaskName = state.TaskName
}
}
func (b mysqlBackend) PurgeState(taskUUID string) error {
return nil
}
func (b mysqlBackend) PurgeGroupMeta(groupUUID string) error {
return nil
}
// getStates returns multiple task states
func (b *mysqlBackend) getStates(taskUUIDs ...string) ([]*tasks.TaskState, error) {
taskStates := make([]*tasks.TaskState, len(taskUUIDs))
return taskStates, nil
}
// updateState saves current task state
func (b *mysqlBackend) updateState(taskState *tasks.TaskState) error {
return nil
}

1
backends/mysql/schema.go Normal file
View File

@ -0,0 +1 @@
package mysql

35
brokers/etcd/etcd.go
View File

@ -51,25 +51,28 @@ func (s *Signature) String() string {
type etcdBroker struct {
common.Broker
globalConf *config.Config
conf clientv3.Config
cli *clientv3.Client
ctx context.Context
client *clientv3.Client
wg sync.WaitGroup
}
// New ..
func New(cnf *config.Config, endpoint string) (iface.Broker, error) {
etcdConf := clientv3.Config{Endpoints: []string{endpoint}}
cli, err := clientv3.New(etcdConf)
func New(ctx context.Context, conf *config.Config) (iface.Broker, error) {
etcdConf := clientv3.Config{
Endpoints: []string{conf.Broker},
Context: ctx,
DialTimeout: time.Second * 5,
}
client, err := clientv3.New(etcdConf)
if err != nil {
return nil, err
}
broker := etcdBroker{
Broker: common.NewBroker(cnf),
globalConf: cnf,
conf: etcdConf,
cli: cli,
Broker: common.NewBroker(conf),
ctx: ctx,
client: client,
}
return &broker, nil
@ -238,17 +241,17 @@ func (b *etcdBroker) Publish(ctx context.Context, signature *tasks.Signature) er
if s.ETA != nil {
if s.ETA.After(now) {
key = fmt.Sprintf("/machinery/v2/broker/delayed_tasks/t%d-%s", s.ETA.UnixNano(), s.UUID)
_, err = b.cli.Put(ctx, key, string(msg))
_, err = b.client.Put(ctx, key, string(msg))
return err
}
}
_, err = b.cli.Put(ctx, key, string(msg))
_, err = b.client.Put(ctx, key, string(msg))
return err
}
func (b *etcdBroker) getTasks(ctx context.Context, key string) ([]*Signature, error) {
resp, err := b.cli.Get(ctx, key, clientv3.WithPrefix())
resp, err := b.client.Get(ctx, key, clientv3.WithPrefix())
if err != nil {
return nil, err
}
@ -310,7 +313,7 @@ func (b *etcdBroker) nextTask(queue string) (*Signature, error) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*2)
defer cancel()
item, err := getFirstItem(ctx, b.cli, keyPrefix)
item, err := getFirstItem(ctx, b.client, keyPrefix)
if err != nil {
return nil, err
}
@ -330,7 +333,7 @@ func (b *etcdBroker) nextDelayedTask() (*Signature, error) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*2)
defer cancel()
item, err := getFirstETAItem(ctx, b.cli, keyPrefix)
item, err := getFirstETAItem(ctx, b.client, keyPrefix)
if err != nil {
return nil, err
}
@ -360,7 +363,7 @@ func (b *etcdBroker) requeueMessage(delivery *Signature, taskProcessor iface.Tas
return
}
_, err = b.cli.KV.Put(b.conf.Context, key, string(body))
_, err = b.client.KV.Put(b.ctx, key, string(body))
if err != nil {
log.ERROR.Print(err)
}

18
examples/etcd/main.go
View File

@ -8,7 +8,6 @@ import (
"time"
"github.com/RichardKnop/machinery/v2"
redisbackend "github.com/RichardKnop/machinery/v2/backends/redis"
"github.com/RichardKnop/machinery/v2/config"
"github.com/RichardKnop/machinery/v2/example/tracers"
"github.com/RichardKnop/machinery/v2/log"
@ -18,6 +17,7 @@ import (
opentracinglog "github.com/opentracing/opentracing-go/log"
"github.com/urfave/cli"
etcdbackend "github.com/ifooth/machinery-plugins/backends/etcd"
etcdbroker "github.com/ifooth/machinery-plugins/brokers/etcd"
exampletasks "github.com/ifooth/machinery-plugins/examples/tasks"
etcdlock "github.com/ifooth/machinery-plugins/locks/etcd"
@ -67,24 +67,30 @@ func main() {
func startServer() (*machinery.Server, error) {
cnf := &config.Config{
DefaultQueue: "machinery_tasks",
Broker: "redis://localhost:6379",
ResultBackend: "redis://localhost:6379",
Broker: "http://127.0.0.1:2379",
ResultBackend: "http://127.0.0.1:2379",
Lock: "http://127.0.0.1:2379",
ResultsExpireIn: 3600,
}
ctx := context.Background()
// Create server instance
// broker := redisbroker.NewGR(cnf, []string{"localhost:6379"}, 1)
broker, err := etcdbroker.New(cnf, "http://127.0.0.1:2379")
broker, err := etcdbroker.New(ctx, cnf)
if err != nil {
return nil, err
}
// lock := redislock.New(cnf, []string{"localhost:6379"}, 3, 2)
lock, err := etcdlock.New(context.Background(), "http://127.0.0.1:2379", 3)
lock, err := etcdlock.New(ctx, cnf, 3)
if err != nil {
return nil, err
}
// backend := redisbackend.NewGR(cnf, []string{"localhost:6379"}, 3)
backend, err := etcdbackend.New(ctx, cnf)
if err != nil {
return nil, err
}
backend := redisbackend.NewGR(cnf, []string{"localhost:6379"}, 3)
server := machinery.NewServer(cnf, broker, backend, lock)
// Register tasks

7
go.mod
View File

@ -5,6 +5,7 @@ go 1.20
require (
github.com/RichardKnop/machinery/v2 v2.0.13
github.com/google/uuid v1.3.1
github.com/jmoiron/sqlx v1.4.0
github.com/opentracing/opentracing-go v1.2.0
github.com/urfave/cli v1.22.5
go.etcd.io/etcd/api/v3 v3.5.13
@ -20,12 +21,9 @@ require (
cloud.google.com/go/pubsub v1.33.0 // indirect
github.com/RichardKnop/logging v0.0.0-20190827224416-1a693bdd4fae // indirect
github.com/aws/aws-sdk-go v1.37.16 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/go-redsync/redsync/v4 v4.8.1 // indirect
github.com/go-stack/stack v1.8.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
@ -35,14 +33,11 @@ require (
github.com/google/s2a-go v0.1.4 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
github.com/googleapis/gax-go/v2 v2.11.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/kelseyhightower/envconfig v1.4.0 // indirect
github.com/klauspost/compress v1.9.5 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/rabbitmq/amqp091-go v1.9.0 // indirect
github.com/redis/go-redis/v9 v9.0.5 // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/russross/blackfriday/v2 v2.0.1 // indirect
github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect

22
go.sum
View File

@ -46,6 +46,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/RichardKnop/logging v0.0.0-20190827224416-1a693bdd4fae h1:DcFpTQBYQ9Ct2d6sC7ol0/ynxc2pO1cpGUM+f4t5adg=
@ -58,15 +60,12 @@ github.com/aws/aws-sdk-go v1.37.16 h1:Q4YOP2s00NpB9wfmTDZArdcLRuG9ijbnoAwTW3ivle
github.com/aws/aws-sdk-go v1.37.16/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA=
github.com/bsm/ginkgo/v2 v2.5.0/go.mod h1:AiKlXPm7ItEHNc/2+OkrNG4E0ITzojb9/xWzvQ9XZ9w=
github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao=
github.com/bsm/ginkgo/v2 v2.7.0/go.mod h1:AiKlXPm7ItEHNc/2+OkrNG4E0ITzojb9/xWzvQ9XZ9w=
github.com/bsm/gomega v1.20.0/go.mod h1:JifAceMQ4crZIWYUKrlGcmbN3bqHogVTADMD2ATsbwk=
github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y=
github.com/bsm/gomega v1.26.0/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
@ -88,7 +87,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@ -103,15 +101,13 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg=
github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
github.com/go-redis/redis/v7 v7.4.0 h1:7obg6wUoj05T0EpY0o8B59S9w5yeMWql7sw2kwNW1x4=
github.com/go-redis/redis/v7 v7.4.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F46Tg=
github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w=
github.com/go-redsync/redsync/v4 v4.8.1 h1:rq2RvdTI0obznMdxKUWGdmmulo7lS9yCzb8fgDKOlbM=
github.com/go-redsync/redsync/v4 v4.8.1/go.mod h1:LmUAsQuQxhzZAoGY7JS6+dNhNmZyonMZiiEDY9plotM=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
@ -222,9 +218,7 @@ github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cU
github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@ -236,6 +230,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
@ -254,8 +250,12 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
@ -280,7 +280,6 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
github.com/rabbitmq/amqp091-go v1.9.0 h1:qrQtyzB4H8BQgEuJwhmVQqVHB9O4+MNDJCCAcpc3Aoo=
github.com/rabbitmq/amqp091-go v1.9.0/go.mod h1:+jPrT9iY2eLjRaMSRHUhc3z14E/l85kv/f+6luSD3pc=
github.com/redis/go-redis/v9 v9.0.2/go.mod h1:/xDTe9EF1LM61hek62Poq2nzQSGj0xSrEtEHbBQevps=
github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl5o=
github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
@ -312,7 +311,6 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203 h1:QVqDTf3h2WHt08YuiTGPZLls0Wq99X9bWd0Q5ZSBesM=
github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203/go.mod h1:oqN97ltKNihBbwlX8dLpwxCl3+HnXKV/R0e+sRLd9C8=
github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=

15
locks/etcd/etcd.go
View File

@ -7,6 +7,7 @@ import (
"strings"
"time"
"github.com/RichardKnop/machinery/v2/config"
"github.com/RichardKnop/machinery/v2/locks/iface"
"github.com/RichardKnop/machinery/v2/log"
clientv3 "go.etcd.io/etcd/client/v3"
@ -19,25 +20,27 @@ var (
)
type etcdLock struct {
cli *clientv3.Client
conf *config.Config
client *clientv3.Client
retries int
}
// New ..
func New(ctx context.Context, endpoint string, retries int) (iface.Lock, error) {
func New(ctx context.Context, conf *config.Config, retries int) (iface.Lock, error) {
etcdConf := clientv3.Config{
Endpoints: []string{endpoint},
Endpoints: []string{conf.Lock},
Context: ctx,
DialTimeout: time.Second * 5,
}
cli, err := clientv3.New(etcdConf)
client, err := clientv3.New(etcdConf)
if err != nil {
return nil, err
}
lock := etcdLock{
cli: cli,
conf: conf,
client: client,
retries: retries,
}
@ -68,7 +71,7 @@ func (r etcdLock) Lock(key string, unixTsToExpireNs int64) error {
ttl := time.Duration(unixTsToExpireNs + 1 - now)
// 创建一个新的session
s, err := concurrency.NewSession(r.cli, concurrency.WithTTL(int(ttl.Seconds())))
s, err := concurrency.NewSession(r.client, concurrency.WithTTL(int(ttl.Seconds())))
if err != nil {
return err
}