add handle more

brokers/etcd/etcd.go

@@ -136,19 +136,20 @@ func (b *etcdBroker) StartConsuming(consumerTag string, concurrency int, taskPro
 		defer b.wg.Done()
 		defer cancel()
 
-		ticker := time.NewTicker(time.Second)
-		defer ticker.Stop()
-
 		for {
 			select {
 			// A way to stop this goroutine from b.StopConsuming
 			case <-b.GetStopChan():
 				return
 
-			case <-ticker.C:
-				err := b.handleDelayedTask(ctx)
+			default:
+				more, err := b.handleDelayedTask(ctx)
 				if err != nil {
 					log.ERROR.Printf("handleDelayedTask err: %s", err)
+					time.Sleep(time.Second)
+				}
+				if !more {
+					time.Sleep(time.Second)
 				}
 			}
 		}
@@ -380,7 +381,7 @@ func (b *etcdBroker) listWatchTasks(ctx context.Context, queue string) error {
 	return nil
 }
 
-func (b *etcdBroker) handleDelayedTask(ctx context.Context) error {
+func (b *etcdBroker) handleDelayedTask(ctx context.Context) (bool, error) {
 	ttl := time.Second * 10
 	ctx, cancel := context.WithTimeout(ctx, ttl)
 	defer cancel()
@@ -388,7 +389,7 @@ func (b *etcdBroker) handleDelayedTask(ctx context.Context) error {
 	// 创建一个新的session
 	s, err := concurrency.NewSession(b.client, concurrency.WithTTL(int(ttl.Seconds())))
 	if err != nil {
-		return err
+		return false, err
 	}
 	defer s.Orphan()
 
@@ -396,17 +397,17 @@ func (b *etcdBroker) handleDelayedTask(ctx context.Context) error {
 	m := concurrency.NewMutex(s, lockKey)
 
 	if err = m.Lock(ctx); err != nil {
-		return err
+		return false, err
 	}
 	defer m.Unlock(ctx) // nolint
 
 	keyPrefix := "/machinery/v2/broker/delayed_tasks/eta-"
 	end := strconv.FormatInt(time.Now().UnixMilli(), 10)
-	kvs, err := b.client.Get(b.ctx, keyPrefix+"0", clientv3.WithRange(keyPrefix+end))
+	resp, err := b.client.Get(b.ctx, keyPrefix+"0", clientv3.WithRange(keyPrefix+end), clientv3.WithLimit(100))
 	if err != nil {
-		return err
+		return false, err
 	}
-	for _, kv := range kvs.Kvs {
+	for _, kv := range resp.Kvs {
 		key := string(kv.Key)
 		parts := strings.Split(key, "/")
 		if len(parts) != 8 {
@@ -419,7 +420,7 @@ func (b *etcdBroker) handleDelayedTask(ctx context.Context) error {
 		putReq := clientv3.OpPut(pendingKey, string(kv.Value))
 		c, err := b.client.Txn(b.ctx).If(cmp).Then(deleteReq, putReq).Commit()
 		if err != nil {
-			return fmt.Errorf("handle delay task %s: %w", key, err)
+			return false, fmt.Errorf("handle delay task %s: %w", key, err)
 		}
 		if !c.Succeeded {
 			log.WARNING.Printf("handle delay task %s not success", key)
@@ -428,7 +429,7 @@ func (b *etcdBroker) handleDelayedTask(ctx context.Context) error {
 		log.DEBUG.Printf("send delay task %s to pending queue done", key)
 	}
 
-	return nil
+	return resp.More, nil
 }
 
 func getQueue(config *config.Config, taskProcessor iface.TaskProcessor) string {