Skip to content

The network connection is reconnected, but the producer and consumer still cannot work #60

Open
Open
The network connection is reconnected, but the producer and consumer still cannot work#60
@dsmlily

Description

@dsmlily
dsmlily
opened on May 11, 2020

Expected behavior

the producer and consumer still worker

Actual behavior

they don't work

Steps to reproduce

producer.go:

package main

import (
	"context"
	"fmt"
	"log"
	"time"

	"github.com/wolfstudy/pulsar-client-go/core/manage"
	"github.com/wolfstudy/pulsar-client-go/core/pub"
	pulsarlog "github.com/wolfstudy/pulsar-client-go/pkg/log"
)

var clientPool = manage.NewClientPool()

func main() {
	pulsarlog.SetLevelByString("debug")
	pulsarlog.SetOutputByName("/tmp/producer/producer.log")

	pulsarlog.Info(pulsarlog.GetLogLevelAsString())
	c := make(chan error, 2)
	ctx := context.Background()
	producerConf := manage.ProducerConfig{
		ClientConfig: manage.ClientConfig{
			Addr: "10.32.69.26:6650",
			Errs: c,
		},

		Topic: "public/default/dsmtest",
		//Topic: "multi-topic-10",

		Router:             pub.RoundRobinDistribution,
		NewProducerTimeout: time.Second * 5,
	}

	go func() {
		for {
			select {
			case got := <-c:
				log.Println("async error:", got)
				pulsarlog.Info("async error:", got)
			}
		}
	}()

	//mp := manage.NewManagedProducer(clientPool, producerConf)

	mp, err := manage.NewManagedPartitionProducer(clientPool, producerConf)
	if err != nil {
		log.Fatal(err)
	}
	i := 0
	timeout := time.NewTicker((time.Duration(10)) * time.Millisecond)
	for {
		select {
		case <-timeout.C:
			payload := fmt.Sprintf("hello-%d", i)
			msgKey := fmt.Sprintf("key-%d", i%7)
			ss, err := mp.Send(ctx, []byte(payload), msgKey)
			if err != nil {
				pulsarlog.Error(err)
				log.Fatal(err)
			}
			fmt.Println(ss.GetMessageId())
			pulsarlog.Info(ss.GetMessageId(), msgKey, payload)
			i++
		}

	}
}

consumer.go

package main

 import (
	 "context"
	 "fmt"
	 "log"
 
	 //"time"
 
	 "github.com/wolfstudy/pulsar-client-go/core/manage"
	 pulsarlog "github.com/wolfstudy/pulsar-client-go/pkg/log"
 )
 
 var clientPool = manage.NewClientPool()
 
 func main() {
	 pulsarlog.SetLevelByString("debug")
	 pulsarlog.SetOutputByName("/tmp/consumer/consumer.log")
 
	 pulsarlog.Info(pulsarlog.GetLogLevelAsString())
	 ctx := context.Background()
	 c := make(chan error, 2)
 
	 consumerConf := manage.ConsumerConfig{
		 ClientConfig: manage.ClientConfig{
			 Addr: "10.32.69.26:6650",
			 Errs: c,
		 },
 
		 Topic: "public/default/dsmtest",
		 //Topic:   "multi-topic-10",
		 Name:    "sub-2",
		 SubMode: manage.SubscriptionModeKeyShared,
		 //SubMode:          manage.SubscriptionModeShard,
		 AckTimeoutMillis: 1000 * 5,
		 QueueSize:        1024,
	 }
 
	 go func() {
		 for {
			 select {
			 case got := <-c:
				 log.Println("async error:", got)
				 pulsarlog.Info("async error:", got)
			 }
		 }
	 }()
	 mp, err := manage.NewPartitionManagedConsumer(clientPool, consumerConf)
	 if err != nil {
		 log.Fatal(err)
	 } else {
		 fmt.Println("consumer create done")
	 }
	 for {
		 msg, err := mp.Receive(ctx)
		 if err != nil {
			 log.Fatal(err)
		 }
 
		 fmt.Printf("consumer1----> msg key is:%s,  msg value is:%s\n", msg.Meta.GetPartitionKey(), string(msg.Payload))
		 err = mp.Ack(ctx, msg)
 
		 if err != nil {
			 log.Fatal(err)
		 }
	}
 
 }

producer log

2020/05/11 17:23:08.035 log.go:75: [debug] choose partition is: 0, msg payload is: hello-19625, msg key is: key-4
2020/05/11 17:23:08.334 producer.go:84: [info] ledgerId:735767 entryId:27158 key-4hello-19625
2020/05/11 17:23:08.335 log.go:75: [debug] choose partition is: 1, msg payload is: hello-19626, msg key is: key-5
2020/05/11 17:53:47.863 producer.go:60: [info] async error:read tcp4 172.23.110.253:51888->10.32.69.26:6650: read: network is down
2020/05/11 17:53:53.248 producer.go:60: [info] async error:read tcp4 172.23.110.253:51889->10.32.69.26:6652: read: connection reset by
peer
2020/05/11 17:53:54.265 producer.go:60: [info] async error:write tcp4 172.23.110.253:51889->10.32.69.26:6652: use of closed network con
nection
2020/05/11 17:53:56.278 producer.go:60: [info] async error:write tcp4 172.23.110.253:51889->10.32.69.26:6652: use of closed network con
nection
2020/05/11 17:54:00.301 producer.go:60: [info] async error:write tcp4 172.23.110.253:51889->10.32.69.26:6652: use of closed network con
nection
2020/05/11 17:54:08.242 producer.go:60: [info] async error:context deadline exceeded
2020/05/11 18:32:30.974 producer.go:60: [info] async error:read tcp4 172.23.110.253:52629->10.32.69.26:6650: read: connection reset by peer
2020/05/11 18:32:31.008 producer.go:60: [info] async error:read tcp4 172.23.110.253:52674->10.32.69.26:6652: read: connection reset by peer
2020/05/11 18:32:31.977 producer.go:60: [info] async error:write tcp4 172.23.110.253:52629->10.32.69.26:6650: use of closed network connection
2020/05/11 18:32:32.009 producer.go:60: [info] async error:write tcp4 172.23.110.253:52629->10.32.69.26:6650: use of closed network connection
2020/05/11 18:32:33.342 producer.go:60: [info] async error:write tcp4 172.23.110.253:52629->10.32.69.26:6650: use of closed network connection
2020/05/11 18:32:33.374 producer.go:60: [info] async error:write tcp4 172.23.110.253:52629->10.32.69.26:6650: use of closed network connection
2020/05/11 18:32:37.347 producer.go:60: [info] async error:write tcp4 172.23.110.253:52629->10.32.69.26:6650: use of closed network connection
2020/05/11 18:32:37.378 producer.go:60: [info] async error:write tcp4 172.23.110.253:52629->10.32.69.26:6650: use of closed network connection
2020/05/11 18:32:45.330 producer.go:60: [info] async error:context deadline exceeded
2020/05/11 18:32:45.361 producer.go:60: [info] async error:context deadline exceeded
2020/05/11 18:57:16.058 producer.go:60: [info] async error:read tcp4 172.23.110.253:55325->10.32.69.26:6652: read: connection reset by peer
2020/05/11 18:57:16.058 producer.go:60: [info] async error:read tcp4 172.23.110.253:55324->10.32.69.26:6650: read: connection reset by peer
2020/05/11 18:57:17.060 producer.go:60: [info] async error:write tcp4 172.23.110.253:55324->10.32.69.26:6650: use of closed network connection
2020/05/11 18:57:17.061 producer.go:60: [info] async error:write tcp4 172.23.110.253:55324->10.32.69.26:6650: use of closed network connection
2020/05/11 18:57:19.061 producer.go:60: [info] async error:write tcp4 172.23.110.253:55324->10.32.69.26:6650: use of closed network connection
2020/05/11 18:57:19.063 producer.go:60: [info] async error:write tcp4 172.23.110.253:55324->10.32.69.26:6650: use of closed network connection
2020/05/11 18:57:23.063 producer.go:60: [info] async error:write tcp4 172.23.110.253:55324->10.32.69.26:6650: use of closed network connection
2020/05/11 18:57:23.064 producer.go:60: [info] async error:write tcp4 172.23.110.253:55324->10.32.69.26:6650: use of closed network connection
2020/05/11 18:57:31.021 producer.go:60: [info] async error:context deadline exceeded
2020/05/11 18:57:31.049 producer.go:60: [info] async error:context deadline exceeded
2020/05/11 19:04:45.027 producer.go:60: [info] async error:read tcp4 172.23.110.253:57751->10.32.69.26:6650: read: connection reset by peer
2020/05/11 19:04:45.059 producer.go:60: [info] async error:read tcp4 172.23.110.253:57752->10.32.69.26:6652: read: connection reset by peer
2020/05/11 19:04:46.031 producer.go:60: [info] async error:write tcp4 172.23.110.253:57751->10.32.69.26:6650: use of closed network connection
2020/05/11 19:04:46.060 producer.go:60: [info] async error:write tcp4 172.23.110.253:57751->10.32.69.26:6650: use of closed network connection
2020/05/11 19:04:48.033 producer.go:60: [info] async error:write tcp4 172.23.110.253:57751->10.32.69.26:6650: use of closed network connection
2020/05/11 19:04:48.063 producer.go:60: [info] async error:write tcp4 172.23.110.253:57751->10.32.69.26:6650: use of closed network connection
2020/05/11 19:04:52.036 producer.go:60: [info] async error:write tcp4 172.23.110.253:57751->10.32.69.26:6650: use of closed network connection
2020/05/11 19:04:52.068 producer.go:60: [info] async error:write tcp4 172.23.110.253:57751->10.32.69.26:6650: use of closed network connection
2020/05/11 19:05:00.022 producer.go:60: [info] async error:context deadline exceeded
2020/05/11 19:05:00.051 producer.go:60: [info] async error:context deadline exceeded

consumer log

2020/05/11 18:57:12.463 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:57:12.463512 +0800 CST m=+2245.005107756
2020/05/11 18:57:17.465 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:57:17.465347 +0800 CST m=+2250.006896772
2020/05/11 18:57:17.497 consumer.go:63: [info] async error:read tcp4 172.23.110.253:55333->10.32.69.26:6650: read: connection reset by
peer
2020/05/11 18:57:17.502 consumer.go:63: [info] async error:consumer connection closed
2020/05/11 18:57:17.502 consumer.go:63: [info] async error:write tcp4 172.23.110.253:55333->10.32.69.26:6650: use of closed network con
nection
2020/05/11 18:57:17.526 consumer.go:63: [info] async error:read tcp4 172.23.110.253:55334->10.32.69.26:6652: read: connection reset by peer
2020/05/11 18:57:17.527 consumer.go:63: [info] async error:consumer connection closed
2020/05/11 18:57:17.527 consumer.go:63: [info] async error:write tcp4 172.23.110.253:55334->10.32.69.26:6652: use of closed network connection
2020/05/11 18:57:18.505 consumer.go:63: [info] async error:write tcp4 172.23.110.253:55333->10.32.69.26:6650: use of closed network connection
2020/05/11 18:57:18.527 consumer.go:63: [info] async error:write tcp4 172.23.110.253:55333->10.32.69.26:6650: use of closed network connection
2020/05/11 18:57:20.505 consumer.go:63: [info] async error:write tcp4 172.23.110.253:55333->10.32.69.26:6650: use of closed network connection
2020/05/11 18:57:20.531 consumer.go:63: [info] async error:write tcp4 172.23.110.253:55333->10.32.69.26:6650: use of closed network connection
2020/05/11 18:57:22.463 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:57:22.463562 +0800 CST m=+2255.005068375
2020/05/11 18:57:24.507 consumer.go:63: [info] async error:write tcp4 172.23.110.253:55333->10.32.69.26:6650: use of closed network connection
2020/05/11 18:57:24.535 consumer.go:63: [info] async error:write tcp4 172.23.110.253:55333->10.32.69.26:6650: use of closed network connection
2020/05/11 18:57:27.464 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:57:27.46455 +0800 CST m=+2260.006012513
2020/05/11 18:57:32.463 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:57:32.463932 +0800 CST m=+2265.005352308
2020/05/11 18:57:32.488 consumer.go:63: [info] async error:context deadline exceeded
2020/05/11 18:57:32.521 consumer.go:63: [info] async error:context deadline exceeded
2020/05/11 18:57:37.464 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:57:37.464294 +0800 CST m=+2270.005671995
2020/05/11 18:57:42.464 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:57:42.464134 +0800 CST m=+2275.005470606
2020/05/11 18:57:47.465 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:57:47.46564 +0800 CST m=+2280.006934806
2020/05/11 18:57:52.465 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:57:52.465642 +0800 CST m=+2285.006895473
2020/05/11 18:57:57.463 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:57:57.463744 +0800 CST m=+2290.004955525
2020/05/11 18:58:02.465 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:58:02.465159 +0800 CST m=+2295.006329275
2020/05/11 18:58:07.462 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:58:07.462466 +0800 CST m=+2300.003595368
2020/05/11 18:58:12.465 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:58:12.465804 +0800 CST m=+2305.006892912
2020/05/11 18:58:17.466 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:58:17.466339 +0800 CST m=+2310.007386604
2020/05/11 18:58:22.463 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:58:22.463773 +0800 CST m=+2315.004779352
2020/05/11 18:58:27.462 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:58:27.462379 +0800 CST m=+2320.003344237
2020/05/11 18:58:32.464 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:58:32.464371 +0800 CST m=+2325.005295347
2020/05/11 18:58:37.466 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:58:37.466078 +0800 CST m=+2330.006961501
2020/05/11 18:58:42.466 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:58:42.466029 +0800 CST m=+2335.006871180
2020/05/11 18:58:47.462 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:58:47.462441 +0800 CST m=+2340.003242424
2020/05/11 18:58:52.466 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:58:52.4661 +0800 CST m=+2345.006860985
2020/05/11 18:58:57.466 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:58:57.466147 +0800 CST m=+2350.006867052
2020/05/11 18:59:02.463 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:59:02.46332 +0800 CST m=+2355.003998456
2020/05/11 18:59:07.466 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:59:07.466251 +0800 CST m=+2360.006889150
2020/05/11 18:59:12.465 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:59:12.465521 +0800 CST m=+2365.006118632
2020/05/11 18:59:17.462 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:59:17.462871 +0800 CST m=+2370.003427040
2020/05/11 18:59:22.466 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:59:22.466848 +0800 CST m=+2375.007362924
2020/05/11 18:59:27.465 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:59:27.465099 +0800 CST m=+2380.005572927
2020/05/11 18:59:32.466 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:59:32.466315 +0800 CST m=+2385.006748639
2020/05/11 18:59:37.464 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:59:37.464439 +0800 CST m=+2390.004831397
2020/05/11 18:59:42.466 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:59:42.466535 +0800 CST m=+2395.006886825
2020/05/11 18:59:47.467 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:59:47.467091 +0800 CST m=+2400.007401889
2020/05/11 18:59:52.464 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:59:52.464738 +0800 CST m=+2405.005007248
2020/05/11 18:59:57.465 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 18:59:57.465314 +0800 CST m=+2410.005542660
2020/05/11 19:00:02.463 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:00:02.463279 +0800 CST m=+2415.003466184
2020/05/11 19:00:07.464 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:00:07.464681 +0800 CST m=+2420.004827176
2020/05/11 19:00:12.466 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:00:12.466668 +0800 CST m=+2425.006773784
2020/05/11 19:00:17.467 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:00:17.467394 +0800 CST m=+2430.007458496
2020/05/11 19:00:22.466 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:00:22.466852 +0800 CST m=+2435.006875698
2020/05/11 19:00:27.465 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:00:27.465771 +0800 CST m=+2440.005753842
2020/05/11 19:00:32.465 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:00:32.465633 +0800 CST m=+2445.005574539
2020/05/11 19:00:37.463 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:00:37.463536 +0800 CST m=+2450.003436845
2020/05/11 19:00:42.466 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:00:42.466118 +0800 CST m=+2455.005978343
2020/05/11 19:00:47.467 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:00:47.467566 +0800 CST m=+2460.007384591
2020/05/11 19:00:52.467 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:00:52.467736 +0800 CST m=+2465.007514054
2020/05/11 19:00:57.465 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:00:57.46525 +0800 CST m=+2470.004987039
2020/05/11 19:01:02.467 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:01:02.467205 +0800 CST m=+2475.006901560
2020/05/11 19:01:07.463 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:01:07.463595 +0800 CST m=+2480.003250346
2020/05/11 19:01:12.467 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:01:12.46729 +0800 CST m=+2485.006904542
2020/05/11 19:01:17.464 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:01:17.464338 +0800 CST m=+2490.003911367
2020/05/11 19:01:22.464 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:01:22.464133 +0800 CST m=+2495.003665274
2020/05/11 19:01:27.466 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:01:27.466727 +0800 CST m=+2500.006218606
2020/05/11 19:01:32.465 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:01:32.465421 +0800 CST m=+2505.004871229
2020/05/11 19:01:37.468 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:01:37.468204 +0800 CST m=+2510.007613561
2020/05/11 19:01:42.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:01:42.469735 +0800 CST m=+2515.009103931
2020/05/11 19:01:47.468 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:01:47.468091 +0800 CST m=+2520.007418610
2020/05/11 19:01:52.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:01:52.468915 +0800 CST m=+2525.008201472
2020/05/11 19:01:57.467 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:01:57.467693 +0800 CST m=+2530.006939241
2020/05/11 19:02:02.468 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:02:02.468202 +0800 CST m=+2535.007407228
2020/05/11 19:02:07.471 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:02:07.471271 +0800 CST m=+2540.010434624
2020/05/11 19:02:12.467 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:02:12.467748 +0800 CST m=+2545.006871429
2020/05/11 19:02:17.474 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:02:17.474265 +0800 CST m=+2550.013347488
2020/05/11 19:02:22.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:02:22.470555 +0800 CST m=+2555.009596492
2020/05/11 19:02:27.466 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:02:27.466678 +0800 CST m=+2560.005678567
2020/05/11 19:02:32.464 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:02:32.464806 +0800 CST m=+2565.003765896
2020/05/11 19:02:37.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:02:37.469049 +0800 CST m=+2570.008181016
2020/05/11 19:02:42.467 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:02:42.467216 +0800 CST m=+2575.006841909
2020/05/11 19:02:47.467 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:02:47.466964 +0800 CST m=+2580.006936773
2020/05/11 19:02:52.468 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:02:52.468323 +0800 CST m=+2585.008535542
2020/05/11 19:04:26.926 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:04:26.926906 +0800 CST m=+2590.007916421
2020/05/11 19:04:31.466 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:04:31.466228 +0800 CST m=+2595.005626428
2020/05/11 19:04:36.468 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:04:36.468187 +0800 CST m=+2600.007329908
2020/05/11 19:04:41.464 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:04:41.464707 +0800 CST m=+2605.003654446
2020/05/11 19:04:46.465 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:04:46.46589 +0800 CST m=+2610.004683874
2020/05/11 19:04:46.491 consumer.go:63: [info] async error:read tcp4 172.23.110.253:57758->10.32.69.26:6650: read: connection reset by peer
2020/05/11 19:04:46.491 consumer.go:63: [info] async error:consumer connection closed
2020/05/11 19:04:46.491 consumer.go:63: [info] async error:write tcp4 172.23.110.253:57758->10.32.69.26:6650: use of closed network connection
2020/05/11 19:04:46.528 consumer.go:63: [info] async error:read tcp4 172.23.110.253:57759->10.32.69.26:6652: read: connection reset by peer
2020/05/11 19:04:46.528 consumer.go:63: [info] async error:consumer connection closed
2020/05/11 19:04:46.531 consumer.go:63: [info] async error:write tcp4 172.23.110.253:57759->10.32.69.26:6652: use of closed network connection
2020/05/11 19:04:47.495 consumer.go:63: [info] async error:write tcp4 172.23.110.253:57758->10.32.69.26:6650: use of closed network connection
2020/05/11 19:04:47.534 consumer.go:63: [info] async error:write tcp4 172.23.110.253:57758->10.32.69.26:6650: use of closed network connection
2020/05/11 19:04:49.495 consumer.go:63: [info] async error:write tcp4 172.23.110.253:57758->10.32.69.26:6650: use of closed network connection
2020/05/11 19:04:49.534 consumer.go:63: [info] async error:write tcp4 172.23.110.253:57758->10.32.69.26:6650: use of closed network connection
2020/05/11 19:04:51.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:04:51.469128 +0800 CST m=+2615.007800211
2020/05/11 19:04:53.499 consumer.go:63: [info] async error:write tcp4 172.23.110.253:57758->10.32.69.26:6650: use of closed network connection
2020/05/11 19:04:53.535 consumer.go:63: [info] async error:write tcp4 172.23.110.253:57758->10.32.69.26:6650: use of closed network connection
2020/05/11 19:04:56.465 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:04:56.465039 +0800 CST m=+2620.003611412
2020/05/11 19:05:01.466 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:05:01.466328 +0800 CST m=+2625.004817176
2020/05/11 19:05:01.483 consumer.go:63: [info] async error:context deadline exceeded
2020/05/11 19:05:01.522 consumer.go:63: [info] async error:context deadline exceeded
2020/05/11 19:05:06.467 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:05:06.4677 +0800 CST m=+2630.006117168
2020/05/11 19:05:11.467 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:05:11.466964 +0800 CST m=+2635.005317743
2020/05/11 19:05:16.465 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:05:16.465143 +0800 CST m=+2640.003440104
2020/05/11 19:05:21.465 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:05:21.465136 +0800 CST m=+2645.003380345
2020/05/11 19:05:26.466 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:05:26.46676 +0800 CST m=+2650.004954442
2020/05/11 19:05:31.467 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:05:31.467959 +0800 CST m=+2655.006106972
2020/05/11 19:05:36.465 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:05:36.465307 +0800 CST m=+2660.003409485
2020/05/11 19:05:41.467 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:05:41.467883 +0800 CST m=+2665.005941855
2020/05/11 19:05:46.467 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:05:46.467891 +0800 CST m=+2670.005906427
2020/05/11 19:05:51.466 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:05:51.466818 +0800 CST m=+2675.004790670
2020/05/11 19:05:56.467 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:05:56.467624 +0800 CST m=+2680.005554473
2020/05/11 19:06:01.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:06:01.469345 +0800 CST m=+2685.007234116
2020/05/11 19:06:06.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:06:06.469553 +0800 CST m=+2690.007400670
2020/05/11 19:06:11.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:06:11.469974 +0800 CST m=+2695.007780451
2020/05/11 19:06:16.467 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:06:16.46704 +0800 CST m=+2700.004805369
2020/05/11 19:06:21.468 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:06:21.468784 +0800 CST m=+2705.006507897
2020/05/11 19:06:26.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:06:26.469744 +0800 CST m=+2710.007426568
2020/05/11 19:06:31.468 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:06:31.468409 +0800 CST m=+2715.006054821
2020/05/11 19:06:36.468 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:06:36.467994 +0800 CST m=+2720.005594442
2020/05/11 19:06:41.466 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:06:41.46598 +0800 CST m=+2725.003539871
2020/05/11 19:06:46.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:06:46.469846 +0800 CST m=+2730.007365149
2020/05/11 19:06:51.467 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:06:51.467596 +0800 CST m=+2735.005073690
2020/05/11 19:06:56.465 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:06:56.465826 +0800 CST m=+2740.003263326
2020/05/11 19:07:01.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:07:01.470424 +0800 CST m=+2745.007819969
2020/05/11 19:07:06.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:07:06.468968 +0800 CST m=+2750.006323949
2020/05/11 19:07:11.471 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:07:11.47088 +0800 CST m=+2755.008194119
2020/05/11 19:07:16.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:07:16.470086 +0800 CST m=+2760.007359369
2020/05/11 19:07:21.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:07:21.470269 +0800 CST m=+2765.007501635
2020/05/11 19:07:26.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:07:26.470343 +0800 CST m=+2770.007534805
2020/05/11 19:07:31.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:07:31.469119 +0800 CST m=+2775.006270154
2020/05/11 19:07:36.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:07:36.469202 +0800 CST m=+2780.006312466
2020/05/11 19:07:41.471 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:07:41.470459 +0800 CST m=+2785.007528501
2020/05/11 19:07:46.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:07:46.468957 +0800 CST m=+2790.005985848
2020/05/11 19:07:51.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:07:51.470344 +0800 CST m=+2795.007331955
2020/05/11 19:07:56.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:07:56.469521 +0800 CST m=+2800.006467765
2020/05/11 19:08:01.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:08:01.469337 +0800 CST m=+2805.006243264
2020/05/11 19:08:06.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:08:06.469917 +0800 CST m=+2810.006782213
2020/05/11 19:08:11.466 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:08:11.466859 +0800 CST m=+2815.003683551
2020/05/11 19:08:16.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:08:16.470011 +0800 CST m=+2820.006795115
2020/05/11 19:08:21.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:08:21.470465 +0800 CST m=+2825.007207323
2020/05/11 19:08:26.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:08:26.470436 +0800 CST m=+2830.007137910
2020/05/11 19:08:31.471 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:08:31.471528 +0800 CST m=+2835.008189445
2020/05/11 19:08:36.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:08:36.470759 +0800 CST m=+2840.007378687
2020/05/11 19:08:41.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:08:41.470721 +0800 CST m=+2845.007299921
2020/05/11 19:08:46.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:08:46.47056 +0800 CST m=+2850.007098068
2020/05/11 19:08:51.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:08:51.470434 +0800 CST m=+2855.006931030
2020/05/11 19:08:56.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:08:56.46999 +0800 CST m=+2860.006446338
2020/05/11 19:09:01.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:09:01.470675 +0800 CST m=+2865.007090632
2020/05/11 19:09:06.468 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:09:06.468282 +0800 CST m=+2870.004657414
2020/05/11 19:09:11.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:09:11.469383 +0800 CST m=+2875.005716985
2020/05/11 19:09:16.467 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:09:16.467506 +0800 CST m=+2880.003799581
2020/05/11 19:09:21.471 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:09:21.471038 +0800 CST m=+2885.007290788
2020/05/11 19:09:26.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:09:26.470579 +0800 CST m=+2890.006790908
2020/05/11 19:09:31.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:09:31.470603 +0800 CST m=+2895.006773386
2020/05/11 19:09:36.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:09:36.46921 +0800 CST m=+2900.005339855
2020/05/11 19:09:41.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:09:41.469526 +0800 CST m=+2905.005614616
2020/05/11 19:09:46.468 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:09:46.468878 +0800 CST m=+2910.004926624
2020/05/11 19:09:51.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:09:51.469482 +0800 CST m=+2915.005489431
2020/05/11 19:09:56.471 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:09:56.470979 +0800 CST m=+2920.006945335
2020/05/11 19:10:01.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:10:01.470278 +0800 CST m=+2925.006203925
2020/05/11 19:10:06.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:10:06.468957 +0800 CST m=+2930.004842012
2020/05/11 19:10:11.471 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:10:11.47112 +0800 CST m=+2935.006963793
2020/05/11 19:10:16.471 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:10:16.471889 +0800 CST m=+2940.007692348
2020/05/11 19:10:21.471 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:10:21.471325 +0800 CST m=+2945.007087519
2020/05/11 19:10:26.467 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:10:26.467594 +0800 CST m=+2950.003314928
2020/05/11 19:10:31.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:10:31.470009 +0800 CST m=+2955.005689512
2020/05/11 19:10:36.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:10:36.4694 +0800 CST m=+2960.005039984
2020/05/11 19:10:41.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:10:41.469513 +0800 CST m=+2965.005111418
2020/05/11 19:10:46.472 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:10:46.472573 +0800 CST m=+2970.008130931
2020/05/11 19:10:51.468 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:10:51.468891 +0800 CST m=+2975.004407861
2020/05/11 19:10:56.471 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:10:56.471946 +0800 CST m=+2980.007422431
2020/05/11 19:11:01.471 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:11:01.471379 +0800 CST m=+2985.006814577
2020/05/11 19:11:06.471 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:11:06.471204 +0800 CST m=+2990.006598258
2020/05/11 19:11:11.471 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:11:11.471067 +0800 CST m=+2995.006420657
2020/05/11 19:11:16.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:11:16.470875 +0800 CST m=+3000.006187749
2020/05/11 19:11:21.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:11:21.469835 +0800 CST m=+3005.005107114
2020/05/11 19:11:26.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:11:26.469434 +0800 CST m=+3010.004665318
2020/05/11 19:11:31.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:11:31.470407 +0800 CST m=+3015.005597632
2020/05/11 19:11:36.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:11:36.470245 +0800 CST m=+3020.005393867
2020/05/11 19:11:41.472 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:11:41.472845 +0800 CST m=+3025.007953067
2020/05/11 19:11:46.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:11:46.469904 +0800 CST m=+3030.004971804
2020/05/11 19:11:51.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:11:51.470052 +0800 CST m=+3035.005079252
2020/05/11 19:11:56.472 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:11:56.472286 +0800 CST m=+3040.007272019
2020/05/11 19:12:01.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:12:01.470943 +0800 CST m=+3045.005888625
2020/05/11 19:12:06.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:12:06.470174 +0800 CST m=+3050.005078532
2020/05/11 19:12:11.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:12:11.470073 +0800 CST m=+3055.004936474
2020/05/11 19:12:16.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:12:16.47007 +0800 CST m=+3060.004892053
2020/05/11 19:12:21.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:12:21.470103 +0800 CST m=+3065.004884653
2020/05/11 19:12:26.472 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:12:26.472374 +0800 CST m=+3070.007114372
2020/05/11 19:12:31.471 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:12:31.471881 +0800 CST m=+3075.006581103
2020/05/11 19:12:36.472 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:12:36.472218 +0800 CST m=+3080.006877333
2020/05/11 19:12:41.472 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:12:41.472325 +0800 CST m=+3085.006942883
2020/05/11 19:12:46.471 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:12:46.471744 +0800 CST m=+3090.006320916
2020/05/11 19:12:51.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:12:51.470653 +0800 CST m=+3095.005189465
2020/05/11 19:12:56.472 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:12:56.472348 +0800 CST m=+3100.006843876
2020/05/11 19:13:01.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:13:01.470357 +0800 CST m=+3105.004811845
2020/05/11 19:13:06.472 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:13:06.472404 +0800 CST m=+3110.006818499
2020/05/11 19:13:11.472 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:13:11.472507 +0800 CST m=+3115.006880402
2020/05/11 19:13:16.472 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:13:16.472582 +0800 CST m=+3120.006914073
2020/05/11 19:13:21.472 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:13:21.472482 +0800 CST m=+3125.006773303
2020/05/11 19:13:26.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:13:26.469151 +0800 CST m=+3130.003401097
2020/05/11 19:13:31.474 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:13:31.473995 +0800 CST m=+3135.008204217
2020/05/11 19:13:36.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:13:36.469569 +0800 CST m=+3140.003737911
2020/05/11 19:13:41.472 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:13:41.472703 +0800 CST m=+3145.006831132
2020/05/11 19:13:46.474 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:13:46.473954 +0800 CST m=+3150.008041213
2020/05/11 19:13:51.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:13:51.473378 +0800 CST m=+3155.007424359
2020/05/11 19:13:56.472 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:13:56.472408 +0800 CST m=+3160.006413226
2020/05/11 19:14:01.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:14:01.47279 +0800 CST m=+3165.006754274
2020/05/11 19:14:06.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:14:06.473498 +0800 CST m=+3170.007421838
2020/05/11 19:14:11.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:14:11.473879 +0800 CST m=+3175.007762147
2020/05/11 19:14:16.474 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:14:16.473936 +0800 CST m=+3180.007778000
2020/05/11 19:14:21.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:14:21.472989 +0800 CST m=+3185.006790588
2020/05/11 19:14:26.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:14:26.472958 +0800 CST m=+3190.006717950
2020/05/11 19:14:31.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:14:31.473053 +0800 CST m=+3195.006772347
2020/05/11 19:14:36.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:14:36.472891 +0800 CST m=+3200.006570116
2020/05/11 19:14:41.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:14:41.472977 +0800 CST m=+3205.006614292
2020/05/11 19:14:46.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:14:46.470125 +0800 CST m=+3210.003722353
2020/05/11 19:14:51.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:14:51.469714 +0800 CST m=+3215.003269824
2020/05/11 19:14:56.472 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:14:56.472486 +0800 CST m=+3220.006001414
2020/05/11 19:15:01.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:15:01.473044 +0800 CST m=+3225.006518136
2020/05/11 19:15:06.472 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:15:06.472021 +0800 CST m=+3230.005454351
2020/05/11 19:15:11.471 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:15:11.471517 +0800 CST m=+3235.004909740
2020/05/11 19:15:16.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:15:16.473835 +0800 CST m=+3240.007186447
2020/05/11 19:15:21.471 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:15:21.47176 +0800 CST m=+3245.005070701
2020/05/11 19:15:26.472 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:15:26.472118 +0800 CST m=+3250.005388453
2020/05/11 19:15:31.474 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:15:31.474007 +0800 CST m=+3255.007236648
2020/05/11 19:15:36.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:15:36.470073 +0800 CST m=+3260.003260979
2020/05/11 19:15:41.472 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:15:41.472152 +0800 CST m=+3265.005299881
2020/05/11 19:15:46.472 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:15:46.472359 +0800 CST m=+3270.005466015
2020/05/11 19:15:51.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:15:51.473269 +0800 CST m=+3275.006334894
2020/05/11 19:15:56.474 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:15:56.474426 +0800 CST m=+3280.007450946
2020/05/11 19:16:01.471 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:16:01.471411 +0800 CST m=+3285.004395605
2020/05/11 19:16:06.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:16:06.470361 +0800 CST m=+3290.003304592
2020/05/11 19:16:11.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:16:11.473431 +0800 CST m=+3295.006333458
2020/05/11 19:16:16.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:16:16.473634 +0800 CST m=+3300.006495924
2020/05/11 19:16:21.474 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:16:21.474467 +0800 CST m=+3305.007287616
2020/05/11 19:16:26.471 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:16:26.471102 +0800 CST m=+3310.003882017
2020/05/11 19:16:31.474 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:16:31.474067 +0800 CST m=+3315.006806000
2020/05/11 19:16:36.472 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:16:36.472133 +0800 CST m=+3320.004831299
2020/05/11 19:16:41.474 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:16:41.473864 +0800 CST m=+3325.006521186
2020/05/11 19:16:46.474 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:16:46.474507 +0800 CST m=+3330.007123648
2020/05/11 19:16:51.471 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:16:51.471427 +0800 CST m=+3335.004002415
2020/05/11 19:16:56.474 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:16:56.474441 +0800 CST m=+3340.006975485
2020/05/11 19:17:01.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:17:01.473241 +0800 CST m=+3345.005734590
2020/05/11 19:17:06.470 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:17:06.470919 +0800 CST m=+3350.003372442
2020/05/11 19:17:11.472 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:17:11.472367 +0800 CST m=+3355.004779347
2020/05/11 19:17:16.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:17:16.473786 +0800 CST m=+3360.006156983
2020/05/11 19:17:21.472 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:17:21.472757 +0800 CST m=+3365.005087047
2020/05/11 19:17:26.471 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:17:26.471363 +0800 CST m=+3370.003652378
2020/05/11 19:17:31.472 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:17:31.472115 +0800 CST m=+3375.004364090
2020/05/11 19:17:36.471 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:17:36.471521 +0800 CST m=+3380.003728463
2020/05/11 19:17:41.474 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:17:41.474586 +0800 CST m=+3385.006753298
2020/05/11 19:17:46.474 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:17:46.474628 +0800 CST m=+3390.006754031
2020/05/11 19:17:51.475 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:17:51.475336 +0800 CST m=+3395.007421305
2020/05/11 19:17:56.474 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:17:56.474668 +0800 CST m=+3400.006712728
2020/05/11 19:18:01.475 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:18:01.475114 +0800 CST m=+3405.007117215
2020/05/11 19:18:06.472 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:18:06.472883 +0800 CST m=+3410.004845864
2020/05/11 19:18:11.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:18:11.47356 +0800 CST m=+3415.005482405
2020/05/11 19:18:16.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:18:16.472982 +0800 CST m=+3420.004862995
2020/05/11 19:18:21.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:18:21.47337 +0800 CST m=+3425.005210529
2020/05/11 19:18:26.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:18:26.472957 +0800 CST m=+3430.004756251
2020/05/11 19:18:31.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:18:31.473162 +0800 CST m=+3435.004919982
2020/05/11 19:18:36.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:18:36.473413 +0800 CST m=+3440.005130857
2020/05/11 19:18:41.475 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:18:41.475825 +0800 CST m=+3445.007501861
2020/05/11 19:18:46.474 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:18:46.474316 +0800 CST m=+3450.005952100
2020/05/11 19:18:51.475 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:18:51.475594 +0800 CST m=+3455.007189361
2020/05/11 19:18:56.475 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:18:56.475835 +0800 CST m=+3460.007389022
2020/05/11 19:19:01.476 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:19:01.4766 +0800 CST m=+3465.008113601
2020/05/11 19:19:06.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:19:06.473595 +0800 CST m=+3470.005067959
2020/05/11 19:19:11.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:19:11.473419 +0800 CST m=+3475.004850751
2020/05/11 19:19:16.472 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:19:16.472755 +0800 CST m=+3480.004145556
2020/05/11 19:19:21.474 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:19:21.474697 +0800 CST m=+3485.006046949
2020/05/11 19:19:26.474 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:19:26.474678 +0800 CST m=+3490.005986946
2020/05/11 19:19:31.474 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:19:31.474229 +0800 CST m=+3495.005497027
2020/05/11 19:19:36.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:19:36.473563 +0800 CST m=+3500.004790539
2020/05/11 19:19:41.476 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:19:41.476812 +0800 CST m=+3505.007998816
2020/05/11 19:19:46.475 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:19:46.475832 +0800 CST m=+3510.006977590
2020/05/11 19:19:51.472 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:19:51.472164 +0800 CST m=+3515.003268554
2020/05/11 19:19:56.472 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:19:56.472231 +0800 CST m=+3520.003294606
2020/05/11 19:20:01.475 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:20:01.475852 +0800 CST m=+3525.006875023
2020/05/11 19:20:06.476 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:20:06.476371 +0800 CST m=+3530.007353194
2020/05/11 19:20:11.476 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:20:11.476182 +0800 CST m=+3535.007123846
2020/05/11 19:20:16.472 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:20:16.472896 +0800 CST m=+3540.003796730
2020/05/11 19:20:21.474 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:20:21.474228 +0800 CST m=+3545.005087588
2020/05/11 19:20:26.475 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:20:26.475406 +0800 CST m=+3550.006225425
2020/05/11 19:20:31.476 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:20:31.47603 +0800 CST m=+3555.006808359
2020/05/11 19:20:36.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:20:36.473746 +0800 CST m=+3560.004482754
2020/05/11 19:20:41.474 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:20:41.474203 +0800 CST m=+3565.004899508
2020/05/11 19:20:46.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:20:46.473861 +0800 CST m=+3570.004516604
2020/05/11 19:20:51.474 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:20:51.474283 +0800 CST m=+3575.004897987
2020/05/11 19:20:56.472 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:20:56.472662 +0800 CST m=+3580.003235394
2020/05/11 19:21:01.474 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:21:01.474281 +0800 CST m=+3585.004814060
2020/05/11 19:21:06.476 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:21:06.476674 +0800 CST m=+3590.007165861
2020/05/11 19:21:11.476 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:21:11.476814 +0800 CST m=+3595.007264823
2020/05/11 19:21:16.476 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:21:16.476499 +0800 CST m=+3600.006909324
2020/05/11 19:21:21.476 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:21:21.47652 +0800 CST m=+3605.006889511
2020/05/11 19:21:26.476 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:21:26.476714 +0800 CST m=+3610.007042958
2020/05/11 19:21:31.476 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:21:31.47652 +0800 CST m=+3615.006807530
2020/05/11 19:21:36.476 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:21:36.476335 +0800 CST m=+3620.006582129
2020/05/11 19:21:41.476 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:21:41.476793 +0800 CST m=+3625.006999212
2020/05/11 19:21:46.476 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:21:46.476803 +0800 CST m=+3630.006968440
2020/05/11 19:21:51.474 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:21:51.474709 +0800 CST m=+3635.004832956
2020/05/11 19:21:56.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:21:56.473157 +0800 CST m=+3640.003240272
2020/05/11 19:22:01.476 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:22:01.476754 +0800 CST m=+3645.006796713
2020/05/11 19:22:06.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:22:06.473378 +0800 CST m=+3650.003379566
2020/05/11 19:22:11.476 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:22:11.476841 +0800 CST m=+3655.006801986
2020/05/11 19:22:16.476 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:22:16.476871 +0800 CST m=+3660.006791308
2020/05/11 19:22:21.475 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:22:21.475778 +0800 CST m=+3665.005657398
2020/05/11 19:22:26.478 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:22:26.478564 +0800 CST m=+3670.008402866
2020/05/11 19:22:31.477 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:22:31.477116 +0800 CST m=+3675.006914120
2020/05/11 19:22:36.473 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:22:36.473703 +0800 CST m=+3680.003460020
2020/05/11 19:22:41.477 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:22:41.47714 +0800 CST m=+3685.006855496
2020/05/11 19:22:46.477 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:22:46.477138 +0800 CST m=+3690.006813425
2020/05/11 19:22:51.477 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:22:51.47781 +0800 CST m=+3695.007444082
2020/05/11 19:22:56.477 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:22:56.477194 +0800 CST m=+3700.006787554
2020/05/11 19:23:01.477 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:23:01.477385 +0800 CST m=+3705.006937727
2020/05/11 19:23:06.477 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:23:06.477386 +0800 CST m=+3710.006897562
2020/05/11 19:23:11.477 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:23:11.477722 +0800 CST m=+3715.007193232
2020/05/11 19:23:16.478 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:23:16.477964 +0800 CST m=+3720.007394312
2020/05/11 19:23:21.477 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:23:21.477409 +0800 CST m=+3725.006797833
2020/05/11 19:23:26.478 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:23:26.477961 +0800 CST m=+3730.007308993
2020/05/11 19:23:31.477 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:23:31.477794 +0800 CST m=+3735.007101825
2020/05/11 19:23:36.477 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:23:36.477635 +0800 CST m=+3740.006901239
2020/05/11 19:23:41.477 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:23:41.476957 +0800 CST m=+3745.006182580
2020/05/11 19:23:46.474 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:23:46.474858 +0800 CST m=+3750.004042536
2020/05/11 19:23:51.479 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:23:51.479223 +0800 CST m=+3755.008366469
2020/05/11 19:23:56.477 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:23:56.477892 +0800 CST m=+3760.006995004
2020/05/11 19:24:01.478 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:24:01.477986 +0800 CST m=+3765.007048681
2020/05/11 19:24:06.477 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:24:06.477823 +0800 CST m=+3770.006844633
2020/05/11 19:24:11.476 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:24:11.476305 +0800 CST m=+3775.005285842
2020/05/11 19:24:16.477 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:24:16.47781 +0800 CST m=+3780.006749643
2020/05/11 19:24:21.478 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:24:21.477891 +0800 CST m=+3785.006789976
2020/05/11 19:24:26.477 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:24:26.47791 +0800 CST m=+3790.006767777
2020/05/11 19:24:31.478 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:24:31.477983 +0800 CST m=+3795.006799755
2020/05/11 19:24:36.478 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:24:36.478134 +0800 CST m=+3800.006910569
2020/05/11 19:24:41.475 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:24:41.475332 +0800 CST m=+3805.004067805
2020/05/11 19:24:46.476 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:24:46.476171 +0800 CST m=+3810.004865227
2020/05/11 19:24:51.476 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:24:51.476155 +0800 CST m=+3815.004809296
2020/05/11 19:24:56.478 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:24:56.478292 +0800 CST m=+3820.006905090
2020/05/11 19:25:01.478 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:25:01.478317 +0800 CST m=+3825.006889332
2020/05/11 19:25:06.480 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:25:06.479874 +0800 CST m=+3830.008405261
2020/05/11 19:25:11.475 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:25:11.474929 +0800 CST m=+3835.003419171
2020/05/11 19:25:16.478 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:25:16.478516 +0800 CST m=+3840.006965257
2020/05/11 19:25:21.478 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:25:21.478533 +0800 CST m=+3845.006941125
2020/05/11 19:25:26.478 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:25:26.478763 +0800 CST m=+3850.007131296
2020/05/11 19:25:31.478 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:25:31.478513 +0800 CST m=+3855.006839508
2020/05/11 19:25:36.478 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:25:36.478129 +0800 CST m=+3860.006414674
2020/05/11 19:25:41.478 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:25:41.478537 +0800 CST m=+3865.006782520
2020/05/11 19:25:46.478 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:25:46.478896 +0800 CST m=+3870.007100719
2020/05/11 19:25:51.478 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:25:51.478436 +0800 CST m=+3875.006599078
2020/05/11 19:25:56.478 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:25:56.478266 +0800 CST m=+3880.006388847
2020/05/11 19:26:01.477 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:26:01.47779 +0800 CST m=+3885.005871362
2020/05/11 19:26:06.475 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:26:06.475364 +0800 CST m=+3890.003404429
2020/05/11 19:26:11.479 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:26:11.47911 +0800 CST m=+3895.007110459
2020/05/11 19:26:16.478 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:26:16.478783 +0800 CST m=+3900.006742751
2020/05/11 19:26:21.478 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:26:21.478252 +0800 CST m=+3905.006170119
2020/05/11 19:26:26.479 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:26:26.47908 +0800 CST m=+3910.006957390
2020/05/11 19:26:31.478 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:26:31.478932 +0800 CST m=+3915.006768157
2020/05/11 19:26:36.479 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:26:36.479483 +0800 CST m=+3920.007278655
2020/05/11 19:26:41.475 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:26:41.475551 +0800 CST m=+3925.003305954
2020/05/11 19:26:46.479 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:26:46.479457 +0800 CST m=+3930.007171082
2020/05/11 19:26:51.478 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:26:51.477986 +0800 CST m=+3935.005658785
2020/05/11 19:26:56.477 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:26:56.477458 +0800 CST m=+3940.005090081
2020/05/11 19:27:01.479 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:27:01.479715 +0800 CST m=+3945.007306151
2020/05/11 19:27:06.479 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:27:06.479849 +0800 CST m=+3950.007400050
2020/05/11 19:27:11.477 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:27:11.477885 +0800 CST m=+3955.005394872
2020/05/11 19:27:16.476 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:27:16.476397 +0800 CST m=+3960.003866080
2020/05/11 19:27:21.480 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:27:21.479985 +0800 CST m=+3965.007412589
2020/05/11 19:27:26.478 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:27:26.477964 +0800 CST m=+3970.005351068
2020/05/11 19:27:31.477 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:27:31.477801 +0800 CST m=+3975.005147255
2020/05/11 19:27:36.479 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:27:36.47891 +0800 CST m=+3980.006215280
2020/05/11 19:27:41.478 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:27:41.478409 +0800 CST m=+3985.005674153
2020/05/11 19:27:46.476 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:27:46.476563 +0800 CST m=+3990.003787270
2020/05/11 19:27:51.476 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:27:51.476083 +0800 CST m=+3995.003265788
2020/05/11 19:27:56.479 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:27:56.479649 +0800 CST m=+4000.006790902
2020/05/11 19:28:01.479 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:28:01.479849 +0800 CST m=+4005.006950460
2020/05/11 19:28:06.480 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:28:06.480073 +0800 CST m=+4010.007133536
2020/05/11 19:28:11.477 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:28:11.477057 +0800 CST m=+4015.004076304
2020/05/11 19:28:16.480 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:28:16.48025 +0800 CST m=+4020.007228393
2020/05/11 19:04:47.495 consumer.go:63: [info] async error:write tcp4 172.23.110.253:57758->10.32.69.26:6650: use of closed network con
nection
2020/05/11 19:04:47.534 consumer.go:63: [info] async error:write tcp4 172.23.110.253:57758->10.32.69.26:6650: use of closed network con
nection
2020/05/11 19:04:49.495 consumer.go:63: [info] async error:write tcp4 172.23.110.253:57758->10.32.69.26:6650: use of closed network con
nection
2020/05/11 19:04:49.534 consumer.go:63: [info] async error:write tcp4 172.23.110.253:57758->10.32.69.26:6650: use of closed network con
nection
2020/05/11 19:04:51.469 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:04:51.469128 +0800 CST m=+2615.007800211
2020/05/11 19:04:53.499 consumer.go:63: [info] async error:write tcp4 172.23.110.253:57758->10.32.69.26:6650: use of closed network con
nection
2020/05/11 19:04:53.535 consumer.go:63: [info] async error:write tcp4 172.23.110.253:57758->10.32.69.26:6650: use of closed network con
nection
2020/05/11 19:04:56.465 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:04:56.465039 +0800 CST m=+2620.003611412
2020/05/11 19:05:01.466 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:05:01.466328 +0800 CST m=+2625.004817176
2020/05/11 19:05:01.483 consumer.go:63: [info] async error:context deadline exceeded
2020/05/11 19:05:01.522 consumer.go:63: [info] async error:context deadline exceeded
2020/05/11 19:05:06.467 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:05:06.4677 +0800 CST m=+2630.006117168
2020/05/11 19:05:11.467 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:05:11.466964 +0800 CST m=+2635.005317743
2020/05/11 19:05:16.465 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:05:16.465143 +0800 CST m=+2640.003440104
2020/05/11 19:05:21.465 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:05:21.465136 +0800 CST m=+2645.003380345
2020/05/11 19:05:26.466 unackedMsgTracker.go:194: [debug] Tick at 2020-05-11 19:05:26.46676 +0800 CST m=+2650.004954442

topics stats

[duansumin]$ ./bin/pulsar-admin topics stats persistent://public/default/dsmtest-partition-0
{
  "msgRateIn" : 0.0,
  "msgThroughputIn" : 0.0,
  "msgRateOut" : 0.0,
  "msgThroughputOut" : 0.0,
  "averageMsgSize" : 0.0,
  "storageSize" : 1664425,
  "publishers" : [ {
    "msgRateIn" : 0.0,
    "msgThroughputIn" : 0.0,
    "averageMsgSize" : 0.0,
    "producerId" : 0,
    "metadata" : { },
    "producerName" : "qa-pulsar-ali-55-107",
    "connectedSince" : "2020-05-11T19:05:01.068+08:00",
    "clientVersion" : "pulsar-client-go",
    "address" : "/172.23.110.253:58738"
  } ],
  "subscriptions" : {
    "sub-2" : {
      "msgRateOut" : 0.0,
      "msgThroughputOut" : 0.0,
      "msgRateRedeliver" : 0.0,
      "msgBacklog" : 0,
      "blockedSubscriptionOnUnackedMsgs" : false,
      "msgDelayed" : 0,
      "unackedMessages" : 0,
      "type" : "Key_Shared",
      "msgRateExpired" : 0.0,
      "consumers" : [ {
        "msgRateOut" : 0.0,
        "msgThroughputOut" : 0.0,
        "msgRateRedeliver" : 0.0,
        "consumerName" : "",
        "availablePermits" : 512,
        "unackedMessages" : 0,
        "blockedConsumerOnUnackedMsgs" : false,
        "metadata" : { },
        "connectedSince" : "2020-05-11T19:05:02.555+08:00",
        "clientVersion" : "pulsar-client-go",
        "address" : "/172.23.110.253:58743"
      } ],
      "isReplicated" : false
    }
  },
  "replication" : { },
  "deduplicationStatus" : "Disabled"
}

[duansumin ]$ ./bin/pulsar-admin topics stats persistent://public/default/dsmtest-partition-1
{
  "msgRateIn" : 0.0,
  "msgThroughputIn" : 0.0,
  "msgRateOut" : 0.0,
  "msgThroughputOut" : 0.0,
  "averageMsgSize" : 0.0,
  "storageSize" : 1268640,
  "publishers" : [ {
    "msgRateIn" : 0.0,
    "msgThroughputIn" : 0.0,
    "averageMsgSize" : 0.0,
    "producerId" : 0,
    "metadata" : { },
    "address" : "/172.23.110.253:57751",
    "producerName" : "qa-pulsar-ali-53-6867",
    "connectedSince" : "2020-05-11T18:57:32.053+08:00",
    "clientVersion" : "pulsar-client-go"
  } ],
  "subscriptions" : {
    "sub-2" : {
      "msgRateOut" : 0.0,
      "msgThroughputOut" : 0.0,
      "msgRateRedeliver" : 0.0,
      "msgBacklog" : 0,
      "blockedSubscriptionOnUnackedMsgs" : false,
      "msgDelayed" : 0,
      "unackedMessages" : 0,
      "type" : "Key_Shared",
      "msgRateExpired" : 0.0,
      "consumers" : [ {
        "msgRateOut" : 0.0,
        "msgThroughputOut" : 0.0,
        "msgRateRedeliver" : 0.0,
        "consumerName" : "",
        "availablePermits" : 512,
        "unackedMessages" : 0,
        "blockedConsumerOnUnackedMsgs" : false,
        "metadata" : { },
        "address" : "/172.23.110.253:57758",
        "connectedSince" : "2020-05-11T18:57:33.517+08:00",
        "clientVersion" : "pulsar-client-go"
      } ],
      "isReplicated" : false
    }
  },
  "replication" : { },
  "deduplicationStatus" : "Disabled"
}

System configuration

Pulsar version: 2.5.0

other information

The topic: "public/default/dsmtest"  has  two partitions

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions