From 338a22e1da984fac09f0f14f92728db0ee13237f Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Mon, 15 Dec 2025 14:25:14 +0800 Subject: [PATCH 1/5] add more logs to debug slow DDL --- downstreamadapter/sink/kafka/sink.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/downstreamadapter/sink/kafka/sink.go b/downstreamadapter/sink/kafka/sink.go index 56580fc029..d1daa8666c 100644 --- a/downstreamadapter/sink/kafka/sink.go +++ b/downstreamadapter/sink/kafka/sink.go @@ -442,9 +442,13 @@ func (s *sink) sendDDLEvent(event *commonEvent.DDLEvent) error { return err } } + + finishedAt := time.UnixMilli(int64(event.GetCommitTs() >> 18)) + duration := time.Since(finishedAt) log.Info("kafka sink send DDL event", zap.String("keyspace", s.changefeedID.Keyspace()), zap.String("changefeed", s.changefeedID.Name()), - zap.Any("startTs", event.GetStartTs()), zap.Any("commitTs", event.GetCommitTs()), zap.Any("event", event.GetDDLQuery()), + zap.Duration("duration", duration), zap.Any("tidbFinishedAt", finishedAt), zap.Any("ticdcFinishedAt", time.Now()), + zap.Any("startTs", event.GetStartTs()), zap.Any("commitTs", event.GetCommitTs()), zap.Any("DDL", event.GetDDLQuery()), zap.String("schema", event.GetSchemaName()), zap.String("table", event.GetTableName())) return nil } From df8f718005d319c10ccbf4fb52991eb03dd78004 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Mon, 15 Dec 2025 15:26:01 +0800 Subject: [PATCH 2/5] add more detailed log to debug debezium slow test --- downstreamadapter/sink/kafka/sink.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/downstreamadapter/sink/kafka/sink.go b/downstreamadapter/sink/kafka/sink.go index d1daa8666c..577db6ec5d 100644 --- a/downstreamadapter/sink/kafka/sink.go +++ b/downstreamadapter/sink/kafka/sink.go @@ -447,8 +447,9 @@ func (s *sink) sendDDLEvent(event *commonEvent.DDLEvent) error { duration := time.Since(finishedAt) log.Info("kafka sink send DDL event", zap.String("keyspace", s.changefeedID.Keyspace()), zap.String("changefeed", s.changefeedID.Name()), - zap.Duration("duration", duration), zap.Any("tidbFinishedAt", finishedAt), zap.Any("ticdcFinishedAt", time.Now()), - zap.Any("startTs", event.GetStartTs()), zap.Any("commitTs", event.GetCommitTs()), zap.Any("DDL", event.GetDDLQuery()), + zap.Any("startTs", event.GetStartTs()), zap.Any("commitTs", event.GetCommitTs()), + zap.Any("tidbFinishedAt", finishedAt), zap.Duration("duration", duration), + zap.Any("DDL", event.GetDDLQuery()), zap.String("schema", event.GetSchemaName()), zap.String("table", event.GetTableName())) return nil } From cf5220da74a43ab7573765bdf93cd9dad2e97333 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Mon, 15 Dec 2025 15:30:16 +0800 Subject: [PATCH 3/5] set timeout to 10s --- tests/integration_tests/debezium01/src/test_cases.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration_tests/debezium01/src/test_cases.go b/tests/integration_tests/debezium01/src/test_cases.go index 50d6373427..554d21f848 100644 --- a/tests/integration_tests/debezium01/src/test_cases.go +++ b/tests/integration_tests/debezium01/src/test_cases.go @@ -36,7 +36,7 @@ import ( "go.uber.org/zap" ) -const timeout = time.Second * 30 +const timeout = time.Second * 10 var ( nFailed = 0 From 94c78c4fcf4f41484f56f026cf70c0f04e1c51a5 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Mon, 15 Dec 2025 19:10:08 +0800 Subject: [PATCH 4/5] set timeout to 20s --- tests/integration_tests/debezium01/src/test_cases.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration_tests/debezium01/src/test_cases.go b/tests/integration_tests/debezium01/src/test_cases.go index 554d21f848..2c6de7a637 100644 --- a/tests/integration_tests/debezium01/src/test_cases.go +++ b/tests/integration_tests/debezium01/src/test_cases.go @@ -36,7 +36,7 @@ import ( "go.uber.org/zap" ) -const timeout = time.Second * 10 +const timeout = time.Second * 20 var ( nFailed = 0 From 82a80272dec5129754762f14030bc73f62a330f5 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 16 Dec 2025 16:15:48 +0800 Subject: [PATCH 5/5] set to 256 --- pkg/config/debug.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/config/debug.go b/pkg/config/debug.go index c8e40c5ea5..0b92705113 100644 --- a/pkg/config/debug.go +++ b/pkg/config/debug.go @@ -78,7 +78,7 @@ func NewDefaultPullerConfig() *PullerConfig { EnableResolvedTsStuckDetection: false, ResolvedTsStuckInterval: TomlDuration(5 * time.Minute), LogRegionDetails: false, - PendingRegionRequestQueueSize: 32, // This value is chosen to reduce the impact of new changefeeds on existing ones. + PendingRegionRequestQueueSize: 256, // This value is chosen to reduce the impact of new changefeeds on existing ones. } }