Skip to content

Commit df14a20

Browse files
JaySon-HuangJaySon-Huang
committed
Refine metrics and comments
Signed-off-by: JaySon-Huang <tshent@qq.com>
1 parent a4b9e53 commit df14a20

12 files changed

Lines changed: 97 additions & 71 deletions

‎dbms/src/Common/TiFlashMetrics.h‎

Lines changed: 21 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -374,7 +374,7 @@ static_assert(RAFT_REGION_BIG_WRITE_THRES * 4 < RAFT_REGION_BIG_WRITE_MAX, "Inva
374374
Histogram, \
375375
F(type_read, {{"type", "read"}}, EqualWidthBuckets{1 * 1024 * 1024, 60, 50 * 1024 * 1024})) \
376376
M(tiflash_storage_io_limiter, \
377-
"Storage I/O limiter metrics", \
377+
"Storage I/O limiter byte flow", \
378378
Counter, \
379379
F(type_fg_read_req_bytes, {"type", "fg_read_req_bytes"}), \
380380
F(type_fg_read_alloc_bytes, {"type", "fg_read_alloc_bytes"}), \
@@ -383,14 +383,17 @@ static_assert(RAFT_REGION_BIG_WRITE_THRES * 4 < RAFT_REGION_BIG_WRITE_MAX, "Inva
383383
F(type_fg_write_req_bytes, {"type", "fg_write_req_bytes"}), \
384384
F(type_fg_write_alloc_bytes, {"type", "fg_write_alloc_bytes"}), \
385385
F(type_bg_write_req_bytes, {"type", "bg_write_req_bytes"}), \
386-
F(type_bg_write_alloc_bytes, {"type", "bg_write_alloc_bytes"})) \
386+
F(type_bg_write_alloc_bytes, {"type", "bg_write_alloc_bytes"}), \
387+
F(type_s3_direct_read_bytes, {{"type", "s3_direct_read_bytes"}}), \
388+
F(type_s3_filecache_download_bytes, {{"type", "s3_filecache_download_bytes"}})) \
387389
M(tiflash_storage_io_limiter_curr, \
388390
"Current limit bytes per second of Storage I/O limiter", \
389391
Gauge, \
390392
F(type_fg_read_bytes, {"type", "fg_read_bytes"}), \
391393
F(type_bg_read_bytes, {"type", "bg_read_bytes"}), \
392394
F(type_fg_write_bytes, {"type", "fg_write_bytes"}), \
393-
F(type_bg_write_bytes, {"type", "bg_write_bytes"})) \
395+
F(type_bg_write_bytes, {"type", "bg_write_bytes"}), \
396+
F(type_s3_read_bytes, {"type", "s3_read_bytes"})) \
394397
M(tiflash_storage_rough_set_filter_rate, \
395398
"Bucketed histogram of rough set filter rate", \
396399
Histogram, \
@@ -791,24 +794,11 @@ static_assert(RAFT_REGION_BIG_WRITE_THRES * 4 < RAFT_REGION_BIG_WRITE_MAX, "Inva
791794
F(type_head_object, {{"type", "head_object"}}, ExpBuckets{0.001, 2, 20}), \
792795
F(type_read_stream, {{"type", "read_stream"}}, ExpBuckets{0.0001, 2, 20}), \
793796
F(type_read_stream_err, {{"type", "read_stream_err"}}, ExpBuckets{0.0001, 2, 20})) \
794-
M(tiflash_storage_s3_read_limiter, \
795-
"S3 read limiter counters", \
796-
Counter, \
797-
F(type_stream_wait_count, {{"type", "stream_wait_count"}}), \
798-
F(type_byte_wait_count, {{"type", "byte_wait_count"}}), \
799-
F(type_direct_read_bytes, {{"type", "direct_read_bytes"}}), \
800-
F(type_filecache_download_bytes, {{"type", "filecache_download_bytes"}})) \
801-
M(tiflash_storage_s3_read_limiter_wait_seconds, \
802-
"S3 read limiter wait duration in seconds", \
803-
Histogram, \
804-
F(type_stream_wait, {{"type", "stream_wait"}}, ExpBuckets{0.0001, 2, 20}), \
805-
F(type_byte_wait, {{"type", "byte_wait"}}, ExpBuckets{0.0001, 2, 20})) \
806797
M(tiflash_storage_s3_read_limiter_status, \
807798
"S3 read limiter status", \
808799
Gauge, \
809800
F(type_active_get_object_streams, {{"type", "active_get_object_streams"}}), \
810-
F(type_max_get_object_streams, {{"type", "max_get_object_streams"}}), \
811-
F(type_max_read_bytes_per_sec, {{"type", "max_read_bytes_per_sec"}})) \
801+
F(type_max_get_object_streams, {{"type", "max_get_object_streams"}})) \
812802
M(tiflash_storage_s3_http_request_seconds, \
813803
"S3 request duration breakdown in seconds", \
814804
Histogram, \
@@ -919,17 +909,28 @@ static_assert(RAFT_REGION_BIG_WRITE_THRES * 4 < RAFT_REGION_BIG_WRITE_MAX, "Inva
919909
F(type_page_evict_bytes, {"type", "page_evict_bytes"}), \
920910
F(type_page_download_bytes, {"type", "page_download_bytes"}), \
921911
F(type_page_read_bytes, {"type", "page_read_bytes"})) \
922-
M(tiflash_storage_remote_cache_status, \
912+
M(tiflash_storage_remote_cache_status, \
923913
"Remote cache status", \
924914
Gauge, \
925-
F(type_bg_downloading_count, {{"type", "bg_downloading_count"}})) \
915+
F(type_bg_downloading_count, {{"type", "bg_downloading_count"}})) \
926916
M(tiflash_storage_io_limiter_pending_seconds, \
927917
"I/O limiter pending duration in seconds", \
928918
Histogram, \
929919
F(type_fg_read, {{"type", "fg_read"}}, ExpBuckets{0.001, 2, 20}), \
930920
F(type_bg_read, {{"type", "bg_read"}}, ExpBuckets{0.001, 2, 20}), \
931921
F(type_fg_write, {{"type", "fg_write"}}, ExpBuckets{0.001, 2, 20}), \
932-
F(type_bg_write, {{"type", "bg_write"}}, ExpBuckets{0.001, 2, 20})) \
922+
F(type_bg_write, {{"type", "bg_write"}}, ExpBuckets{0.001, 2, 20}), \
923+
F(type_s3_read_stream, {{"type", "s3_read_stream"}}, ExpBuckets{0.001, 2, 20}), \
924+
F(type_s3_read_byte, {{"type", "s3_read_byte"}}, ExpBuckets{0.001, 2, 20})) \
925+
M(tiflash_storage_io_limiter_pending_count, \
926+
"I/O limiter pending count", \
927+
Counter, \
928+
F(type_fg_read, {"type", "fg_read"}), \
929+
F(type_bg_read, {"type", "bg_read"}), \
930+
F(type_fg_write, {"type", "fg_write"}), \
931+
F(type_bg_write, {"type", "bg_write"}), \
932+
F(type_s3_read_byte, {"type", "s3_read_byte"}), \
933+
F(type_s3_read_stream, {"type", "s3_read_stream"})) \
933934
M(tiflash_system_seconds, \
934935
"system calls duration in seconds", \
935936
Histogram, \
@@ -1021,13 +1022,6 @@ static_assert(RAFT_REGION_BIG_WRITE_THRES * 4 < RAFT_REGION_BIG_WRITE_MAX, "Inva
10211022
F(type_load_dmfile_local, {{"type", "load_dmfile_local"}}, ExpBuckets{0.001, 2, 20}), \
10221023
F(type_load_dmfile_s3, {{"type", "load_dmfile_s3"}}, ExpBuckets{0.001, 2, 20}), \
10231024
F(type_search, {{"type", "search"}}, ExpBuckets{0.001, 2, 20})) \
1024-
M(tiflash_storage_io_limiter_pending_count, \
1025-
"I/O limiter pending count", \
1026-
Counter, \
1027-
F(type_fg_read, {"type", "fg_read"}), \
1028-
F(type_bg_read, {"type", "bg_read"}), \
1029-
F(type_fg_write, {"type", "fg_write"}), \
1030-
F(type_bg_write, {"type", "bg_write"})) \
10311025
M(tiflash_read_thread_internal_us, \
10321026
"Durations of read thread internal components", \
10331027
Histogram, \

‎dbms/src/IO/BaseFile/IORateLimitConfig.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -172,12 +172,12 @@ bool IORateLimitConfig::operator==(const IORateLimitConfig & config) const
172172
return config.max_bytes_per_sec == max_bytes_per_sec && config.max_read_bytes_per_sec == max_read_bytes_per_sec
173173
&& config.max_write_bytes_per_sec == max_write_bytes_per_sec
174174
&& config.s3_max_read_bytes_per_sec == s3_max_read_bytes_per_sec
175-
&& config.s3_max_get_object_streams == s3_max_get_object_streams
175+
&& config.s3_max_get_object_streams == s3_max_get_object_streams //
176176
&& config.bg_write_weight == bg_write_weight && config.fg_write_weight == fg_write_weight
177177
&& config.bg_read_weight == bg_read_weight && config.fg_read_weight == fg_read_weight
178-
&& config.emergency_pct == emergency_pct
179-
&& config.high_pct == high_pct && config.medium_pct == medium_pct && config.tune_base == tune_base
180-
&& config.min_bytes_per_sec == min_bytes_per_sec && config.auto_tune_sec == auto_tune_sec;
178+
&& config.emergency_pct == emergency_pct && config.high_pct == high_pct && config.medium_pct == medium_pct
179+
&& config.tune_base == tune_base && config.min_bytes_per_sec == min_bytes_per_sec
180+
&& config.auto_tune_sec == auto_tune_sec;
181181
}
182182

183183
} // namespace DB

‎dbms/src/IO/BaseFile/IORateLimitConfig.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,9 @@ struct IORateLimitConfig
2828
// For disk that read bandwidth and write bandwith are calculated separately, such as GCP's persistent disks.
2929
UInt64 max_read_bytes_per_sec;
3030
UInt64 max_write_bytes_per_sec;
31+
// Node-level byte budget shared by all S3 direct reads and FileCache downloads. `0` disables byte throttling.
3132
UInt64 s3_max_read_bytes_per_sec;
33+
// Node-level cap for concurrently active `GetObject` response bodies. `0` disables stream throttling.
3234
UInt64 s3_max_get_object_streams;
3335

3436
// only true when both max_read_bytes_per_sec and max_write_bytes_per_sec are 0

‎dbms/src/IO/BaseFile/RateLimiter.cpp‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -525,15 +525,16 @@ void IORateLimiter::updateLimiterByConfig(const IORateLimitConfig & cfg)
525525
std::lock_guard lock(limiter_mtx);
526526
updateReadLimiter(cfg.getBgReadMaxBytesPerSec(), cfg.getFgReadMaxBytesPerSec());
527527
updateWriteLimiter(cfg.getBgWriteMaxBytesPerSec(), cfg.getFgWriteMaxBytesPerSec());
528+
529+
// updateS3ReadLimiter
528530
if (cfg.s3_max_read_bytes_per_sec == 0 && cfg.s3_max_get_object_streams == 0)
529531
{
530532
s3_read_limiter = nullptr;
531533
}
532534
else if (s3_read_limiter == nullptr)
533535
{
534-
s3_read_limiter = std::make_shared<S3::S3ReadLimiter>(
535-
cfg.s3_max_read_bytes_per_sec,
536-
cfg.s3_max_get_object_streams);
536+
s3_read_limiter
537+
= std::make_shared<S3::S3ReadLimiter>(cfg.s3_max_read_bytes_per_sec, cfg.s3_max_get_object_streams);
537538
}
538539
else
539540
{

‎dbms/src/Server/Server.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -942,6 +942,8 @@ try
942942

943943
/// Initialize RateLimiter.
944944
global_context->initializeRateLimiter(config(), bg_pool, blockable_bg_pool);
945+
// ClientFactory keeps the process-wide shared S3 client. Publish the latest limiter explicitly so
946+
// every existing and future `TiFlashS3Client` observes the same node-level S3 read budget.
945947
S3::ClientFactory::instance().setS3ReadLimiter(global_context->getIORateLimiter().getS3ReadLimiter());
946948

947949
global_context->setServerInfo(server_info);
@@ -972,6 +974,7 @@ try
972974
buildLoggers(*config);
973975
global_context->getTMTContext().reloadConfig(*config);
974976
global_context->getIORateLimiter().updateConfig(*config);
977+
// Config reload may replace the limiter instance or disable it. Re-publish it to the shared S3 client.
975978
S3::ClientFactory::instance().setS3ReadLimiter(global_context->getIORateLimiter().getS3ReadLimiter());
976979
global_context->reloadDeltaTreeConfig(*config);
977980
DM::SegmentReadTaskScheduler::instance().updateConfig(global_context->getSettingsRef());

0 commit comments

Comments
 (0)