Skip to content

Commit 00c0b07

Browse files
ztlpnztlpn
committed
more test helpers refactoring (passes existing tests)
1 parent 0b48dcc commit 00c0b07

File tree

5 files changed

+136
-99
lines changed

5 files changed

+136
-99
lines changed

‎ydb/core/statistics/aggregator/ut/ut_analyze_datashard.cpp‎

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,26 @@
1010
namespace NKikimr {
1111
namespace NStat {
1212

13+
namespace {
14+
15+
void PrepareTable(TTestEnv& env, const TString& tableName) {
16+
CreateUniformTable(env, "Database", tableName);
17+
InsertDataIntoTable(env, "Database", tableName, RowsWithFewDistinctValues(1000));
18+
}
19+
20+
void ValidateCountMinSketch(TTestActorRuntime& runtime, const TPathId& pathId) {
21+
std::vector<TCountMinSketchProbes> expected = {
22+
{
23+
.Tag = 2, // Value column
24+
.Probes{ {"1", 100}, {"2", 100}, {"10", 0} }
25+
}
26+
};
27+
28+
CheckCountMinSketch(runtime, pathId, expected);
29+
}
30+
31+
} // namespace
32+
1333
Y_UNIT_TEST_SUITE(AnalyzeDatashard) {
1434

1535
Y_UNIT_TEST(AnalyzeOneTable) {
@@ -18,14 +38,14 @@ Y_UNIT_TEST_SUITE(AnalyzeDatashard) {
1838
auto& runtime = *env.GetServer().GetRuntime();
1939

2040
CreateDatabase(env, "Database");
21-
PrepareUniformTable(env, "Database", "Table");
41+
PrepareTable(env, "Table");
2242

2343
ui64 saTabletId;
2444
auto pathId = ResolvePathId(runtime, "/Root/Database/Table", nullptr, &saTabletId);
2545

2646
Analyze(runtime, saTabletId, {{pathId}});
2747

28-
ValidateCountMinDatashard(runtime, pathId);
48+
ValidateCountMinSketch(runtime, pathId);
2949
}
3050

3151
Y_UNIT_TEST(AnalyzeTwoTables) {
@@ -34,17 +54,17 @@ Y_UNIT_TEST_SUITE(AnalyzeDatashard) {
3454
auto& runtime = *env.GetServer().GetRuntime();
3555

3656
CreateDatabase(env, "Database");
37-
PrepareUniformTable(env, "Database", "Table1");
38-
PrepareUniformTable(env, "Database", "Table2");
57+
PrepareTable(env, "Table1");
58+
PrepareTable(env, "Table2");
3959

4060
ui64 saTabletId1;
4161
auto pathId1 = ResolvePathId(runtime, "/Root/Database/Table1", nullptr, &saTabletId1);
4262
auto pathId2 = ResolvePathId(runtime, "/Root/Database/Table2");
4363

4464
Analyze(runtime, saTabletId1, {pathId1, pathId2});
4565

46-
ValidateCountMinDatashard(runtime, pathId1);
47-
ValidateCountMinDatashard(runtime, pathId2);
66+
ValidateCountMinSketch(runtime, pathId1);
67+
ValidateCountMinSketch(runtime, pathId2);
4868
}
4969

5070
Y_UNIT_TEST(DropTableNavigateError) {
@@ -53,7 +73,7 @@ Y_UNIT_TEST_SUITE(AnalyzeDatashard) {
5373
auto& runtime = *env.GetServer().GetRuntime();
5474

5575
CreateDatabase(env, "Database");
56-
PrepareUniformTable(env, "Database", "Table");
76+
PrepareTable(env, "Table");
5777

5878
ui64 saTabletId = 0;
5979
auto pathId = ResolvePathId(runtime, "/Root/Database/Table", nullptr, &saTabletId);

‎ydb/core/statistics/service/ut/ut_column_statistics.cpp‎

Lines changed: 22 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -13,73 +13,32 @@
1313
namespace NKikimr {
1414
namespace NStat {
1515

16-
struct TColumnStatisticsProbes {
17-
struct TProbe {
18-
ui64 Value;
19-
ui64 Probe;
20-
};
21-
22-
ui16 Tag;
23-
std::vector<TProbe> Probes;
24-
};
25-
26-
void CheckColumnStatistics(
27-
TTestActorRuntime& runtime, const TPathId& pathId, const TActorId& sender, const std::vector<TColumnStatisticsProbes>& expected
28-
) {
29-
auto evGet = std::make_unique<TEvStatistics::TEvGetStatistics>();
30-
evGet->StatType = NStat::EStatType::COUNT_MIN_SKETCH;
31-
32-
for (auto item : expected) {
33-
NStat::TRequest req;
34-
req.PathId = pathId;
35-
req.ColumnTag = item.Tag;
36-
evGet->StatRequests.push_back(req);
37-
}
38-
39-
auto statServiceId = NStat::MakeStatServiceID(runtime.GetNodeId(0));
40-
runtime.Send(statServiceId, sender, evGet.release(), 0, true);
41-
42-
auto res = runtime.GrabEdgeEventRethrow<TEvStatistics::TEvGetStatisticsResult>(sender);
43-
auto msg = res->Get();
44-
45-
UNIT_ASSERT(msg->Success);
46-
UNIT_ASSERT( msg->StatResponses.size() == expected.size());
47-
48-
for (size_t i = 0; i < msg->StatResponses.size(); ++i) {
49-
const auto& stat = msg->StatResponses[i];
50-
UNIT_ASSERT(stat.Success);
16+
namespace {
5117

52-
auto countMin = stat.CountMinSketch.CountMin.get();
53-
UNIT_ASSERT(countMin != nullptr);
54-
55-
for (const auto& item : expected[i].Probes) {
56-
ui64 value = item.Value;
57-
auto probe = countMin->Probe((const char*)&value, sizeof(ui64));
58-
UNIT_ASSERT_VALUES_EQUAL(item.Probe, probe);
59-
}
60-
}
18+
TTableInfo PrepareTable(TTestEnv& env, const TString& databaseName, const TString& tableName) {
19+
auto tableInfo = CreateColumnTable(env, databaseName, tableName, 1);
20+
InsertDataIntoTable(env, databaseName, tableName, RowsWithFewDistinctValues(1000));
21+
return tableInfo;
6122
}
6223

24+
} // namespace
25+
6326
Y_UNIT_TEST_SUITE(ColumnStatistics) {
6427
Y_UNIT_TEST(CountMinSketchStatistics) {
6528
TTestEnv env(1, 1);
6629
auto& runtime = *env.GetServer().GetRuntime();
6730

6831
CreateDatabase(env, "Database");
69-
PrepareColumnTable(env, "Database", "Table1", 1);
70-
ui64 saTabletId = 0;
71-
auto pathId = ResolvePathId(runtime, "/Root/Database/Table1", nullptr, &saTabletId);
32+
const auto tableInfo = PrepareTable(env, "Database", "Table1");
33+
Analyze(runtime, tableInfo.SaTabletId, {tableInfo.PathId});
7234

73-
Analyze(runtime, saTabletId, {pathId});
74-
75-
std::vector<TColumnStatisticsProbes> expected = {
35+
std::vector<TCountMinSketchProbes> expected = {
7636
{
77-
.Tag = 1, // Key column
78-
.Probes{ {1, 4}, {2, 4} }
37+
.Tag = 2, // Key column
38+
.Probes{ {"1", 100}, {"2", 100} }
7939
}
8040
};
81-
auto sender = runtime.AllocateEdgeActor();
82-
CheckColumnStatistics(runtime, pathId, sender, expected);
41+
CheckCountMinSketch(runtime, tableInfo.PathId, expected);
8342
}
8443

8544
Y_UNIT_TEST(CountMinSketchServerlessStatistics) {
@@ -90,27 +49,21 @@ Y_UNIT_TEST_SUITE(ColumnStatistics) {
9049
CreateServerlessDatabase(env, "Serverless1", "/Root/Shared", 1);
9150
CreateServerlessDatabase(env, "Serverless2", "/Root/Shared", 1);
9251

93-
PrepareColumnTable(env, "Serverless1", "Table1", 1);
94-
PrepareColumnTable(env, "Serverless2", "Table2", 1);
95-
96-
// Same SA tablet for both serverless databases
97-
ui64 saTabletId = 0;
98-
auto pathId1 = ResolvePathId(runtime, "/Root/Serverless1/Table1", nullptr, &saTabletId);
99-
auto pathId2 = ResolvePathId(runtime, "/Root/Serverless2/Table2");
52+
const auto table1 = PrepareTable(env, "Serverless1", "Table1");
53+
const auto table2 = PrepareTable(env, "Serverless2", "Table2");
10054

101-
Analyze(runtime, saTabletId, {pathId1}, "opId1", "/Root/Serverless1");
102-
Analyze(runtime, saTabletId, {pathId2}, "opId1", "/Root/Serverless2");
55+
Analyze(runtime, table1.SaTabletId, {table1.PathId}, "opId1", "/Root/Serverless1");
56+
Analyze(runtime, table2.SaTabletId, {table2.PathId}, "opId1", "/Root/Serverless2");
10357

104-
auto sender = runtime.AllocateEdgeActor();
105-
std::vector<TColumnStatisticsProbes> expected = {
58+
std::vector<TCountMinSketchProbes> expected = {
10659
{
107-
.Tag = 1, // Key column
108-
.Probes{ {1, 4}, {2, 4} }
60+
.Tag = 2, // Value column
61+
.Probes{ {"1", 100}, {"2", 100} }
10962
}
11063
};
11164

112-
CheckColumnStatistics(runtime, pathId1, sender, expected);
113-
CheckColumnStatistics(runtime, pathId2, sender, expected);
65+
CheckCountMinSketch(runtime, table1.PathId, expected);
66+
CheckCountMinSketch(runtime, table2.PathId, expected);
11467
}
11568
}
11669

‎ydb/core/statistics/service/ut/ut_http_request.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@ TTableInfo PrepareDatabaseAndTable(TTestEnv& env, bool isServerless) {
1717
} else {
1818
CreateDatabase(env, "Database");
1919
}
20-
return PrepareColumnTable(env, "Database", "Table", 10);
20+
auto info = PrepareColumnTable(env, "Database", "Table", 10);
21+
InsertDataIntoTable(env, "Database", "Table", RowsWithFewDistinctValues(1000));
22+
return info;
2123
}
2224

2325
void AnalyzeTest(bool isServerless) {

‎ydb/core/statistics/ut_common/ut_common.cpp‎

Lines changed: 60 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -248,7 +248,7 @@ void CreateUniformTable(TTestEnv& env, const TString& databaseName, const TStrin
248248
ExecuteYqlScript(env, Sprintf(R"(
249249
CREATE TABLE `Root/%s/%s` (
250250
Key Uint64,
251-
Value Uint64,
251+
Value String,
252252
PRIMARY KEY (Key)
253253
)
254254
WITH ( UNIFORM_PARTITIONS = 4 );
@@ -266,7 +266,7 @@ void PrepareUniformTable(TTestEnv& env, const TString& databaseName, const TStri
266266
replace << ", ";
267267
}
268268
ui64 value = 4000000000000000000ull * (i + 1);
269-
replace << Sprintf("(%" PRIu64 "ul, %" PRIu64 "ul)", value, value);
269+
replace << Sprintf("(%" PRIu64 "ul, \"%" PRIu64 "\")", value, value);
270270
}
271271
replace << ";";
272272
ExecuteYqlScript(env, replace);
@@ -299,7 +299,9 @@ TTableInfo CreateColumnTable(TTestEnv& env, const TString& databaseName, const T
299299
return tableInfo;
300300
}
301301

302-
void InsertDataIntoTable(TTestEnv& env, const TString& databaseName, const TString& tableName, size_t rowCount) {
302+
void InsertDataIntoTable(
303+
TTestEnv& env, const TString& databaseName, const TString& tableName,
304+
std::vector<TInsertedRow> insertedRows) {
303305
auto fullTableName = Sprintf("Root/%s/%s", databaseName.c_str(), tableName.c_str());
304306
auto& runtime = *env.GetServer().GetRuntime();
305307

@@ -319,11 +321,10 @@ void InsertDataIntoTable(TTestEnv& env, const TString& databaseName, const TStri
319321
reqValueType->mutable_type()->mutable_optional_type()->mutable_item()->set_type_id(Ydb::Type::STRING);
320322

321323
auto* reqRows = rows->mutable_value();
322-
323-
for (size_t i = 0; i < rowCount; ++i) {
324+
for (const auto& inserted : insertedRows) {
324325
auto* row = reqRows->add_items();
325-
row->add_items()->set_uint64_value(i);
326-
row->add_items()->set_bytes_value(ToString(i));
326+
row->add_items()->set_uint64_value(inserted.Key);
327+
row->add_items()->set_bytes_value(inserted.Value);
327328
}
328329

329330
auto future = NRpcService::DoLocalRpc<TEvBulkUpsertRequest>(
@@ -335,12 +336,16 @@ void InsertDataIntoTable(TTestEnv& env, const TString& databaseName, const TStri
335336
env.GetController()->WaitActualization(TDuration::Seconds(1));
336337
}
337338

338-
339339
TTableInfo PrepareColumnTable(TTestEnv& env, const TString& databaseName, const TString& tableName,
340340
int shardCount)
341341
{
342342
auto info = CreateColumnTable(env, databaseName, tableName, shardCount);
343-
InsertDataIntoTable(env, databaseName, tableName, ColumnTableRowsNumber);
343+
344+
std::vector<TInsertedRow> rows;
345+
for (size_t i = 0; i < ColumnTableRowsNumber; ++i) {
346+
rows.push_back(TInsertedRow{.Key = i, .Value = ToString(i)});
347+
}
348+
InsertDataIntoTable(env, databaseName, tableName, rows);
344349
return info;
345350
}
346351

@@ -399,7 +404,7 @@ TTableInfo PrepareColumnTableWithIndexes(TTestEnv& env, const TString& databaseN
399404
auto future = NRpcService::DoLocalRpc<TEvBulkUpsertRequest>(
400405
std::move(request), "", "", runtime.GetActorSystem(0));
401406
auto response = runtime.WaitFuture(std::move(future));
402-
407+
403408
UNIT_ASSERT(response.operation().ready());
404409
UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS);
405410
}
@@ -409,6 +414,14 @@ TTableInfo PrepareColumnTableWithIndexes(TTestEnv& env, const TString& databaseN
409414
return info;
410415
}
411416

417+
std::vector<TInsertedRow> RowsWithFewDistinctValues(size_t count) {
418+
std::vector<TInsertedRow> rows;
419+
for (size_t i = 0; i < count; ++i) {
420+
rows.push_back(TInsertedRow{.Key = i, .Value = ToString(i % 10)});
421+
}
422+
return rows;
423+
}
424+
412425
void DropTable(TTestEnv& env, const TString& databaseName, const TString& tableName) {
413426
ExecuteYqlScript(env, Sprintf(R"(
414427
DROP TABLE `Root/%s/%s`;
@@ -442,16 +455,6 @@ std::shared_ptr<TCountMinSketch> ExtractCountMin(TTestActorRuntime& runtime, con
442455
return stat.CountMin;
443456
}
444457

445-
void ValidateCountMinDatashard(TTestActorRuntime& runtime, TPathId pathId) {
446-
auto countMin = ExtractCountMin(runtime, pathId);
447-
448-
for (ui32 i = 0; i < 4; ++i) {
449-
ui64 value = 4000000000000000000ull * (i + 1);
450-
auto probe = countMin->Probe((const char *)&value, sizeof(ui64));
451-
UNIT_ASSERT_VALUES_EQUAL(probe, 1);
452-
}
453-
}
454-
455458
void ValidateCountMinAbsence(TTestActorRuntime& runtime, TPathId pathId) {
456459
auto statServiceId = NStat::MakeStatServiceID(runtime.GetNodeId(1));
457460

@@ -475,6 +478,43 @@ void ValidateCountMinAbsence(TTestActorRuntime& runtime, TPathId pathId) {
475478
UNIT_ASSERT(!rsp.Success);
476479
}
477480

481+
void CheckCountMinSketch(
482+
TTestActorRuntime& runtime, const TPathId& pathId,
483+
const std::vector<TCountMinSketchProbes>& expected) {
484+
auto evGet = std::make_unique<TEvStatistics::TEvGetStatistics>();
485+
evGet->StatType = NStat::EStatType::COUNT_MIN_SKETCH;
486+
487+
for (auto item : expected) {
488+
NStat::TRequest req;
489+
req.PathId = pathId;
490+
req.ColumnTag = item.Tag;
491+
evGet->StatRequests.push_back(req);
492+
}
493+
494+
auto sender = runtime.AllocateEdgeActor();
495+
auto statServiceId = NStat::MakeStatServiceID(runtime.GetNodeId(0));
496+
runtime.Send(statServiceId, sender, evGet.release(), 0, true);
497+
498+
auto res = runtime.GrabEdgeEventRethrow<TEvStatistics::TEvGetStatisticsResult>(sender);
499+
auto msg = res->Get();
500+
501+
UNIT_ASSERT(msg->Success);
502+
UNIT_ASSERT( msg->StatResponses.size() == expected.size());
503+
504+
for (size_t i = 0; i < msg->StatResponses.size(); ++i) {
505+
const auto& stat = msg->StatResponses[i];
506+
UNIT_ASSERT(stat.Success);
507+
508+
auto countMin = stat.CountMinSketch.CountMin.get();
509+
UNIT_ASSERT(countMin != nullptr);
510+
511+
for (const auto& item : expected[i].Probes) {
512+
auto probe = countMin->Probe(item.Value.data(), item.Value.size());
513+
UNIT_ASSERT_VALUES_EQUAL(item.Expected, probe);
514+
}
515+
}
516+
}
517+
478518
TAnalyzedTable::TAnalyzedTable(const TPathId& pathId)
479519
: PathId(pathId)
480520
{}

0 commit comments

Comments
 (0)