Skip to content

Commit dae3888

Browse files
authored
support sum_int in TiFlash (#10695)
close #10720 Signed-off-by: xufei <xufeixw@mail.ustc.edu.cn>
1 parent e7d70a3 commit dae3888

12 files changed

Lines changed: 135 additions & 15 deletions

File tree

dbms/src/Debug/MockExecutor/AggregationBinder.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -202,7 +202,7 @@ void AggregationBinder::buildAggFunc(tipb::Expr * agg_func, const ASTFunction *
202202
auto agg_sig = agg_sig_it->second;
203203
agg_func->set_tp(agg_sig);
204204

205-
if (agg_sig == tipb::ExprType::Count || agg_sig == tipb::ExprType::Sum)
205+
if (agg_sig == tipb::ExprType::Count || agg_sig == tipb::ExprType::Sum || agg_sig == tipb::ExprType::SumInt)
206206
{
207207
auto * ft = agg_func->mutable_field_type();
208208
ft->set_tp(TiDB::TypeLongLong);

dbms/src/Debug/MockExecutor/FuncSigMap.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ std::unordered_map<String, tipb::ExprType> agg_func_name_to_sig({
8989
{"max", tipb::ExprType::Max},
9090
{"count", tipb::ExprType::Count},
9191
{"sum", tipb::ExprType::Sum},
92+
{"sum_int", tipb::ExprType::SumInt},
9293
{"first_row", tipb::ExprType::First},
9394
{"uniqRawRes", tipb::ExprType::ApproxCountDistinct},
9495
{"group_concat", tipb::ExprType::GroupConcat},

dbms/src/Flash/Coprocessor/DAGUtils.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ const std::unordered_map<tipb::ExprType, String> window_func_map({
5353
const std::unordered_map<tipb::ExprType, String> agg_func_map({
5454
{tipb::ExprType::Count, "count"},
5555
{tipb::ExprType::Sum, "sum"},
56+
{tipb::ExprType::SumInt, "sum"},
5657
{tipb::ExprType::Min, "min"},
5758
{tipb::ExprType::Max, "max"},
5859
{tipb::ExprType::First, "first_row"},
@@ -994,6 +995,7 @@ String exprToString(const tipb::Expr & expr, const std::vector<NameAndTypePair>
994995
return getColumnNameForColumnExpr(expr, input_col);
995996
case tipb::ExprType::Count:
996997
case tipb::ExprType::Sum:
998+
case tipb::ExprType::SumInt:
997999
case tipb::ExprType::Avg:
9981000
case tipb::ExprType::Min:
9991001
case tipb::ExprType::Max:
@@ -1050,6 +1052,7 @@ bool isAggFunctionExpr(const tipb::Expr & expr)
10501052
{
10511053
case tipb::ExprType::Count:
10521054
case tipb::ExprType::Sum:
1055+
case tipb::ExprType::SumInt:
10531056
case tipb::ExprType::Avg:
10541057
case tipb::ExprType::Min:
10551058
case tipb::ExprType::Max:
Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
// Copyright 2023 PingCAP, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#include <AggregateFunctions/AggregateFunctionNull.h>
16+
#include <AggregateFunctions/AggregateFunctionSum.h>
17+
#include <Core/Types.h>
18+
#include <DataTypes/DataTypesNumber.h>
19+
#include <Flash/Coprocessor/AggregationInterpreterHelper.h>
20+
#include <Flash/Coprocessor/DAGCodec.h>
21+
#include <Flash/Coprocessor/DAGUtils.h>
22+
#include <IO/Buffer/WriteBufferFromString.h>
23+
#include <gtest/gtest.h>
24+
25+
namespace DB
26+
{
27+
namespace tests
28+
{
29+
namespace
30+
{
31+
template <typename T>
32+
using AggregateFunctionSumSimple = AggregateFunctionSum<
33+
T,
34+
typename NearestFieldType<T>::Type,
35+
AggregateFunctionSumData<typename NearestFieldType<T>::Type>>;
36+
37+
template <typename TInput, typename TExpectedOutput>
38+
void checkSumIntReturnType(const String & expected_output, const String & expected_output_nullable)
39+
{
40+
using NestedFunc = AggregateFunctionSumSimple<TInput>;
41+
auto nested = std::make_shared<NestedFunc>();
42+
ASSERT_EQ(nested->getReturnType()->getName(), expected_output);
43+
44+
auto wrapped
45+
= std::make_shared<AggregateFunctionNullUnary</*result_is_nullable*/ true, /*input_is_nullable*/ true>>(nested);
46+
ASSERT_EQ(wrapped->getReturnType()->getName(), expected_output_nullable);
47+
}
48+
} // namespace
49+
50+
TEST(SumIntAggFuncTest, DagUtilsMappedToSum)
51+
{
52+
tipb::Expr expr;
53+
expr.set_tp(tipb::ExprType::SumInt);
54+
55+
ASSERT_TRUE(isAggFunctionExpr(expr));
56+
ASSERT_EQ(getAggFunctionName(expr), "sum");
57+
ASSERT_EQ(getFunctionName(expr), "sum");
58+
}
59+
60+
TEST(SumIntAggFuncTest, ExprToStringIsSum)
61+
{
62+
std::vector<NameAndTypePair> input_cols;
63+
input_cols.emplace_back("col0", std::make_shared<DataTypeInt32>());
64+
65+
tipb::Expr col_ref;
66+
col_ref.set_tp(tipb::ExprType::ColumnRef);
67+
WriteBufferFromOwnString ss;
68+
encodeDAGInt64(0, ss);
69+
col_ref.set_val(ss.releaseStr());
70+
71+
tipb::Expr sum_int;
72+
sum_int.set_tp(tipb::ExprType::SumInt);
73+
*sum_int.add_children() = col_ref;
74+
75+
ASSERT_EQ(exprToString(sum_int, input_cols), "sum(col0)");
76+
}
77+
78+
TEST(SumIntAggFuncTest, PartialStageIsTreatedAsSum)
79+
{
80+
tipb::Expr expr;
81+
expr.set_tp(tipb::ExprType::SumInt);
82+
83+
ASSERT_FALSE(AggregationInterpreterHelper::isSumOnPartialResults(expr));
84+
85+
expr.set_aggfuncmode(tipb::AggFunctionMode::Partial1Mode);
86+
ASSERT_FALSE(AggregationInterpreterHelper::isSumOnPartialResults(expr));
87+
88+
expr.set_aggfuncmode(tipb::AggFunctionMode::Partial2Mode);
89+
ASSERT_TRUE(AggregationInterpreterHelper::isSumOnPartialResults(expr));
90+
91+
expr.set_aggfuncmode(tipb::AggFunctionMode::FinalMode);
92+
ASSERT_TRUE(AggregationInterpreterHelper::isSumOnPartialResults(expr));
93+
}
94+
95+
TEST(SumIntAggFuncTest, ReturnTypeForIntegerInputs)
96+
{
97+
checkSumIntReturnType<Int8, Int64>("Int64", "Nullable(Int64)");
98+
checkSumIntReturnType<Int16, Int64>("Int64", "Nullable(Int64)");
99+
checkSumIntReturnType<Int32, Int64>("Int64", "Nullable(Int64)");
100+
checkSumIntReturnType<Int64, Int64>("Int64", "Nullable(Int64)");
101+
102+
checkSumIntReturnType<UInt8, UInt64>("UInt64", "Nullable(UInt64)");
103+
checkSumIntReturnType<UInt16, UInt64>("UInt64", "Nullable(UInt64)");
104+
checkSumIntReturnType<UInt32, UInt64>("UInt64", "Nullable(UInt64)");
105+
checkSumIntReturnType<UInt64, UInt64>("UInt64", "Nullable(UInt64)");
106+
}
107+
108+
} // namespace tests
109+
} // namespace DB

tests/docker/cluster.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ version: '2.3'
1616

1717
services:
1818
pd0:
19-
image: hub.pingcap.net/qa/pd:${PD_BRANCH:-master}
19+
image: us-docker.pkg.dev/pingcap-testing-account/hub/tikv/pd/image:${PD_BRANCH:-master}
2020
security_opt:
2121
- seccomp:unconfined
2222
volumes:
@@ -35,7 +35,7 @@ services:
3535
- --log-file=/log/pd.log
3636
restart: on-failure
3737
tikv0:
38-
image: hub.pingcap.net/qa/tikv:${TIKV_BRANCH:-master}
38+
image: us-docker.pkg.dev/pingcap-testing-account/hub/tikv/tikv/image:${TIKV_BRANCH:-master}
3939
security_opt:
4040
- seccomp:unconfined
4141
volumes:
@@ -55,7 +55,7 @@ services:
5555
- "pd0"
5656
restart: on-failure
5757
tidb0:
58-
image: hub.pingcap.net/qa/tidb:${TIDB_BRANCH:-master}
58+
image: us-docker.pkg.dev/pingcap-testing-account/hub/pingcap/tidb/images/tidb-server:${TIDB_BRANCH:-master}
5959
security_opt:
6060
- seccomp:unconfined
6161
volumes:

tests/docker/cluster_disable_new_collation.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ version: '2.3'
1616

1717
services:
1818
pd0:
19-
image: hub.pingcap.net/qa/pd:${PD_BRANCH:-master}
19+
image: us-docker.pkg.dev/pingcap-testing-account/hub/tikv/pd/image:${PD_BRANCH:-master}
2020
security_opt:
2121
- seccomp:unconfined
2222
volumes:
@@ -35,7 +35,7 @@ services:
3535
- --log-file=/log/pd.log
3636
restart: on-failure
3737
tikv0:
38-
image: hub.pingcap.net/qa/tikv:${TIKV_BRANCH:-master}
38+
image: us-docker.pkg.dev/pingcap-testing-account/hub/tikv/tikv/image:${TIKV_BRANCH:-master}
3939
security_opt:
4040
- seccomp:unconfined
4141
volumes:
@@ -55,7 +55,7 @@ services:
5555
- "pd0"
5656
restart: on-failure
5757
tidb0:
58-
image: hub.pingcap.net/qa/tidb:${TIDB_BRANCH:-master}
58+
image: us-docker.pkg.dev/pingcap-testing-account/hub/pingcap/tidb/images/tidb-server:${TIDB_BRANCH:-master}
5959
security_opt:
6060
- seccomp:unconfined
6161
volumes:

tests/docker/cluster_new_collation.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ version: '2.3'
1616

1717
services:
1818
pd0:
19-
image: hub.pingcap.net/qa/pd:${PD_BRANCH:-master}
19+
image: us-docker.pkg.dev/pingcap-testing-account/hub/tikv/pd/image:${PD_BRANCH:-master}
2020
security_opt:
2121
- seccomp:unconfined
2222
volumes:
@@ -35,7 +35,7 @@ services:
3535
- --log-file=/log/pd.log
3636
restart: on-failure
3737
tikv0:
38-
image: hub.pingcap.net/qa/tikv:${TIKV_BRANCH:-master}
38+
image: us-docker.pkg.dev/pingcap-testing-account/hub/tikv/tikv/image:${TIKV_BRANCH:-master}
3939
security_opt:
4040
- seccomp:unconfined
4141
volumes:
@@ -55,7 +55,7 @@ services:
5555
- "pd0"
5656
restart: on-failure
5757
tidb0:
58-
image: hub.pingcap.net/qa/tidb:${TIDB_BRANCH:-master}
58+
image: us-docker.pkg.dev/pingcap-testing-account/hub/pingcap/tidb/images/tidb-server:${TIDB_BRANCH:-master}
5959
security_opt:
6060
- seccomp:unconfined
6161
volumes:

tests/docker/cluster_tidb_fail_point.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ version: '2.3'
1616

1717
services:
1818
pd0:
19-
image: hub.pingcap.net/qa/pd:${PD_BRANCH:-master}
19+
image: us-docker.pkg.dev/pingcap-testing-account/hub/tikv/pd/image:${PD_BRANCH:-master}
2020
security_opt:
2121
- seccomp:unconfined
2222
volumes:
@@ -35,7 +35,7 @@ services:
3535
- --log-file=/log/pd.log
3636
restart: on-failure
3737
tikv0:
38-
image: hub.pingcap.net/qa/tikv:${TIKV_BRANCH:-master}
38+
image: us-docker.pkg.dev/pingcap-testing-account/hub/tikv/tikv/image:${TIKV_BRANCH:-master}
3939
security_opt:
4040
- seccomp:unconfined
4141
volumes:
@@ -55,7 +55,7 @@ services:
5555
- "pd0"
5656
restart: on-failure
5757
tidb0:
58-
image: hub.pingcap.net/qa/tidb:${TIDB_BRANCH:-master}-failpoint
58+
image: us-docker.pkg.dev/pingcap-testing-account/hub/pingcap/tidb/images/tidb-server:${TIDB_BRANCH:-master}-failpoint
5959
security_opt:
6060
- seccomp:unconfined
6161
environment:

tests/docker/run.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,10 @@ if [ -n "$BRANCH" ]; then
7171
[ -z "$TIKV_BRANCH" ] && export TIKV_BRANCH="$BRANCH"
7272
[ -z "$TIDB_BRANCH" ] && export TIDB_BRANCH="$BRANCH"
7373
fi
74+
export PD_BRANCH="${PD_BRANCH//\//-}"
75+
export TIKV_BRANCH="${TIKV_BRANCH//\//-}"
76+
export TIDB_BRANCH="${TIDB_BRANCH//\//-}"
77+
export TAG="${TAG//\//-}"
7478

7579

7680
# Stop all docker instances if exist.

0 commit comments

Comments
 (0)