From cc2639f04b557b6e7fb51b464d364c2c4ff5e717 Mon Sep 17 00:00:00 2001 From: wenmo <32723967+aiwenmo@users.noreply.github.com> Date: Thu, 6 Mar 2025 20:37:21 +0800 Subject: [PATCH 1/4] [Feature-4280][CI] add flinkjar task on k8s application in CI --- .../tools/dinky_task/flink_jar_sql/mysql2doris.sql | 9 --------- e2e_test/tools/dinky_task/flink_jar_sql/word-count.sql | 7 +++++++ e2e_test/tools/main.py | 10 +++++----- 3 files changed, 12 insertions(+), 14 deletions(-) delete mode 100644 e2e_test/tools/dinky_task/flink_jar_sql/mysql2doris.sql create mode 100644 e2e_test/tools/dinky_task/flink_jar_sql/word-count.sql diff --git a/e2e_test/tools/dinky_task/flink_jar_sql/mysql2doris.sql b/e2e_test/tools/dinky_task/flink_jar_sql/mysql2doris.sql deleted file mode 100644 index e2e109cb60..0000000000 --- a/e2e_test/tools/dinky_task/flink_jar_sql/mysql2doris.sql +++ /dev/null @@ -1,9 +0,0 @@ -set 'execution.checkpointing.interval'='5 s'; -ADD CUSTOMJAR 'rs:/flink-sql-connector-mysql-cdc.jar'; -ADD CUSTOMJAR 'rs:/mysql-connector-java.jar'; -EXECUTE JAR WITH ( -'uri'='rs:/flink-doris-connector.jar', -'main-class'='org.apache.doris.flink.tools.cdc.CdcTools', -'args'='base64@bXlzcWwtc3luYy1kYXRhYmFzZSAgICAgLS1kYXRhYmFzZSBkaW5reSAgICAgLS1teXNxbC1jb25mIGhvc3RuYW1lPW15c3FsICAgICAtLW15c3FsLWNvbmYgcG9ydD0zMzA2ICAgICAtLW15c3FsLWNvbmYgdXNlcm5hbWU9cm9vdCAgICAgLS1teXNxbC1jb25mIHBhc3N3b3JkPWRpbmt5ICAgICAtLW15c3FsLWNvbmYgZGF0YWJhc2UtbmFtZT1kaW5reSAgICAgLS1teXNxbC1jb25mIHNlcnZlci10aW1lLXpvbmU9QXNpYS9TaGFuZ2hhaSAgICAgLS1pbmNsdWRpbmctdGFibGVzICJkaW5reV90YXNrIiAgICAgLS1zaW5rLWNvbmYgZmVub2Rlcz1kb3Jpcy1mZTo4MDMwICAgICAtLXNpbmstY29uZiB1c2VybmFtZT1yb290ICAgICAtLXNpbmstY29uZiBqZGJjLXVybD1qZGJjOm15c3FsOi8vZG9yaXMtZmU6OTAzMCAgICAgLS1zaW5rLWNvbmYgc2luay5sYWJlbC1wcmVmaXg9bGFiZWwtMSAgICAgLS10YWJsZS1jb25mIHJlcGxpY2F0aW9uX251bT0xIA==', -'allowNonRestoredState'='false' -); diff --git a/e2e_test/tools/dinky_task/flink_jar_sql/word-count.sql b/e2e_test/tools/dinky_task/flink_jar_sql/word-count.sql new file mode 100644 index 0000000000..2569d2a819 --- /dev/null +++ b/e2e_test/tools/dinky_task/flink_jar_sql/word-count.sql @@ -0,0 +1,7 @@ +set 'execution.checkpointing.interval' = '5s'; +EXECUTE JAR WITH ( +'uri'='rs:/opt/flink/examples/streaming/WordCount.jar', +'main-class'='org.apache.flink.streaming.examples.wordcount.WordCount', +'args'='', +'allowNonRestoredState'='false' +); diff --git a/e2e_test/tools/main.py b/e2e_test/tools/main.py index d82644957a..9ec1842ea3 100644 --- a/e2e_test/tools/main.py +++ b/e2e_test/tools/main.py @@ -28,8 +28,8 @@ def traverse_files(directory) -> list[str]: task_name = file_name.split(".")[0] flink_sql_datagen_test.runFlinkTask(sql, task_name, is_async=True) - # flink_jar_sql_task_path = "dinky_task/flink_jar_sql" - # for file_name in traverse_files(flink_jar_sql_task_path): - # sql = open(os.path.join(flink_jar_sql_task_path, file_name)).read() - # task_name = file_name.split(".")[0] - # flink_sql_datagen_test.runFlinkTask(sql, task_name,"FlinkJar", is_async=True) + flink_jar_sql_task_path = "dinky_task/flink_jar_sql" + for file_name in traverse_files(flink_jar_sql_task_path): + sql = open(os.path.join(flink_jar_sql_task_path, file_name)).read() + task_name = file_name.split(".")[0] + flink_sql_datagen_test.runFlinkTask(sql, task_name,"FlinkJar", is_async=True) From 1997951f2d862e4daded309cf17bad33adc09446 Mon Sep 17 00:00:00 2001 From: wenmo <32723967+aiwenmo@users.noreply.github.com> Date: Thu, 13 Mar 2025 21:10:04 +0800 Subject: [PATCH 2/4] Modify jar path --- e2e_test/tools/dinky_task/flink_jar_sql/word-count.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e_test/tools/dinky_task/flink_jar_sql/word-count.sql b/e2e_test/tools/dinky_task/flink_jar_sql/word-count.sql index 2569d2a819..1c3f1f515d 100644 --- a/e2e_test/tools/dinky_task/flink_jar_sql/word-count.sql +++ b/e2e_test/tools/dinky_task/flink_jar_sql/word-count.sql @@ -1,6 +1,6 @@ set 'execution.checkpointing.interval' = '5s'; EXECUTE JAR WITH ( -'uri'='rs:/opt/flink/examples/streaming/WordCount.jar', +'uri'='file:/opt/flink/examples/streaming/WordCount.jar', 'main-class'='org.apache.flink.streaming.examples.wordcount.WordCount', 'args'='', 'allowNonRestoredState'='false' From 586a4db3e62573604a637104f3437aebc6c376d2 Mon Sep 17 00:00:00 2001 From: wenmo <32723967+aiwenmo@users.noreply.github.com> Date: Thu, 13 Mar 2025 22:05:31 +0800 Subject: [PATCH 3/4] try local --- e2e_test/tools/dinky_task/flink_jar_sql/word-count.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e_test/tools/dinky_task/flink_jar_sql/word-count.sql b/e2e_test/tools/dinky_task/flink_jar_sql/word-count.sql index 1c3f1f515d..e792f239b5 100644 --- a/e2e_test/tools/dinky_task/flink_jar_sql/word-count.sql +++ b/e2e_test/tools/dinky_task/flink_jar_sql/word-count.sql @@ -1,6 +1,6 @@ set 'execution.checkpointing.interval' = '5s'; EXECUTE JAR WITH ( -'uri'='file:/opt/flink/examples/streaming/WordCount.jar', +'uri'='local:/opt/flink/examples/streaming/WordCount.jar', 'main-class'='org.apache.flink.streaming.examples.wordcount.WordCount', 'args'='', 'allowNonRestoredState'='false' From cb1303607a1ee671d7541ee8372bc87690f009d2 Mon Sep 17 00:00:00 2001 From: wenmo <32723967+aiwenmo@users.noreply.github.com> Date: Thu, 13 Mar 2025 23:27:37 +0800 Subject: [PATCH 4/4] try hdfs --- e2e_test/tools/dinky_task/flink_jar_sql/word-count.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e_test/tools/dinky_task/flink_jar_sql/word-count.sql b/e2e_test/tools/dinky_task/flink_jar_sql/word-count.sql index e792f239b5..b20f56d5bc 100644 --- a/e2e_test/tools/dinky_task/flink_jar_sql/word-count.sql +++ b/e2e_test/tools/dinky_task/flink_jar_sql/word-count.sql @@ -1,6 +1,6 @@ set 'execution.checkpointing.interval' = '5s'; EXECUTE JAR WITH ( -'uri'='local:/opt/flink/examples/streaming/WordCount.jar', +'uri'='hdfs:/opt/flink/examples/streaming/WordCount.jar', 'main-class'='org.apache.flink.streaming.examples.wordcount.WordCount', 'args'='', 'allowNonRestoredState'='false'