Skip to content

Commit 70c77bd

Browse files
sijiesijie
authored
Add a Flink connector example (#14)
1 parent 3b811fa commit 70c77bd

File tree

7 files changed

+509
-1
lines changed

7 files changed

+509
-1
lines changed

‎README.md‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,8 @@ This is a curated list of demos that showcase Apache Pulsar® messaging and even
88

99
# Clients
1010

11-
- [PubSub Client Examples](clients/README.md)
11+
- [PubSub Client Examples](clients/README.md)
12+
13+
# Flink
14+
15+
- [Pulsar Flink Connector](pulsar-flink/README.md)

‎pulsar-flink/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
dependency-reduced-pom.xml

‎pulsar-flink/README.md‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
# Pulsar Flink Connector Examples
2+
3+
This is a curated list of examples that demonstrate how to process event streams in Apache Pulsar using Apache Flink.
4+
5+
## Prerequisites
6+
7+
- Java 1.8 or higher to run the demo application
8+
- Maven to compile the demo application
9+
- Pulsar 2.5.2 or higher
10+
- Flink 1.10.1
11+
12+
## Pulsar Streaming Word Count
13+
14+
This example demonstrates a Flink streaming job that reads events from Pulsar, processes them and produces the word count results
15+
back to Pulsar.
16+
17+
```bash
18+
export INPUT_TOPIC=wordcount_input
19+
export OUTPUT_TOPIC=wordcount_output
20+
```
21+
22+
### Steps
23+
24+
1. Download Pulsar 2.5.1 and start Pulsar standalone. Assume `PULSAR_HOME` is the root directory of pulsar distribution.
25+
26+
```bash
27+
${PULSAR_HOME}/bin/pulsar standalone
28+
```
29+
30+
2. Download Flink 1.10.1 and start Flink locally. Assume `FLINK_HOME` is the root directory of flink distribution.
31+
32+
```bash
33+
${FLINK_HOME}/bin/start-cluster.sh
34+
```
35+
36+
3. Clone the examples repo and build the flink examples. Assume `EXAMPLES_HOME` is the root directory of the cloned `streamnative/pulsar-examples` repo.
37+
38+
```bash
39+
git clone https://github.com/streamnative/pulsar-examples.git
40+
```
41+
42+
```bash
43+
cd pulsar-examples/pulsar-flink
44+
```
45+
46+
```bash
47+
mvn clean install
48+
```
49+
50+
4. Open a terminal to subscribe to the output topic `${OUTPUT_TOPIC}` to receive word count results from it.
51+
52+
```bash
53+
${PULSAR_HOME}/bin/pulsar-client consume -s sub -n 0 ${OUTPUT_TOPIC}
54+
```
55+
56+
5. Open a terminal to submit the PulsarStreamingWordCount job to Flink.
57+
58+
```bash
59+
${FLINK_HOME}/bin/flink run ${EXAMPLES_HOME}/pulsar-flink/target/pulsar-flink-examples-0.0.0-SNAPSHOT.jar \
60+
--broker-service-url pulsar://localhost:6650 \
61+
--admin-service-url http://localhost:8080 \
62+
--input-topic ${INPUT_TOPIC} --output-topic ${OUTPUT_TOPIC}
63+
```
64+
65+
6. Open a terminal to produce a stream of sentences to the input topic `${INPUT_TOPIC}`.
66+
67+
```bash
68+
${PULSAR_HOME}/bin/pulsar-client produce -m "test flink streaming word count" -n 100 ${INPUT_TOPIC}
69+
```
70+
71+
7. In the terminal of `step 4`, you should see a stream of wordcount results similar as below. The wordcount results are saved in AVRO format in the output topic.
72+
73+
```bash
74+
----- got message -----
75+
test�
76+
----- got message -----
77+
78+
count�
79+
----- got message -----
80+
word�
81+
----- got message -----
82+
streaming�
83+
----- got message -----
84+
85+
flink�
86+
```

‎pulsar-flink/pom.xml‎

Lines changed: 278 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,278 @@
1+
<!--
2+
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License.
14+
15+
-->
16+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
17+
<modelVersion>4.0.0</modelVersion>
18+
<groupId>io.streamnative.examples</groupId>
19+
<version>0.0.0-SNAPSHOT</version>
20+
<artifactId>pulsar-flink-examples</artifactId>
21+
<packaging>jar</packaging>
22+
<name>Pulsar Flink Examples</name>
23+
<url>http://github.com/streamnative/examples</url>
24+
<inceptionYear>2020</inceptionYear>
25+
<licenses>
26+
<license>
27+
<name>Apache License, Version 2.0</name>
28+
<url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
29+
<distribution>repo</distribution>
30+
</license>
31+
</licenses>
32+
<scm>
33+
<connection>scm:git:https://github.com/streamnative/examples.git</connection>
34+
<developerConnection>scm:git:https://github.com/streamnative/examples.git</developerConnection>
35+
<url>https://github.com/streamnative/examples</url>
36+
<tag>branch-0.0.1</tag>
37+
</scm>
38+
<issueManagement>
39+
<system>Github</system>
40+
<url>https://github.com/streamnative/examples/issues</url>
41+
</issueManagement>
42+
<developers>
43+
<developer>
44+
<name>The StreamNative Team</name>
45+
<url>http://github.com/streamnative</url>
46+
</developer>
47+
</developers>
48+
<properties>
49+
<maven.compiler.source>1.8</maven.compiler.source>
50+
<maven.compiler.target>1.8</maven.compiler.target>
51+
<encoding>UTF-8</encoding>
52+
<javac.target>1.8</javac.target>
53+
<pulsar.version>2.5.2</pulsar.version>
54+
<scala.version>2.11.12</scala.version>
55+
<scala.compat.version>2.11</scala.compat.version>
56+
<flink.version>1.10.1</flink.version>
57+
<pulsar-flink-connector.version>2.4.17</pulsar-flink-connector.version>
58+
59+
<!-- plugin dependencies -->
60+
<license-maven-plugin.version>3.0.rc1</license-maven-plugin.version>
61+
<maven-checkstyle-plugin.version>3.0.0</maven-checkstyle-plugin.version>
62+
<maven-compiler-plugin.version>3.7.0</maven-compiler-plugin.version>
63+
<maven-dependency-plugin.version>3.1.1</maven-dependency-plugin.version>
64+
<maven-shade-plugin.version>3.1.0</maven-shade-plugin.version>
65+
<maven-source-plugin.version>2.2.1</maven-source-plugin.version>
66+
<maven-surefire-plugin.version>2.21.0</maven-surefire-plugin.version>
67+
<os-maven-plugin.version>1.4.1.Final</os-maven-plugin.version>
68+
<puppycrawl.checkstyle.version>6.19</puppycrawl.checkstyle.version>
69+
<spotbugs-maven-plugin.version>3.1.8</spotbugs-maven-plugin.version>
70+
</properties>
71+
72+
<dependencies>
73+
<dependency>
74+
<groupId>org.scala-lang</groupId>
75+
<artifactId>scala-library</artifactId>
76+
<version>${scala.version}</version>
77+
</dependency>
78+
79+
<dependency>
80+
<groupId>io.streamnative.connectors</groupId>
81+
<artifactId>pulsar-flink-connector_${scala.compat.version}</artifactId>
82+
<version>${pulsar-flink-connector.version}</version>
83+
</dependency>
84+
85+
<dependency>
86+
<groupId>org.apache.flink</groupId>
87+
<artifactId>flink-streaming-scala_${scala.compat.version}</artifactId>
88+
<version>${flink.version}</version>
89+
<scope>provided</scope>
90+
</dependency>
91+
92+
<dependency>
93+
<groupId>org.apache.flink</groupId>
94+
<artifactId>flink-table-api-scala-bridge_${scala.compat.version}</artifactId>
95+
<version>${flink.version}</version>
96+
<scope>provided</scope>
97+
</dependency>
98+
</dependencies>
99+
100+
<build>
101+
<extensions>
102+
<extension>
103+
<groupId>kr.motd.maven</groupId>
104+
<artifactId>os-maven-plugin</artifactId>
105+
<version>${os-maven-plugin.version}</version>
106+
</extension>
107+
</extensions>
108+
<pluginManagement>
109+
<plugins>
110+
<plugin>
111+
<groupId>org.apache.maven.plugins</groupId>
112+
<artifactId>maven-checkstyle-plugin</artifactId>
113+
<version>${maven-checkstyle-plugin.version}</version>
114+
<dependencies>
115+
<dependency>
116+
<groupId>com.puppycrawl.tools</groupId>
117+
<artifactId>checkstyle</artifactId>
118+
<version>${puppycrawl.checkstyle.version}</version>
119+
</dependency>
120+
</dependencies>
121+
<configuration>
122+
<configLocation>../clients/buildtools/src/main/resources/streamnative/checkstyle.xml</configLocation>
123+
<suppressionsLocation>../clients/buildtools/src/main/resources/streamnative/suppressions.xml</suppressionsLocation>
124+
<encoding>UTF-8</encoding>
125+
<consoleOutput>true</consoleOutput>
126+
<failOnViolation>true</failOnViolation>
127+
<includeResources>false</includeResources>
128+
<includeTestSourceDirectory>true</includeTestSourceDirectory>
129+
</configuration>
130+
<executions>
131+
<execution>
132+
<id>checkstyle</id>
133+
<phase>validate</phase>
134+
<goals>
135+
<goal>check</goal>
136+
</goals>
137+
</execution>
138+
</executions>
139+
</plugin>
140+
</plugins>
141+
</pluginManagement>
142+
<plugins>
143+
<plugin>
144+
<groupId>com.github.spotbugs</groupId>
145+
<artifactId>spotbugs-maven-plugin</artifactId>
146+
<version>${spotbugs-maven-plugin.version}</version>
147+
<configuration>
148+
<excludeFilterFile>${session.executionRootDirectory}/../buildtools/src/main/resources/streamnative/findbugsExclude.xml</excludeFilterFile>
149+
</configuration>
150+
</plugin>
151+
<plugin>
152+
<artifactId>maven-compiler-plugin</artifactId>
153+
<version>${maven-compiler-plugin.version}</version>
154+
<configuration>
155+
<source>${javac.target}</source>
156+
<target>${javac.target}</target>
157+
<compilerArgs>
158+
<compilerArg>-Werror</compilerArg>
159+
<compilerArg>-Xlint:deprecation</compilerArg>
160+
<compilerArg>-Xlint:unchecked</compilerArg>
161+
<!-- https://issues.apache.org/jira/browse/MCOMPILER-205 -->
162+
<compilerArg>-Xpkginfo:always</compilerArg>
163+
</compilerArgs>
164+
</configuration>
165+
</plugin>
166+
<plugin>
167+
<groupId>org.apache.maven.plugins</groupId>
168+
<artifactId>maven-surefire-plugin</artifactId>
169+
<version>${maven-surefire-plugin.version}</version>
170+
<configuration>
171+
<argLine>-Xmx2G -Djava.net.preferIPv4Stack=true -Dio.netty.leakDetection.level=paranoid</argLine>
172+
<redirectTestOutputToFile>${redirectTestOutputToFile}</redirectTestOutputToFile>
173+
<reuseForks>false</reuseForks>
174+
<forkedProcessTimeoutInSeconds>1800</forkedProcessTimeoutInSeconds>
175+
<rerunFailingTestsCount>${testRetryCount}</rerunFailingTestsCount>
176+
</configuration>
177+
</plugin>
178+
<plugin>
179+
<groupId>org.apache.maven.plugins</groupId>
180+
<artifactId>maven-source-plugin</artifactId>
181+
<version>${maven-source-plugin.version}</version>
182+
<executions>
183+
<execution>
184+
<id>attach-sources</id>
185+
<goals>
186+
<goal>jar</goal>
187+
</goals>
188+
</execution>
189+
</executions>
190+
</plugin>
191+
<plugin>
192+
<groupId>com.mycila</groupId>
193+
<artifactId>license-maven-plugin</artifactId>
194+
<version>${license-maven-plugin.version}</version>
195+
<configuration>
196+
<header>src/resources/license.template</header>
197+
198+
<excludes>
199+
<exclude>LICENSE</exclude>
200+
<exclude>NOTICE</exclude>
201+
</excludes>
202+
<mapping>
203+
<proto>JAVADOC_STYLE</proto>
204+
<go>DOUBLESLASH_STYLE</go>
205+
<conf>SCRIPT_STYLE</conf>
206+
<ini>SCRIPT_STYLE</ini>
207+
<yaml>SCRIPT_STYLE</yaml>
208+
<tf>SCRIPT_STYLE</tf>
209+
<cfg>SCRIPT_STYLE</cfg>
210+
<Makefile>SCRIPT_STYLE</Makefile>
211+
<service>SCRIPT_STYLE</service>
212+
<cc>JAVADOC_STYLE</cc>
213+
<md>XML_STYLE</md>
214+
<txt>SCRIPT_STYLE</txt>
215+
<scss>JAVADOC_STYLE</scss>
216+
<Doxyfile>SCRIPT_STYLE</Doxyfile>
217+
<tfvars>SCRIPT_STYLE</tfvars>
218+
</mapping>
219+
</configuration>
220+
</plugin>
221+
<plugin>
222+
<!-- Shade all the dependencies to avoid conflicts -->
223+
<groupId>org.apache.maven.plugins</groupId>
224+
<artifactId>maven-shade-plugin</artifactId>
225+
<version>${maven-shade-plugin.version}</version>
226+
<executions>
227+
<execution>
228+
<phase>package</phase>
229+
<goals>
230+
<goal>shade</goal>
231+
</goals>
232+
<configuration>
233+
<createDependencyReducedPom>true</createDependencyReducedPom>
234+
<promoteTransitiveDependencies>true</promoteTransitiveDependencies>
235+
<minimizeJar>false</minimizeJar>
236+
<artifactSet>
237+
<includes>
238+
<include>io.streamnative.connectors:*</include>
239+
</includes>
240+
</artifactSet>
241+
<filters>
242+
<filter>
243+
<artifact>*:*</artifact>
244+
<excludes>
245+
<exclude>META-INF/*.SF</exclude>
246+
<exclude>META-INF/*.DSA</exclude>
247+
<exclude>META-INF/*.RSA</exclude>
248+
</excludes>
249+
</filter>
250+
</filters>
251+
<transformers>
252+
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
253+
<transformer implementation="org.apache.maven.plugins.shade.resource.PluginXmlResourceTransformer" />
254+
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
255+
<mainClass>io.streamnative.examples.flink.PulsarStreamingWordCount</mainClass>
256+
</transformer>
257+
</transformers>
258+
</configuration>
259+
</execution>
260+
</executions>
261+
</plugin>
262+
</plugins>
263+
</build>
264+
<profiles>
265+
</profiles>
266+
<repositories>
267+
<repository>
268+
<id>central</id>
269+
<layout>default</layout>
270+
<url>https://repo1.maven.org/maven2</url>
271+
</repository>
272+
<repository>
273+
<id>bintray-streamnative-maven</id>
274+
<name>bintray</name>
275+
<url>https://dl.bintray.com/streamnative/maven</url>
276+
</repository>
277+
</repositories>
278+
</project>

0 commit comments

Comments
 (0)