cyberpartner-state-processor/Dockerfile at main · lockfale/cyberpartner-state-processor · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
FROM flink:1.20.1-scala_2.12-java17

# Install Python 3.11, JDK, and other dependencies
RUN apt-get update && \
    apt-get install -y python3.11 python3.11-dev python3-pip build-essential git maven openjdk-21-jdk nano vim jq && \
    apt-get clean && \
    rm -rf /var/lib/apt/lists/* && \
    ln -sf /usr/bin/python3.11 /usr/local/bin/python3 && \
    ln -sf /usr/bin/python3.11 /usr/local/bin/python && \
    ln -sf /usr/bin/pip3 /usr/local/bin/pip

ENV JAVA_HOME=/usr/lib/jvm/java-21-openjdk-amd64

# Update pip and install Poetry v1.8.5 explicitly
RUN pip install --no-cache-dir --upgrade pip && \
    pip install --no-cache-dir poetry==1.8.5

# https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-kafka/3.4.0-1.20/flink-sql-connector-kafka-3.4.0-1.20.jar
# Download the Kafka connector
RUN mkdir -p /tmp/downloads && \
    wget -P /tmp/downloads https://repo.maven.apache.org/maven2/org/apache/flink/flink-connector-kafka/3.4.0-1.20/flink-connector-kafka-3.4.0-1.20.jar && \
    wget -P /tmp/downloads https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-kafka/3.4.0-1.20/flink-sql-connector-kafka-3.4.0-1.20.jar && \
    wget -P /tmp/downloads https://repo.maven.apache.org/maven2/org/apache/kafka/kafka-clients/3.4.0/kafka-clients-3.4.0.jar && \
    wget -P /tmp/downloads https://repo.maven.apache.org/maven2/org/apache/flink/flink-python/1.20.1/flink-python-1.20.1.jar && \
    mv /tmp/downloads/flink-connector-kafka-3.4.0-1.20.jar /opt/flink/lib/ && \
    mv /tmp/downloads/flink-sql-connector-kafka-3.4.0-1.20.jar /opt/flink/lib/ && \
    mv /tmp/downloads/kafka-clients-3.4.0.jar /opt/flink/lib/ && \
    mv /tmp/downloads/flink-python-1.20.1.jar /opt/flink/opt/ && \
    echo "Kafka connector and client JARs have been installed" && \
    rm -rf /tmp/downloads

RUN chmod 644 /opt/flink/lib/*.jar && \
    chmod 644 /opt/flink/opt/*.jar

# Copy project files
COPY pyproject.toml poetry.lock* /opt/app/

# Set working directory
WORKDIR /opt/app

# Install dependencies with Poetry
RUN poetry export --without-hashes --without dev -f requirements.txt > requirements.txt && \
    pip install --no-cache-dir -r requirements.txt

# honestly have no idea why _this_ works but is fucked in the requirements...
RUN python -m pip install apache-flink==1.20.1 kafka-python redis async_timeout

COPY deploy_jobs.sh log.ini /opt/app/
RUN chmod +x /opt/app/deploy_jobs.sh

ENV PYFLINK_PYTHON=/usr/local/bin/python
ENV PYFLINK_CLIENT_EXECUTABLE=/usr/local/bin/python

# Copy the job code
COPY src/ /opt/app/src/

WORKDIR /opt/app