my-data-stack/docker-compose.yaml at master · minhthong582000/my-data-stack · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
version: '3.7'

services:
    hadoop:
        container_name: hadoop
        build:
            context: ./hadoop
        restart: always
        ports:
            - "8088:8088"
            - "50070:50070"
            - "50075:50075"
        volumes:
            - ./sample:/var/log/sample
            - ./configs/hosts:/etc/hosts
            # - hadoop-node:/root/hdfs/
        networks:
            hadoop:
                ipv4_address: 172.28.1.10

    jupyterlab:
        container_name: jupyterlab
        build:
            context: ./jupiter
        ports:
            - 8888:8888
        volumes:
            - ./scripts:/opt/workspace
        networks:
            - hadoop

    spark-master:
        build:
            context: ./spark/master
        container_name: spark-master
        ports:
            - 8080:8080
            - 7077:7077
        networks:
            - hadoop

    spark-worker-1:
        build:
            context: ./spark/worker
        container_name: spark-worker-1
        environment:
            - SPARK_WORKER_CORES=1
            - SPARK_WORKER_MEMORY=512m
        ports:
            - 8081:8081
        depends_on:
            - spark-master
        networks:
            - hadoop

    db:
        image: mysql:8.0.22
        container_name: db
        command: --default-authentication-plugin=mysql_native_password
        restart: always
        network_mode: 'host'
        volumes:
            - mysql:/var/lib/mysql
        env_file:
            - .env.example

# Volumes
volumes:
    mysql:
    # hadoop-node:

# Network
networks:
    hadoop:
        ipam:
            driver: default
            config:
                - subnet: 172.28.0.0/16