ExperimentsWithDocker/docker-compose.yml at master · miciav/ExperimentsWithDocker · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
version: '2'

networks:
  prodnetwork:
    driver: bridge

services:
  zookeeper:
    image: wurstmeister/zookeeper
    container_name: zookeeper
    ports:
      - "2181:2181"
    networks:
      - prodnetwork
  kafka:
    image: wurstmeister/kafka
    container_name: kafka
    ports:
      - "9092:9092"
    environment:
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_PORT: 9092
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
    networks:
      - prodnetwork
    depends_on:
      - zookeeper
  producer:
    container_name: producer
    image: miciav/producer
    environment:
       KAFKA_ADDRESS: kafka:9092
       ZK_ADDRESS: zookeeper:2181
    depends_on:
      - kafka
      - zookeeper
    networks:
      - prodnetwork
    links:
      - kafka
    restart: on-failure:10
  master:
    image: gettyimages/spark
    command: bin/spark-class org.apache.spark.deploy.master.Master -h master
    container_name: master
    environment:
      MASTER: spark://master:7077
      SPARK_CONF_DIR: /conf
      SPARK_PUBLIC_DNS: localhost
    expose:
      - 7001
      - 7002
      - 7003
      - 7004
      - 7005
      - 7006
      - 7077
      - 6066
    ports:
      - 4040:4040
      - 6066:6066
      - 7077:7077
      - 9000:8080
    volumes:
      - ./spark/conf/master:/conf
      - ./data:/tmp/data
    networks:
      - prodnetwork
    links:
     - cassandra
  worker:
    image: gettyimages/spark
    command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://master:7077
    container_name: worker
    environment:
      SPARK_CONF_DIR: /conf
      SPARK_WORKER_CORES: 2
      SPARK_WORKER_MEMORY: 1g
      SPARK_WORKER_PORT: 8881
      SPARK_WORKER_WEBUI_PORT: 8081
      SPARK_PUBLIC_DNS: localhost
    links:
      - master
    expose:
      - 7012
      - 7013
      - 7014
      - 7015
      - 7016
      - 8881
    ports:
      - 9001:8081
    volumes:
      - ./spark/conf/worker:/conf
      - ./data:/tmp/data
    links:
     - cassandra
    networks:
      - prodnetwork
  zeppelin:
    container_name: zeppelin
    image: dylanmei/zeppelin
    environment:
      ZEPPELIN_PORT: 8080
      MASTER: spark://master:7077
      KAFKA_ADDRESS: kafka:9092
    depends_on:
      - kafka
      - zookeeper
      - cassandra
    links:
      - cassandra
      - master
    ports:
      - "9100:8080"
    volumes:
      - ./zeppelin/about.json:/usr/zeppelin/notebook/about.json
      - ./zeppelin/notebooks:/usr/zeppelin/notebook
      - ./zeppelin/conf:/usr/zeppelin/conf
    networks:
      - prodnetwork
  cassandra:
    container_name: cassandra
    image: spotify/cassandra
    ports:
      - "7001:7001"
      - "7199:7199"
      - "9160:9160"
      - "7000:7000"
      - "9042:9042"
    volumes:
      - ./cassandra:/var/lib/cassandra
    networks:
      - prodnetwork