stream-data-processing/storage-docker-compose.yaml at main · VuBacktracking/stream-data-processing · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
version: "3.8"
services:
  metastore_db:
    container_name: metastoredb-datalake
    image: postgres:11
    hostname: metastore_db
    ports:
      - "5433:5432"
    environment:
      POSTGRES_USER: hive
      POSTGRES_PASSWORD: hive
      POSTGRES_DB: metastore

  hive-metastore:
    container_name: datalake-hive-metastore
    image: "starburstdata/hive:3.1.2-e.18"
    hostname: hive-metastore
    ports:
      - "9083:9083"
    environment:
      HIVE_METASTORE_DRIVER: org.postgresql.Driver
      HIVE_METASTORE_JDBC_URL: jdbc:postgresql://metastore_db:5432/metastore
      HIVE_METASTORE_USER: hive
      HIVE_METASTORE_PASSWORD: hive
      HIVE_METASTORE_WAREHOUSE_DIR: s3://datalake/
      HIVE_METASTORE_USERS_IN_ADMIN_ROLE: "admin"
      S3_ENDPOINT: http://minio:9000
      S3_ACCESS_KEY: minio_access_key
      S3_SECRET_KEY: minio_secret_key
      S3_PATH_STYLE_ACCESS: "true"
      REGION: ""
      GOOGLE_CLOUD_KEY_FILE_PATH: ""
      AZURE_ADL_CLIENT_ID: ""
      AZURE_ADL_CREDENTIAL: ""
      AZURE_ADL_REFRESH_URL: ""
      AZURE_ABFS_STORAGE_ACCOUNT: ""
      AZURE_ABFS_ACCESS_KEY: ""
      AZURE_WASB_STORAGE_ACCOUNT: ""
      AZURE_ABFS_OAUTH: ""
      AZURE_ABFS_OAUTH_TOKEN_PROVIDER: ""
      AZURE_ABFS_OAUTH_CLIENT_ID: ""
      AZURE_ABFS_OAUTH_SECRET: ""
      AZURE_ABFS_OAUTH_ENDPOINT: ""
      AZURE_WASB_ACCESS_KEY: ""
    depends_on:
      - metastore_db

  trino:
    container_name: trinodb-datalake
    image: "trinodb/trino:410"
    hostname: trino
    ports:
      - "8084:8080"
    volumes:
      - /home/vuphan/stream-data-processing/trino/etc:/usr/lib/trino/etc:ro
      - /home/vuphan/stream-data-processing/trino/catalog:/etc/trino/catalog
    depends_on:
      - hive-metastore

  minio:
    image: minio/minio
    container_name: minio-datalake
    hostname: minio
    ports:
      - "9000:9000"
      - "9001:9001"
    volumes:
      - minio_storage:/data
    environment:
      - MINIO_ACCESS_KEY=minio_access_key
      - MINIO_SECRET_KEY=minio_secret_key
    command: server --console-address ":9001" /data

volumes:
  minio_storage: