60 lines
2.9 KiB
YAML
Executable File
60 lines
2.9 KiB
YAML
Executable File
version: '3.7'
|
|
services:
|
|
|
|
postgres: # create postgres container
|
|
image: postgres:9.6
|
|
container_name: postgres_container
|
|
environment:
|
|
- POSTGRES_USER=airflow
|
|
- POSTGRES_PASSWORD=airflow
|
|
- POSTGRES_DB=airflow
|
|
|
|
airflow: # create airflow container
|
|
build: './airflow_docker' # construct the container along the Dockerfile in this folder
|
|
container_name: airflow_container
|
|
restart: always
|
|
depends_on:
|
|
- postgres
|
|
environment:
|
|
- LOAD_EX=n
|
|
- EXECUTOR=Local
|
|
volumes: # mount the following local folders
|
|
- ./dags:/usr/local/airflow/dags
|
|
- ./data:/usr/local/airflow/data
|
|
- ./models:/usr/local/airflow/models
|
|
ports:
|
|
- "8080:8080" # expose port
|
|
command: webserver
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"]
|
|
interval: 30s
|
|
timeout: 30s
|
|
retries: 3
|
|
|
|
zookeeper: # create zookeeper container
|
|
image: wurstmeister/zookeeper
|
|
container_name: zookeeper_container
|
|
ports:
|
|
- "2181:2181" # expose port
|
|
|
|
kafka: # create an instance of a Kafka broker in a container
|
|
image: wurstmeister/kafka
|
|
container_name: kafka_container
|
|
ports:
|
|
- "9092:9092" # expose port
|
|
environment:
|
|
KAFKA_ADVERTISED_HOST_NAME: kafka # specify the docker host IP at which other containers can reach the broker
|
|
KAFKA_CREATE_TOPICS: "TopicA:1:1" # create a topic called 'TopicA" with 1 partition and 1 replica
|
|
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 # specify where the broker can reach Zookeeper
|
|
KAFKA_LISTENERS: PLAINTEXT://:9092 # the list of addresses on which the Kafka broker will listen on for incoming connections.
|
|
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 # Kafka sends the value of this variable to clients during their connection. After receiving that value, the clients use it for sending/consuming records to/from the Kafka broker.y connect to it.
|
|
volumes:
|
|
- /var/run/docker.sock:/var/run/docker.sock
|
|
mlflow: # create a MLFlow container
|
|
build: './mlflow_docker' # construct the container along the Dockerfile in this folder
|
|
container_name: mlflow_container
|
|
ports:
|
|
- "5000:5000" # expose port
|
|
command: 'mlflow server --backend-store-uri ./mlflow --host 0.0.0.0 --port 5000'
|
|
|