Feature dockerfile for cortex (#1014)

Co-authored-by: Hien To <tominhhien97@gmail.com>

Author: hiento09

Dockerfile

@@ -0,0 +1,36 @@
+# Please change the base image to the appropriate CUDA version base on NVIDIA Driver Compatibility
+# Run nvidia-smi to check the CUDA version and the corresponding driver version
+# Then update the base image to the appropriate CUDA version refer https://catalog.ngc.nvidia.com/orgs/nvidia/containers/cuda/tags
+
+FROM nvidia/cuda:12.4.1-runtime-ubuntu22.04 AS base 
+
+# 1. Install dependencies only when needed
+FROM base AS devel
+
+# Install g++ 11
+RUN apt update && apt install -y gcc-11 g++-11 cpp-11 jq xsel curl gnupg make python3-dev && curl -sL https://deb.nodesource.com/setup_20.x | bash - && apt install nodejs -y && rm -rf /var/lib/apt/lists/*
+
+# Update alternatives for GCC and related tools
+RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 110 \
+                         --slave /usr/bin/g++ g++ /usr/bin/g++-11 \
+                         --slave /usr/bin/gcov gcov /usr/bin/gcov-11 \
+                         --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-11 \
+                         --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-11 && \
+    update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-11 110
+
+RUN npm install -g yarn
+
+WORKDIR /app
+
+FROM devel AS release
+
+EXPOSE 1337
+
+COPY ./common/entrypoint.sh /usr/local/bin/entrypoint.sh
+
+RUN chmod +x /usr/local/bin/entrypoint.sh
+
+HEALTHCHECK --interval=300s --timeout=30s --start-period=10s --retries=3 \
+  CMD curl -f http://127.0.0.1:1337/api/system || exit 1
+
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

docker-compose.yml

@@ -0,0 +1,45 @@
+# Please change the base image to the appropriate CUDA version base on NVIDIA Driver Compatibility
+# Run nvidia-smi to check the CUDA version and the corresponding driver version
+# Then update the base image to the appropriate CUDA version refer https://catalog.ngc.nvidia.com/orgs/nvidia/containers/cuda/tags
+
+FROM nvidia/cuda:12.4.1-runtime-ubuntu22.04 AS base 
+
+# 1. Install dependencies only when needed
+FROM base AS devel
+
+# Install g++ 11
+RUN apt update && apt install -y gcc-11 g++-11 cpp-11 jq xsel curl gnupg make python3-dev dnsmasq nginx iproute2 && curl -sL https://deb.nodesource.com/setup_20.x | bash - && apt install nodejs -y && rm -rf /var/lib/apt/lists/*
+
+# Update alternatives for GCC and related tools
+RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 110 \
+                         --slave /usr/bin/g++ g++ /usr/bin/g++-11 \
+                         --slave /usr/bin/gcov gcov /usr/bin/gcov-11 \
+                         --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-11 \
+                         --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-11 && \
+    update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-11 110
+
+RUN npm install -g yarn
+
+RUN mkdir -p /etc/dnsmasq.d/
+
+WORKDIR /app
+
+FROM devel AS release
+
+EXPOSE 80
+
+COPY ./common/dnsmasq.conf /etc/dnsmasq.conf
+COPY ./common/blocked-domains.txt /etc/dnsmasq.d/blocked-domains.txt
+
+COPY ./common/entrypoint-firewall.sh /usr/local/bin/entrypoint.sh
+
+COPY ./common/routes.txt /app/routes.txt
+COPY ./common/generate_nginx_conf.sh /usr/local/bin/generate_nginx_conf.sh
+COPY ./common/nginx.conf /etc/nginx/nginx.conf
+
+RUN chmod +x /usr/local/bin/entrypoint.sh
+
+HEALTHCHECK --interval=300s --timeout=30s --start-period=10s --retries=3 \
+  CMD curl -f http://127.0.0.1/api/system || exit 1
+
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

docker/Dockerfile

@@ -0,0 +1,46 @@
+# Docker with cortex
+
+We offer two methods for deploying the Cortex environment on Docker.
+
+## Method 1: Use the default Dockerfile with Cortex.
+
+To use this method, you need to follow these steps:
+```bash
+git clone https://github.com/janhq/cortex.git
+cd cortex/docker
+docker build -t cortex:latest .
+
+# Run the container with GPU support
+docker run -it --gpus all -d -p 1337:1337 cortex:latest
+
+# Run the container with CPU support
+docker run -it -d -p 1337:1337 cortex:latest
+
+# After starting, you can access Swagger at http://localhost:1337/api and the API server at http://localhost:1337.
+# Additionally, you can exec into the container and use cortex-cli to perform other operations.
+```
+
+## Method 2: Use Dockerfile.firewall with the feature to block outbound connections by domain and block inbound connections by API path.
+
+The use case for this method is when you want to host the Cortex API 100% offline, preventing access to remote models like the OpenAI API. Alternatively, you might want to block inbound connections by restricting clients from calling the API to load models `/v1/models/start`.
+
+To use this method, you need to follow these steps:
+
+- Step 1: Edit the contents of the [blocked-domains.txt](./docker/common/blocked-domains.txt) file according to your requirements. Refer to the provided examples in the file. The goal is to block outbound connections to the domains you do not want to allow.
+- Step 2: Edit the contents of the [blocked-paths.txt](./docker/common/blocked-paths.txt) file according to your requirements. Refer to the provided examples in the file. The goal is to block inbound connections to the paths you do not want to allow.
+- Step 3: Build the image with Dockerfile.firewall following the instructions below:
+
+    ```bash
+    git clone https://github.com/janhq/cortex.git
+    cd cortex/docker
+    docker build -f Dockerfile.firewall -t cortex-with-firewall:latest .
+
+    # Run the container with GPU support
+    docker run -it --gpus all -d -p 1337:1337 cortex:latest
+
+    # Run the container with CPU support
+    docker run -it -d -p 1337:1337 cortex:latest
+
+    # After starting, you can access Swagger at http://localhost:1337/api and the API server at http://localhost:1337.
+    # Additionally, you can exec into the container and use cortex-cli to perform other operations.
+    ```

docker/Dockerfile.firewall

@@ -0,0 +1,5 @@
+# Block IPv4 of domain  openai.com and all subdomains *.openai.com
+address=/openai.com/0.0.0.0
+
+# Block IPv6 of domain  openai.com and all subdomains *.openai.com
+address=/openai.com/::

docker/README.md

@@ -0,0 +1,5 @@
+server=8.8.8.8
+
+no-resolv
+
+conf-file=/etc/dnsmasq.d/blocked-domains.txt

docker/common/blocked-domains.txt

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+# Setup DNS resolution with dnsmasq
+echo "nameserver 127.0.0.1" > /etc/resolv.conf
+dnsmasq -k &
+
+# Generate Nginx configuration from routes.txt
+/usr/local/bin/generate_nginx_conf.sh
+
+# Install cortex
+npm install -g cortexso
+
+# Start cortex
+cortex -a 127.0.0.1
+
+cortex engines llamacpp init
+cortex engines tensorrt-llm init
+
+# Start nginx
+nginx -g 'daemon off;' &
+
+# Keep the container running by tailing the log file
+tail -f /root/cortex/cortex.log

docker/common/dnsmasq.conf

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+npm install -g cortexso
+# Run cortex
+cortex  -a 0.0.0.0
+
+cortex engines llamacpp init
+cortex engines tensorrt-llm init
+
+# Keep the container running by tailing the log file
+tail -f /root/cortex/cortex.log

docker/common/entrypoint-firewall.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+NGINX_CONF="/etc/nginx/conf.d/generated_routes.conf"
+
+rm -f $NGINX_CONF
+
+touch $NGINX_CONF
+
+while IFS= read -r line || [ -n "$line" ]
+do
+    route=$(echo $line | awk '{print $1}')
+    action=$(echo $line | awk '{print $2}')
+
+    echo "location $route {" >> $NGINX_CONF
+    if [ "$action" = "allow" ]; then
+        echo "    allow all;" >> $NGINX_CONF
+    else
+        echo "    deny all;" >> $NGINX_CONF
+    fi
+    echo "}" >> $NGINX_CONF
+done < /app/routes.txt