diff --git a/docker/Dockerfile b/docker/Dockerfile index 6d67fcf4df..52669259e6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,48 +1,52 @@ ARG CUDA_VERSION=12.6.1 FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu22.04 ARG PYTHON_VERSION=3.10 -ARG MAMBA_VERSION=24.7.1-0 ARG TARGETPLATFORM -ENV PATH=/opt/conda/bin:$PATH \ - CONDA_PREFIX=/opt/conda +ENV PATH=/root/.local/bin:/root/.cargo/bin:$PATH \ + UV_SYSTEM_PYTHON=1 \ + UV_COMPILE_BYTECODE=0 \ + UV_HTTP_TIMEOUT=300 + +# Install system dependencies RUN chmod 777 -R /tmp && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ ca-certificates \ libssl-dev \ curl \ g++ \ make \ - git && \ + git \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ + python${PYTHON_VERSION}-venv \ + python3-pip \ + libnuma-dev && \ rm -rf /var/lib/apt/lists/* -RUN case ${TARGETPLATFORM} in \ - "linux/arm64") MAMBA_ARCH=aarch64 ;; \ - *) MAMBA_ARCH=x86_64 ;; \ - esac && \ - curl -fsSL -o ~/mambaforge.sh "https://github.com/conda-forge/miniforge/releases/download/${MAMBA_VERSION}/Mambaforge-${MAMBA_VERSION}-Linux-${MAMBA_ARCH}.sh" && \ - bash ~/mambaforge.sh -b -p /opt/conda && \ - rm ~/mambaforge.sh - -RUN case ${TARGETPLATFORM} in \ - "linux/arm64") exit 1 ;; \ - *) /opt/conda/bin/conda update -y conda && \ - /opt/conda/bin/conda install -y "python=${PYTHON_VERSION}" ;; \ - esac && \ - /opt/conda/bin/conda clean -ya +# Set python alias +RUN update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1 && \ + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 +# Install uv package manager +RUN curl -LsSf https://astral.sh/uv/install.sh | sh -WORKDIR /root +WORKDIR /lightllm -COPY ./requirements.txt /lightllm/requirements.txt -RUN pip install -U pip -RUN pip install -r /lightllm/requirements.txt --no-cache-dir +# Copy project files for dependency installation +COPY pyproject.toml README.md ./ +COPY lightllm/__init__.py lightllm/ -RUN pip install --no-cache-dir vllm --pre --extra-index-url https://wheels.vllm.ai/nightly +# Install dependencies using uv (faster than pip) +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e . --no-cache-dir -# TODO: offline compile -# RUN git clone https://github.com/ModelTC/LightKernel.git && cd LightKernel && pip install --no-deps -v . +# Install vllm for additional kernels +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system vllm --pre --index-strategy unsafe-best-match --extra-index-url https://wheels.vllm.ai/nightly -RUN apt-get update && apt-get install -y libnuma-dev # for sgl_kernel +# Copy full source code and install in editable mode +COPY . . +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e . --no-deps -COPY . /lightllm -RUN pip install -e /lightllm --no-cache-dir +WORKDIR /root diff --git a/docker/Dockerfile.deepep b/docker/Dockerfile.deepep index e765978b91..3de2cea0fd 100644 --- a/docker/Dockerfile.deepep +++ b/docker/Dockerfile.deepep @@ -1,68 +1,83 @@ ARG CUDA_VERSION=12.6.1 FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu22.04 ARG PYTHON_VERSION=3.10 -ARG MAMBA_VERSION=24.7.1-0 ARG TARGETPLATFORM -ENV PATH=/opt/conda/bin:$PATH \ - CONDA_PREFIX=/opt/conda +ARG DEEPEP_COMMIT=b6ce310bb0b75079682d09bc2ebc063a074fbd58 + +ENV PATH=/root/.local/bin:/root/.cargo/bin:$PATH \ + UV_SYSTEM_PYTHON=1 \ + UV_COMPILE_BYTECODE=0 \ + UV_HTTP_TIMEOUT=300 \ + CUDA_HOME=/usr/local/cuda \ + GDRCOPY_HOME=/usr/src/gdrdrv-2.4.4/ +# Install system dependencies RUN chmod 777 -R /tmp && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ ca-certificates \ libssl-dev \ curl \ g++ \ make \ - git && \ + git \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ + python${PYTHON_VERSION}-venv \ + python3-pip \ + libnuma-dev \ + wget \ + devscripts \ + debhelper \ + dh-make \ + build-essential \ + dkms \ + ibverbs-providers \ + infiniband-diags \ + perftest \ + rdma-core \ + libibverbs-dev \ + librdmacm-dev \ + cmake && \ rm -rf /var/lib/apt/lists/* -RUN case ${TARGETPLATFORM} in \ - "linux/arm64") MAMBA_ARCH=aarch64 ;; \ - *) MAMBA_ARCH=x86_64 ;; \ - esac && \ - curl -fsSL -o ~/mambaforge.sh "https://github.com/conda-forge/miniforge/releases/download/${MAMBA_VERSION}/Mambaforge-${MAMBA_VERSION}-Linux-${MAMBA_ARCH}.sh" && \ - bash ~/mambaforge.sh -b -p /opt/conda && \ - rm ~/mambaforge.sh - -RUN case ${TARGETPLATFORM} in \ - "linux/arm64") exit 1 ;; \ - *) /opt/conda/bin/conda update -y conda && \ - /opt/conda/bin/conda install -y "python=${PYTHON_VERSION}" ;; \ - esac && \ - /opt/conda/bin/conda clean -ya - - -WORKDIR /root +# Set python alias +RUN update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1 && \ + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 -COPY ./requirements.txt /lightllm/requirements.txt -RUN pip install -U pip -RUN pip install -r /lightllm/requirements.txt --no-cache-dir +# Install uv package manager +RUN curl -LsSf https://astral.sh/uv/install.sh | sh -RUN pip install --no-cache-dir vllm --pre --extra-index-url https://wheels.vllm.ai/nightly +WORKDIR /lightllm -# TODO: offline compile -# RUN git clone https://github.com/ModelTC/LightKernel.git && cd LightKernel && pip install --no-deps -v . +# Copy project files for dependency installation +COPY pyproject.toml README.md ./ +COPY lightllm/__init__.py lightllm/ -RUN apt-get update && apt-get install -y libnuma-dev wget devscripts debhelper dh-make build-essential dkms -RUN apt-get install -y ibverbs-providers infiniband-diags perftest rdma-core libibverbs-dev librdmacm-dev +# Install dependencies using uv (faster than pip) +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e . --no-cache-dir -ENV CUDA_HOME=/usr/local/cuda \ - GDRCOPY_HOME=/usr/src/gdrdrv-2.4.4/ +# Install vllm for additional kernels +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system vllm --pre --index-strategy unsafe-best-match --extra-index-url https://wheels.vllm.ai/nightly -RUN mkdir -p /tmp/gdrcopy && cd /tmp \ - && git clone https://github.com/NVIDIA/gdrcopy.git -b v2.4.4 \ - && cd gdrcopy/packages \ - && CUDA=/usr/local/cuda ./build-deb-packages.sh \ - && dpkg -i gdrdrv-dkms_*.deb libgdrapi_*.deb gdrcopy-tests_*.deb gdrcopy_*.deb \ - && cd / && rm -rf /tmp/gdrcopy +# Build and install gdrcopy +RUN mkdir -p /tmp/gdrcopy && cd /tmp && \ + git clone https://github.com/NVIDIA/gdrcopy.git -b v2.4.4 && \ + cd gdrcopy/packages && \ + CUDA=/usr/local/cuda ./build-deb-packages.sh && \ + dpkg -i gdrdrv-dkms_*.deb libgdrapi_*.deb gdrcopy-tests_*.deb gdrcopy_*.deb && \ + cd / && rm -rf /tmp/gdrcopy - # Fix DeepEP IBGDA symlink +# Fix DeepEP IBGDA symlink RUN ln -sf /usr/lib/x86_64-linux-gnu/libmlx5.so.1 /usr/lib/x86_64-linux-gnu/libmlx5.so -RUN wget https://developer.download.nvidia.com/compute/redist/nvshmem/3.3.9/source/nvshmem_src_cuda12-all-all-3.3.9.tar.gz \ - && tar -xf nvshmem_src_cuda12-all-all-3.3.9.tar.gz && mv nvshmem_src nvshmem \ - && cd nvshmem \ - && rm -f /root/nvshmem_src_cuda12-all-all-3.3.9.tar.gz \ - && NVSHMEM_SHMEM_SUPPORT=0 \ +# Build and install NVSHMEM +RUN wget https://developer.download.nvidia.com/compute/redist/nvshmem/3.3.9/source/nvshmem_src_cuda12-all-all-3.3.9.tar.gz && \ + tar -xf nvshmem_src_cuda12-all-all-3.3.9.tar.gz && \ + mv nvshmem_src nvshmem && \ + cd nvshmem && \ + rm -f /root/nvshmem_src_cuda12-all-all-3.3.9.tar.gz && \ + NVSHMEM_SHMEM_SUPPORT=0 \ NVSHMEM_UCX_SUPPORT=0 \ NVSHMEM_USE_NCCL=0 \ NVSHMEM_MPI_SUPPORT=0 \ @@ -70,15 +85,19 @@ RUN wget https://developer.download.nvidia.com/compute/redist/nvshmem/3.3.9/sour NVSHMEM_PMIX_SUPPORT=0 \ NVSHMEM_TIMEOUT_DEVICE_POLLING=0 \ NVSHMEM_USE_GDRCOPY=1 \ - cmake -S . -B build/ -DCMAKE_INSTALL_PREFIX=/root/nvshmem/install -DCMAKE_CUDA_ARCHITECTURES=90 \ - && cmake --build build --target install -j64 - -ARG DEEPEP_COMMIT=b6ce310bb0b75079682d09bc2ebc063a074fbd58 -RUN git clone https://github.com/deepseek-ai/DeepEP.git && cd DeepEP && git checkout ${DEEPEP_COMMIT} && cd .. + cmake -S . -B build/ -DCMAKE_INSTALL_PREFIX=/root/nvshmem/install -DCMAKE_CUDA_ARCHITECTURES=90 && \ + cmake --build build --target install -j$(nproc) -WORKDIR /root/DeepEP +# Build and install DeepEP ENV NVSHMEM_DIR=/root/nvshmem/install -RUN NVSHMEM_DIR=/root/nvshmem/install python setup.py install +RUN git clone https://github.com/deepseek-ai/DeepEP.git /root/DeepEP && \ + cd /root/DeepEP && \ + git checkout ${DEEPEP_COMMIT} && \ + NVSHMEM_DIR=/root/nvshmem/install python setup.py install +# Copy full source code and install in editable mode COPY . /lightllm -RUN pip install -e /lightllm --no-cache-dir \ No newline at end of file +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e /lightllm --no-deps + +WORKDIR /root diff --git a/docker/Dockerfile.nixl b/docker/Dockerfile.nixl index b8047bbd03..2f9259b3ad 100644 --- a/docker/Dockerfile.nixl +++ b/docker/Dockerfile.nixl @@ -1,85 +1,117 @@ ARG CUDA_VERSION=12.6.1 FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu22.04 ARG PYTHON_VERSION=3.10 -ARG MAMBA_VERSION=24.7.1-0 ARG TARGETPLATFORM -ENV PATH=/opt/conda/bin:$PATH \ - CONDA_PREFIX=/opt/conda +ENV PATH=/root/.local/bin:/root/.cargo/bin:$PATH \ + UV_SYSTEM_PYTHON=1 \ + UV_COMPILE_BYTECODE=0 \ + UV_HTTP_TIMEOUT=300 \ + CUDA_HOME=/usr/local/cuda \ + GDRCOPY_HOME=/usr/src/gdrdrv-2.4.4/ + +# Install system dependencies RUN chmod 777 -R /tmp && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ ca-certificates \ libssl-dev \ curl \ g++ \ make \ - git && \ + git \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ + python${PYTHON_VERSION}-venv \ + python3-pip \ + libnuma-dev \ + wget \ + devscripts \ + debhelper \ + dh-make \ + build-essential \ + dkms \ + ibverbs-providers \ + infiniband-diags \ + perftest \ + rdma-core \ + libibverbs-dev \ + librdmacm-dev \ + cmake \ + automake \ + autotools-dev \ + libtool \ + libz-dev \ + pkg-config \ + tmux \ + net-tools && \ rm -rf /var/lib/apt/lists/* -RUN case ${TARGETPLATFORM} in \ - "linux/arm64") MAMBA_ARCH=aarch64 ;; \ - *) MAMBA_ARCH=x86_64 ;; \ - esac && \ - curl -fsSL -o ~/mambaforge.sh -v "https://github.com/conda-forge/miniforge/releases/download/${MAMBA_VERSION}/Mambaforge-${MAMBA_VERSION}-Linux-${MAMBA_ARCH}.sh" && \ - bash ~/mambaforge.sh -b -p /opt/conda && \ - rm ~/mambaforge.sh +# Reinstall RDMA packages +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + apt-get -y install --reinstall libibverbs-dev rdma-core ibverbs-utils libibumad-dev && \ + rm -rf /var/lib/apt/lists/* -RUN case ${TARGETPLATFORM} in \ - "linux/arm64") exit 1 ;; \ - *) /opt/conda/bin/conda update -y conda && \ - /opt/conda/bin/conda install -y "python=${PYTHON_VERSION}" ;; \ - esac && \ - /opt/conda/bin/conda clean -ya +# Set python alias +RUN update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1 && \ + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 +# Install uv package manager +RUN curl -LsSf https://astral.sh/uv/install.sh | sh -WORKDIR /root +WORKDIR /lightllm -COPY ./requirements.txt /lightllm/requirements.txt -RUN --mount=type=cache,target=/root/.cache/pip pip install -r /lightllm/requirements.txt --ignore-installed --extra-index-url https://download.pytorch.org/whl/cu124 +# Copy project files for dependency installation +COPY pyproject.toml README.md ./ +COPY lightllm/__init__.py lightllm/ -RUN --mount=type=cache,target=/root/.cache/pip pip install vllm --pre --extra-index-url https://wheels.vllm.ai/nightly -RUN --mount=type=cache,target=/root/.cache/pip git clone https://github.com/ModelTC/LightKernel.git && cd LightKernel && pip install --no-deps -v . +# Install dependencies using uv (faster than pip) +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e . --no-cache-dir -RUN apt-get update && apt-get install -y libnuma-dev wget devscripts debhelper dh-make build-essential dkms -RUN apt-get install -y ibverbs-providers infiniband-diags perftest rdma-core libibverbs-dev librdmacm-dev +# Install vllm for additional kernels +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system vllm --pre --index-strategy unsafe-best-match --extra-index-url https://wheels.vllm.ai/nightly -ENV CUDA_HOME=/usr/local/cuda \ - GDRCOPY_HOME=/usr/src/gdrdrv-2.4.4/ +# Install LightKernel +RUN git clone https://github.com/ModelTC/LightKernel.git /tmp/LightKernel && \ + cd /tmp/LightKernel && \ + uv pip install --system --no-deps -v . && \ + rm -rf /tmp/LightKernel -RUN mkdir -p /tmp/gdrcopy && cd /tmp \ - && git clone https://github.com/NVIDIA/gdrcopy.git -b v2.4.4 \ - && cd gdrcopy/packages \ - && CUDA=/usr/local/cuda ./build-deb-packages.sh \ - && dpkg -i gdrdrv-dkms_*.deb libgdrapi_*.deb gdrcopy-tests_*.deb gdrcopy_*.deb \ - && cd / && rm -rf /tmp/gdrcopy +# Build and install gdrcopy +RUN mkdir -p /tmp/gdrcopy && cd /tmp && \ + git clone https://github.com/NVIDIA/gdrcopy.git -b v2.4.4 && \ + cd gdrcopy/packages && \ + CUDA=/usr/local/cuda ./build-deb-packages.sh && \ + dpkg -i gdrdrv-dkms_*.deb libgdrapi_*.deb gdrcopy-tests_*.deb gdrcopy_*.deb && \ + cd / && rm -rf /tmp/gdrcopy -RUN apt-get update && apt-get install -y cmake automake autotools-dev libtool libz-dev && \ - DEBIAN_FRONTEND=noninteractive apt-get -y install --reinstall libibverbs-dev rdma-core ibverbs-utils libibumad-dev; \ - rm -rf /usr/lib/ucx && \ - rm -rf /opt/hpcx/ucx && \ +# Build and install UCX with CUDA and gdrcopy support +RUN rm -rf /usr/lib/ucx /opt/hpcx/ucx && \ cd /usr/local/src && \ git clone https://github.com/openucx/ucx.git && \ - cd ucx && \ - git checkout v1.19.x && \ - ./autogen.sh && ./configure \ - --enable-shared \ - --disable-static \ - --disable-doxygen-doc \ - --enable-optimizations \ - --enable-cma \ - --enable-devel-headers \ - --with-cuda=/usr/local/cuda \ - --with-verbs=yes \ - --with-dm \ - --with-gdrcopy=/usr/local \ - --with-efa \ - --enable-mt && \ - make -j && \ - make -j install-strip && \ - ldconfig; + cd ucx && \ + git checkout v1.19.x && \ + ./autogen.sh && ./configure \ + --enable-shared \ + --disable-static \ + --disable-doxygen-doc \ + --enable-optimizations \ + --enable-cma \ + --enable-devel-headers \ + --with-cuda=/usr/local/cuda \ + --with-verbs=yes \ + --with-dm \ + --with-gdrcopy=/usr/local \ + --with-efa \ + --enable-mt && \ + make -j$(nproc) && \ + make -j install-strip && \ + ldconfig && \ + rm -rf /usr/local/src/ucx -RUN apt-get update && apt-get install -y pkg-config tmux net-tools ; \ - cd /usr/local/src; \ - pip install --upgrade meson pybind11 patchelf; \ +# Build and install NixL +RUN uv pip install --system --upgrade meson pybind11 patchelf && \ + cd /usr/local/src && \ git clone https://github.com/ai-dynamo/nixl.git -b main && \ cd nixl && \ rm -rf build && \ @@ -88,7 +120,13 @@ RUN apt-get update && apt-get install -y pkg-config tmux net-tools ; \ cd build && \ ninja && \ ninja install && \ - cd .. && pip install . --no-deps; + cd .. && \ + uv pip install --system . --no-deps && \ + rm -rf /usr/local/src/nixl +# Copy full source code and install in editable mode COPY . /lightllm -RUN pip install -e /lightllm --no-cache-dir +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e /lightllm --no-deps + +WORKDIR /root diff --git a/docker/Dockerfile.nixl.deepep b/docker/Dockerfile.nixl.deepep index 8ca06e1094..5a533b7a80 100644 --- a/docker/Dockerfile.nixl.deepep +++ b/docker/Dockerfile.nixl.deepep @@ -1,67 +1,101 @@ ARG CUDA_VERSION=12.6.1 FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu22.04 - ARG PYTHON_VERSION=3.10 -ARG MAMBA_VERSION=24.7.1-0 ARG TARGETPLATFORM +ARG DEEPEP_COMMIT=b6ce310bb0b75079682d09bc2ebc063a074fbd58 -ENV PATH=/opt/conda/bin:$PATH \ - CONDA_PREFIX=/opt/conda +ENV PATH=/root/.local/bin:/root/.cargo/bin:$PATH \ + UV_SYSTEM_PYTHON=1 \ + UV_COMPILE_BYTECODE=0 \ + UV_HTTP_TIMEOUT=300 \ + CUDA_HOME=/usr/local/cuda \ + GDRCOPY_HOME=/usr/src/gdrdrv-2.4.4/ +# Install system dependencies RUN chmod 777 -R /tmp && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ ca-certificates \ libssl-dev \ curl \ g++ \ make \ - git && \ + git \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ + python${PYTHON_VERSION}-venv \ + python3-pip \ + libnuma-dev \ + wget \ + devscripts \ + debhelper \ + dh-make \ + build-essential \ + dkms \ + ibverbs-providers \ + infiniband-diags \ + perftest \ + rdma-core \ + libibverbs-dev \ + librdmacm-dev \ + cmake \ + automake \ + autotools-dev \ + libtool \ + libz-dev \ + pkg-config \ + tmux \ + net-tools && \ rm -rf /var/lib/apt/lists/* -RUN case ${TARGETPLATFORM} in \ - "linux/arm64") MAMBA_ARCH=aarch64 ;; \ - *) MAMBA_ARCH=x86_64 ;; \ - esac && \ - curl -fsSL -o ~/mambaforge.sh -v "https://github.com/conda-forge/miniforge/releases/download/${MAMBA_VERSION}/Mambaforge-${MAMBA_VERSION}-Linux-${MAMBA_ARCH}.sh" && \ - bash ~/mambaforge.sh -b -p /opt/conda && \ - rm ~/mambaforge.sh +# Reinstall RDMA packages +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + apt-get -y install --reinstall libibverbs-dev rdma-core ibverbs-utils libibumad-dev && \ + rm -rf /var/lib/apt/lists/* -RUN case ${TARGETPLATFORM} in \ - "linux/arm64") exit 1 ;; \ - *) /opt/conda/bin/conda update -y conda && \ - /opt/conda/bin/conda install -y "python=${PYTHON_VERSION}" ;; \ - esac && \ - /opt/conda/bin/conda clean -ya +# Set python alias +RUN update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1 && \ + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 +# Install uv package manager +RUN curl -LsSf https://astral.sh/uv/install.sh | sh -WORKDIR /root +WORKDIR /lightllm -COPY ./requirements.txt /lightllm/requirements.txt -RUN --mount=type=cache,target=/root/.cache/pip pip install -r /lightllm/requirements.txt --ignore-installed --extra-index-url https://download.pytorch.org/whl/cu124 +# Copy project files for dependency installation +COPY pyproject.toml README.md ./ +COPY lightllm/__init__.py lightllm/ -RUN --mount=type=cache,target=/root/.cache/pip pip install vllm --pre --extra-index-url https://wheels.vllm.ai/nightly -RUN --mount=type=cache,target=/root/.cache/pip git clone https://github.com/ModelTC/LightKernel.git && cd LightKernel && pip install --no-deps -v . +# Install dependencies using uv (faster than pip) +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e . --no-cache-dir -RUN apt-get update && apt-get install -y libnuma-dev wget devscripts debhelper dh-make build-essential dkms -RUN apt-get install -y ibverbs-providers infiniband-diags perftest rdma-core libibverbs-dev librdmacm-dev +# Install vllm for additional kernels +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system vllm --pre --index-strategy unsafe-best-match --extra-index-url https://wheels.vllm.ai/nightly -ENV CUDA_HOME=/usr/local/cuda \ - GDRCOPY_HOME=/usr/src/gdrdrv-2.4.4/ +# Install LightKernel +RUN git clone https://github.com/ModelTC/LightKernel.git /tmp/LightKernel && \ + cd /tmp/LightKernel && \ + uv pip install --system --no-deps -v . && \ + rm -rf /tmp/LightKernel -RUN mkdir -p /tmp/gdrcopy && cd /tmp \ - && git clone https://github.com/NVIDIA/gdrcopy.git -b v2.4.4 \ - && cd gdrcopy/packages \ - && CUDA=/usr/local/cuda ./build-deb-packages.sh \ - && dpkg -i gdrdrv-dkms_*.deb libgdrapi_*.deb gdrcopy-tests_*.deb gdrcopy_*.deb \ - && cd / && rm -rf /tmp/gdrcopy +# Build and install gdrcopy +RUN mkdir -p /tmp/gdrcopy && cd /tmp && \ + git clone https://github.com/NVIDIA/gdrcopy.git -b v2.4.4 && \ + cd gdrcopy/packages && \ + CUDA=/usr/local/cuda ./build-deb-packages.sh && \ + dpkg -i gdrdrv-dkms_*.deb libgdrapi_*.deb gdrcopy-tests_*.deb gdrcopy_*.deb && \ + cd / && rm -rf /tmp/gdrcopy - # Fix DeepEP IBGDA symlink +# Fix DeepEP IBGDA symlink RUN ln -sf /usr/lib/x86_64-linux-gnu/libmlx5.so.1 /usr/lib/x86_64-linux-gnu/libmlx5.so -RUN wget https://developer.download.nvidia.com/compute/redist/nvshmem/3.3.9/source/nvshmem_src_cuda12-all-all-3.3.9.tar.gz \ - && tar -xf nvshmem_src_cuda12-all-all-3.3.9.tar.gz && mv nvshmem_src nvshmem \ - && cd nvshmem \ - && rm -f /root/nvshmem_src_cuda12-all-all-3.3.9.tar.gz \ - && NVSHMEM_SHMEM_SUPPORT=0 \ +# Build and install NVSHMEM +RUN wget https://developer.download.nvidia.com/compute/redist/nvshmem/3.3.9/source/nvshmem_src_cuda12-all-all-3.3.9.tar.gz && \ + tar -xf nvshmem_src_cuda12-all-all-3.3.9.tar.gz && \ + mv nvshmem_src nvshmem && \ + cd nvshmem && \ + rm -f /root/nvshmem_src_cuda12-all-all-3.3.9.tar.gz && \ + NVSHMEM_SHMEM_SUPPORT=0 \ NVSHMEM_UCX_SUPPORT=0 \ NVSHMEM_USE_NCCL=0 \ NVSHMEM_MPI_SUPPORT=0 \ @@ -69,44 +103,43 @@ RUN wget https://developer.download.nvidia.com/compute/redist/nvshmem/3.3.9/sour NVSHMEM_PMIX_SUPPORT=0 \ NVSHMEM_TIMEOUT_DEVICE_POLLING=0 \ NVSHMEM_USE_GDRCOPY=1 \ - cmake -S . -B build/ -DCMAKE_INSTALL_PREFIX=/root/nvshmem/install -DCMAKE_CUDA_ARCHITECTURES=90 \ - && cmake --build build --target install -j64 + cmake -S . -B build/ -DCMAKE_INSTALL_PREFIX=/root/nvshmem/install -DCMAKE_CUDA_ARCHITECTURES=90 && \ + cmake --build build --target install -j$(nproc) -ARG DEEPEP_COMMIT=b6ce310bb0b75079682d09bc2ebc063a074fbd58 -RUN git clone https://github.com/deepseek-ai/DeepEP.git && cd DeepEP && git checkout ${DEEPEP_COMMIT} && cd .. - -WORKDIR /root/DeepEP +# Build and install DeepEP ENV NVSHMEM_DIR=/root/nvshmem/install -RUN NVSHMEM_DIR=/root/nvshmem/install python setup.py install +RUN git clone https://github.com/deepseek-ai/DeepEP.git /root/DeepEP && \ + cd /root/DeepEP && \ + git checkout ${DEEPEP_COMMIT} && \ + NVSHMEM_DIR=/root/nvshmem/install python setup.py install -RUN apt-get update && apt-get install -y cmake automake autotools-dev libtool libz-dev && \ - DEBIAN_FRONTEND=noninteractive apt-get -y install --reinstall libibverbs-dev rdma-core ibverbs-utils libibumad-dev; \ - rm -rf /usr/lib/ucx && \ - rm -rf /opt/hpcx/ucx && \ +# Build and install UCX with CUDA and gdrcopy support +RUN rm -rf /usr/lib/ucx /opt/hpcx/ucx && \ cd /usr/local/src && \ git clone https://github.com/openucx/ucx.git && \ - cd ucx && \ - git checkout v1.19.x && \ - ./autogen.sh && ./configure \ - --enable-shared \ - --disable-static \ - --disable-doxygen-doc \ - --enable-optimizations \ - --enable-cma \ - --enable-devel-headers \ - --with-cuda=/usr/local/cuda \ - --with-verbs=yes \ - --with-dm \ - --with-gdrcopy=/usr/local \ - --with-efa \ - --enable-mt && \ - make -j && \ - make -j install-strip && \ - ldconfig; - -RUN apt-get update && apt-get install -y pkg-config tmux net-tools ; \ - cd /usr/local/src; \ - pip install --upgrade meson pybind11 patchelf; \ + cd ucx && \ + git checkout v1.19.x && \ + ./autogen.sh && ./configure \ + --enable-shared \ + --disable-static \ + --disable-doxygen-doc \ + --enable-optimizations \ + --enable-cma \ + --enable-devel-headers \ + --with-cuda=/usr/local/cuda \ + --with-verbs=yes \ + --with-dm \ + --with-gdrcopy=/usr/local \ + --with-efa \ + --enable-mt && \ + make -j$(nproc) && \ + make -j install-strip && \ + ldconfig && \ + rm -rf /usr/local/src/ucx + +# Build and install NixL +RUN uv pip install --system --upgrade meson pybind11 patchelf && \ + cd /usr/local/src && \ git clone https://github.com/ai-dynamo/nixl.git -b main && \ cd nixl && \ rm -rf build && \ @@ -115,7 +148,13 @@ RUN apt-get update && apt-get install -y pkg-config tmux net-tools ; \ cd build && \ ninja && \ ninja install && \ - cd .. && pip install . --no-deps; + cd .. && \ + uv pip install --system . --no-deps && \ + rm -rf /usr/local/src/nixl +# Copy full source code and install in editable mode COPY . /lightllm -RUN pip install -e /lightllm --no-cache-dir +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e /lightllm --no-deps + +WORKDIR /root diff --git a/docker/cuda_version_12.6.1/Dockerfile b/docker/cuda_version_12.6.1/Dockerfile index 6d67fcf4df..52669259e6 100644 --- a/docker/cuda_version_12.6.1/Dockerfile +++ b/docker/cuda_version_12.6.1/Dockerfile @@ -1,48 +1,52 @@ ARG CUDA_VERSION=12.6.1 FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu22.04 ARG PYTHON_VERSION=3.10 -ARG MAMBA_VERSION=24.7.1-0 ARG TARGETPLATFORM -ENV PATH=/opt/conda/bin:$PATH \ - CONDA_PREFIX=/opt/conda +ENV PATH=/root/.local/bin:/root/.cargo/bin:$PATH \ + UV_SYSTEM_PYTHON=1 \ + UV_COMPILE_BYTECODE=0 \ + UV_HTTP_TIMEOUT=300 + +# Install system dependencies RUN chmod 777 -R /tmp && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ ca-certificates \ libssl-dev \ curl \ g++ \ make \ - git && \ + git \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ + python${PYTHON_VERSION}-venv \ + python3-pip \ + libnuma-dev && \ rm -rf /var/lib/apt/lists/* -RUN case ${TARGETPLATFORM} in \ - "linux/arm64") MAMBA_ARCH=aarch64 ;; \ - *) MAMBA_ARCH=x86_64 ;; \ - esac && \ - curl -fsSL -o ~/mambaforge.sh "https://github.com/conda-forge/miniforge/releases/download/${MAMBA_VERSION}/Mambaforge-${MAMBA_VERSION}-Linux-${MAMBA_ARCH}.sh" && \ - bash ~/mambaforge.sh -b -p /opt/conda && \ - rm ~/mambaforge.sh - -RUN case ${TARGETPLATFORM} in \ - "linux/arm64") exit 1 ;; \ - *) /opt/conda/bin/conda update -y conda && \ - /opt/conda/bin/conda install -y "python=${PYTHON_VERSION}" ;; \ - esac && \ - /opt/conda/bin/conda clean -ya +# Set python alias +RUN update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1 && \ + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 +# Install uv package manager +RUN curl -LsSf https://astral.sh/uv/install.sh | sh -WORKDIR /root +WORKDIR /lightllm -COPY ./requirements.txt /lightllm/requirements.txt -RUN pip install -U pip -RUN pip install -r /lightllm/requirements.txt --no-cache-dir +# Copy project files for dependency installation +COPY pyproject.toml README.md ./ +COPY lightllm/__init__.py lightllm/ -RUN pip install --no-cache-dir vllm --pre --extra-index-url https://wheels.vllm.ai/nightly +# Install dependencies using uv (faster than pip) +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e . --no-cache-dir -# TODO: offline compile -# RUN git clone https://github.com/ModelTC/LightKernel.git && cd LightKernel && pip install --no-deps -v . +# Install vllm for additional kernels +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system vllm --pre --index-strategy unsafe-best-match --extra-index-url https://wheels.vllm.ai/nightly -RUN apt-get update && apt-get install -y libnuma-dev # for sgl_kernel +# Copy full source code and install in editable mode +COPY . . +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e . --no-deps -COPY . /lightllm -RUN pip install -e /lightllm --no-cache-dir +WORKDIR /root diff --git a/docker/cuda_version_12.6.1/Dockerfile.deepep b/docker/cuda_version_12.6.1/Dockerfile.deepep index e765978b91..3de2cea0fd 100644 --- a/docker/cuda_version_12.6.1/Dockerfile.deepep +++ b/docker/cuda_version_12.6.1/Dockerfile.deepep @@ -1,68 +1,83 @@ ARG CUDA_VERSION=12.6.1 FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu22.04 ARG PYTHON_VERSION=3.10 -ARG MAMBA_VERSION=24.7.1-0 ARG TARGETPLATFORM -ENV PATH=/opt/conda/bin:$PATH \ - CONDA_PREFIX=/opt/conda +ARG DEEPEP_COMMIT=b6ce310bb0b75079682d09bc2ebc063a074fbd58 + +ENV PATH=/root/.local/bin:/root/.cargo/bin:$PATH \ + UV_SYSTEM_PYTHON=1 \ + UV_COMPILE_BYTECODE=0 \ + UV_HTTP_TIMEOUT=300 \ + CUDA_HOME=/usr/local/cuda \ + GDRCOPY_HOME=/usr/src/gdrdrv-2.4.4/ +# Install system dependencies RUN chmod 777 -R /tmp && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ ca-certificates \ libssl-dev \ curl \ g++ \ make \ - git && \ + git \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ + python${PYTHON_VERSION}-venv \ + python3-pip \ + libnuma-dev \ + wget \ + devscripts \ + debhelper \ + dh-make \ + build-essential \ + dkms \ + ibverbs-providers \ + infiniband-diags \ + perftest \ + rdma-core \ + libibverbs-dev \ + librdmacm-dev \ + cmake && \ rm -rf /var/lib/apt/lists/* -RUN case ${TARGETPLATFORM} in \ - "linux/arm64") MAMBA_ARCH=aarch64 ;; \ - *) MAMBA_ARCH=x86_64 ;; \ - esac && \ - curl -fsSL -o ~/mambaforge.sh "https://github.com/conda-forge/miniforge/releases/download/${MAMBA_VERSION}/Mambaforge-${MAMBA_VERSION}-Linux-${MAMBA_ARCH}.sh" && \ - bash ~/mambaforge.sh -b -p /opt/conda && \ - rm ~/mambaforge.sh - -RUN case ${TARGETPLATFORM} in \ - "linux/arm64") exit 1 ;; \ - *) /opt/conda/bin/conda update -y conda && \ - /opt/conda/bin/conda install -y "python=${PYTHON_VERSION}" ;; \ - esac && \ - /opt/conda/bin/conda clean -ya - - -WORKDIR /root +# Set python alias +RUN update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1 && \ + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 -COPY ./requirements.txt /lightllm/requirements.txt -RUN pip install -U pip -RUN pip install -r /lightllm/requirements.txt --no-cache-dir +# Install uv package manager +RUN curl -LsSf https://astral.sh/uv/install.sh | sh -RUN pip install --no-cache-dir vllm --pre --extra-index-url https://wheels.vllm.ai/nightly +WORKDIR /lightllm -# TODO: offline compile -# RUN git clone https://github.com/ModelTC/LightKernel.git && cd LightKernel && pip install --no-deps -v . +# Copy project files for dependency installation +COPY pyproject.toml README.md ./ +COPY lightllm/__init__.py lightllm/ -RUN apt-get update && apt-get install -y libnuma-dev wget devscripts debhelper dh-make build-essential dkms -RUN apt-get install -y ibverbs-providers infiniband-diags perftest rdma-core libibverbs-dev librdmacm-dev +# Install dependencies using uv (faster than pip) +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e . --no-cache-dir -ENV CUDA_HOME=/usr/local/cuda \ - GDRCOPY_HOME=/usr/src/gdrdrv-2.4.4/ +# Install vllm for additional kernels +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system vllm --pre --index-strategy unsafe-best-match --extra-index-url https://wheels.vllm.ai/nightly -RUN mkdir -p /tmp/gdrcopy && cd /tmp \ - && git clone https://github.com/NVIDIA/gdrcopy.git -b v2.4.4 \ - && cd gdrcopy/packages \ - && CUDA=/usr/local/cuda ./build-deb-packages.sh \ - && dpkg -i gdrdrv-dkms_*.deb libgdrapi_*.deb gdrcopy-tests_*.deb gdrcopy_*.deb \ - && cd / && rm -rf /tmp/gdrcopy +# Build and install gdrcopy +RUN mkdir -p /tmp/gdrcopy && cd /tmp && \ + git clone https://github.com/NVIDIA/gdrcopy.git -b v2.4.4 && \ + cd gdrcopy/packages && \ + CUDA=/usr/local/cuda ./build-deb-packages.sh && \ + dpkg -i gdrdrv-dkms_*.deb libgdrapi_*.deb gdrcopy-tests_*.deb gdrcopy_*.deb && \ + cd / && rm -rf /tmp/gdrcopy - # Fix DeepEP IBGDA symlink +# Fix DeepEP IBGDA symlink RUN ln -sf /usr/lib/x86_64-linux-gnu/libmlx5.so.1 /usr/lib/x86_64-linux-gnu/libmlx5.so -RUN wget https://developer.download.nvidia.com/compute/redist/nvshmem/3.3.9/source/nvshmem_src_cuda12-all-all-3.3.9.tar.gz \ - && tar -xf nvshmem_src_cuda12-all-all-3.3.9.tar.gz && mv nvshmem_src nvshmem \ - && cd nvshmem \ - && rm -f /root/nvshmem_src_cuda12-all-all-3.3.9.tar.gz \ - && NVSHMEM_SHMEM_SUPPORT=0 \ +# Build and install NVSHMEM +RUN wget https://developer.download.nvidia.com/compute/redist/nvshmem/3.3.9/source/nvshmem_src_cuda12-all-all-3.3.9.tar.gz && \ + tar -xf nvshmem_src_cuda12-all-all-3.3.9.tar.gz && \ + mv nvshmem_src nvshmem && \ + cd nvshmem && \ + rm -f /root/nvshmem_src_cuda12-all-all-3.3.9.tar.gz && \ + NVSHMEM_SHMEM_SUPPORT=0 \ NVSHMEM_UCX_SUPPORT=0 \ NVSHMEM_USE_NCCL=0 \ NVSHMEM_MPI_SUPPORT=0 \ @@ -70,15 +85,19 @@ RUN wget https://developer.download.nvidia.com/compute/redist/nvshmem/3.3.9/sour NVSHMEM_PMIX_SUPPORT=0 \ NVSHMEM_TIMEOUT_DEVICE_POLLING=0 \ NVSHMEM_USE_GDRCOPY=1 \ - cmake -S . -B build/ -DCMAKE_INSTALL_PREFIX=/root/nvshmem/install -DCMAKE_CUDA_ARCHITECTURES=90 \ - && cmake --build build --target install -j64 - -ARG DEEPEP_COMMIT=b6ce310bb0b75079682d09bc2ebc063a074fbd58 -RUN git clone https://github.com/deepseek-ai/DeepEP.git && cd DeepEP && git checkout ${DEEPEP_COMMIT} && cd .. + cmake -S . -B build/ -DCMAKE_INSTALL_PREFIX=/root/nvshmem/install -DCMAKE_CUDA_ARCHITECTURES=90 && \ + cmake --build build --target install -j$(nproc) -WORKDIR /root/DeepEP +# Build and install DeepEP ENV NVSHMEM_DIR=/root/nvshmem/install -RUN NVSHMEM_DIR=/root/nvshmem/install python setup.py install +RUN git clone https://github.com/deepseek-ai/DeepEP.git /root/DeepEP && \ + cd /root/DeepEP && \ + git checkout ${DEEPEP_COMMIT} && \ + NVSHMEM_DIR=/root/nvshmem/install python setup.py install +# Copy full source code and install in editable mode COPY . /lightllm -RUN pip install -e /lightllm --no-cache-dir \ No newline at end of file +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e /lightllm --no-deps + +WORKDIR /root diff --git a/docker/cuda_version_12.6.1/Dockerfile.nixl b/docker/cuda_version_12.6.1/Dockerfile.nixl index b8047bbd03..2f9259b3ad 100644 --- a/docker/cuda_version_12.6.1/Dockerfile.nixl +++ b/docker/cuda_version_12.6.1/Dockerfile.nixl @@ -1,85 +1,117 @@ ARG CUDA_VERSION=12.6.1 FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu22.04 ARG PYTHON_VERSION=3.10 -ARG MAMBA_VERSION=24.7.1-0 ARG TARGETPLATFORM -ENV PATH=/opt/conda/bin:$PATH \ - CONDA_PREFIX=/opt/conda +ENV PATH=/root/.local/bin:/root/.cargo/bin:$PATH \ + UV_SYSTEM_PYTHON=1 \ + UV_COMPILE_BYTECODE=0 \ + UV_HTTP_TIMEOUT=300 \ + CUDA_HOME=/usr/local/cuda \ + GDRCOPY_HOME=/usr/src/gdrdrv-2.4.4/ + +# Install system dependencies RUN chmod 777 -R /tmp && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ ca-certificates \ libssl-dev \ curl \ g++ \ make \ - git && \ + git \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ + python${PYTHON_VERSION}-venv \ + python3-pip \ + libnuma-dev \ + wget \ + devscripts \ + debhelper \ + dh-make \ + build-essential \ + dkms \ + ibverbs-providers \ + infiniband-diags \ + perftest \ + rdma-core \ + libibverbs-dev \ + librdmacm-dev \ + cmake \ + automake \ + autotools-dev \ + libtool \ + libz-dev \ + pkg-config \ + tmux \ + net-tools && \ rm -rf /var/lib/apt/lists/* -RUN case ${TARGETPLATFORM} in \ - "linux/arm64") MAMBA_ARCH=aarch64 ;; \ - *) MAMBA_ARCH=x86_64 ;; \ - esac && \ - curl -fsSL -o ~/mambaforge.sh -v "https://github.com/conda-forge/miniforge/releases/download/${MAMBA_VERSION}/Mambaforge-${MAMBA_VERSION}-Linux-${MAMBA_ARCH}.sh" && \ - bash ~/mambaforge.sh -b -p /opt/conda && \ - rm ~/mambaforge.sh +# Reinstall RDMA packages +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + apt-get -y install --reinstall libibverbs-dev rdma-core ibverbs-utils libibumad-dev && \ + rm -rf /var/lib/apt/lists/* -RUN case ${TARGETPLATFORM} in \ - "linux/arm64") exit 1 ;; \ - *) /opt/conda/bin/conda update -y conda && \ - /opt/conda/bin/conda install -y "python=${PYTHON_VERSION}" ;; \ - esac && \ - /opt/conda/bin/conda clean -ya +# Set python alias +RUN update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1 && \ + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 +# Install uv package manager +RUN curl -LsSf https://astral.sh/uv/install.sh | sh -WORKDIR /root +WORKDIR /lightllm -COPY ./requirements.txt /lightllm/requirements.txt -RUN --mount=type=cache,target=/root/.cache/pip pip install -r /lightllm/requirements.txt --ignore-installed --extra-index-url https://download.pytorch.org/whl/cu124 +# Copy project files for dependency installation +COPY pyproject.toml README.md ./ +COPY lightllm/__init__.py lightllm/ -RUN --mount=type=cache,target=/root/.cache/pip pip install vllm --pre --extra-index-url https://wheels.vllm.ai/nightly -RUN --mount=type=cache,target=/root/.cache/pip git clone https://github.com/ModelTC/LightKernel.git && cd LightKernel && pip install --no-deps -v . +# Install dependencies using uv (faster than pip) +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e . --no-cache-dir -RUN apt-get update && apt-get install -y libnuma-dev wget devscripts debhelper dh-make build-essential dkms -RUN apt-get install -y ibverbs-providers infiniband-diags perftest rdma-core libibverbs-dev librdmacm-dev +# Install vllm for additional kernels +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system vllm --pre --index-strategy unsafe-best-match --extra-index-url https://wheels.vllm.ai/nightly -ENV CUDA_HOME=/usr/local/cuda \ - GDRCOPY_HOME=/usr/src/gdrdrv-2.4.4/ +# Install LightKernel +RUN git clone https://github.com/ModelTC/LightKernel.git /tmp/LightKernel && \ + cd /tmp/LightKernel && \ + uv pip install --system --no-deps -v . && \ + rm -rf /tmp/LightKernel -RUN mkdir -p /tmp/gdrcopy && cd /tmp \ - && git clone https://github.com/NVIDIA/gdrcopy.git -b v2.4.4 \ - && cd gdrcopy/packages \ - && CUDA=/usr/local/cuda ./build-deb-packages.sh \ - && dpkg -i gdrdrv-dkms_*.deb libgdrapi_*.deb gdrcopy-tests_*.deb gdrcopy_*.deb \ - && cd / && rm -rf /tmp/gdrcopy +# Build and install gdrcopy +RUN mkdir -p /tmp/gdrcopy && cd /tmp && \ + git clone https://github.com/NVIDIA/gdrcopy.git -b v2.4.4 && \ + cd gdrcopy/packages && \ + CUDA=/usr/local/cuda ./build-deb-packages.sh && \ + dpkg -i gdrdrv-dkms_*.deb libgdrapi_*.deb gdrcopy-tests_*.deb gdrcopy_*.deb && \ + cd / && rm -rf /tmp/gdrcopy -RUN apt-get update && apt-get install -y cmake automake autotools-dev libtool libz-dev && \ - DEBIAN_FRONTEND=noninteractive apt-get -y install --reinstall libibverbs-dev rdma-core ibverbs-utils libibumad-dev; \ - rm -rf /usr/lib/ucx && \ - rm -rf /opt/hpcx/ucx && \ +# Build and install UCX with CUDA and gdrcopy support +RUN rm -rf /usr/lib/ucx /opt/hpcx/ucx && \ cd /usr/local/src && \ git clone https://github.com/openucx/ucx.git && \ - cd ucx && \ - git checkout v1.19.x && \ - ./autogen.sh && ./configure \ - --enable-shared \ - --disable-static \ - --disable-doxygen-doc \ - --enable-optimizations \ - --enable-cma \ - --enable-devel-headers \ - --with-cuda=/usr/local/cuda \ - --with-verbs=yes \ - --with-dm \ - --with-gdrcopy=/usr/local \ - --with-efa \ - --enable-mt && \ - make -j && \ - make -j install-strip && \ - ldconfig; + cd ucx && \ + git checkout v1.19.x && \ + ./autogen.sh && ./configure \ + --enable-shared \ + --disable-static \ + --disable-doxygen-doc \ + --enable-optimizations \ + --enable-cma \ + --enable-devel-headers \ + --with-cuda=/usr/local/cuda \ + --with-verbs=yes \ + --with-dm \ + --with-gdrcopy=/usr/local \ + --with-efa \ + --enable-mt && \ + make -j$(nproc) && \ + make -j install-strip && \ + ldconfig && \ + rm -rf /usr/local/src/ucx -RUN apt-get update && apt-get install -y pkg-config tmux net-tools ; \ - cd /usr/local/src; \ - pip install --upgrade meson pybind11 patchelf; \ +# Build and install NixL +RUN uv pip install --system --upgrade meson pybind11 patchelf && \ + cd /usr/local/src && \ git clone https://github.com/ai-dynamo/nixl.git -b main && \ cd nixl && \ rm -rf build && \ @@ -88,7 +120,13 @@ RUN apt-get update && apt-get install -y pkg-config tmux net-tools ; \ cd build && \ ninja && \ ninja install && \ - cd .. && pip install . --no-deps; + cd .. && \ + uv pip install --system . --no-deps && \ + rm -rf /usr/local/src/nixl +# Copy full source code and install in editable mode COPY . /lightllm -RUN pip install -e /lightllm --no-cache-dir +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e /lightllm --no-deps + +WORKDIR /root diff --git a/docker/cuda_version_12.6.1/Dockerfile.nixl.deepep b/docker/cuda_version_12.6.1/Dockerfile.nixl.deepep index 8ca06e1094..5a533b7a80 100644 --- a/docker/cuda_version_12.6.1/Dockerfile.nixl.deepep +++ b/docker/cuda_version_12.6.1/Dockerfile.nixl.deepep @@ -1,67 +1,101 @@ ARG CUDA_VERSION=12.6.1 FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu22.04 - ARG PYTHON_VERSION=3.10 -ARG MAMBA_VERSION=24.7.1-0 ARG TARGETPLATFORM +ARG DEEPEP_COMMIT=b6ce310bb0b75079682d09bc2ebc063a074fbd58 -ENV PATH=/opt/conda/bin:$PATH \ - CONDA_PREFIX=/opt/conda +ENV PATH=/root/.local/bin:/root/.cargo/bin:$PATH \ + UV_SYSTEM_PYTHON=1 \ + UV_COMPILE_BYTECODE=0 \ + UV_HTTP_TIMEOUT=300 \ + CUDA_HOME=/usr/local/cuda \ + GDRCOPY_HOME=/usr/src/gdrdrv-2.4.4/ +# Install system dependencies RUN chmod 777 -R /tmp && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ ca-certificates \ libssl-dev \ curl \ g++ \ make \ - git && \ + git \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ + python${PYTHON_VERSION}-venv \ + python3-pip \ + libnuma-dev \ + wget \ + devscripts \ + debhelper \ + dh-make \ + build-essential \ + dkms \ + ibverbs-providers \ + infiniband-diags \ + perftest \ + rdma-core \ + libibverbs-dev \ + librdmacm-dev \ + cmake \ + automake \ + autotools-dev \ + libtool \ + libz-dev \ + pkg-config \ + tmux \ + net-tools && \ rm -rf /var/lib/apt/lists/* -RUN case ${TARGETPLATFORM} in \ - "linux/arm64") MAMBA_ARCH=aarch64 ;; \ - *) MAMBA_ARCH=x86_64 ;; \ - esac && \ - curl -fsSL -o ~/mambaforge.sh -v "https://github.com/conda-forge/miniforge/releases/download/${MAMBA_VERSION}/Mambaforge-${MAMBA_VERSION}-Linux-${MAMBA_ARCH}.sh" && \ - bash ~/mambaforge.sh -b -p /opt/conda && \ - rm ~/mambaforge.sh +# Reinstall RDMA packages +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + apt-get -y install --reinstall libibverbs-dev rdma-core ibverbs-utils libibumad-dev && \ + rm -rf /var/lib/apt/lists/* -RUN case ${TARGETPLATFORM} in \ - "linux/arm64") exit 1 ;; \ - *) /opt/conda/bin/conda update -y conda && \ - /opt/conda/bin/conda install -y "python=${PYTHON_VERSION}" ;; \ - esac && \ - /opt/conda/bin/conda clean -ya +# Set python alias +RUN update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1 && \ + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 +# Install uv package manager +RUN curl -LsSf https://astral.sh/uv/install.sh | sh -WORKDIR /root +WORKDIR /lightllm -COPY ./requirements.txt /lightllm/requirements.txt -RUN --mount=type=cache,target=/root/.cache/pip pip install -r /lightllm/requirements.txt --ignore-installed --extra-index-url https://download.pytorch.org/whl/cu124 +# Copy project files for dependency installation +COPY pyproject.toml README.md ./ +COPY lightllm/__init__.py lightllm/ -RUN --mount=type=cache,target=/root/.cache/pip pip install vllm --pre --extra-index-url https://wheels.vllm.ai/nightly -RUN --mount=type=cache,target=/root/.cache/pip git clone https://github.com/ModelTC/LightKernel.git && cd LightKernel && pip install --no-deps -v . +# Install dependencies using uv (faster than pip) +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e . --no-cache-dir -RUN apt-get update && apt-get install -y libnuma-dev wget devscripts debhelper dh-make build-essential dkms -RUN apt-get install -y ibverbs-providers infiniband-diags perftest rdma-core libibverbs-dev librdmacm-dev +# Install vllm for additional kernels +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system vllm --pre --index-strategy unsafe-best-match --extra-index-url https://wheels.vllm.ai/nightly -ENV CUDA_HOME=/usr/local/cuda \ - GDRCOPY_HOME=/usr/src/gdrdrv-2.4.4/ +# Install LightKernel +RUN git clone https://github.com/ModelTC/LightKernel.git /tmp/LightKernel && \ + cd /tmp/LightKernel && \ + uv pip install --system --no-deps -v . && \ + rm -rf /tmp/LightKernel -RUN mkdir -p /tmp/gdrcopy && cd /tmp \ - && git clone https://github.com/NVIDIA/gdrcopy.git -b v2.4.4 \ - && cd gdrcopy/packages \ - && CUDA=/usr/local/cuda ./build-deb-packages.sh \ - && dpkg -i gdrdrv-dkms_*.deb libgdrapi_*.deb gdrcopy-tests_*.deb gdrcopy_*.deb \ - && cd / && rm -rf /tmp/gdrcopy +# Build and install gdrcopy +RUN mkdir -p /tmp/gdrcopy && cd /tmp && \ + git clone https://github.com/NVIDIA/gdrcopy.git -b v2.4.4 && \ + cd gdrcopy/packages && \ + CUDA=/usr/local/cuda ./build-deb-packages.sh && \ + dpkg -i gdrdrv-dkms_*.deb libgdrapi_*.deb gdrcopy-tests_*.deb gdrcopy_*.deb && \ + cd / && rm -rf /tmp/gdrcopy - # Fix DeepEP IBGDA symlink +# Fix DeepEP IBGDA symlink RUN ln -sf /usr/lib/x86_64-linux-gnu/libmlx5.so.1 /usr/lib/x86_64-linux-gnu/libmlx5.so -RUN wget https://developer.download.nvidia.com/compute/redist/nvshmem/3.3.9/source/nvshmem_src_cuda12-all-all-3.3.9.tar.gz \ - && tar -xf nvshmem_src_cuda12-all-all-3.3.9.tar.gz && mv nvshmem_src nvshmem \ - && cd nvshmem \ - && rm -f /root/nvshmem_src_cuda12-all-all-3.3.9.tar.gz \ - && NVSHMEM_SHMEM_SUPPORT=0 \ +# Build and install NVSHMEM +RUN wget https://developer.download.nvidia.com/compute/redist/nvshmem/3.3.9/source/nvshmem_src_cuda12-all-all-3.3.9.tar.gz && \ + tar -xf nvshmem_src_cuda12-all-all-3.3.9.tar.gz && \ + mv nvshmem_src nvshmem && \ + cd nvshmem && \ + rm -f /root/nvshmem_src_cuda12-all-all-3.3.9.tar.gz && \ + NVSHMEM_SHMEM_SUPPORT=0 \ NVSHMEM_UCX_SUPPORT=0 \ NVSHMEM_USE_NCCL=0 \ NVSHMEM_MPI_SUPPORT=0 \ @@ -69,44 +103,43 @@ RUN wget https://developer.download.nvidia.com/compute/redist/nvshmem/3.3.9/sour NVSHMEM_PMIX_SUPPORT=0 \ NVSHMEM_TIMEOUT_DEVICE_POLLING=0 \ NVSHMEM_USE_GDRCOPY=1 \ - cmake -S . -B build/ -DCMAKE_INSTALL_PREFIX=/root/nvshmem/install -DCMAKE_CUDA_ARCHITECTURES=90 \ - && cmake --build build --target install -j64 + cmake -S . -B build/ -DCMAKE_INSTALL_PREFIX=/root/nvshmem/install -DCMAKE_CUDA_ARCHITECTURES=90 && \ + cmake --build build --target install -j$(nproc) -ARG DEEPEP_COMMIT=b6ce310bb0b75079682d09bc2ebc063a074fbd58 -RUN git clone https://github.com/deepseek-ai/DeepEP.git && cd DeepEP && git checkout ${DEEPEP_COMMIT} && cd .. - -WORKDIR /root/DeepEP +# Build and install DeepEP ENV NVSHMEM_DIR=/root/nvshmem/install -RUN NVSHMEM_DIR=/root/nvshmem/install python setup.py install +RUN git clone https://github.com/deepseek-ai/DeepEP.git /root/DeepEP && \ + cd /root/DeepEP && \ + git checkout ${DEEPEP_COMMIT} && \ + NVSHMEM_DIR=/root/nvshmem/install python setup.py install -RUN apt-get update && apt-get install -y cmake automake autotools-dev libtool libz-dev && \ - DEBIAN_FRONTEND=noninteractive apt-get -y install --reinstall libibverbs-dev rdma-core ibverbs-utils libibumad-dev; \ - rm -rf /usr/lib/ucx && \ - rm -rf /opt/hpcx/ucx && \ +# Build and install UCX with CUDA and gdrcopy support +RUN rm -rf /usr/lib/ucx /opt/hpcx/ucx && \ cd /usr/local/src && \ git clone https://github.com/openucx/ucx.git && \ - cd ucx && \ - git checkout v1.19.x && \ - ./autogen.sh && ./configure \ - --enable-shared \ - --disable-static \ - --disable-doxygen-doc \ - --enable-optimizations \ - --enable-cma \ - --enable-devel-headers \ - --with-cuda=/usr/local/cuda \ - --with-verbs=yes \ - --with-dm \ - --with-gdrcopy=/usr/local \ - --with-efa \ - --enable-mt && \ - make -j && \ - make -j install-strip && \ - ldconfig; - -RUN apt-get update && apt-get install -y pkg-config tmux net-tools ; \ - cd /usr/local/src; \ - pip install --upgrade meson pybind11 patchelf; \ + cd ucx && \ + git checkout v1.19.x && \ + ./autogen.sh && ./configure \ + --enable-shared \ + --disable-static \ + --disable-doxygen-doc \ + --enable-optimizations \ + --enable-cma \ + --enable-devel-headers \ + --with-cuda=/usr/local/cuda \ + --with-verbs=yes \ + --with-dm \ + --with-gdrcopy=/usr/local \ + --with-efa \ + --enable-mt && \ + make -j$(nproc) && \ + make -j install-strip && \ + ldconfig && \ + rm -rf /usr/local/src/ucx + +# Build and install NixL +RUN uv pip install --system --upgrade meson pybind11 patchelf && \ + cd /usr/local/src && \ git clone https://github.com/ai-dynamo/nixl.git -b main && \ cd nixl && \ rm -rf build && \ @@ -115,7 +148,13 @@ RUN apt-get update && apt-get install -y pkg-config tmux net-tools ; \ cd build && \ ninja && \ ninja install && \ - cd .. && pip install . --no-deps; + cd .. && \ + uv pip install --system . --no-deps && \ + rm -rf /usr/local/src/nixl +# Copy full source code and install in editable mode COPY . /lightllm -RUN pip install -e /lightllm --no-cache-dir +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e /lightllm --no-deps + +WORKDIR /root diff --git a/docker/cuda_version_12.8.0/Dockerfile b/docker/cuda_version_12.8.0/Dockerfile index 439031ce48..194bed68c3 100644 --- a/docker/cuda_version_12.8.0/Dockerfile +++ b/docker/cuda_version_12.8.0/Dockerfile @@ -1,48 +1,52 @@ ARG CUDA_VERSION=12.8.0 FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu22.04 ARG PYTHON_VERSION=3.10 -ARG MAMBA_VERSION=24.7.1-0 ARG TARGETPLATFORM -ENV PATH=/opt/conda/bin:$PATH \ - CONDA_PREFIX=/opt/conda +ENV PATH=/root/.local/bin:/root/.cargo/bin:$PATH \ + UV_SYSTEM_PYTHON=1 \ + UV_COMPILE_BYTECODE=0 \ + UV_HTTP_TIMEOUT=300 + +# Install system dependencies RUN chmod 777 -R /tmp && apt-get update --allow-insecure-repositories && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ ca-certificates \ libssl-dev \ curl \ g++ \ make \ - git && \ + git \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ + python${PYTHON_VERSION}-venv \ + python3-pip \ + libnuma-dev && \ rm -rf /var/lib/apt/lists/* -RUN case ${TARGETPLATFORM} in \ - "linux/arm64") MAMBA_ARCH=aarch64 ;; \ - *) MAMBA_ARCH=x86_64 ;; \ - esac && \ - curl -fsSL -o ~/mambaforge.sh "https://github.com/conda-forge/miniforge/releases/download/${MAMBA_VERSION}/Mambaforge-${MAMBA_VERSION}-Linux-${MAMBA_ARCH}.sh" && \ - bash ~/mambaforge.sh -b -p /opt/conda && \ - rm ~/mambaforge.sh - -RUN case ${TARGETPLATFORM} in \ - "linux/arm64") exit 1 ;; \ - *) /opt/conda/bin/conda update -y conda && \ - /opt/conda/bin/conda install -y "python=${PYTHON_VERSION}" ;; \ - esac && \ - /opt/conda/bin/conda clean -ya +# Set python alias +RUN update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1 && \ + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 +# Install uv package manager +RUN curl -LsSf https://astral.sh/uv/install.sh | sh -WORKDIR /root +WORKDIR /lightllm -RUN pip install --no-cache-dir vllm==0.11.0 --pre --extra-index-url https://wheels.vllm.ai/nightly +# Copy project files for dependency installation +COPY pyproject.toml README.md ./ +COPY lightllm/__init__.py lightllm/ -COPY ./requirements.txt /lightllm/requirements.txt -RUN pip install -U pip -RUN --mount=type=cache,target=/root/.cache/pip pip install -r /lightllm/requirements.txt --extra-index-url https://download.pytorch.org/whl/cu128 +# Install dependencies using uv (faster than pip) +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e . --no-cache-dir --index-strategy unsafe-best-match --extra-index-url https://download.pytorch.org/whl/cu128 -# TODO: offline compile -# RUN git clone https://github.com/ModelTC/LightKernel.git && cd LightKernel && pip install --no-deps -v . +# Install vllm for additional kernels +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system vllm --pre --index-strategy unsafe-best-match --extra-index-url https://wheels.vllm.ai/nightly -RUN apt-get update && apt-get install -y libnuma-dev # for sgl_kernel +# Copy full source code and install in editable mode +COPY . . +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e . --no-deps -COPY . /lightllm -RUN pip install -e /lightllm --no-cache-dir +WORKDIR /root diff --git a/docker/cuda_version_12.8.0/Dockerfile.deepep b/docker/cuda_version_12.8.0/Dockerfile.deepep index 99997fc5bd..acf8c905ce 100644 --- a/docker/cuda_version_12.8.0/Dockerfile.deepep +++ b/docker/cuda_version_12.8.0/Dockerfile.deepep @@ -1,67 +1,83 @@ ARG CUDA_VERSION=12.8.0 FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu22.04 ARG PYTHON_VERSION=3.10 -ARG MAMBA_VERSION=24.7.1-0 ARG TARGETPLATFORM -ENV PATH=/opt/conda/bin:$PATH \ - CONDA_PREFIX=/opt/conda +ARG DEEPEP_COMMIT=b6ce310bb0b75079682d09bc2ebc063a074fbd58 + +ENV PATH=/root/.local/bin:/root/.cargo/bin:$PATH \ + UV_SYSTEM_PYTHON=1 \ + UV_COMPILE_BYTECODE=0 \ + UV_HTTP_TIMEOUT=300 \ + CUDA_HOME=/usr/local/cuda \ + GDRCOPY_HOME=/usr/src/gdrdrv-2.4.4/ -RUN chmod 777 -R /tmp && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ +# Install system dependencies +RUN chmod 777 -R /tmp && apt-get update --allow-insecure-repositories && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ ca-certificates \ libssl-dev \ curl \ g++ \ make \ - git && \ + git \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ + python${PYTHON_VERSION}-venv \ + python3-pip \ + libnuma-dev \ + wget \ + devscripts \ + debhelper \ + dh-make \ + build-essential \ + dkms \ + ibverbs-providers \ + infiniband-diags \ + perftest \ + rdma-core \ + libibverbs-dev \ + librdmacm-dev \ + cmake && \ rm -rf /var/lib/apt/lists/* -RUN case ${TARGETPLATFORM} in \ - "linux/arm64") MAMBA_ARCH=aarch64 ;; \ - *) MAMBA_ARCH=x86_64 ;; \ - esac && \ - curl -fsSL -o ~/mambaforge.sh "https://github.com/conda-forge/miniforge/releases/download/${MAMBA_VERSION}/Mambaforge-${MAMBA_VERSION}-Linux-${MAMBA_ARCH}.sh" && \ - bash ~/mambaforge.sh -b -p /opt/conda && \ - rm ~/mambaforge.sh +# Set python alias +RUN update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1 && \ + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 -RUN case ${TARGETPLATFORM} in \ - "linux/arm64") exit 1 ;; \ - *) /opt/conda/bin/conda update -y conda && \ - /opt/conda/bin/conda install -y "python=${PYTHON_VERSION}" ;; \ - esac && \ - /opt/conda/bin/conda clean -ya +# Install uv package manager +RUN curl -LsSf https://astral.sh/uv/install.sh | sh +WORKDIR /lightllm -WORKDIR /root - -RUN pip install --no-cache-dir vllm==0.11.0 --pre --extra-index-url https://wheels.vllm.ai/nightly +# Copy project files for dependency installation +COPY pyproject.toml README.md ./ +COPY lightllm/__init__.py lightllm/ -COPY ./requirements.txt /lightllm/requirements.txt -RUN pip install -U pip -RUN --mount=type=cache,target=/root/.cache/pip pip install -r /lightllm/requirements.txt --extra-index-url https://download.pytorch.org/whl/cu128 -# TODO: offline compile -# RUN git clone https://github.com/ModelTC/LightKernel.git && cd LightKernel && pip install --no-deps -v . +# Install dependencies using uv (faster than pip) +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e . --no-cache-dir --index-strategy unsafe-best-match --extra-index-url https://download.pytorch.org/whl/cu128 -RUN apt-get update --allow-insecure-repositories && apt-get install -y libnuma-dev wget devscripts debhelper dh-make build-essential dkms -RUN apt-get install -y ibverbs-providers infiniband-diags perftest rdma-core libibverbs-dev librdmacm-dev +# Install vllm for additional kernels +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system vllm --pre --index-strategy unsafe-best-match --extra-index-url https://wheels.vllm.ai/nightly -ENV CUDA_HOME=/usr/local/cuda \ - GDRCOPY_HOME=/usr/src/gdrdrv-2.4.4/ +# Build and install gdrcopy +RUN mkdir -p /tmp/gdrcopy && cd /tmp && \ + git clone https://github.com/NVIDIA/gdrcopy.git -b v2.4.4 && \ + cd gdrcopy/packages && \ + CUDA=/usr/local/cuda ./build-deb-packages.sh && \ + dpkg -i gdrdrv-dkms_*.deb libgdrapi_*.deb gdrcopy-tests_*.deb gdrcopy_*.deb && \ + cd / && rm -rf /tmp/gdrcopy -RUN mkdir -p /tmp/gdrcopy && cd /tmp \ - && git clone https://github.com/NVIDIA/gdrcopy.git -b v2.4.4 \ - && cd gdrcopy/packages \ - && CUDA=/usr/local/cuda ./build-deb-packages.sh \ - && dpkg -i gdrdrv-dkms_*.deb libgdrapi_*.deb gdrcopy-tests_*.deb gdrcopy_*.deb \ - && cd / && rm -rf /tmp/gdrcopy - - # Fix DeepEP IBGDA symlink +# Fix DeepEP IBGDA symlink RUN ln -sf /usr/lib/x86_64-linux-gnu/libmlx5.so.1 /usr/lib/x86_64-linux-gnu/libmlx5.so -RUN wget https://developer.download.nvidia.com/compute/redist/nvshmem/3.3.9/source/nvshmem_src_cuda12-all-all-3.3.9.tar.gz \ - && tar -xf nvshmem_src_cuda12-all-all-3.3.9.tar.gz && mv nvshmem_src nvshmem \ - && cd nvshmem \ - && rm -f /root/nvshmem_src_cuda12-all-all-3.3.9.tar.gz \ - && NVSHMEM_SHMEM_SUPPORT=0 \ +# Build and install NVSHMEM +RUN wget https://developer.download.nvidia.com/compute/redist/nvshmem/3.3.9/source/nvshmem_src_cuda12-all-all-3.3.9.tar.gz && \ + tar -xf nvshmem_src_cuda12-all-all-3.3.9.tar.gz && \ + mv nvshmem_src nvshmem && \ + cd nvshmem && \ + rm -f /root/nvshmem_src_cuda12-all-all-3.3.9.tar.gz && \ + NVSHMEM_SHMEM_SUPPORT=0 \ NVSHMEM_UCX_SUPPORT=0 \ NVSHMEM_USE_NCCL=0 \ NVSHMEM_MPI_SUPPORT=0 \ @@ -69,15 +85,19 @@ RUN wget https://developer.download.nvidia.com/compute/redist/nvshmem/3.3.9/sour NVSHMEM_PMIX_SUPPORT=0 \ NVSHMEM_TIMEOUT_DEVICE_POLLING=0 \ NVSHMEM_USE_GDRCOPY=1 \ - cmake -S . -B build/ -DCMAKE_INSTALL_PREFIX=/root/nvshmem/install -DCMAKE_CUDA_ARCHITECTURES=90 \ - && cmake --build build --target install -j64 - -ARG DEEPEP_COMMIT=b6ce310bb0b75079682d09bc2ebc063a074fbd58 -RUN git clone https://github.com/deepseek-ai/DeepEP.git && cd DeepEP && git checkout ${DEEPEP_COMMIT} && cd .. + cmake -S . -B build/ -DCMAKE_INSTALL_PREFIX=/root/nvshmem/install -DCMAKE_CUDA_ARCHITECTURES=90 && \ + cmake --build build --target install -j$(nproc) -WORKDIR /root/DeepEP +# Build and install DeepEP ENV NVSHMEM_DIR=/root/nvshmem/install -RUN NVSHMEM_DIR=/root/nvshmem/install python setup.py install +RUN git clone https://github.com/deepseek-ai/DeepEP.git /root/DeepEP && \ + cd /root/DeepEP && \ + git checkout ${DEEPEP_COMMIT} && \ + NVSHMEM_DIR=/root/nvshmem/install python setup.py install +# Copy full source code and install in editable mode COPY . /lightllm -RUN pip install -e /lightllm --no-cache-dir \ No newline at end of file +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e /lightllm --no-deps + +WORKDIR /root diff --git a/docker/cuda_version_12.8.0/Dockerfile.nixl b/docker/cuda_version_12.8.0/Dockerfile.nixl index 4bcb66af56..0a896e0ec2 100644 --- a/docker/cuda_version_12.8.0/Dockerfile.nixl +++ b/docker/cuda_version_12.8.0/Dockerfile.nixl @@ -1,86 +1,116 @@ ARG CUDA_VERSION=12.8.0 FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu22.04 ARG PYTHON_VERSION=3.10 -ARG MAMBA_VERSION=24.7.1-0 ARG TARGETPLATFORM -ENV PATH=/opt/conda/bin:$PATH \ - CONDA_PREFIX=/opt/conda +ENV PATH=/root/.local/bin:/root/.cargo/bin:$PATH \ + UV_SYSTEM_PYTHON=1 \ + UV_COMPILE_BYTECODE=0 \ + UV_HTTP_TIMEOUT=300 \ + CUDA_HOME=/usr/local/cuda \ + GDRCOPY_HOME=/usr/src/gdrdrv-2.4.4/ + +# Install system dependencies RUN chmod 777 -R /tmp && apt-get update --allow-insecure-repositories && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ ca-certificates \ libssl-dev \ curl \ g++ \ make \ - git && \ + git \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ + python${PYTHON_VERSION}-venv \ + python3-pip \ + libnuma-dev \ + wget \ + devscripts \ + debhelper \ + dh-make \ + build-essential \ + dkms \ + ibverbs-providers \ + infiniband-diags \ + perftest \ + rdma-core \ + libibverbs-dev \ + librdmacm-dev \ + cmake \ + automake \ + autotools-dev \ + libtool \ + libz-dev \ + pkg-config \ + tmux \ + net-tools \ + libaio-dev && \ rm -rf /var/lib/apt/lists/* -RUN case ${TARGETPLATFORM} in \ - "linux/arm64") MAMBA_ARCH=aarch64 ;; \ - *) MAMBA_ARCH=x86_64 ;; \ - esac && \ - curl -fsSL -o ~/mambaforge.sh -v "https://github.com/conda-forge/miniforge/releases/download/${MAMBA_VERSION}/Mambaforge-${MAMBA_VERSION}-Linux-${MAMBA_ARCH}.sh" && \ - bash ~/mambaforge.sh -b -p /opt/conda && \ - rm ~/mambaforge.sh +# Reinstall RDMA packages +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + apt-get -y install --reinstall libibverbs-dev rdma-core ibverbs-utils libibumad-dev && \ + rm -rf /var/lib/apt/lists/* -RUN case ${TARGETPLATFORM} in \ - "linux/arm64") exit 1 ;; \ - *) /opt/conda/bin/conda update -y conda && \ - /opt/conda/bin/conda install -y "python=${PYTHON_VERSION}" ;; \ - esac && \ - /opt/conda/bin/conda clean -ya +# Set python alias +RUN update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1 && \ + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 +# Install uv package manager +RUN curl -LsSf https://astral.sh/uv/install.sh | sh -WORKDIR /root +WORKDIR /lightllm -RUN --mount=type=cache,target=/root/.cache/pip pip install vllm==0.11.0 --pre --extra-index-url https://wheels.vllm.ai/nightly +# Copy project files for dependency installation +COPY pyproject.toml README.md ./ +COPY lightllm/__init__.py lightllm/ -COPY ./requirements.txt /lightllm/requirements.txt -RUN --mount=type=cache,target=/root/.cache/pip pip install -r /lightllm/requirements.txt --extra-index-url https://download.pytorch.org/whl/cu128 +# Install dependencies using uv (faster than pip) +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e . --no-cache-dir --index-strategy unsafe-best-match --extra-index-url https://download.pytorch.org/whl/cu128 -RUN --mount=type=cache,target=/root/.cache/pip pip install --no-deps -v 'git+https://github.com/ModelTC/LightKernel.git@07f2f62af5deb41f10a22660f9f42dba9273361e#egg=lightllm_kernel' +# Install vllm for additional kernels +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system vllm --pre --index-strategy unsafe-best-match --extra-index-url https://wheels.vllm.ai/nightly -RUN apt-get update && apt-get install -y libnuma-dev wget devscripts debhelper dh-make build-essential dkms -RUN apt-get install -y ibverbs-providers infiniband-diags perftest rdma-core libibverbs-dev librdmacm-dev +# Install LightKernel +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system --no-deps -v 'git+https://github.com/ModelTC/LightKernel.git@07f2f62af5deb41f10a22660f9f42dba9273361e#egg=lightllm_kernel' -ENV CUDA_HOME=/usr/local/cuda \ - GDRCOPY_HOME=/usr/src/gdrdrv-2.4.4/ +# Build and install gdrcopy +RUN mkdir -p /tmp/gdrcopy && cd /tmp && \ + git clone https://github.com/NVIDIA/gdrcopy.git -b v2.4.4 && \ + cd gdrcopy/packages && \ + CUDA=/usr/local/cuda ./build-deb-packages.sh && \ + dpkg -i gdrdrv-dkms_*.deb libgdrapi_*.deb gdrcopy-tests_*.deb gdrcopy_*.deb && \ + cd / && rm -rf /tmp/gdrcopy -RUN mkdir -p /tmp/gdrcopy && cd /tmp \ - && git clone https://github.com/NVIDIA/gdrcopy.git -b v2.4.4 \ - && cd gdrcopy/packages \ - && CUDA=/usr/local/cuda ./build-deb-packages.sh \ - && dpkg -i gdrdrv-dkms_*.deb libgdrapi_*.deb gdrcopy-tests_*.deb gdrcopy_*.deb \ - && cd / && rm -rf /tmp/gdrcopy - -RUN apt-get update && apt-get install -y cmake automake autotools-dev libtool libz-dev && \ - DEBIAN_FRONTEND=noninteractive apt-get -y install --reinstall libibverbs-dev rdma-core ibverbs-utils libibumad-dev; \ - rm -rf /usr/lib/ucx && \ - rm -rf /opt/hpcx/ucx && \ +# Build and install UCX with CUDA and gdrcopy support +RUN rm -rf /usr/lib/ucx /opt/hpcx/ucx && \ cd /usr/local/src && \ git clone https://github.com/openucx/ucx.git && \ - cd ucx && \ - git checkout v1.19.x && \ - ./autogen.sh && ./configure \ - --enable-shared \ - --disable-static \ - --disable-doxygen-doc \ - --enable-optimizations \ - --enable-cma \ - --enable-devel-headers \ - --with-cuda=/usr/local/cuda \ - --with-verbs=yes \ - --with-dm \ - --with-gdrcopy=/usr/local \ - --with-efa \ - --enable-mt && \ - make -j && \ - make -j install-strip && \ - ldconfig; + cd ucx && \ + git checkout v1.19.x && \ + ./autogen.sh && ./configure \ + --enable-shared \ + --disable-static \ + --disable-doxygen-doc \ + --enable-optimizations \ + --enable-cma \ + --enable-devel-headers \ + --with-cuda=/usr/local/cuda \ + --with-verbs=yes \ + --with-dm \ + --with-gdrcopy=/usr/local \ + --with-efa \ + --enable-mt && \ + make -j$(nproc) && \ + make -j install-strip && \ + ldconfig && \ + rm -rf /usr/local/src/ucx -RUN apt-get update && apt-get install -y pkg-config tmux net-tools libaio-dev ; \ - cd /usr/local/src; \ - pip install --upgrade meson pybind11 patchelf; \ +# Build and install NixL +RUN uv pip install --system --upgrade meson pybind11 patchelf && \ + cd /usr/local/src && \ git clone https://github.com/ai-dynamo/nixl.git -b 0.8.0 && \ cd nixl && \ rm -rf build && \ @@ -89,7 +119,13 @@ RUN apt-get update && apt-get install -y pkg-config tmux net-tools libaio-dev ; cd build && \ ninja && \ ninja install && \ - cd .. && pip install . --no-deps; + cd .. && \ + uv pip install --system . --no-deps && \ + rm -rf /usr/local/src/nixl +# Copy full source code and install in editable mode COPY . /lightllm -RUN pip install -e /lightllm --no-cache-dir +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e /lightllm --no-deps + +WORKDIR /root diff --git a/docker/cuda_version_12.8.0/Dockerfile.nixl.deepep b/docker/cuda_version_12.8.0/Dockerfile.nixl.deepep index 96461dcc1b..83995a6a79 100644 --- a/docker/cuda_version_12.8.0/Dockerfile.nixl.deepep +++ b/docker/cuda_version_12.8.0/Dockerfile.nixl.deepep @@ -1,68 +1,100 @@ ARG CUDA_VERSION=12.8.0 FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu22.04 - ARG PYTHON_VERSION=3.10 -ARG MAMBA_VERSION=24.7.1-0 ARG TARGETPLATFORM +ARG DEEPEP_COMMIT=b6ce310bb0b75079682d09bc2ebc063a074fbd58 -ENV PATH=/opt/conda/bin:$PATH \ - CONDA_PREFIX=/opt/conda +ENV PATH=/root/.local/bin:/root/.cargo/bin:$PATH \ + UV_SYSTEM_PYTHON=1 \ + UV_COMPILE_BYTECODE=0 \ + UV_HTTP_TIMEOUT=300 \ + CUDA_HOME=/usr/local/cuda \ + GDRCOPY_HOME=/usr/src/gdrdrv-2.4.4/ +# Install system dependencies RUN chmod 777 -R /tmp && apt-get update --allow-insecure-repositories && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ ca-certificates \ libssl-dev \ curl \ g++ \ make \ - git && \ + git \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ + python${PYTHON_VERSION}-venv \ + python3-pip \ + libnuma-dev \ + wget \ + devscripts \ + debhelper \ + dh-make \ + build-essential \ + dkms \ + ibverbs-providers \ + infiniband-diags \ + perftest \ + rdma-core \ + libibverbs-dev \ + librdmacm-dev \ + cmake \ + automake \ + autotools-dev \ + libtool \ + libz-dev \ + pkg-config \ + tmux \ + net-tools \ + libaio-dev && \ rm -rf /var/lib/apt/lists/* -RUN case ${TARGETPLATFORM} in \ - "linux/arm64") MAMBA_ARCH=aarch64 ;; \ - *) MAMBA_ARCH=x86_64 ;; \ - esac && \ - curl -fsSL -o ~/mambaforge.sh -v "https://github.com/conda-forge/miniforge/releases/download/${MAMBA_VERSION}/Mambaforge-${MAMBA_VERSION}-Linux-${MAMBA_ARCH}.sh" && \ - bash ~/mambaforge.sh -b -p /opt/conda && \ - rm ~/mambaforge.sh - -RUN case ${TARGETPLATFORM} in \ - "linux/arm64") exit 1 ;; \ - *) /opt/conda/bin/conda update -y conda && \ - /opt/conda/bin/conda install -y "python=${PYTHON_VERSION}" ;; \ - esac && \ - /opt/conda/bin/conda clean -ya +# Reinstall RDMA packages +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + apt-get -y install --reinstall libibverbs-dev rdma-core ibverbs-utils libibumad-dev && \ + rm -rf /var/lib/apt/lists/* +# Set python alias +RUN update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1 && \ + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 -WORKDIR /root +# Install uv package manager +RUN curl -LsSf https://astral.sh/uv/install.sh | sh -RUN --mount=type=cache,target=/root/.cache/pip pip install vllm==0.11.0 --pre --extra-index-url https://wheels.vllm.ai/nightly +WORKDIR /lightllm -COPY ./requirements.txt /lightllm/requirements.txt -RUN --mount=type=cache,target=/root/.cache/pip pip install -r /lightllm/requirements.txt --extra-index-url https://download.pytorch.org/whl/cu128 +# Copy project files for dependency installation +COPY pyproject.toml README.md ./ +COPY lightllm/__init__.py lightllm/ -RUN --mount=type=cache,target=/root/.cache/pip pip install --no-deps -v 'git+https://github.com/ModelTC/LightKernel.git@07f2f62af5deb41f10a22660f9f42dba9273361e#egg=lightllm_kernel' +# Install dependencies using uv (faster than pip) +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e . --no-cache-dir --index-strategy unsafe-best-match --extra-index-url https://download.pytorch.org/whl/cu128 -RUN apt-get update && apt-get install -y libnuma-dev wget devscripts debhelper dh-make build-essential dkms -RUN apt-get install -y ibverbs-providers infiniband-diags perftest rdma-core libibverbs-dev librdmacm-dev +# Install vllm for additional kernels +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system vllm --pre --index-strategy unsafe-best-match --extra-index-url https://wheels.vllm.ai/nightly -ENV CUDA_HOME=/usr/local/cuda \ - GDRCOPY_HOME=/usr/src/gdrdrv-2.4.4/ +# Install LightKernel +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system --no-deps -v 'git+https://github.com/ModelTC/LightKernel.git@07f2f62af5deb41f10a22660f9f42dba9273361e#egg=lightllm_kernel' -RUN mkdir -p /tmp/gdrcopy && cd /tmp \ - && git clone https://github.com/NVIDIA/gdrcopy.git -b v2.4.4 \ - && cd gdrcopy/packages \ - && CUDA=/usr/local/cuda ./build-deb-packages.sh \ - && dpkg -i gdrdrv-dkms_*.deb libgdrapi_*.deb gdrcopy-tests_*.deb gdrcopy_*.deb \ - && cd / && rm -rf /tmp/gdrcopy +# Build and install gdrcopy +RUN mkdir -p /tmp/gdrcopy && cd /tmp && \ + git clone https://github.com/NVIDIA/gdrcopy.git -b v2.4.4 && \ + cd gdrcopy/packages && \ + CUDA=/usr/local/cuda ./build-deb-packages.sh && \ + dpkg -i gdrdrv-dkms_*.deb libgdrapi_*.deb gdrcopy-tests_*.deb gdrcopy_*.deb && \ + cd / && rm -rf /tmp/gdrcopy - # Fix DeepEP IBGDA symlink +# Fix DeepEP IBGDA symlink RUN ln -sf /usr/lib/x86_64-linux-gnu/libmlx5.so.1 /usr/lib/x86_64-linux-gnu/libmlx5.so -RUN wget https://developer.download.nvidia.com/compute/redist/nvshmem/3.3.9/source/nvshmem_src_cuda12-all-all-3.3.9.tar.gz \ - && tar -xf nvshmem_src_cuda12-all-all-3.3.9.tar.gz && mv nvshmem_src nvshmem \ - && cd nvshmem \ - && rm -f /root/nvshmem_src_cuda12-all-all-3.3.9.tar.gz \ - && NVSHMEM_SHMEM_SUPPORT=0 \ +# Build and install NVSHMEM +RUN wget https://developer.download.nvidia.com/compute/redist/nvshmem/3.3.9/source/nvshmem_src_cuda12-all-all-3.3.9.tar.gz && \ + tar -xf nvshmem_src_cuda12-all-all-3.3.9.tar.gz && \ + mv nvshmem_src nvshmem && \ + cd nvshmem && \ + rm -f /root/nvshmem_src_cuda12-all-all-3.3.9.tar.gz && \ + NVSHMEM_SHMEM_SUPPORT=0 \ NVSHMEM_UCX_SUPPORT=0 \ NVSHMEM_USE_NCCL=0 \ NVSHMEM_MPI_SUPPORT=0 \ @@ -70,44 +102,43 @@ RUN wget https://developer.download.nvidia.com/compute/redist/nvshmem/3.3.9/sour NVSHMEM_PMIX_SUPPORT=0 \ NVSHMEM_TIMEOUT_DEVICE_POLLING=0 \ NVSHMEM_USE_GDRCOPY=1 \ - cmake -S . -B build/ -DCMAKE_INSTALL_PREFIX=/root/nvshmem/install -DCMAKE_CUDA_ARCHITECTURES=90 \ - && cmake --build build --target install -j64 - -ARG DEEPEP_COMMIT=b6ce310bb0b75079682d09bc2ebc063a074fbd58 -RUN git clone https://github.com/deepseek-ai/DeepEP.git && cd DeepEP && git checkout ${DEEPEP_COMMIT} && cd .. + cmake -S . -B build/ -DCMAKE_INSTALL_PREFIX=/root/nvshmem/install -DCMAKE_CUDA_ARCHITECTURES=90 && \ + cmake --build build --target install -j$(nproc) -WORKDIR /root/DeepEP +# Build and install DeepEP ENV NVSHMEM_DIR=/root/nvshmem/install -RUN NVSHMEM_DIR=/root/nvshmem/install python setup.py install +RUN git clone https://github.com/deepseek-ai/DeepEP.git /root/DeepEP && \ + cd /root/DeepEP && \ + git checkout ${DEEPEP_COMMIT} && \ + NVSHMEM_DIR=/root/nvshmem/install python setup.py install -RUN apt-get update && apt-get install -y cmake automake autotools-dev libtool libz-dev && \ - DEBIAN_FRONTEND=noninteractive apt-get -y install --reinstall libibverbs-dev rdma-core ibverbs-utils libibumad-dev; \ - rm -rf /usr/lib/ucx && \ - rm -rf /opt/hpcx/ucx && \ +# Build and install UCX with CUDA and gdrcopy support +RUN rm -rf /usr/lib/ucx /opt/hpcx/ucx && \ cd /usr/local/src && \ git clone https://github.com/openucx/ucx.git && \ - cd ucx && \ - git checkout v1.19.x && \ - ./autogen.sh && ./configure \ - --enable-shared \ - --disable-static \ - --disable-doxygen-doc \ - --enable-optimizations \ - --enable-cma \ - --enable-devel-headers \ - --with-cuda=/usr/local/cuda \ - --with-verbs=yes \ - --with-dm \ - --with-gdrcopy=/usr/local \ - --with-efa \ - --enable-mt && \ - make -j && \ - make -j install-strip && \ - ldconfig; - -RUN apt-get update && apt-get install -y pkg-config tmux net-tools libaio-dev ; \ - cd /usr/local/src; \ - pip install --upgrade meson pybind11 patchelf; \ + cd ucx && \ + git checkout v1.19.x && \ + ./autogen.sh && ./configure \ + --enable-shared \ + --disable-static \ + --disable-doxygen-doc \ + --enable-optimizations \ + --enable-cma \ + --enable-devel-headers \ + --with-cuda=/usr/local/cuda \ + --with-verbs=yes \ + --with-dm \ + --with-gdrcopy=/usr/local \ + --with-efa \ + --enable-mt && \ + make -j$(nproc) && \ + make -j install-strip && \ + ldconfig && \ + rm -rf /usr/local/src/ucx + +# Build and install NixL +RUN uv pip install --system --upgrade meson pybind11 patchelf && \ + cd /usr/local/src && \ git clone https://github.com/ai-dynamo/nixl.git -b 0.8.0 && \ cd nixl && \ rm -rf build && \ @@ -116,7 +147,13 @@ RUN apt-get update && apt-get install -y pkg-config tmux net-tools libaio-dev ; cd build && \ ninja && \ ninja install && \ - cd .. && pip install . --no-deps; + cd .. && \ + uv pip install --system . --no-deps && \ + rm -rf /usr/local/src/nixl +# Copy full source code and install in editable mode COPY . /lightllm -RUN pip install -e /lightllm --no-cache-dir +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e /lightllm --no-deps + +WORKDIR /root diff --git a/docker/cuda_version_12.9.0/Dockerfile b/docker/cuda_version_12.9.0/Dockerfile new file mode 100644 index 0000000000..49566ce42f --- /dev/null +++ b/docker/cuda_version_12.9.0/Dockerfile @@ -0,0 +1,52 @@ +ARG CUDA_VERSION=12.9.0 +FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu22.04 +ARG PYTHON_VERSION=3.10 +ARG TARGETPLATFORM + +ENV PATH=/root/.local/bin:/root/.cargo/bin:$PATH \ + UV_SYSTEM_PYTHON=1 \ + UV_COMPILE_BYTECODE=0 \ + UV_HTTP_TIMEOUT=300 + +# Install system dependencies +RUN chmod 777 -R /tmp && apt-get update --allow-insecure-repositories && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates \ + libssl-dev \ + curl \ + g++ \ + make \ + git \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ + python${PYTHON_VERSION}-venv \ + python3-pip \ + libnuma-dev && \ + rm -rf /var/lib/apt/lists/* + +# Set python alias +RUN update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1 && \ + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 + +# Install uv package manager +RUN curl -LsSf https://astral.sh/uv/install.sh | sh + +WORKDIR /lightllm + +# Copy project files for dependency installation +COPY pyproject.toml README.md ./ +COPY lightllm/__init__.py lightllm/ + +# Install dependencies using uv (faster than pip) +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e . --no-cache-dir --index-strategy unsafe-best-match --extra-index-url https://download.pytorch.org/whl/cu129 + +# Install vllm for additional kernels +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system vllm --pre --index-strategy unsafe-best-match --extra-index-url https://wheels.vllm.ai/nightly + +# Copy full source code and install in editable mode +COPY . . +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e . --no-deps + +WORKDIR /root diff --git a/docker/cuda_version_12.9.0/Dockerfile.deepep b/docker/cuda_version_12.9.0/Dockerfile.deepep new file mode 100644 index 0000000000..442fcac802 --- /dev/null +++ b/docker/cuda_version_12.9.0/Dockerfile.deepep @@ -0,0 +1,103 @@ +ARG CUDA_VERSION=12.9.0 +FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu22.04 +ARG PYTHON_VERSION=3.10 +ARG TARGETPLATFORM +ARG DEEPEP_COMMIT=b6ce310bb0b75079682d09bc2ebc063a074fbd58 + +ENV PATH=/root/.local/bin:/root/.cargo/bin:$PATH \ + UV_SYSTEM_PYTHON=1 \ + UV_COMPILE_BYTECODE=0 \ + UV_HTTP_TIMEOUT=300 \ + CUDA_HOME=/usr/local/cuda \ + GDRCOPY_HOME=/usr/src/gdrdrv-2.4.4/ + +# Install system dependencies +RUN chmod 777 -R /tmp && apt-get update --allow-insecure-repositories && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates \ + libssl-dev \ + curl \ + g++ \ + make \ + git \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ + python${PYTHON_VERSION}-venv \ + python3-pip \ + libnuma-dev \ + wget \ + devscripts \ + debhelper \ + dh-make \ + build-essential \ + dkms \ + ibverbs-providers \ + infiniband-diags \ + perftest \ + rdma-core \ + libibverbs-dev \ + librdmacm-dev \ + cmake && \ + rm -rf /var/lib/apt/lists/* + +# Set python alias +RUN update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1 && \ + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 + +# Install uv package manager +RUN curl -LsSf https://astral.sh/uv/install.sh | sh + +WORKDIR /lightllm + +# Copy project files for dependency installation +COPY pyproject.toml README.md ./ +COPY lightllm/__init__.py lightllm/ + +# Install dependencies using uv (faster than pip) +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e . --no-cache-dir --index-strategy unsafe-best-match --extra-index-url https://download.pytorch.org/whl/cu129 + +# Install vllm for additional kernels +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system vllm --pre --index-strategy unsafe-best-match --extra-index-url https://wheels.vllm.ai/nightly + +# Build and install gdrcopy +RUN mkdir -p /tmp/gdrcopy && cd /tmp && \ + git clone https://github.com/NVIDIA/gdrcopy.git -b v2.4.4 && \ + cd gdrcopy/packages && \ + CUDA=/usr/local/cuda ./build-deb-packages.sh && \ + dpkg -i gdrdrv-dkms_*.deb libgdrapi_*.deb gdrcopy-tests_*.deb gdrcopy_*.deb && \ + cd / && rm -rf /tmp/gdrcopy + +# Fix DeepEP IBGDA symlink +RUN ln -sf /usr/lib/x86_64-linux-gnu/libmlx5.so.1 /usr/lib/x86_64-linux-gnu/libmlx5.so + +# Build and install NVSHMEM +RUN wget https://developer.download.nvidia.com/compute/redist/nvshmem/3.3.9/source/nvshmem_src_cuda12-all-all-3.3.9.tar.gz && \ + tar -xf nvshmem_src_cuda12-all-all-3.3.9.tar.gz && \ + mv nvshmem_src nvshmem && \ + cd nvshmem && \ + rm -f /root/nvshmem_src_cuda12-all-all-3.3.9.tar.gz && \ + NVSHMEM_SHMEM_SUPPORT=0 \ + NVSHMEM_UCX_SUPPORT=0 \ + NVSHMEM_USE_NCCL=0 \ + NVSHMEM_MPI_SUPPORT=0 \ + NVSHMEM_IBGDA_SUPPORT=1 \ + NVSHMEM_PMIX_SUPPORT=0 \ + NVSHMEM_TIMEOUT_DEVICE_POLLING=0 \ + NVSHMEM_USE_GDRCOPY=1 \ + cmake -S . -B build/ -DCMAKE_INSTALL_PREFIX=/root/nvshmem/install -DCMAKE_CUDA_ARCHITECTURES=90 && \ + cmake --build build --target install -j$(nproc) + +# Build and install DeepEP +ENV NVSHMEM_DIR=/root/nvshmem/install +RUN git clone https://github.com/deepseek-ai/DeepEP.git /root/DeepEP && \ + cd /root/DeepEP && \ + git checkout ${DEEPEP_COMMIT} && \ + NVSHMEM_DIR=/root/nvshmem/install python setup.py install + +# Copy full source code and install in editable mode +COPY . /lightllm +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e /lightllm --no-deps + +WORKDIR /root diff --git a/docker/cuda_version_12.9.0/Dockerfile.nixl b/docker/cuda_version_12.9.0/Dockerfile.nixl new file mode 100644 index 0000000000..f0cf983504 --- /dev/null +++ b/docker/cuda_version_12.9.0/Dockerfile.nixl @@ -0,0 +1,131 @@ +ARG CUDA_VERSION=12.9.0 +FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu22.04 +ARG PYTHON_VERSION=3.10 +ARG TARGETPLATFORM + +ENV PATH=/root/.local/bin:/root/.cargo/bin:$PATH \ + UV_SYSTEM_PYTHON=1 \ + UV_COMPILE_BYTECODE=0 \ + UV_HTTP_TIMEOUT=300 \ + CUDA_HOME=/usr/local/cuda \ + GDRCOPY_HOME=/usr/src/gdrdrv-2.4.4/ + +# Install system dependencies +RUN chmod 777 -R /tmp && apt-get update --allow-insecure-repositories && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates \ + libssl-dev \ + curl \ + g++ \ + make \ + git \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ + python${PYTHON_VERSION}-venv \ + python3-pip \ + libnuma-dev \ + wget \ + devscripts \ + debhelper \ + dh-make \ + build-essential \ + dkms \ + ibverbs-providers \ + infiniband-diags \ + perftest \ + rdma-core \ + libibverbs-dev \ + librdmacm-dev \ + cmake \ + automake \ + autotools-dev \ + libtool \ + libz-dev \ + pkg-config \ + tmux \ + net-tools \ + libaio-dev && \ + rm -rf /var/lib/apt/lists/* + +# Reinstall RDMA packages +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + apt-get -y install --reinstall libibverbs-dev rdma-core ibverbs-utils libibumad-dev && \ + rm -rf /var/lib/apt/lists/* + +# Set python alias +RUN update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1 && \ + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 + +# Install uv package manager +RUN curl -LsSf https://astral.sh/uv/install.sh | sh + +WORKDIR /lightllm + +# Copy project files for dependency installation +COPY pyproject.toml README.md ./ +COPY lightllm/__init__.py lightllm/ + +# Install dependencies using uv (faster than pip) +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e . --no-cache-dir --index-strategy unsafe-best-match --extra-index-url https://download.pytorch.org/whl/cu129 + +# Install vllm for additional kernels +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system vllm --pre --index-strategy unsafe-best-match --extra-index-url https://wheels.vllm.ai/nightly + +# Install LightKernel +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system --no-deps -v 'git+https://github.com/ModelTC/LightKernel.git#egg=lightllm_kernel' + +# Build and install gdrcopy +RUN mkdir -p /tmp/gdrcopy && cd /tmp && \ + git clone https://github.com/NVIDIA/gdrcopy.git -b v2.4.4 && \ + cd gdrcopy/packages && \ + CUDA=/usr/local/cuda ./build-deb-packages.sh && \ + dpkg -i gdrdrv-dkms_*.deb libgdrapi_*.deb gdrcopy-tests_*.deb gdrcopy_*.deb && \ + cd / && rm -rf /tmp/gdrcopy + +# Build and install UCX with CUDA and gdrcopy support +RUN rm -rf /usr/lib/ucx /opt/hpcx/ucx && \ + cd /usr/local/src && \ + git clone https://github.com/openucx/ucx.git && \ + cd ucx && \ + git checkout v1.19.x && \ + ./autogen.sh && ./configure \ + --enable-shared \ + --disable-static \ + --disable-doxygen-doc \ + --enable-optimizations \ + --enable-cma \ + --enable-devel-headers \ + --with-cuda=/usr/local/cuda \ + --with-verbs=yes \ + --with-dm \ + --with-gdrcopy=/usr/local \ + --with-efa \ + --enable-mt && \ + make -j$(nproc) && \ + make -j install-strip && \ + ldconfig && \ + rm -rf /usr/local/src/ucx + +# Build and install NixL +RUN uv pip install --system --upgrade meson pybind11 patchelf && \ + cd /usr/local/src && \ + git clone https://github.com/ai-dynamo/nixl.git -b main && \ + cd nixl && \ + rm -rf build && \ + mkdir build && \ + meson setup build/ --prefix=/usr/local/nixl --buildtype=release && \ + cd build && \ + ninja && \ + ninja install && \ + cd .. && \ + uv pip install --system . --no-deps && \ + rm -rf /usr/local/src/nixl + +# Copy full source code and install in editable mode +COPY . /lightllm +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e /lightllm --no-deps + +WORKDIR /root diff --git a/docker/cuda_version_12.9.0/Dockerfile.nixl.deepep b/docker/cuda_version_12.9.0/Dockerfile.nixl.deepep new file mode 100644 index 0000000000..4bc8e2f47f --- /dev/null +++ b/docker/cuda_version_12.9.0/Dockerfile.nixl.deepep @@ -0,0 +1,159 @@ +ARG CUDA_VERSION=12.9.0 +FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu22.04 +ARG PYTHON_VERSION=3.10 +ARG TARGETPLATFORM +ARG DEEPEP_COMMIT=b6ce310bb0b75079682d09bc2ebc063a074fbd58 + +ENV PATH=/root/.local/bin:/root/.cargo/bin:$PATH \ + UV_SYSTEM_PYTHON=1 \ + UV_COMPILE_BYTECODE=0 \ + UV_HTTP_TIMEOUT=300 \ + CUDA_HOME=/usr/local/cuda \ + GDRCOPY_HOME=/usr/src/gdrdrv-2.4.4/ + +# Install system dependencies +RUN chmod 777 -R /tmp && apt-get update --allow-insecure-repositories && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates \ + libssl-dev \ + curl \ + g++ \ + make \ + git \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ + python${PYTHON_VERSION}-venv \ + python3-pip \ + libnuma-dev \ + wget \ + devscripts \ + debhelper \ + dh-make \ + build-essential \ + dkms \ + ibverbs-providers \ + infiniband-diags \ + perftest \ + rdma-core \ + libibverbs-dev \ + librdmacm-dev \ + cmake \ + automake \ + autotools-dev \ + libtool \ + libz-dev \ + pkg-config \ + tmux \ + net-tools \ + libaio-dev && \ + rm -rf /var/lib/apt/lists/* + +# Reinstall RDMA packages +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + apt-get -y install --reinstall libibverbs-dev rdma-core ibverbs-utils libibumad-dev && \ + rm -rf /var/lib/apt/lists/* + +# Set python alias +RUN update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1 && \ + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 + +# Install uv package manager +RUN curl -LsSf https://astral.sh/uv/install.sh | sh + +WORKDIR /lightllm + +# Copy project files for dependency installation +COPY pyproject.toml README.md ./ +COPY lightllm/__init__.py lightllm/ + +# Install dependencies using uv (faster than pip) +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e . --no-cache-dir --index-strategy unsafe-best-match --extra-index-url https://download.pytorch.org/whl/cu129 + +# Install vllm for additional kernels +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system vllm --pre --index-strategy unsafe-best-match --extra-index-url https://wheels.vllm.ai/nightly + +# Install LightKernel +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system --no-deps -v 'git+https://github.com/ModelTC/LightKernel.git#egg=lightllm_kernel' + +# Build and install gdrcopy +RUN mkdir -p /tmp/gdrcopy && cd /tmp && \ + git clone https://github.com/NVIDIA/gdrcopy.git -b v2.4.4 && \ + cd gdrcopy/packages && \ + CUDA=/usr/local/cuda ./build-deb-packages.sh && \ + dpkg -i gdrdrv-dkms_*.deb libgdrapi_*.deb gdrcopy-tests_*.deb gdrcopy_*.deb && \ + cd / && rm -rf /tmp/gdrcopy + +# Fix DeepEP IBGDA symlink +RUN ln -sf /usr/lib/x86_64-linux-gnu/libmlx5.so.1 /usr/lib/x86_64-linux-gnu/libmlx5.so + +# Build and install NVSHMEM +RUN wget https://developer.download.nvidia.com/compute/redist/nvshmem/3.3.9/source/nvshmem_src_cuda12-all-all-3.3.9.tar.gz && \ + tar -xf nvshmem_src_cuda12-all-all-3.3.9.tar.gz && \ + mv nvshmem_src nvshmem && \ + cd nvshmem && \ + rm -f /root/nvshmem_src_cuda12-all-all-3.3.9.tar.gz && \ + NVSHMEM_SHMEM_SUPPORT=0 \ + NVSHMEM_UCX_SUPPORT=0 \ + NVSHMEM_USE_NCCL=0 \ + NVSHMEM_MPI_SUPPORT=0 \ + NVSHMEM_IBGDA_SUPPORT=1 \ + NVSHMEM_PMIX_SUPPORT=0 \ + NVSHMEM_TIMEOUT_DEVICE_POLLING=0 \ + NVSHMEM_USE_GDRCOPY=1 \ + cmake -S . -B build/ -DCMAKE_INSTALL_PREFIX=/root/nvshmem/install -DCMAKE_CUDA_ARCHITECTURES=90 && \ + cmake --build build --target install -j$(nproc) + +# Build and install DeepEP +ENV NVSHMEM_DIR=/root/nvshmem/install +RUN git clone https://github.com/deepseek-ai/DeepEP.git /root/DeepEP && \ + cd /root/DeepEP && \ + git checkout ${DEEPEP_COMMIT} && \ + NVSHMEM_DIR=/root/nvshmem/install python setup.py install + +# Build and install UCX with CUDA and gdrcopy support +RUN rm -rf /usr/lib/ucx /opt/hpcx/ucx && \ + cd /usr/local/src && \ + git clone https://github.com/openucx/ucx.git && \ + cd ucx && \ + git checkout v1.19.x && \ + ./autogen.sh && ./configure \ + --enable-shared \ + --disable-static \ + --disable-doxygen-doc \ + --enable-optimizations \ + --enable-cma \ + --enable-devel-headers \ + --with-cuda=/usr/local/cuda \ + --with-verbs=yes \ + --with-dm \ + --with-gdrcopy=/usr/local \ + --with-efa \ + --enable-mt && \ + make -j$(nproc) && \ + make -j install-strip && \ + ldconfig && \ + rm -rf /usr/local/src/ucx + +# Build and install NixL +RUN uv pip install --system --upgrade meson pybind11 patchelf && \ + cd /usr/local/src && \ + git clone https://github.com/ai-dynamo/nixl.git -b main && \ + cd nixl && \ + rm -rf build && \ + mkdir build && \ + meson setup build/ --prefix=/usr/local/nixl --buildtype=release && \ + cd build && \ + ninja && \ + ninja install && \ + cd .. && \ + uv pip install --system . --no-deps && \ + rm -rf /usr/local/src/nixl + +# Copy full source code and install in editable mode +COPY . /lightllm +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -e /lightllm --no-deps + +WORKDIR /root diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..64b3ce24d5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,201 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "lightllm" +version = "1.1.0" +description = "LightLLM - A lightweight, high-performance LLM inference and serving framework" +readme = "README.md" +license = { text = "Apache-2.0" } +requires-python = ">=3.9.16" +authors = [ + { name = "model toolchain" } +] +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Operating System :: POSIX :: Linux", + "License :: OSI Approved :: Apache Software License", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] +keywords = ["llm", "inference", "serving", "deep-learning", "pytorch", "triton"] + +# Full dependencies (default installation includes all features) +# Versions pinned to tested Docker environment +dependencies = [ + # Communication & async + "pyzmq==27.1.0", + "uvloop==0.22.1", + "rpyc==6.0.2", + + # ML frameworks + "torch==2.9.1", + "triton==3.5.1", + "transformers>=4.50.0", + "safetensors==0.7.0", + + # Numeric & data processing + "numpy==2.2.6", + "einops==0.8.1", + + # Web server + "fastapi==0.128.0", + "uvicorn==0.40.0", + "pydantic==2.12.5", + "httpx==0.28.1", + "ujson==5.11.0", + "gunicorn==23.0.0", + "websockets==16.0", + + # Utilities + "packaging==25.0", + "ninja==1.13.0", + "orjson==3.11.5", + "tqdm==4.67.1", + "requests==2.32.5", + "filelock==3.20.2", + "fsspec==2025.10.0", + "Pillow==12.1.0", + "psutil==7.2.1", + "prometheus_client==0.23.1", + "setproctitle==1.3.7", + "PyYAML==6.0.3", + "easydict==1.13", + + # Core server components + "sortedcontainers==2.4.0", + "atomics==1.0.3", + "frozendict==2.4.7", + "xxhash==3.6.0", + + # Tokenizers + "sentencepiece==0.2.1", + "tiktoken==0.12.0", + + # Multimodal support + "torchvision==0.24.1", + "librosa==0.11.0", + "numba==0.61.2", + "matplotlib==3.10.8", + + # CUDA optimizations + "flashinfer-python==0.6.1", + "sgl-kernel==0.3.21", + "cupy-cuda12x==13.6.0", + "cuda-bindings==12.9.5", + "torch-c-dlpack-ext==0.1.5", + + # Constrained decoding + "interegular==0.3.3", + "partial-json-parser==0.2.1.1.post7", +] + +[project.optional-dependencies] +# Minimal installation (core functionality only, no CUDA optimizations or multimodal) +minimal = [ + "pyzmq==27.1.0", + "uvloop==0.22.1", + "rpyc==6.0.2", + "torch==2.9.1", + "triton==3.5.1", + "transformers>=4.50.0", + "safetensors==0.7.0", + "numpy==2.2.6", + "einops==0.8.1", + "fastapi==0.128.0", + "uvicorn==0.40.0", + "pydantic==2.12.5", + "httpx==0.28.1", + "ujson==5.11.0", + "packaging==25.0", + "ninja==1.13.0", + "orjson==3.11.5", + "tqdm==4.67.1", + "requests==2.32.5", + "filelock==3.20.2", + "fsspec==2025.10.0", + "Pillow==12.1.0", + "psutil==7.2.1", + "prometheus_client==0.23.1", + "setproctitle==1.3.7", + "PyYAML==6.0.3", + "sortedcontainers==2.4.0", + "atomics==1.0.3", + "frozendict==2.4.7", + "xxhash==3.6.0", +] + +# Development dependencies +dev = [ + "pytest>=7.0.0", + "pytest-cov>=4.0.0", + "black>=23.0.0", + "flake8>=6.0.0", + "pre-commit>=3.0.0", + "mypy>=1.0.0", +] + +# Documentation dependencies +docs = [ + "sphinx>=7.0.0", + "sphinx-rtd-theme>=2.0.0", + "myst-parser>=2.0.0", +] + +# DeepSeek optimizations (requires manual installation due to build complexity) +# Install: uv pip install ".[deepseek]" --no-build-isolation +deepseek = [ + "deep-gemm @ git+https://github.com/deepseek-ai/DeepGEMM.git@v2.1.1.post3", +] + +[project.urls] +Homepage = "https://github.com/ModelTC/lightllm" +Documentation = "https://lightllm-docs.readthedocs.io/" +Repository = "https://github.com/ModelTC/lightllm" +Issues = "https://github.com/ModelTC/lightllm/issues" + +[tool.setuptools] +packages = { find = { exclude = ["build", "include", "test", "dist", "docs", "benchmarks", "lightllm.egg-info", "unit_tests"] } } + +[tool.setuptools.package-data] +lightllm = [ + "common/all_kernel_configs/*/*.json", + "common/triton_utils/*/*/*/*/*.json", +] + +[tool.black] +line-length = 120 +target-version = ["py39", "py310", "py311", "py312"] +exclude = ''' +/( + \.git + | \.venv + | build + | dist + | \.eggs +)/ +''' + +[tool.flake8] +max-line-length = 120 +extend-ignore = ["E203", "E501", "W503"] +exclude = [".git", "__pycache__", "build", "dist", ".eggs", "*.egg-info"] + +[tool.pytest.ini_options] +testpaths = ["test", "unit_tests"] +python_files = ["test_*.py", "*_test.py"] +addopts = "-v --tb=short" + +[tool.mypy] +python_version = "3.9" +warn_return_any = true +warn_unused_configs = true +ignore_missing_imports = true + +[tool.uv] +# Disable build isolation for packages that need torch during build +no-build-isolation-package = ["deep-gemm"]