Branch of nemotron using script submitter to run jobs

singularity/Singularity.vllm

@@ -1,11 +1,52 @@
 Bootstrap: docker
 From: vllm/vllm-openai:latest
 
+%post
+    set -eux
+
+    apt-get update
+    apt-get install -y --no-install-recommends \
+        clang lld llvm \
+        build-essential \
+        git ca-certificates curl \
+        pkg-config cmake ninja-build
+    rm -rf /var/lib/apt/lists/*
+
+    # Pick a Python interpreter that actually exists in the base image
+    if command -v python >/dev/null 2>&1; then
+        PY=python
+    elif command -v python3 >/dev/null 2>&1; then
+        PY=python3
+    elif [ -x /opt/conda/bin/python ]; then
+        PY=/opt/conda/bin/python
+    else
+        echo "No Python interpreter found (python/python3/conda)."; exit 1
+    fi
+
+    # Prefer clang for any builds happening inside the container
+    echo "export CC=clang"  >> /etc/profile.d/clang.sh
+    echo "export CXX=clang++" >> /etc/profile.d/clang.sh
+
+    # Upgrade packaging tooling
+    $PY -m pip install -U pip setuptools wheel
+
+    # Ensure Transformers has the ministral3 config mapping
+    $PY -m pip uninstall -y transformers || true
+    $PY -m pip install -U git+https://github.com/huggingface/transformers
+
+    $PY - << 'PY'
+import transformers
+from transformers.models.auto import CONFIG_MAPPING
+print("Transformers:", transformers.__version__)
+print("ministral3 in CONFIG_MAPPING:", "ministral3" in CONFIG_MAPPING)
+PY
+
 %runscript
-mkdir -p /app
-cd /app
-exec /bin/bash -lc "$@"
+    mkdir -p /app
+    cd /app
+    exec /bin/bash -lc "$@"
+
 %startscript
-mkdir -p /app
-cd /app
-exec /bin/bash -lc "$@"
+    mkdir -p /app
+    cd /app
+    exec /bin/bash -lc "$@"

singularity/env.sh.example

@@ -23,7 +23,7 @@ export HF_HOME="/root/.cache/huggingface"
 
 # Recommended on T4/V100 and for mistral tokenizer
 export DOCS_DIR=./docs
-export VLLM_ATTENTION_BACKEND=TRITON_ATTN_VLLM_V1
+export VLLM_ATTENTION_BACKEND=TRITON_ATTN
 export VLLM_EXTRA_ARGS="__VLLM_EXTRA_ARGS__"
 export TRITON_CC=gcc
 export CC=/usr/bin/gcc
@@ -32,7 +32,7 @@ export CXX=/usr/bin/g++
 export TMPDIR=${PWD}/tmp
 export CUDA_CACHE_PATH=${TMPDIR}/cuda_cache
 export TORCH_EXTENSIONS_DIR=${TMPDIR}/torch_extensions
-export FLASHINFER_JIT_DIR=${TMPDIR}/flashinfer_jitß
+export FLASHINFER_JIT_DIR=${TMPDIR}/flashinfer_jit
 
 # Other VLLM tuning settings
 export VLLM_LOGGING_LEVEL=INFO

singularity/singularity-compose.yml

@@ -20,6 +20,7 @@ instances:
     volumes:
       - ./logs:/logs
       - ./cache:/root/.cache
+      - ./cache/sagemaker_sessions:/dev/shm/sagemaker_sessions
       - ./env.sh:/.singularity.d/env/env.sh
       - __MODEL_PATH__:/__MODEL_BASE__
 

start_service.sh

@@ -76,6 +76,12 @@ start_rootless_docker() {
 echo '#!/bin/bash' > cancel.sh
 chmod +x cancel.sh
 
+
+# Indicates job started running
+echo "touch job.started" >> run.sh
+echo "hostname > HOSTNAME" >> run.sh
+
+
 if [ "$RUNMODE" == "docker" ];then
 
     # Ensure docker service is installed
@@ -185,6 +191,16 @@ if [ "$RUNMODE" == "docker" ];then
 
 elif [ "$RUNMODE" == "singularity" ]; then
 
+    # load singularity via module if not already in path
+    # Ensure Singularity is available
+    if ! command -v singularity >/dev/null 2>&1; then
+        if command -v module >/dev/null 2>&1; then
+            module load singularity || module load apptainer
+        else
+            echo "ERROR: singularity/apptainer not found" >&2
+        fi
+    fi
+
     # Check if singularity is installed
     if ! command -v singularity >/dev/null 2>&1; then
         echo "$(date) ERROR: singularity is not installed"
@@ -203,7 +219,7 @@ elif [ "$RUNMODE" == "singularity" ]; then
 
     cp singularity/* ./ -Rf
     cp env.sh.example env.sh
-    
+
     VLLM_SERVER_PORT=$(findAvailablePort)
     RAG_PORT=$(findAvailablePort)
     PROXY_PORT=$(findAvailablePort)
@@ -245,6 +261,13 @@ elif [ "$RUNMODE" == "singularity" ]; then
 
     mkdir -p logs cache cache/chroma $DOCS_DIR
 
+    # fixing updated vllm sagemarker sessions issue
+    mkdir -p cache/sagemaker_sessions
+    chmod 700 cache/sagemaker_sessions
+
+    mkdir -p /dev/shm/sagemaker_sessions
+    chmod 700 /dev/shm/sagemaker_sessions
+
     # singularity-compose does not support env variables in the yml config file
     if [ "$DOCS_DIR" != "./docs" ];then
         ln -s $DOCS_DIR ./docs
@@ -263,7 +286,18 @@ elif [ "$RUNMODE" == "singularity" ]; then
         [ "$BUILD" = "true" ] && singularity-compose build "${RUNTYPE}1"
         singularity-compose up "${RUNTYPE}1"
     fi
-    # Follow the logs
-    tail -f logs/*
+
+    # Only follow logs if up succeeded
+    # Make tail die when this script dies (and don't explode if logs don't exist yet)
+    shopt -s nullglob
+    logs=(logs/*)
+    if ((${#logs[@]} > 0)); then
+        tail -F "${logs[@]}" &
+        tail_pid=$!
+        trap 'kill "$tail_pid" >/dev/null 2>&1 || true; cleanup' EXIT
+        wait "$tail_pid"
+    else
+        echo "No logs found under logs/. Skipping tail."
+    fi
 
 fi