Merge remote-tracking branch 'main' into feat/1635/decouple-nitro-inference-engine-into-a-library

Author: InNoobWeTrust

.github/scripts/e2e-test-llama-windows.bat

@@ -25,10 +25,6 @@ set /a max=11000
 set /a range=max-min+1
 set /a PORT=%min% + %RANDOM% %% %range%
 
-rem Kill any existing Nitro processes
-echo Killing any existing Nitro processes...
-taskkill /f /im nitro.exe 2>nul
-
 rem Start the binary file
 start /B "" "%BINARY_PATH%" 1 "127.0.0.1" %PORT% > %TEMP%\nitro.log 2>&1
 
@@ -64,9 +60,9 @@ echo curl_data1=%curl_data1%
 echo curl_data2=%curl_data2%
 
 rem Run the curl commands and capture the status code
-curl.exe -o "%TEMP%\response1.log" -s -w "%%{http_code}" --location "http://127.0.0.1:%PORT%/inferences/llamacpp/loadModel" --header "Content-Type: application/json" --data "%curl_data1%" > %TEMP%\response1.log 2>&1
+curl.exe --connect-timeout 60 -o "%TEMP%\response1.log" -s -w "%%{http_code}" --location "http://127.0.0.1:%PORT%/inferences/llamacpp/loadModel" --header "Content-Type: application/json" --data "%curl_data1%" > %TEMP%\response1.log 2>&1
 
-curl.exe -o "%TEMP%\response2.log" -s -w "%%{http_code}" --location "http://127.0.0.1:%PORT%/inferences/llamacpp/chat_completion" ^
+curl.exe --connect-timeout 60 -o "%TEMP%\response2.log" -s -w "%%{http_code}" --location "http://127.0.0.1:%PORT%/inferences/llamacpp/chat_completion" ^
 --header "Content-Type: application/json" ^
 --header "Accept: text/event-stream" ^
 --header "Access-Control-Allow-Origin: *" ^
@@ -78,13 +74,13 @@ rem Read the status codes from the log files
 for /f %%a in (%TEMP%\response1.log) do set "response1=%%a"
 for /f %%a in (%TEMP%\response2.log) do set "response2=%%a"
 
-if "%response1%" neq "000" (
+if "%response1%" neq "200" (
     echo The first curl command failed with status code: %response1%
     type %TEMP%\response1.log
     set "error_occurred=1"
 )
 
-if "%response2%" neq "000" (
+if "%response2%" neq "200" (
     echo The second curl command failed with status code: %response2%
     type %TEMP%\response2.log
     set "error_occurred=1"
@@ -111,4 +107,4 @@ echo Nitro test run successfully!
 
 rem Kill the server process
 @REM taskkill /f /pid %pid%
-taskkill /f /im nitro.exe 2>nul || exit /B 0
+taskkill /f /im nitro.exe 2>nul || exit /B 0

.github/scripts/e2e-test-whisper-windows.bat

@@ -25,10 +25,6 @@ set /a max=11000
 set /a range=max-min+1
 set /a PORT=%min% + %RANDOM% %% %range%
 
-rem Kill any existing Nitro processes
-echo Killing any existing Nitro processes...
-taskkill /f /im nitro.exe 2>nul
-
 rem Start the binary file
 start /B "" "%BINARY_PATH%" 1 "127.0.0.1" %PORT% > %TEMP%\nitro.log 2>&1
 
@@ -49,28 +45,21 @@ if not defined pid (
 
 rem Wait for a few seconds to let the server start
 
-rem Check if %TEMP%\testmodel exists, if not, download it
+rem Check if %TEMP%\testwhisper exists, if not, download it
 if not exist "%MODEL_PATH%" (
     bitsadmin.exe /transfer "DownloadTestModel" %DOWNLOAD_URL% "%MODEL_PATH%"
 )
 
 rem Define JSON strings for curl data
 call set "MODEL_PATH_STRING=%%MODEL_PATH:\=\\%%"
-set "curl_data1={\"model_path\":\"%MODEL_PATH_STRING%\",\"model_id\":\"whisper.cpp\"}"
-
-rem Print the values of curl_data1 for debugging
-echo curl_data1=%curl_data1%
+set "curl_data1={\"model_path\":\"%MODEL_PATH_STRING%\",\"model_id\":\"whisper\"}"
 
 rem Run the curl commands and capture the status code
-curl.exe -o %TEMP%\response1_code.log -s -w "%%{http_code}" --location "http://127.0.0.1:%PORT%/v1/audio/load_model" --header "Content-Type: application/json" --data "%curl_data1%" > %TEMP%\response1_code.log 2>&1
+curl.exe -o %TEMP%\response1.log -s -w "%%{http_code}" --location "http://127.0.0.1:%PORT%/v1/audio/load_model" --header "Content-Type: application/json" --data "%curl_data1%" > %TEMP%\response1_code.log 2>&1
 
-curl.exe -o %TEMP%\response2_code.log -s -w "%%{http_code}" --location "http://127.0.0.1:%PORT%/v1/audio/transcriptions" ^
---header "Access-Control-Allow-Origin: *" ^
---form 'model_id="whisper.cpp"' ^
---form 'file=@"..\whisper.cpp\samples\jfk.wav"' ^
---form 'temperature="0.0"' ^
---form 'prompt="The transcript is about OpenAI which makes technology like DALL·E, GPT-3, and ChatGPT with the hope of one day building an AGI system that benefits all of humanity. The president is trying to raly people to support the cause."' ^
-> %TEMP%\response2_code.log 2>&1
+curl -o %TEMP%\response2.log -s -w "%%{http_code}" --location "http://localhost:%PORT%/v1/audio/transcriptions" ^
+--form "file=@../..//whisper.cpp/samples/jfk.wav" ^
+--form "model_id=whisper" > %TEMP%\response2_code.log 2>&1
 
 set "error_occurred=0"
 
@@ -80,13 +69,13 @@ for /f %%a in (%TEMP%\response2_code.log) do set "response2=%%a"
 
 if "%response1%" neq "200" (
     echo The first curl command failed with status code: %response1%
-    type %TEMP%\response1_code.log
+    type %TEMP%\response1.log
     set "error_occurred=1"
 )
 
-if "%response2%" neq "000" (
+if "%response2%" neq "200" (
     echo The second curl command failed with status code: %response2%
-    type %TEMP%\response2_code.log
+    type %TEMP%\response2.log
     set "error_occurred=1"
 )
 
@@ -101,14 +90,13 @@ if "%error_occurred%"=="1" (
 
 echo ----------------------
 echo Log load model:
-type %TEMP%\response1_code.log
+type %TEMP%\response1.log
 
 echo ----------------------
 echo "Log run test:"
-type %TEMP%\response2_code.log
+type %TEMP%\response2.log
 
 echo Nitro test run successfully!
 
 rem Kill the server process
-@REM taskkill /f /pid %pid%
 taskkill /f /im nitro.exe 2>nul || exit /B 0

.github/workflows/build.yml

@@ -50,7 +50,7 @@ on:
 env:
   BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
   LLM_MODEL_URL: https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v0.3-GGUF/resolve/main/tinyllama-1.1b-chat-v0.3.Q2_K.gguf
-  WHISPER_MODEL_URL: https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.bin
+  WHISPER_MODEL_URL: https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny-q5_1.bin
 
 jobs:
   create-draft-release:
@@ -377,11 +377,11 @@ jobs:
         run: |
           # To test with CoreML
           if [[ ! -f "/tmp/testwhisper-encoder.mlmodelc" ]]; then
-            wget https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en-encoder.mlmodelc.zip
-            unzip ggml-tiny.en-encoder.mlmodelc.zip
-            rm ggml-tiny.en-encoder.mlmodelc.zip
+            wget https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny-encoder.mlmodelc.zip
+            unzip ggml-tiny-encoder.mlmodelc.zip
+            rm ggml-tiny-encoder.mlmodelc.zip
             rm -rf /tmp/testwhisper-encoder.mlmodelc
-            mv ggml-tiny.en-encoder.mlmodelc /tmp/testwhisper-encoder.mlmodelc
+            mv ggml-tiny-encoder.mlmodelc /tmp/testwhisper-encoder.mlmodelc
           fi
           # run e2e testing
           cd nitro
@@ -586,7 +586,7 @@ jobs:
           cmake --build ./build_deps/nitro_deps --config Release
           mkdir -p build
           cd build
-          cmake .. -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_BLAS=ON -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=RELEASE -DWHISPER_SDL2=ON -DNITRO_VERSION=${{ needs.set-nitro-version.outputs.version }}
+          cmake .. -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_BLAS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=RELEASE -DWHISPER_SDL2=ON -DNITRO_VERSION=${{ needs.set-nitro-version.outputs.version }}
           cmake --build . --config Release -j "%NUMBER_OF_PROCESSORS%"
 
       - name: Pack artifacts
@@ -602,15 +602,19 @@ jobs:
           7z a -ttar temp.tar .\build\Release\*
           7z a -tgzip nitro.tar.gz temp.tar
 
-      # - name: Run e2e testing - Llama.cpp
-      #   shell: cmd
-      #   run: |
-      #     .\.github\scripts\e2e-test-llama-windows.bat .\build\Release\nitro.exe ${{ env.LLM_MODEL_URL }}
+      - name: Run e2e testing - Llama.cpp
+        shell: cmd
+        run: |
+          cd build\Release
+          ..\..\.github\scripts\e2e-test-llama-windows.bat nitro.exe ${{ env.LLM_MODEL_URL }}
+          rmdir /S /Q .\build\Release\uploads
 
-      # - name: Run e2e testing - Whisper.cpp
-      #   shell: cmd
-      #   run: |
-      #     .\.github\scripts\e2e-test-whisper-windows.bat .\build\Release\nitro.exe ${{ env.WHISPER_MODEL_URL }}
+      - name: Run e2e testing - Whisper.cpp
+        shell: cmd
+        run: |
+          cd build\Release
+          ..\..\.github\scripts\e2e-test-whisper-windows.bat nitro.exe ${{ env.WHISPER_MODEL_URL }}
+          rmdir /S /Q .\build\Release\uploads
 
       - name: Upload Artifact
         uses: actions/upload-artifact@v2
@@ -679,7 +683,7 @@ jobs:
           cmake --build ./build_deps/nitro_deps --config Release
           mkdir -p build
           cd build
-          cmake .. -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_VULKAN=ON -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=RELEASE -DWHISPER_SDL2=ON -DNITRO_VERSION=${{ needs.set-nitro-version.outputs.version }}
+          cmake .. -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_VULKAN=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=RELEASE -DWHISPER_SDL2=ON -DNITRO_VERSION=${{ needs.set-nitro-version.outputs.version }}
           cmake --build . --config Release -j "%NUMBER_OF_PROCESSORS%"
 
       - name: Pack artifacts
@@ -770,7 +774,7 @@ jobs:
           cmake --build ./build_deps/nitro_deps --config Release
           mkdir -p build
           cd build
-          cmake .. -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_CUBLAS=ON -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=RELEASE -DWHISPER_SDL2=ON -DWHISPER_CUBLAS=ON -DNITRO_VERSION=${{ needs.set-nitro-version.outputs.version }}
+          cmake .. -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_CUBLAS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=RELEASE -DWHISPER_SDL2=ON -DWHISPER_CUBLAS=ON -DNITRO_VERSION=${{ needs.set-nitro-version.outputs.version }}
           cmake --build . --config Release -j "%NUMBER_OF_PROCESSORS%"
 
       - name: Pack artifacts

CMakeLists.txt

@@ -59,7 +59,7 @@ add_executable(${PROJECT_NAME} main.cc)
 #
 # and comment out the following lines
 find_package(Drogon CONFIG REQUIRED)
-target_link_libraries(${PROJECT_NAME} PRIVATE Drogon::Drogon common llama llava whisper
+target_link_libraries(${PROJECT_NAME} PRIVATE Drogon::Drogon common llama whisper llava
                                               ${CMAKE_THREAD_LIBS_INIT})
 
 # ##############################################################################

README.md

@@ -92,21 +92,6 @@ curl http://localhost:3928/v1/chat/completions \
   }'
 ```
 
-***OPTIONAL***: You can constrain the sampling using GBNF grammars by providing path to a grammar file
-```bash title="Nitro Inference With Grammar"
-curl http://localhost:3928/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -d '{
-    "messages": [
-      {
-        "role": "user",
-        "content": "Who won the world series in 2020?"
-      },
-    ],
-    "grammar_file": "/path/to/grammarfile"
-  }'
-```
-
 Table of parameters
 
 | Parameter        | Type    | Description                                                  |
@@ -128,6 +113,7 @@ Table of parameters
 |`grp_attn_n`|Integer|Group attention factor in self-extend|
 |`grp_attn_w`|Integer|Group attention width in self-extend|
 |`mlock`|Boolean|Prevent system swapping of the model to disk in macOS|
+|`grammar_file`| String |You can constrain the sampling using GBNF grammars by providing path to a grammar file|
 
 ***OPTIONAL***: You can run Nitro on a different port like 5000 instead of 3928 by running it manually in terminal
 ```zsh
@@ -155,37 +141,37 @@ To compile nitro please visit [Compile from source](docs/docs/new/build-source.m
   <tr>
     <td style="text-align:center"><b>Stable (Recommended)</b></td>
     <td style="text-align:center">
-      <a href='https://github.com/janhq/nitro/releases/download/v0.3.1/nitro-0.3.1-win-amd64.tar.gz'>
+      <a href='https://github.com/janhq/nitro/releases/download/v0.3.6/nitro-0.3.6-win-amd64.tar.gz'>
         <img src='./docs/static/img/windows.png' style="height:15px; width: 15px" />
         <b>CPU</b>
       </a>
     </td>
     <td style="text-align:center">
-      <a href='https://github.com/janhq/nitro/releases/download/v0.3.1/nitro-0.3.1-win-amd64-cuda.tar.gz'>
+      <a href='https://github.com/janhq/nitro/releases/download/v0.3.6/nitro-0.3.6-win-amd64-cuda.tar.gz'>
         <img src='./docs/static/img/windows.png' style="height:15px; width: 15px" />
         <b>CUDA</b>
       </a>
     </td>
     <td style="text-align:center">
-      <a href='https://github.com/janhq/nitro/releases/download/v0.3.1/nitro-0.3.1-mac-amd64.tar.gz'>
+      <a href='https://github.com/janhq/nitro/releases/download/v0.3.6/nitro-0.3.6-mac-amd64.tar.gz'>
         <img src='./docs/static/img/mac.png' style="height:15px; width: 15px" />
         <b>Intel</b>
       </a>
     </td>
     <td style="text-align:center">
-      <a href='https://github.com/janhq/nitro/releases/download/v0.3.1/nitro-0.3.1-mac-arm64.tar.gz'>
+      <a href='https://github.com/janhq/nitro/releases/download/v0.3.6/nitro-0.3.6-mac-arm64.tar.gz'>
         <img src='./docs/static/img/mac.png' style="height:15px; width: 15px" />
         <b>M1/M2</b>
       </a>
     </td>
     <td style="text-align:center">
-      <a href='https://github.com/janhq/nitro/releases/download/v0.3.1/nitro-0.3.1-linux-amd64.tar.gz'>
+      <a href='https://github.com/janhq/nitro/releases/download/v0.3.6/nitro-0.3.6-linux-amd64.tar.gz'>
         <img src='./docs/static/img/linux.png' style="height:15px; width: 15px" />
         <b>CPU</b>
       </a>
     </td>
     <td style="text-align:center">
-      <a href='https://github.com/janhq/nitro/releases/download/v0.3.1/nitro-0.3.1-linux-amd64-cuda.tar.gz'>
+      <a href='https://github.com/janhq/nitro/releases/download/v0.3.6/nitro-0.3.6-linux-amd64-cuda.tar.gz'>
         <img src='./docs/static/img/linux.png' style="height:15px; width: 15px" />
         <b>CUDA</b>
       </a>
@@ -194,7 +180,7 @@ To compile nitro please visit [Compile from source](docs/docs/new/build-source.m
   <tr style="text-align: center">
     <td style="text-align:center"><b>Experimental (Nighlty Build)</b></td>
     <td style="text-align:center" colspan="6">
-      <a href='https://github.com/janhq/nitro/actions/runs/7701765982'>
+      <a href='https://github.com/janhq/nitro/actions/runs/7790169166'>
         <b>GitHub action artifactory</b>
       </a>
     </td>