From 6d3e229577380cfa5187cb2916417c9914ff682e Mon Sep 17 00:00:00 2001 From: Dzmitry Pihulski Date: Tue, 24 Feb 2026 15:00:04 +0100 Subject: [PATCH 1/3] leaderboard data added --- .../5fewshots/inference_script.py | 25 +++ .../5fewshots/requirements.txt | 172 ++++++++++++++++++ .../Llama-3.2-1B-Instruct/5fewshots/run.yaml | 57 ++++++ .../5fewshots/inference_script.py | 25 +++ .../5fewshots/requirements.txt | 172 ++++++++++++++++++ .../Llama-3.2-3B-Instruct/5fewshots/run.yaml | 57 ++++++ .../5fewshots/inference_script.py | 25 +++ .../5fewshots/requirements.txt | 172 ++++++++++++++++++ .../Llama-3.3-70B-Instruct/5fewshots/run.yaml | 57 ++++++ .../5fewshots/inference_script.py | 25 +++ .../5fewshots/requirements.txt | 172 ++++++++++++++++++ .../5fewshots/run.yaml | 56 ++++++ .../5fewshots/inference_script.py | 26 +++ .../5fewshots/requirements.txt | 172 ++++++++++++++++++ .../5fewshots/run.yaml | 59 ++++++ .../5fewshots/inference_script.py | 25 +++ .../PLLuM-12B-chat/5fewshots/requirements.txt | 172 ++++++++++++++++++ leaderboard/PLLuM-12B-chat/5fewshots/run.yaml | 57 ++++++ .../5fewshots/inference_script.py | 25 +++ .../5fewshots/requirements.txt | 172 ++++++++++++++++++ .../PLLuM-12B-nc-chat/5fewshots/run.yaml | 57 ++++++ .../5fewshots/inference_script.py | 25 +++ .../5fewshots/requirements.txt | 172 ++++++++++++++++++ .../Qwen2.5-1.5B-Instruct/5fewshots/run.yaml | 57 ++++++ .../5fewshots/inference_script.py | 25 +++ .../5fewshots/requirements.txt | 172 ++++++++++++++++++ .../Qwen2.5-7B-Instruct/5fewshots/run.yaml | 56 ++++++ .../Qwen3-0.6B/5fewshots/inference_script.py | 26 +++ .../Qwen3-0.6B/5fewshots/requirements.txt | 172 ++++++++++++++++++ leaderboard/Qwen3-0.6B/5fewshots/run.yaml | 60 ++++++ .../5fewshots/inference_script.py | 26 +++ .../gpt-oss-120b/5fewshots/requirements.txt | 172 ++++++++++++++++++ leaderboard/gpt-oss-120b/5fewshots/run.yaml | 60 ++++++ .../gpt-oss-20b/5fewshots/inference_script.py | 26 +++ .../gpt-oss-20b/5fewshots/requirements.txt | 172 ++++++++++++++++++ leaderboard/gpt-oss-20b/5fewshots/run.yaml | 59 ++++++ .../5fewshots/inference_script.py | 25 +++ .../5fewshots/requirements.txt | 172 ++++++++++++++++++ .../5fewshots/run.yaml | 56 ++++++ 39 files changed, 3313 insertions(+) create mode 100644 leaderboard/Llama-3.2-1B-Instruct/5fewshots/inference_script.py create mode 100644 leaderboard/Llama-3.2-1B-Instruct/5fewshots/requirements.txt create mode 100644 leaderboard/Llama-3.2-1B-Instruct/5fewshots/run.yaml create mode 100644 leaderboard/Llama-3.2-3B-Instruct/5fewshots/inference_script.py create mode 100644 leaderboard/Llama-3.2-3B-Instruct/5fewshots/requirements.txt create mode 100644 leaderboard/Llama-3.2-3B-Instruct/5fewshots/run.yaml create mode 100644 leaderboard/Llama-3.3-70B-Instruct/5fewshots/inference_script.py create mode 100644 leaderboard/Llama-3.3-70B-Instruct/5fewshots/requirements.txt create mode 100644 leaderboard/Llama-3.3-70B-Instruct/5fewshots/run.yaml create mode 100644 leaderboard/Mistral-Nemo-Instruct-2407/5fewshots/inference_script.py create mode 100644 leaderboard/Mistral-Nemo-Instruct-2407/5fewshots/requirements.txt create mode 100644 leaderboard/Mistral-Nemo-Instruct-2407/5fewshots/run.yaml create mode 100644 leaderboard/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/5fewshots/inference_script.py create mode 100644 leaderboard/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/5fewshots/requirements.txt create mode 100644 leaderboard/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/5fewshots/run.yaml create mode 100644 leaderboard/PLLuM-12B-chat/5fewshots/inference_script.py create mode 100644 leaderboard/PLLuM-12B-chat/5fewshots/requirements.txt create mode 100644 leaderboard/PLLuM-12B-chat/5fewshots/run.yaml create mode 100644 leaderboard/PLLuM-12B-nc-chat/5fewshots/inference_script.py create mode 100644 leaderboard/PLLuM-12B-nc-chat/5fewshots/requirements.txt create mode 100644 leaderboard/PLLuM-12B-nc-chat/5fewshots/run.yaml create mode 100644 leaderboard/Qwen2.5-1.5B-Instruct/5fewshots/inference_script.py create mode 100644 leaderboard/Qwen2.5-1.5B-Instruct/5fewshots/requirements.txt create mode 100644 leaderboard/Qwen2.5-1.5B-Instruct/5fewshots/run.yaml create mode 100644 leaderboard/Qwen2.5-7B-Instruct/5fewshots/inference_script.py create mode 100644 leaderboard/Qwen2.5-7B-Instruct/5fewshots/requirements.txt create mode 100644 leaderboard/Qwen2.5-7B-Instruct/5fewshots/run.yaml create mode 100644 leaderboard/Qwen3-0.6B/5fewshots/inference_script.py create mode 100644 leaderboard/Qwen3-0.6B/5fewshots/requirements.txt create mode 100644 leaderboard/Qwen3-0.6B/5fewshots/run.yaml create mode 100644 leaderboard/gpt-oss-120b/5fewshots/inference_script.py create mode 100644 leaderboard/gpt-oss-120b/5fewshots/requirements.txt create mode 100644 leaderboard/gpt-oss-120b/5fewshots/run.yaml create mode 100644 leaderboard/gpt-oss-20b/5fewshots/inference_script.py create mode 100644 leaderboard/gpt-oss-20b/5fewshots/requirements.txt create mode 100644 leaderboard/gpt-oss-20b/5fewshots/run.yaml create mode 100644 leaderboard/pllum-12b-nc-chat-250715/5fewshots/inference_script.py create mode 100644 leaderboard/pllum-12b-nc-chat-250715/5fewshots/requirements.txt create mode 100644 leaderboard/pllum-12b-nc-chat-250715/5fewshots/run.yaml diff --git a/leaderboard/Llama-3.2-1B-Instruct/5fewshots/inference_script.py b/leaderboard/Llama-3.2-1B-Instruct/5fewshots/inference_script.py new file mode 100644 index 0000000..1049e44 --- /dev/null +++ b/leaderboard/Llama-3.2-1B-Instruct/5fewshots/inference_script.py @@ -0,0 +1,25 @@ +import os + +from dotenv import load_dotenv + +from llmsql import evaluate, inference_vllm + +load_dotenv() + +MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct" + +results = inference_vllm( + model_name=MODEL_NAME, + output_file=f"{MODEL_NAME}_outputs.jsonl", + batch_size=20000, + tensor_parallel_size=4, + do_sample=False, + hf_token=os.environ["HF_TOKEN"], + max_new_tokens=256, + temperature=0.0, + num_fewshots=5, + seed=42, + llm_kwargs={"dtype": "bfloat16"}, +) + +evaluate(results) diff --git a/leaderboard/Llama-3.2-1B-Instruct/5fewshots/requirements.txt b/leaderboard/Llama-3.2-1B-Instruct/5fewshots/requirements.txt new file mode 100644 index 0000000..929f583 --- /dev/null +++ b/leaderboard/Llama-3.2-1B-Instruct/5fewshots/requirements.txt @@ -0,0 +1,172 @@ +accelerate==1.12.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.13.3 +aiosignal==1.4.0 +annotated-doc==0.0.4 +annotated-types==0.7.0 +anthropic==0.83.0 +anyio==4.12.1 +apache-tvm-ffi==0.1.8.post2 +astor==0.8.1 +attrs==25.4.0 +blake3==1.0.8 +cachetools==7.0.1 +cbor2==5.8.0 +certifi==2026.1.4 +cffi==2.0.0 +charset-normalizer==3.4.4 +click==8.3.1 +cloudpickle==3.1.2 +compressed-tensors==0.13.0 +cryptography==46.0.5 +cuda-bindings==13.1.1 +cuda-pathfinder==1.3.5 +cuda-python==13.1.1 +cupy-cuda12x==14.0.1 +datasets==4.5.0 +depyf==0.20.0 +dill==0.4.0 +diskcache==5.6.3 +distro==1.9.0 +dnspython==2.8.0 +docstring_parser==0.17.0 +einops==0.8.2 +email-validator==2.3.0 +fastapi==0.132.0 +fastapi-cli==0.0.23 +fastapi-cloud-cli==0.13.0 +fastar==0.8.0 +filelock==3.24.3 +flashinfer-python==0.6.1 +frozenlist==1.8.0 +fsspec==2025.10.0 +gguf==0.17.1 +grpcio==1.78.1 +grpcio-reflection==1.78.1 +h11==0.16.0 +hf-xet==1.3.0 +httpcore==1.0.9 +httptools==0.7.1 +httpx==0.28.1 +httpx-sse==0.4.3 +huggingface_hub==0.36.2 +idna==3.11 +ijson==3.5.0 +interegular==0.3.3 +Jinja2==3.1.6 +jiter==0.13.0 +jmespath==1.1.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +lark==1.2.2 +llguidance==1.3.0 +llmsql==0.1.15 +llvmlite==0.44.0 +lm-format-enforcer==0.11.3 +loguru==0.7.3 +markdown-it-py==4.0.0 +MarkupSafe==3.0.3 +mcp==1.26.0 +mdurl==0.1.2 +mistral_common==1.9.1 +model-hosting-container-standards==0.1.13 +mpmath==1.3.0 +msgpack==1.1.2 +msgspec==0.20.0 +multidict==6.7.1 +multiprocess==0.70.18 +networkx==3.6.1 +ninja==1.13.0 +numba==0.61.2 +numpy==2.2.6 +nvidia-cublas-cu12==12.8.4.1 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cudnn-cu12==9.10.2.21 +nvidia-cudnn-frontend==1.18.0 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cufile-cu12==1.13.1.3 +nvidia-curand-cu12==10.3.9.90 +nvidia-cusolver-cu12==11.7.3.90 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cusparselt-cu12==0.7.1 +nvidia-cutlass-dsl==4.4.0 +nvidia-cutlass-dsl-libs-base==4.4.0 +nvidia-ml-py==13.590.48 +nvidia-nccl-cu12==2.27.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nvshmem-cu12==3.3.20 +nvidia-nvtx-cu12==12.8.90 +openai==2.23.0 +openai-harmony==0.0.8 +opencv-python-headless==4.13.0.92 +outlines_core==0.2.11 +packaging==26.0 +pandas==3.0.1 +partial-json-parser==0.2.1.1.post7 +pillow==12.1.1 +prometheus-fastapi-instrumentator==7.1.0 +prometheus_client==0.24.1 +propcache==0.4.1 +protobuf==6.33.5 +psutil==7.2.2 +py-cpuinfo==9.0.0 +pyarrow==23.0.1 +pybase64==1.4.3 +pycountry==26.2.16 +pycparser==3.0 +pydantic==2.12.5 +pydantic-extra-types==2.11.0 +pydantic-settings==2.13.1 +pydantic_core==2.41.5 +Pygments==2.19.2 +PyJWT==2.11.0 +python-dateutil==2.9.0.post0 +python-dotenv==1.2.1 +python-json-logger==4.0.0 +python-multipart==0.0.22 +PyYAML==6.0.3 +pyzmq==27.1.0 +ray==2.54.0 +referencing==0.37.0 +regex==2026.2.19 +requests==2.32.5 +rich==14.3.3 +rich-toolkit==0.19.4 +rignore==0.7.6 +rpds-py==0.30.0 +safetensors==0.7.0 +sentencepiece==0.2.1 +sentry-sdk==2.53.0 +setproctitle==1.3.7 +setuptools==80.10.2 +shellingham==1.5.4 +six==1.17.0 +sniffio==1.3.1 +sse-starlette==3.2.0 +starlette==0.52.1 +supervisor==4.3.0 +sympy==1.14.0 +tabulate==0.9.0 +tiktoken==0.12.0 +tokenizers==0.22.2 +torch==2.9.1 +torchaudio==2.9.1 +torchvision==0.24.1 +tqdm==4.67.3 +transformers==4.57.6 +triton==3.5.1 +typer==0.24.1 +typer-slim==0.24.0 +typing-inspection==0.4.2 +typing_extensions==4.15.0 +urllib3==2.6.3 +uvicorn==0.41.0 +uvloop==0.22.1 +vllm==0.15.1 +watchfiles==1.1.1 +websockets==16.0 +xgrammar==0.1.29 +xxhash==3.6.0 +yarl==1.22.0 diff --git a/leaderboard/Llama-3.2-1B-Instruct/5fewshots/run.yaml b/leaderboard/Llama-3.2-1B-Instruct/5fewshots/run.yaml new file mode 100644 index 0000000..7aaea2e --- /dev/null +++ b/leaderboard/Llama-3.2-1B-Instruct/5fewshots/run.yaml @@ -0,0 +1,57 @@ +date: 2026-02-24 +# ===================== +# Model Information +# ===================== +model: + name: meta-llama/Llama-3.2-1B-Instruct + revision: main + commit_hash: 9213176726f574b556790deb65791e0c5aa438b6 + parameter_count: 1B + dtype: bfloat16 + thinking: false + +type: open-source # open-source | proprietary + +# ===================== +# Package Information +# ===================== +llmsql: + version: 0.1.15 + commit_hash: 79175212c90b1fc094abd2c9666c23d903060014 + +# ===================== +# Benchmark Information +# ===================== +version: 2.0 + +# ===================== +# Environment Information +# ===================== +os_name: Ubuntu 24.04.3 LTS +python_version: 3.12.12 +pip_freeze: requirements.txt +device: 4xH200 + +# ===================== +# Function Inputs / Inference Backend +# ===================== +inference: + backend: vllm # vllm | transformers + arguments: + batch_size: 20000 + tetensor_parallel_size: 4 + do_sample: false + max_new_tokens: 256 + temperature: 0.0 + num_fewshots: 5 + seed: 42 + llm_kwargs: + dtype: bfloat16 + + +# ===================== +# Results +# ===================== +results: + execution_accuracy: 0.2678 + answers_path: https://huggingface.co/datasets/llmsql-bench/benchmark-evaluation-results/blob/main/Llama-3.2-1B-Instruct/5fewshots/Llama-3.2-1B-Instruct_outputs.jsonl diff --git a/leaderboard/Llama-3.2-3B-Instruct/5fewshots/inference_script.py b/leaderboard/Llama-3.2-3B-Instruct/5fewshots/inference_script.py new file mode 100644 index 0000000..a3400c8 --- /dev/null +++ b/leaderboard/Llama-3.2-3B-Instruct/5fewshots/inference_script.py @@ -0,0 +1,25 @@ +import os + +from dotenv import load_dotenv + +from llmsql import evaluate, inference_vllm + +load_dotenv() + +MODEL_NAME = "meta-llama/Llama-3.2-3B-Instruct" + +results = inference_vllm( + model_name=MODEL_NAME, + output_file=f"{MODEL_NAME}_outputs.jsonl", + batch_size=20000, + tensor_parallel_size=4, + do_sample=False, + hf_token=os.environ["HF_TOKEN"], + max_new_tokens=256, + temperature=0.0, + num_fewshots=5, + seed=42, + llm_kwargs={"dtype": "bfloat16"}, +) + +evaluate(results) diff --git a/leaderboard/Llama-3.2-3B-Instruct/5fewshots/requirements.txt b/leaderboard/Llama-3.2-3B-Instruct/5fewshots/requirements.txt new file mode 100644 index 0000000..929f583 --- /dev/null +++ b/leaderboard/Llama-3.2-3B-Instruct/5fewshots/requirements.txt @@ -0,0 +1,172 @@ +accelerate==1.12.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.13.3 +aiosignal==1.4.0 +annotated-doc==0.0.4 +annotated-types==0.7.0 +anthropic==0.83.0 +anyio==4.12.1 +apache-tvm-ffi==0.1.8.post2 +astor==0.8.1 +attrs==25.4.0 +blake3==1.0.8 +cachetools==7.0.1 +cbor2==5.8.0 +certifi==2026.1.4 +cffi==2.0.0 +charset-normalizer==3.4.4 +click==8.3.1 +cloudpickle==3.1.2 +compressed-tensors==0.13.0 +cryptography==46.0.5 +cuda-bindings==13.1.1 +cuda-pathfinder==1.3.5 +cuda-python==13.1.1 +cupy-cuda12x==14.0.1 +datasets==4.5.0 +depyf==0.20.0 +dill==0.4.0 +diskcache==5.6.3 +distro==1.9.0 +dnspython==2.8.0 +docstring_parser==0.17.0 +einops==0.8.2 +email-validator==2.3.0 +fastapi==0.132.0 +fastapi-cli==0.0.23 +fastapi-cloud-cli==0.13.0 +fastar==0.8.0 +filelock==3.24.3 +flashinfer-python==0.6.1 +frozenlist==1.8.0 +fsspec==2025.10.0 +gguf==0.17.1 +grpcio==1.78.1 +grpcio-reflection==1.78.1 +h11==0.16.0 +hf-xet==1.3.0 +httpcore==1.0.9 +httptools==0.7.1 +httpx==0.28.1 +httpx-sse==0.4.3 +huggingface_hub==0.36.2 +idna==3.11 +ijson==3.5.0 +interegular==0.3.3 +Jinja2==3.1.6 +jiter==0.13.0 +jmespath==1.1.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +lark==1.2.2 +llguidance==1.3.0 +llmsql==0.1.15 +llvmlite==0.44.0 +lm-format-enforcer==0.11.3 +loguru==0.7.3 +markdown-it-py==4.0.0 +MarkupSafe==3.0.3 +mcp==1.26.0 +mdurl==0.1.2 +mistral_common==1.9.1 +model-hosting-container-standards==0.1.13 +mpmath==1.3.0 +msgpack==1.1.2 +msgspec==0.20.0 +multidict==6.7.1 +multiprocess==0.70.18 +networkx==3.6.1 +ninja==1.13.0 +numba==0.61.2 +numpy==2.2.6 +nvidia-cublas-cu12==12.8.4.1 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cudnn-cu12==9.10.2.21 +nvidia-cudnn-frontend==1.18.0 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cufile-cu12==1.13.1.3 +nvidia-curand-cu12==10.3.9.90 +nvidia-cusolver-cu12==11.7.3.90 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cusparselt-cu12==0.7.1 +nvidia-cutlass-dsl==4.4.0 +nvidia-cutlass-dsl-libs-base==4.4.0 +nvidia-ml-py==13.590.48 +nvidia-nccl-cu12==2.27.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nvshmem-cu12==3.3.20 +nvidia-nvtx-cu12==12.8.90 +openai==2.23.0 +openai-harmony==0.0.8 +opencv-python-headless==4.13.0.92 +outlines_core==0.2.11 +packaging==26.0 +pandas==3.0.1 +partial-json-parser==0.2.1.1.post7 +pillow==12.1.1 +prometheus-fastapi-instrumentator==7.1.0 +prometheus_client==0.24.1 +propcache==0.4.1 +protobuf==6.33.5 +psutil==7.2.2 +py-cpuinfo==9.0.0 +pyarrow==23.0.1 +pybase64==1.4.3 +pycountry==26.2.16 +pycparser==3.0 +pydantic==2.12.5 +pydantic-extra-types==2.11.0 +pydantic-settings==2.13.1 +pydantic_core==2.41.5 +Pygments==2.19.2 +PyJWT==2.11.0 +python-dateutil==2.9.0.post0 +python-dotenv==1.2.1 +python-json-logger==4.0.0 +python-multipart==0.0.22 +PyYAML==6.0.3 +pyzmq==27.1.0 +ray==2.54.0 +referencing==0.37.0 +regex==2026.2.19 +requests==2.32.5 +rich==14.3.3 +rich-toolkit==0.19.4 +rignore==0.7.6 +rpds-py==0.30.0 +safetensors==0.7.0 +sentencepiece==0.2.1 +sentry-sdk==2.53.0 +setproctitle==1.3.7 +setuptools==80.10.2 +shellingham==1.5.4 +six==1.17.0 +sniffio==1.3.1 +sse-starlette==3.2.0 +starlette==0.52.1 +supervisor==4.3.0 +sympy==1.14.0 +tabulate==0.9.0 +tiktoken==0.12.0 +tokenizers==0.22.2 +torch==2.9.1 +torchaudio==2.9.1 +torchvision==0.24.1 +tqdm==4.67.3 +transformers==4.57.6 +triton==3.5.1 +typer==0.24.1 +typer-slim==0.24.0 +typing-inspection==0.4.2 +typing_extensions==4.15.0 +urllib3==2.6.3 +uvicorn==0.41.0 +uvloop==0.22.1 +vllm==0.15.1 +watchfiles==1.1.1 +websockets==16.0 +xgrammar==0.1.29 +xxhash==3.6.0 +yarl==1.22.0 diff --git a/leaderboard/Llama-3.2-3B-Instruct/5fewshots/run.yaml b/leaderboard/Llama-3.2-3B-Instruct/5fewshots/run.yaml new file mode 100644 index 0000000..57616a3 --- /dev/null +++ b/leaderboard/Llama-3.2-3B-Instruct/5fewshots/run.yaml @@ -0,0 +1,57 @@ +date: 2026-02-24 +# ===================== +# Model Information +# ===================== +model: + name: meta-llama/Llama-3.2-3B-Instruct + revision: main + commit_hash: 0cb88a4f764b7a12671c53f0838cd831a0843b95 + parameter_count: 3B + dtype: bfloat16 + thinking: false + +type: open-source # open-source | proprietary + +# ===================== +# Package Information +# ===================== +llmsql: + version: 0.1.15 + commit_hash: 79175212c90b1fc094abd2c9666c23d903060014 + +# ===================== +# Benchmark Information +# ===================== +version: 2.0 + +# ===================== +# Environment Information +# ===================== +os_name: Ubuntu 24.04.3 LTS +python_version: 3.12.12 +pip_freeze: requirements.txt +device: 4xH200 + +# ===================== +# Function Inputs / Inference Backend +# ===================== +inference: + backend: vllm # vllm | transformers + arguments: + batch_size: 20000 + tetensor_parallel_size: 4 + do_sample: false + max_new_tokens: 256 + temperature: 0.0 + num_fewshots: 5 + seed: 42 + llm_kwargs: + dtype: bfloat16 + + +# ===================== +# Results +# ===================== +results: + execution_accuracy: 0.5415 + answers_path: https://huggingface.co/datasets/llmsql-bench/benchmark-evaluation-results/blob/main/Llama-3.2-3B-Instruct/5fewshots/Llama-3.2-3B-Instruct_outputs.jsonl diff --git a/leaderboard/Llama-3.3-70B-Instruct/5fewshots/inference_script.py b/leaderboard/Llama-3.3-70B-Instruct/5fewshots/inference_script.py new file mode 100644 index 0000000..4efc97e --- /dev/null +++ b/leaderboard/Llama-3.3-70B-Instruct/5fewshots/inference_script.py @@ -0,0 +1,25 @@ +import os + +from dotenv import load_dotenv + +from llmsql import evaluate, inference_vllm + +load_dotenv() + +MODEL_NAME = "meta-llama/Llama-3.3-70B-Instruct" + +results = inference_vllm( + model_name=MODEL_NAME, + output_file=f"{MODEL_NAME}_outputs.jsonl", + batch_size=20000, + tensor_parallel_size=4, + do_sample=False, + hf_token=os.environ["HF_TOKEN"], + max_new_tokens=256, + temperature=0.0, + num_fewshots=5, + seed=42, + llm_kwargs={"dtype": "bfloat16"}, +) + +evaluate(results) diff --git a/leaderboard/Llama-3.3-70B-Instruct/5fewshots/requirements.txt b/leaderboard/Llama-3.3-70B-Instruct/5fewshots/requirements.txt new file mode 100644 index 0000000..929f583 --- /dev/null +++ b/leaderboard/Llama-3.3-70B-Instruct/5fewshots/requirements.txt @@ -0,0 +1,172 @@ +accelerate==1.12.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.13.3 +aiosignal==1.4.0 +annotated-doc==0.0.4 +annotated-types==0.7.0 +anthropic==0.83.0 +anyio==4.12.1 +apache-tvm-ffi==0.1.8.post2 +astor==0.8.1 +attrs==25.4.0 +blake3==1.0.8 +cachetools==7.0.1 +cbor2==5.8.0 +certifi==2026.1.4 +cffi==2.0.0 +charset-normalizer==3.4.4 +click==8.3.1 +cloudpickle==3.1.2 +compressed-tensors==0.13.0 +cryptography==46.0.5 +cuda-bindings==13.1.1 +cuda-pathfinder==1.3.5 +cuda-python==13.1.1 +cupy-cuda12x==14.0.1 +datasets==4.5.0 +depyf==0.20.0 +dill==0.4.0 +diskcache==5.6.3 +distro==1.9.0 +dnspython==2.8.0 +docstring_parser==0.17.0 +einops==0.8.2 +email-validator==2.3.0 +fastapi==0.132.0 +fastapi-cli==0.0.23 +fastapi-cloud-cli==0.13.0 +fastar==0.8.0 +filelock==3.24.3 +flashinfer-python==0.6.1 +frozenlist==1.8.0 +fsspec==2025.10.0 +gguf==0.17.1 +grpcio==1.78.1 +grpcio-reflection==1.78.1 +h11==0.16.0 +hf-xet==1.3.0 +httpcore==1.0.9 +httptools==0.7.1 +httpx==0.28.1 +httpx-sse==0.4.3 +huggingface_hub==0.36.2 +idna==3.11 +ijson==3.5.0 +interegular==0.3.3 +Jinja2==3.1.6 +jiter==0.13.0 +jmespath==1.1.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +lark==1.2.2 +llguidance==1.3.0 +llmsql==0.1.15 +llvmlite==0.44.0 +lm-format-enforcer==0.11.3 +loguru==0.7.3 +markdown-it-py==4.0.0 +MarkupSafe==3.0.3 +mcp==1.26.0 +mdurl==0.1.2 +mistral_common==1.9.1 +model-hosting-container-standards==0.1.13 +mpmath==1.3.0 +msgpack==1.1.2 +msgspec==0.20.0 +multidict==6.7.1 +multiprocess==0.70.18 +networkx==3.6.1 +ninja==1.13.0 +numba==0.61.2 +numpy==2.2.6 +nvidia-cublas-cu12==12.8.4.1 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cudnn-cu12==9.10.2.21 +nvidia-cudnn-frontend==1.18.0 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cufile-cu12==1.13.1.3 +nvidia-curand-cu12==10.3.9.90 +nvidia-cusolver-cu12==11.7.3.90 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cusparselt-cu12==0.7.1 +nvidia-cutlass-dsl==4.4.0 +nvidia-cutlass-dsl-libs-base==4.4.0 +nvidia-ml-py==13.590.48 +nvidia-nccl-cu12==2.27.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nvshmem-cu12==3.3.20 +nvidia-nvtx-cu12==12.8.90 +openai==2.23.0 +openai-harmony==0.0.8 +opencv-python-headless==4.13.0.92 +outlines_core==0.2.11 +packaging==26.0 +pandas==3.0.1 +partial-json-parser==0.2.1.1.post7 +pillow==12.1.1 +prometheus-fastapi-instrumentator==7.1.0 +prometheus_client==0.24.1 +propcache==0.4.1 +protobuf==6.33.5 +psutil==7.2.2 +py-cpuinfo==9.0.0 +pyarrow==23.0.1 +pybase64==1.4.3 +pycountry==26.2.16 +pycparser==3.0 +pydantic==2.12.5 +pydantic-extra-types==2.11.0 +pydantic-settings==2.13.1 +pydantic_core==2.41.5 +Pygments==2.19.2 +PyJWT==2.11.0 +python-dateutil==2.9.0.post0 +python-dotenv==1.2.1 +python-json-logger==4.0.0 +python-multipart==0.0.22 +PyYAML==6.0.3 +pyzmq==27.1.0 +ray==2.54.0 +referencing==0.37.0 +regex==2026.2.19 +requests==2.32.5 +rich==14.3.3 +rich-toolkit==0.19.4 +rignore==0.7.6 +rpds-py==0.30.0 +safetensors==0.7.0 +sentencepiece==0.2.1 +sentry-sdk==2.53.0 +setproctitle==1.3.7 +setuptools==80.10.2 +shellingham==1.5.4 +six==1.17.0 +sniffio==1.3.1 +sse-starlette==3.2.0 +starlette==0.52.1 +supervisor==4.3.0 +sympy==1.14.0 +tabulate==0.9.0 +tiktoken==0.12.0 +tokenizers==0.22.2 +torch==2.9.1 +torchaudio==2.9.1 +torchvision==0.24.1 +tqdm==4.67.3 +transformers==4.57.6 +triton==3.5.1 +typer==0.24.1 +typer-slim==0.24.0 +typing-inspection==0.4.2 +typing_extensions==4.15.0 +urllib3==2.6.3 +uvicorn==0.41.0 +uvloop==0.22.1 +vllm==0.15.1 +watchfiles==1.1.1 +websockets==16.0 +xgrammar==0.1.29 +xxhash==3.6.0 +yarl==1.22.0 diff --git a/leaderboard/Llama-3.3-70B-Instruct/5fewshots/run.yaml b/leaderboard/Llama-3.3-70B-Instruct/5fewshots/run.yaml new file mode 100644 index 0000000..1e966f9 --- /dev/null +++ b/leaderboard/Llama-3.3-70B-Instruct/5fewshots/run.yaml @@ -0,0 +1,57 @@ +date: 2026-02-24 +# ===================== +# Model Information +# ===================== +model: + name: meta-llama/Llama-3.3-70B-Instruct + revision: main + commit_hash: 6f6073b423013f6a7d4d9f39144961bfbfbc386b + parameter_count: 70B + dtype: bfloat16 + thinking: false + +type: open-source # open-source | proprietary + +# ===================== +# Package Information +# ===================== +llmsql: + version: 0.1.15 + commit_hash: 79175212c90b1fc094abd2c9666c23d903060014 + +# ===================== +# Benchmark Information +# ===================== +version: 2.0 + +# ===================== +# Environment Information +# ===================== +os_name: Ubuntu 24.04.3 LTS +python_version: 3.12.12 +pip_freeze: requirements.txt +device: 4xH200 + +# ===================== +# Function Inputs / Inference Backend +# ===================== +inference: + backend: vllm # vllm | transformers + arguments: + batch_size: 20000 + tetensor_parallel_size: 4 + do_sample: false + max_new_tokens: 256 + temperature: 0.0 + num_fewshots: 5 + seed: 42 + llm_kwargs: + dtype: bfloat16 + + +# ===================== +# Results +# ===================== +results: + execution_accuracy: 0.8607 + answers_path: https://huggingface.co/datasets/llmsql-bench/benchmark-evaluation-results/blob/main/Llama-3.3-70B-Instruct/5fewshots/Llama-3.3-70B-Instruct_outputs.jsonl diff --git a/leaderboard/Mistral-Nemo-Instruct-2407/5fewshots/inference_script.py b/leaderboard/Mistral-Nemo-Instruct-2407/5fewshots/inference_script.py new file mode 100644 index 0000000..ef0562c --- /dev/null +++ b/leaderboard/Mistral-Nemo-Instruct-2407/5fewshots/inference_script.py @@ -0,0 +1,25 @@ +import os + +from dotenv import load_dotenv + +from llmsql import evaluate, inference_vllm + +load_dotenv() + +MODEL_NAME = "mistralai/Mistral-Nemo-Instruct-2407" + +results = inference_vllm( + model_name=MODEL_NAME, + output_file=f"{MODEL_NAME}_outputs.jsonl", + batch_size=20000, + tensor_parallel_size=4, + do_sample=False, + hf_token=os.environ["HF_TOKEN"], + max_new_tokens=256, + temperature=0.0, + num_fewshots=5, + seed=42, + llm_kwargs={"dtype": "bfloat16"}, +) + +evaluate(results) diff --git a/leaderboard/Mistral-Nemo-Instruct-2407/5fewshots/requirements.txt b/leaderboard/Mistral-Nemo-Instruct-2407/5fewshots/requirements.txt new file mode 100644 index 0000000..929f583 --- /dev/null +++ b/leaderboard/Mistral-Nemo-Instruct-2407/5fewshots/requirements.txt @@ -0,0 +1,172 @@ +accelerate==1.12.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.13.3 +aiosignal==1.4.0 +annotated-doc==0.0.4 +annotated-types==0.7.0 +anthropic==0.83.0 +anyio==4.12.1 +apache-tvm-ffi==0.1.8.post2 +astor==0.8.1 +attrs==25.4.0 +blake3==1.0.8 +cachetools==7.0.1 +cbor2==5.8.0 +certifi==2026.1.4 +cffi==2.0.0 +charset-normalizer==3.4.4 +click==8.3.1 +cloudpickle==3.1.2 +compressed-tensors==0.13.0 +cryptography==46.0.5 +cuda-bindings==13.1.1 +cuda-pathfinder==1.3.5 +cuda-python==13.1.1 +cupy-cuda12x==14.0.1 +datasets==4.5.0 +depyf==0.20.0 +dill==0.4.0 +diskcache==5.6.3 +distro==1.9.0 +dnspython==2.8.0 +docstring_parser==0.17.0 +einops==0.8.2 +email-validator==2.3.0 +fastapi==0.132.0 +fastapi-cli==0.0.23 +fastapi-cloud-cli==0.13.0 +fastar==0.8.0 +filelock==3.24.3 +flashinfer-python==0.6.1 +frozenlist==1.8.0 +fsspec==2025.10.0 +gguf==0.17.1 +grpcio==1.78.1 +grpcio-reflection==1.78.1 +h11==0.16.0 +hf-xet==1.3.0 +httpcore==1.0.9 +httptools==0.7.1 +httpx==0.28.1 +httpx-sse==0.4.3 +huggingface_hub==0.36.2 +idna==3.11 +ijson==3.5.0 +interegular==0.3.3 +Jinja2==3.1.6 +jiter==0.13.0 +jmespath==1.1.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +lark==1.2.2 +llguidance==1.3.0 +llmsql==0.1.15 +llvmlite==0.44.0 +lm-format-enforcer==0.11.3 +loguru==0.7.3 +markdown-it-py==4.0.0 +MarkupSafe==3.0.3 +mcp==1.26.0 +mdurl==0.1.2 +mistral_common==1.9.1 +model-hosting-container-standards==0.1.13 +mpmath==1.3.0 +msgpack==1.1.2 +msgspec==0.20.0 +multidict==6.7.1 +multiprocess==0.70.18 +networkx==3.6.1 +ninja==1.13.0 +numba==0.61.2 +numpy==2.2.6 +nvidia-cublas-cu12==12.8.4.1 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cudnn-cu12==9.10.2.21 +nvidia-cudnn-frontend==1.18.0 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cufile-cu12==1.13.1.3 +nvidia-curand-cu12==10.3.9.90 +nvidia-cusolver-cu12==11.7.3.90 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cusparselt-cu12==0.7.1 +nvidia-cutlass-dsl==4.4.0 +nvidia-cutlass-dsl-libs-base==4.4.0 +nvidia-ml-py==13.590.48 +nvidia-nccl-cu12==2.27.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nvshmem-cu12==3.3.20 +nvidia-nvtx-cu12==12.8.90 +openai==2.23.0 +openai-harmony==0.0.8 +opencv-python-headless==4.13.0.92 +outlines_core==0.2.11 +packaging==26.0 +pandas==3.0.1 +partial-json-parser==0.2.1.1.post7 +pillow==12.1.1 +prometheus-fastapi-instrumentator==7.1.0 +prometheus_client==0.24.1 +propcache==0.4.1 +protobuf==6.33.5 +psutil==7.2.2 +py-cpuinfo==9.0.0 +pyarrow==23.0.1 +pybase64==1.4.3 +pycountry==26.2.16 +pycparser==3.0 +pydantic==2.12.5 +pydantic-extra-types==2.11.0 +pydantic-settings==2.13.1 +pydantic_core==2.41.5 +Pygments==2.19.2 +PyJWT==2.11.0 +python-dateutil==2.9.0.post0 +python-dotenv==1.2.1 +python-json-logger==4.0.0 +python-multipart==0.0.22 +PyYAML==6.0.3 +pyzmq==27.1.0 +ray==2.54.0 +referencing==0.37.0 +regex==2026.2.19 +requests==2.32.5 +rich==14.3.3 +rich-toolkit==0.19.4 +rignore==0.7.6 +rpds-py==0.30.0 +safetensors==0.7.0 +sentencepiece==0.2.1 +sentry-sdk==2.53.0 +setproctitle==1.3.7 +setuptools==80.10.2 +shellingham==1.5.4 +six==1.17.0 +sniffio==1.3.1 +sse-starlette==3.2.0 +starlette==0.52.1 +supervisor==4.3.0 +sympy==1.14.0 +tabulate==0.9.0 +tiktoken==0.12.0 +tokenizers==0.22.2 +torch==2.9.1 +torchaudio==2.9.1 +torchvision==0.24.1 +tqdm==4.67.3 +transformers==4.57.6 +triton==3.5.1 +typer==0.24.1 +typer-slim==0.24.0 +typing-inspection==0.4.2 +typing_extensions==4.15.0 +urllib3==2.6.3 +uvicorn==0.41.0 +uvloop==0.22.1 +vllm==0.15.1 +watchfiles==1.1.1 +websockets==16.0 +xgrammar==0.1.29 +xxhash==3.6.0 +yarl==1.22.0 diff --git a/leaderboard/Mistral-Nemo-Instruct-2407/5fewshots/run.yaml b/leaderboard/Mistral-Nemo-Instruct-2407/5fewshots/run.yaml new file mode 100644 index 0000000..7914a99 --- /dev/null +++ b/leaderboard/Mistral-Nemo-Instruct-2407/5fewshots/run.yaml @@ -0,0 +1,56 @@ +date: 2026-02-24 +# ===================== +# Model Information +# ===================== +model: + name: mistralai/Mistral-Nemo-Instruct-2407 + revision: main + commit_hash: 04d8a90549d23fc6bd7f642064003592df51e9b3 + parameter_count: 12B + dtype: bfloat16 + thinking: false + +type: open-source # open-source | proprietary + +# ===================== +# Package Information +# ===================== +llmsql: + version: 0.1.15 + commit_hash: 79175212c90b1fc094abd2c9666c23d903060014 + +# ===================== +# Benchmark Information +# ===================== +version: 2.0 + +# ===================== +# Environment Information +# ===================== +os_name: Ubuntu 24.04.3 LTS +python_version: 3.12.12 +pip_freeze: requirements.txt +device: 4xH200 + +# ===================== +# Function Inputs / Inference Backend +# ===================== +inference: + backend: vllm # vllm | transformers + arguments: + batch_size: 20000 + tetensor_parallel_size: 4 + do_sample: false + max_new_tokens: 256 + temperature: 0.0 + num_fewshots: 5 + seed: 42 + llm_kwargs: + dtype: bfloat16 + +# ===================== +# Results +# ===================== +results: + execution_accuracy: 0.7599 + answers_path: https://huggingface.co/datasets/llmsql-bench/benchmark-evaluation-results/blob/main/Mistral-Nemo-Instruct-2407/5fewshots/Mistral-Nemo-Instruct-2407_outputs.jsonl diff --git a/leaderboard/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/5fewshots/inference_script.py b/leaderboard/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/5fewshots/inference_script.py new file mode 100644 index 0000000..fa7cfb7 --- /dev/null +++ b/leaderboard/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/5fewshots/inference_script.py @@ -0,0 +1,26 @@ +import os + +from dotenv import load_dotenv + +from llmsql import evaluate, inference_vllm + +load_dotenv() + +MODEL_NAME = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" + +results = inference_vllm( + model_name=MODEL_NAME, + output_file=f"{MODEL_NAME}_outputs.jsonl", + batch_size=20000, + tensor_parallel_size=4, + do_sample=True, + hf_token=os.environ["HF_TOKEN"], + max_new_tokens=1024, + temperature=1.0, + sampling_kwargs={"top_p": 1.0}, + num_fewshots=5, + seed=42, + llm_kwargs={"dtype": "bfloat16"}, +) + +evaluate(results) diff --git a/leaderboard/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/5fewshots/requirements.txt b/leaderboard/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/5fewshots/requirements.txt new file mode 100644 index 0000000..929f583 --- /dev/null +++ b/leaderboard/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/5fewshots/requirements.txt @@ -0,0 +1,172 @@ +accelerate==1.12.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.13.3 +aiosignal==1.4.0 +annotated-doc==0.0.4 +annotated-types==0.7.0 +anthropic==0.83.0 +anyio==4.12.1 +apache-tvm-ffi==0.1.8.post2 +astor==0.8.1 +attrs==25.4.0 +blake3==1.0.8 +cachetools==7.0.1 +cbor2==5.8.0 +certifi==2026.1.4 +cffi==2.0.0 +charset-normalizer==3.4.4 +click==8.3.1 +cloudpickle==3.1.2 +compressed-tensors==0.13.0 +cryptography==46.0.5 +cuda-bindings==13.1.1 +cuda-pathfinder==1.3.5 +cuda-python==13.1.1 +cupy-cuda12x==14.0.1 +datasets==4.5.0 +depyf==0.20.0 +dill==0.4.0 +diskcache==5.6.3 +distro==1.9.0 +dnspython==2.8.0 +docstring_parser==0.17.0 +einops==0.8.2 +email-validator==2.3.0 +fastapi==0.132.0 +fastapi-cli==0.0.23 +fastapi-cloud-cli==0.13.0 +fastar==0.8.0 +filelock==3.24.3 +flashinfer-python==0.6.1 +frozenlist==1.8.0 +fsspec==2025.10.0 +gguf==0.17.1 +grpcio==1.78.1 +grpcio-reflection==1.78.1 +h11==0.16.0 +hf-xet==1.3.0 +httpcore==1.0.9 +httptools==0.7.1 +httpx==0.28.1 +httpx-sse==0.4.3 +huggingface_hub==0.36.2 +idna==3.11 +ijson==3.5.0 +interegular==0.3.3 +Jinja2==3.1.6 +jiter==0.13.0 +jmespath==1.1.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +lark==1.2.2 +llguidance==1.3.0 +llmsql==0.1.15 +llvmlite==0.44.0 +lm-format-enforcer==0.11.3 +loguru==0.7.3 +markdown-it-py==4.0.0 +MarkupSafe==3.0.3 +mcp==1.26.0 +mdurl==0.1.2 +mistral_common==1.9.1 +model-hosting-container-standards==0.1.13 +mpmath==1.3.0 +msgpack==1.1.2 +msgspec==0.20.0 +multidict==6.7.1 +multiprocess==0.70.18 +networkx==3.6.1 +ninja==1.13.0 +numba==0.61.2 +numpy==2.2.6 +nvidia-cublas-cu12==12.8.4.1 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cudnn-cu12==9.10.2.21 +nvidia-cudnn-frontend==1.18.0 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cufile-cu12==1.13.1.3 +nvidia-curand-cu12==10.3.9.90 +nvidia-cusolver-cu12==11.7.3.90 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cusparselt-cu12==0.7.1 +nvidia-cutlass-dsl==4.4.0 +nvidia-cutlass-dsl-libs-base==4.4.0 +nvidia-ml-py==13.590.48 +nvidia-nccl-cu12==2.27.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nvshmem-cu12==3.3.20 +nvidia-nvtx-cu12==12.8.90 +openai==2.23.0 +openai-harmony==0.0.8 +opencv-python-headless==4.13.0.92 +outlines_core==0.2.11 +packaging==26.0 +pandas==3.0.1 +partial-json-parser==0.2.1.1.post7 +pillow==12.1.1 +prometheus-fastapi-instrumentator==7.1.0 +prometheus_client==0.24.1 +propcache==0.4.1 +protobuf==6.33.5 +psutil==7.2.2 +py-cpuinfo==9.0.0 +pyarrow==23.0.1 +pybase64==1.4.3 +pycountry==26.2.16 +pycparser==3.0 +pydantic==2.12.5 +pydantic-extra-types==2.11.0 +pydantic-settings==2.13.1 +pydantic_core==2.41.5 +Pygments==2.19.2 +PyJWT==2.11.0 +python-dateutil==2.9.0.post0 +python-dotenv==1.2.1 +python-json-logger==4.0.0 +python-multipart==0.0.22 +PyYAML==6.0.3 +pyzmq==27.1.0 +ray==2.54.0 +referencing==0.37.0 +regex==2026.2.19 +requests==2.32.5 +rich==14.3.3 +rich-toolkit==0.19.4 +rignore==0.7.6 +rpds-py==0.30.0 +safetensors==0.7.0 +sentencepiece==0.2.1 +sentry-sdk==2.53.0 +setproctitle==1.3.7 +setuptools==80.10.2 +shellingham==1.5.4 +six==1.17.0 +sniffio==1.3.1 +sse-starlette==3.2.0 +starlette==0.52.1 +supervisor==4.3.0 +sympy==1.14.0 +tabulate==0.9.0 +tiktoken==0.12.0 +tokenizers==0.22.2 +torch==2.9.1 +torchaudio==2.9.1 +torchvision==0.24.1 +tqdm==4.67.3 +transformers==4.57.6 +triton==3.5.1 +typer==0.24.1 +typer-slim==0.24.0 +typing-inspection==0.4.2 +typing_extensions==4.15.0 +urllib3==2.6.3 +uvicorn==0.41.0 +uvloop==0.22.1 +vllm==0.15.1 +watchfiles==1.1.1 +websockets==16.0 +xgrammar==0.1.29 +xxhash==3.6.0 +yarl==1.22.0 diff --git a/leaderboard/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/5fewshots/run.yaml b/leaderboard/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/5fewshots/run.yaml new file mode 100644 index 0000000..a2d5154 --- /dev/null +++ b/leaderboard/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/5fewshots/run.yaml @@ -0,0 +1,59 @@ +date: 2026-02-24 +# ===================== +# Model Information +# ===================== +model: + name: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 + revision: main + commit_hash: 5a48de7e98cce824b3456eb9857ded839c3b6475 + parameter_count: 30B + dtype: bfloat16 + thinking: true + +type: open-source # open-source | proprietary + +# ===================== +# Package Information +# ===================== +llmsql: + version: 0.1.15 + commit_hash: 79175212c90b1fc094abd2c9666c23d903060014 + +# ===================== +# Benchmark Information +# ===================== +version: 2.0 + +# ===================== +# Environment Information +# ===================== +os_name: Ubuntu 24.04.3 LTS +python_version: 3.12.12 +pip_freeze: requirements.txt +device: 4xH200 + +# ===================== +# Function Inputs / Inference Backend +# ===================== +inference: + backend: vllm # vllm | transformers + arguments: + batch_size: 20000 + tetensor_parallel_size: 4 + do_sample: true + max_new_tokens: 1024 + temperature: 1.0 + sampling_kwargs: + top_p: 1.0 + num_fewshots: 5 + seed: 42 + llm_kwargs: + dtype: bfloat16 + + +# ===================== +# Results +# ===================== +results: + execution_accuracy: 0.8519 + answers_path: https://huggingface.co/datasets/llmsql-bench/benchmark-evaluation-results/blob/main/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/5fewshots/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16_outputs.jsonl diff --git a/leaderboard/PLLuM-12B-chat/5fewshots/inference_script.py b/leaderboard/PLLuM-12B-chat/5fewshots/inference_script.py new file mode 100644 index 0000000..59bfa27 --- /dev/null +++ b/leaderboard/PLLuM-12B-chat/5fewshots/inference_script.py @@ -0,0 +1,25 @@ +import os + +from dotenv import load_dotenv + +from llmsql import evaluate, inference_vllm + +load_dotenv() + +MODEL_NAME = "CYFRAGOVPL/PLLuM-12B-chat" + +results = inference_vllm( + model_name=MODEL_NAME, + output_file=f"{MODEL_NAME}_outputs.jsonl", + batch_size=20000, + tensor_parallel_size=4, + do_sample=False, + hf_token=os.environ["HF_TOKEN"], + max_new_tokens=256, + temperature=0.0, + num_fewshots=5, + seed=42, + llm_kwargs={"dtype": "bfloat16"}, +) + +evaluate(results) diff --git a/leaderboard/PLLuM-12B-chat/5fewshots/requirements.txt b/leaderboard/PLLuM-12B-chat/5fewshots/requirements.txt new file mode 100644 index 0000000..929f583 --- /dev/null +++ b/leaderboard/PLLuM-12B-chat/5fewshots/requirements.txt @@ -0,0 +1,172 @@ +accelerate==1.12.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.13.3 +aiosignal==1.4.0 +annotated-doc==0.0.4 +annotated-types==0.7.0 +anthropic==0.83.0 +anyio==4.12.1 +apache-tvm-ffi==0.1.8.post2 +astor==0.8.1 +attrs==25.4.0 +blake3==1.0.8 +cachetools==7.0.1 +cbor2==5.8.0 +certifi==2026.1.4 +cffi==2.0.0 +charset-normalizer==3.4.4 +click==8.3.1 +cloudpickle==3.1.2 +compressed-tensors==0.13.0 +cryptography==46.0.5 +cuda-bindings==13.1.1 +cuda-pathfinder==1.3.5 +cuda-python==13.1.1 +cupy-cuda12x==14.0.1 +datasets==4.5.0 +depyf==0.20.0 +dill==0.4.0 +diskcache==5.6.3 +distro==1.9.0 +dnspython==2.8.0 +docstring_parser==0.17.0 +einops==0.8.2 +email-validator==2.3.0 +fastapi==0.132.0 +fastapi-cli==0.0.23 +fastapi-cloud-cli==0.13.0 +fastar==0.8.0 +filelock==3.24.3 +flashinfer-python==0.6.1 +frozenlist==1.8.0 +fsspec==2025.10.0 +gguf==0.17.1 +grpcio==1.78.1 +grpcio-reflection==1.78.1 +h11==0.16.0 +hf-xet==1.3.0 +httpcore==1.0.9 +httptools==0.7.1 +httpx==0.28.1 +httpx-sse==0.4.3 +huggingface_hub==0.36.2 +idna==3.11 +ijson==3.5.0 +interegular==0.3.3 +Jinja2==3.1.6 +jiter==0.13.0 +jmespath==1.1.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +lark==1.2.2 +llguidance==1.3.0 +llmsql==0.1.15 +llvmlite==0.44.0 +lm-format-enforcer==0.11.3 +loguru==0.7.3 +markdown-it-py==4.0.0 +MarkupSafe==3.0.3 +mcp==1.26.0 +mdurl==0.1.2 +mistral_common==1.9.1 +model-hosting-container-standards==0.1.13 +mpmath==1.3.0 +msgpack==1.1.2 +msgspec==0.20.0 +multidict==6.7.1 +multiprocess==0.70.18 +networkx==3.6.1 +ninja==1.13.0 +numba==0.61.2 +numpy==2.2.6 +nvidia-cublas-cu12==12.8.4.1 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cudnn-cu12==9.10.2.21 +nvidia-cudnn-frontend==1.18.0 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cufile-cu12==1.13.1.3 +nvidia-curand-cu12==10.3.9.90 +nvidia-cusolver-cu12==11.7.3.90 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cusparselt-cu12==0.7.1 +nvidia-cutlass-dsl==4.4.0 +nvidia-cutlass-dsl-libs-base==4.4.0 +nvidia-ml-py==13.590.48 +nvidia-nccl-cu12==2.27.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nvshmem-cu12==3.3.20 +nvidia-nvtx-cu12==12.8.90 +openai==2.23.0 +openai-harmony==0.0.8 +opencv-python-headless==4.13.0.92 +outlines_core==0.2.11 +packaging==26.0 +pandas==3.0.1 +partial-json-parser==0.2.1.1.post7 +pillow==12.1.1 +prometheus-fastapi-instrumentator==7.1.0 +prometheus_client==0.24.1 +propcache==0.4.1 +protobuf==6.33.5 +psutil==7.2.2 +py-cpuinfo==9.0.0 +pyarrow==23.0.1 +pybase64==1.4.3 +pycountry==26.2.16 +pycparser==3.0 +pydantic==2.12.5 +pydantic-extra-types==2.11.0 +pydantic-settings==2.13.1 +pydantic_core==2.41.5 +Pygments==2.19.2 +PyJWT==2.11.0 +python-dateutil==2.9.0.post0 +python-dotenv==1.2.1 +python-json-logger==4.0.0 +python-multipart==0.0.22 +PyYAML==6.0.3 +pyzmq==27.1.0 +ray==2.54.0 +referencing==0.37.0 +regex==2026.2.19 +requests==2.32.5 +rich==14.3.3 +rich-toolkit==0.19.4 +rignore==0.7.6 +rpds-py==0.30.0 +safetensors==0.7.0 +sentencepiece==0.2.1 +sentry-sdk==2.53.0 +setproctitle==1.3.7 +setuptools==80.10.2 +shellingham==1.5.4 +six==1.17.0 +sniffio==1.3.1 +sse-starlette==3.2.0 +starlette==0.52.1 +supervisor==4.3.0 +sympy==1.14.0 +tabulate==0.9.0 +tiktoken==0.12.0 +tokenizers==0.22.2 +torch==2.9.1 +torchaudio==2.9.1 +torchvision==0.24.1 +tqdm==4.67.3 +transformers==4.57.6 +triton==3.5.1 +typer==0.24.1 +typer-slim==0.24.0 +typing-inspection==0.4.2 +typing_extensions==4.15.0 +urllib3==2.6.3 +uvicorn==0.41.0 +uvloop==0.22.1 +vllm==0.15.1 +watchfiles==1.1.1 +websockets==16.0 +xgrammar==0.1.29 +xxhash==3.6.0 +yarl==1.22.0 diff --git a/leaderboard/PLLuM-12B-chat/5fewshots/run.yaml b/leaderboard/PLLuM-12B-chat/5fewshots/run.yaml new file mode 100644 index 0000000..6a2ab5f --- /dev/null +++ b/leaderboard/PLLuM-12B-chat/5fewshots/run.yaml @@ -0,0 +1,57 @@ +date: 2026-02-24 +# ===================== +# Model Information +# ===================== +model: + name: CYFRAGOVPL/PLLuM-12B-chat + revision: main + commit_hash: 74d80ff96552d9555f6f6f28321433da3895d2ec + parameter_count: 12B + dtype: bfloat16 + thinking: false + +type: open-source # open-source | proprietary + +# ===================== +# Package Information +# ===================== +llmsql: + version: 0.1.15 + commit_hash: 79175212c90b1fc094abd2c9666c23d903060014 + +# ===================== +# Benchmark Information +# ===================== +version: 2.0 + +# ===================== +# Environment Information +# ===================== +os_name: Ubuntu 24.04.3 LTS +python_version: 3.12.12 +pip_freeze: requirements.txt +device: 4xH200 + +# ===================== +# Function Inputs / Inference Backend +# ===================== +inference: + backend: vllm # vllm | transformers + arguments: + batch_size: 20000 + tetensor_parallel_size: 4 + do_sample: false + max_new_tokens: 256 + temperature: 0.0 + num_fewshots: 5 + seed: 42 + llm_kwargs: + dtype: bfloat16 + + +# ===================== +# Results +# ===================== +results: + execution_accuracy: 0.5224 + answers_path: https://huggingface.co/datasets/llmsql-bench/benchmark-evaluation-results/blob/main/PLLuM-12B-chat/5fewshots/PLLuM-12B-chat_outputs.jsonl diff --git a/leaderboard/PLLuM-12B-nc-chat/5fewshots/inference_script.py b/leaderboard/PLLuM-12B-nc-chat/5fewshots/inference_script.py new file mode 100644 index 0000000..372e696 --- /dev/null +++ b/leaderboard/PLLuM-12B-nc-chat/5fewshots/inference_script.py @@ -0,0 +1,25 @@ +import os + +from dotenv import load_dotenv + +from llmsql import evaluate, inference_vllm + +load_dotenv() + +MODEL_NAME = "CYFRAGOVPL/PLLuM-12B-nc-chat" + +results = inference_vllm( + model_name=MODEL_NAME, + output_file=f"{MODEL_NAME}_outputs.jsonl", + batch_size=20000, + tensor_parallel_size=4, + do_sample=False, + hf_token=os.environ["HF_TOKEN"], + max_new_tokens=256, + temperature=0.0, + num_fewshots=5, + seed=42, + llm_kwargs={"dtype": "bfloat16"}, +) + +evaluate(results) diff --git a/leaderboard/PLLuM-12B-nc-chat/5fewshots/requirements.txt b/leaderboard/PLLuM-12B-nc-chat/5fewshots/requirements.txt new file mode 100644 index 0000000..929f583 --- /dev/null +++ b/leaderboard/PLLuM-12B-nc-chat/5fewshots/requirements.txt @@ -0,0 +1,172 @@ +accelerate==1.12.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.13.3 +aiosignal==1.4.0 +annotated-doc==0.0.4 +annotated-types==0.7.0 +anthropic==0.83.0 +anyio==4.12.1 +apache-tvm-ffi==0.1.8.post2 +astor==0.8.1 +attrs==25.4.0 +blake3==1.0.8 +cachetools==7.0.1 +cbor2==5.8.0 +certifi==2026.1.4 +cffi==2.0.0 +charset-normalizer==3.4.4 +click==8.3.1 +cloudpickle==3.1.2 +compressed-tensors==0.13.0 +cryptography==46.0.5 +cuda-bindings==13.1.1 +cuda-pathfinder==1.3.5 +cuda-python==13.1.1 +cupy-cuda12x==14.0.1 +datasets==4.5.0 +depyf==0.20.0 +dill==0.4.0 +diskcache==5.6.3 +distro==1.9.0 +dnspython==2.8.0 +docstring_parser==0.17.0 +einops==0.8.2 +email-validator==2.3.0 +fastapi==0.132.0 +fastapi-cli==0.0.23 +fastapi-cloud-cli==0.13.0 +fastar==0.8.0 +filelock==3.24.3 +flashinfer-python==0.6.1 +frozenlist==1.8.0 +fsspec==2025.10.0 +gguf==0.17.1 +grpcio==1.78.1 +grpcio-reflection==1.78.1 +h11==0.16.0 +hf-xet==1.3.0 +httpcore==1.0.9 +httptools==0.7.1 +httpx==0.28.1 +httpx-sse==0.4.3 +huggingface_hub==0.36.2 +idna==3.11 +ijson==3.5.0 +interegular==0.3.3 +Jinja2==3.1.6 +jiter==0.13.0 +jmespath==1.1.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +lark==1.2.2 +llguidance==1.3.0 +llmsql==0.1.15 +llvmlite==0.44.0 +lm-format-enforcer==0.11.3 +loguru==0.7.3 +markdown-it-py==4.0.0 +MarkupSafe==3.0.3 +mcp==1.26.0 +mdurl==0.1.2 +mistral_common==1.9.1 +model-hosting-container-standards==0.1.13 +mpmath==1.3.0 +msgpack==1.1.2 +msgspec==0.20.0 +multidict==6.7.1 +multiprocess==0.70.18 +networkx==3.6.1 +ninja==1.13.0 +numba==0.61.2 +numpy==2.2.6 +nvidia-cublas-cu12==12.8.4.1 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cudnn-cu12==9.10.2.21 +nvidia-cudnn-frontend==1.18.0 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cufile-cu12==1.13.1.3 +nvidia-curand-cu12==10.3.9.90 +nvidia-cusolver-cu12==11.7.3.90 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cusparselt-cu12==0.7.1 +nvidia-cutlass-dsl==4.4.0 +nvidia-cutlass-dsl-libs-base==4.4.0 +nvidia-ml-py==13.590.48 +nvidia-nccl-cu12==2.27.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nvshmem-cu12==3.3.20 +nvidia-nvtx-cu12==12.8.90 +openai==2.23.0 +openai-harmony==0.0.8 +opencv-python-headless==4.13.0.92 +outlines_core==0.2.11 +packaging==26.0 +pandas==3.0.1 +partial-json-parser==0.2.1.1.post7 +pillow==12.1.1 +prometheus-fastapi-instrumentator==7.1.0 +prometheus_client==0.24.1 +propcache==0.4.1 +protobuf==6.33.5 +psutil==7.2.2 +py-cpuinfo==9.0.0 +pyarrow==23.0.1 +pybase64==1.4.3 +pycountry==26.2.16 +pycparser==3.0 +pydantic==2.12.5 +pydantic-extra-types==2.11.0 +pydantic-settings==2.13.1 +pydantic_core==2.41.5 +Pygments==2.19.2 +PyJWT==2.11.0 +python-dateutil==2.9.0.post0 +python-dotenv==1.2.1 +python-json-logger==4.0.0 +python-multipart==0.0.22 +PyYAML==6.0.3 +pyzmq==27.1.0 +ray==2.54.0 +referencing==0.37.0 +regex==2026.2.19 +requests==2.32.5 +rich==14.3.3 +rich-toolkit==0.19.4 +rignore==0.7.6 +rpds-py==0.30.0 +safetensors==0.7.0 +sentencepiece==0.2.1 +sentry-sdk==2.53.0 +setproctitle==1.3.7 +setuptools==80.10.2 +shellingham==1.5.4 +six==1.17.0 +sniffio==1.3.1 +sse-starlette==3.2.0 +starlette==0.52.1 +supervisor==4.3.0 +sympy==1.14.0 +tabulate==0.9.0 +tiktoken==0.12.0 +tokenizers==0.22.2 +torch==2.9.1 +torchaudio==2.9.1 +torchvision==0.24.1 +tqdm==4.67.3 +transformers==4.57.6 +triton==3.5.1 +typer==0.24.1 +typer-slim==0.24.0 +typing-inspection==0.4.2 +typing_extensions==4.15.0 +urllib3==2.6.3 +uvicorn==0.41.0 +uvloop==0.22.1 +vllm==0.15.1 +watchfiles==1.1.1 +websockets==16.0 +xgrammar==0.1.29 +xxhash==3.6.0 +yarl==1.22.0 diff --git a/leaderboard/PLLuM-12B-nc-chat/5fewshots/run.yaml b/leaderboard/PLLuM-12B-nc-chat/5fewshots/run.yaml new file mode 100644 index 0000000..819021e --- /dev/null +++ b/leaderboard/PLLuM-12B-nc-chat/5fewshots/run.yaml @@ -0,0 +1,57 @@ +date: 2026-02-24 +# ===================== +# Model Information +# ===================== +model: + name: CYFRAGOVPL/PLLuM-12B-nc-chat + revision: main + commit_hash: 7089352cfc2efbd2d3c64cc8cd5c97cd2c4fc013 + parameter_count: 12B + dtype: bfloat16 + thinking: false + +type: open-source # open-source | proprietary + +# ===================== +# Package Information +# ===================== +llmsql: + version: 0.1.15 + commit_hash: 79175212c90b1fc094abd2c9666c23d903060014 + +# ===================== +# Benchmark Information +# ===================== +version: 2.0 + + +# ===================== +# Environment Information +# ===================== +os_name: Ubuntu 24.04.3 LTS +python_version: 3.12.12 +pip_freeze: requirements.txt +device: 4xH200 + +# ===================== +# Function Inputs / Inference Backend +# ===================== +inference: + backend: vllm # vllm | transformers + arguments: + batch_size: 20000 + tetensor_parallel_size: 4 + do_sample: false + max_new_tokens: 256 + temperature: 0.0 + num_fewshots: 5 + seed: 42 + llm_kwargs: + dtype: bfloat16 + +# ===================== +# Results +# ===================== +results: + execution_accuracy: 0.4044 + answers_path: https://huggingface.co/datasets/llmsql-bench/benchmark-evaluation-results/blob/main/PLLuM-12B-nc-chat/5fewshots/PLLuM-12B-nc-chat_outputs.jsonl diff --git a/leaderboard/Qwen2.5-1.5B-Instruct/5fewshots/inference_script.py b/leaderboard/Qwen2.5-1.5B-Instruct/5fewshots/inference_script.py new file mode 100644 index 0000000..c8af571 --- /dev/null +++ b/leaderboard/Qwen2.5-1.5B-Instruct/5fewshots/inference_script.py @@ -0,0 +1,25 @@ +import os + +from dotenv import load_dotenv + +from llmsql import evaluate, inference_vllm + +load_dotenv() + +MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct" + +results = inference_vllm( + model_name=MODEL_NAME, + output_file=f"{MODEL_NAME}_outputs.jsonl", + batch_size=20000, + tensor_parallel_size=4, + do_sample=False, + hf_token=os.environ["HF_TOKEN"], + max_new_tokens=256, + temperature=0.0, + num_fewshots=5, + seed=42, + llm_kwargs={"dtype": "bfloat16"}, +) + +evaluate(results) diff --git a/leaderboard/Qwen2.5-1.5B-Instruct/5fewshots/requirements.txt b/leaderboard/Qwen2.5-1.5B-Instruct/5fewshots/requirements.txt new file mode 100644 index 0000000..929f583 --- /dev/null +++ b/leaderboard/Qwen2.5-1.5B-Instruct/5fewshots/requirements.txt @@ -0,0 +1,172 @@ +accelerate==1.12.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.13.3 +aiosignal==1.4.0 +annotated-doc==0.0.4 +annotated-types==0.7.0 +anthropic==0.83.0 +anyio==4.12.1 +apache-tvm-ffi==0.1.8.post2 +astor==0.8.1 +attrs==25.4.0 +blake3==1.0.8 +cachetools==7.0.1 +cbor2==5.8.0 +certifi==2026.1.4 +cffi==2.0.0 +charset-normalizer==3.4.4 +click==8.3.1 +cloudpickle==3.1.2 +compressed-tensors==0.13.0 +cryptography==46.0.5 +cuda-bindings==13.1.1 +cuda-pathfinder==1.3.5 +cuda-python==13.1.1 +cupy-cuda12x==14.0.1 +datasets==4.5.0 +depyf==0.20.0 +dill==0.4.0 +diskcache==5.6.3 +distro==1.9.0 +dnspython==2.8.0 +docstring_parser==0.17.0 +einops==0.8.2 +email-validator==2.3.0 +fastapi==0.132.0 +fastapi-cli==0.0.23 +fastapi-cloud-cli==0.13.0 +fastar==0.8.0 +filelock==3.24.3 +flashinfer-python==0.6.1 +frozenlist==1.8.0 +fsspec==2025.10.0 +gguf==0.17.1 +grpcio==1.78.1 +grpcio-reflection==1.78.1 +h11==0.16.0 +hf-xet==1.3.0 +httpcore==1.0.9 +httptools==0.7.1 +httpx==0.28.1 +httpx-sse==0.4.3 +huggingface_hub==0.36.2 +idna==3.11 +ijson==3.5.0 +interegular==0.3.3 +Jinja2==3.1.6 +jiter==0.13.0 +jmespath==1.1.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +lark==1.2.2 +llguidance==1.3.0 +llmsql==0.1.15 +llvmlite==0.44.0 +lm-format-enforcer==0.11.3 +loguru==0.7.3 +markdown-it-py==4.0.0 +MarkupSafe==3.0.3 +mcp==1.26.0 +mdurl==0.1.2 +mistral_common==1.9.1 +model-hosting-container-standards==0.1.13 +mpmath==1.3.0 +msgpack==1.1.2 +msgspec==0.20.0 +multidict==6.7.1 +multiprocess==0.70.18 +networkx==3.6.1 +ninja==1.13.0 +numba==0.61.2 +numpy==2.2.6 +nvidia-cublas-cu12==12.8.4.1 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cudnn-cu12==9.10.2.21 +nvidia-cudnn-frontend==1.18.0 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cufile-cu12==1.13.1.3 +nvidia-curand-cu12==10.3.9.90 +nvidia-cusolver-cu12==11.7.3.90 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cusparselt-cu12==0.7.1 +nvidia-cutlass-dsl==4.4.0 +nvidia-cutlass-dsl-libs-base==4.4.0 +nvidia-ml-py==13.590.48 +nvidia-nccl-cu12==2.27.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nvshmem-cu12==3.3.20 +nvidia-nvtx-cu12==12.8.90 +openai==2.23.0 +openai-harmony==0.0.8 +opencv-python-headless==4.13.0.92 +outlines_core==0.2.11 +packaging==26.0 +pandas==3.0.1 +partial-json-parser==0.2.1.1.post7 +pillow==12.1.1 +prometheus-fastapi-instrumentator==7.1.0 +prometheus_client==0.24.1 +propcache==0.4.1 +protobuf==6.33.5 +psutil==7.2.2 +py-cpuinfo==9.0.0 +pyarrow==23.0.1 +pybase64==1.4.3 +pycountry==26.2.16 +pycparser==3.0 +pydantic==2.12.5 +pydantic-extra-types==2.11.0 +pydantic-settings==2.13.1 +pydantic_core==2.41.5 +Pygments==2.19.2 +PyJWT==2.11.0 +python-dateutil==2.9.0.post0 +python-dotenv==1.2.1 +python-json-logger==4.0.0 +python-multipart==0.0.22 +PyYAML==6.0.3 +pyzmq==27.1.0 +ray==2.54.0 +referencing==0.37.0 +regex==2026.2.19 +requests==2.32.5 +rich==14.3.3 +rich-toolkit==0.19.4 +rignore==0.7.6 +rpds-py==0.30.0 +safetensors==0.7.0 +sentencepiece==0.2.1 +sentry-sdk==2.53.0 +setproctitle==1.3.7 +setuptools==80.10.2 +shellingham==1.5.4 +six==1.17.0 +sniffio==1.3.1 +sse-starlette==3.2.0 +starlette==0.52.1 +supervisor==4.3.0 +sympy==1.14.0 +tabulate==0.9.0 +tiktoken==0.12.0 +tokenizers==0.22.2 +torch==2.9.1 +torchaudio==2.9.1 +torchvision==0.24.1 +tqdm==4.67.3 +transformers==4.57.6 +triton==3.5.1 +typer==0.24.1 +typer-slim==0.24.0 +typing-inspection==0.4.2 +typing_extensions==4.15.0 +urllib3==2.6.3 +uvicorn==0.41.0 +uvloop==0.22.1 +vllm==0.15.1 +watchfiles==1.1.1 +websockets==16.0 +xgrammar==0.1.29 +xxhash==3.6.0 +yarl==1.22.0 diff --git a/leaderboard/Qwen2.5-1.5B-Instruct/5fewshots/run.yaml b/leaderboard/Qwen2.5-1.5B-Instruct/5fewshots/run.yaml new file mode 100644 index 0000000..47c8f25 --- /dev/null +++ b/leaderboard/Qwen2.5-1.5B-Instruct/5fewshots/run.yaml @@ -0,0 +1,57 @@ +date: 2026-02-24 +# ===================== +# Model Information +# ===================== +model: + name: Qwen/Qwen2.5-1.5B-Instruct + revision: main + commit_hash: 989aa7980e4cf806f80c7fef2b1adb7bc71aa306 + parameter_count: 1.5B + dtype: bfloat16 + thinking: false + +type: open-source # open-source | proprietary + +# ===================== +# Package Information +# ===================== +llmsql: + version: 0.1.15 + commit_hash: 79175212c90b1fc094abd2c9666c23d903060014 + +# ===================== +# Benchmark Information +# ===================== +version: 2.0 + +# ===================== +# Environment Information +# ===================== +os_name: Ubuntu 24.04.3 LTS +python_version: 3.12.12 +pip_freeze: requirements.txt +device: 4xH200 + +# ===================== +# Function Inputs / Inference Backend +# ===================== +inference: + backend: vllm # vllm | transformers + arguments: + batch_size: 20000 + tetensor_parallel_size: 4 + do_sample: false + max_new_tokens: 256 + temperature: 0.0 + num_fewshots: 5 + seed: 42 + llm_kwargs: + dtype: bfloat16 + + +# ===================== +# Results +# ===================== +results: + execution_accuracy: 0.6401 + answers_path: https://huggingface.co/datasets/llmsql-bench/benchmark-evaluation-results/blob/main/Qwen2.5-1.5B-Instruct/5fewshots/Qwen2.5-1.5B-Instruct_outputs.jsonl diff --git a/leaderboard/Qwen2.5-7B-Instruct/5fewshots/inference_script.py b/leaderboard/Qwen2.5-7B-Instruct/5fewshots/inference_script.py new file mode 100644 index 0000000..e463467 --- /dev/null +++ b/leaderboard/Qwen2.5-7B-Instruct/5fewshots/inference_script.py @@ -0,0 +1,25 @@ +import os + +from dotenv import load_dotenv + +from llmsql import evaluate, inference_vllm + +load_dotenv() + +MODEL_NAME = "Qwen/Qwen2.5-7B-Instruct" + +results = inference_vllm( + model_name=MODEL_NAME, + output_file=f"{MODEL_NAME}_outputs.jsonl", + batch_size=20000, + tensor_parallel_size=4, + do_sample=False, + hf_token=os.environ["HF_TOKEN"], + max_new_tokens=256, + temperature=0.0, + num_fewshots=5, + seed=42, + llm_kwargs={"dtype": "bfloat16"}, +) + +evaluate(results) diff --git a/leaderboard/Qwen2.5-7B-Instruct/5fewshots/requirements.txt b/leaderboard/Qwen2.5-7B-Instruct/5fewshots/requirements.txt new file mode 100644 index 0000000..929f583 --- /dev/null +++ b/leaderboard/Qwen2.5-7B-Instruct/5fewshots/requirements.txt @@ -0,0 +1,172 @@ +accelerate==1.12.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.13.3 +aiosignal==1.4.0 +annotated-doc==0.0.4 +annotated-types==0.7.0 +anthropic==0.83.0 +anyio==4.12.1 +apache-tvm-ffi==0.1.8.post2 +astor==0.8.1 +attrs==25.4.0 +blake3==1.0.8 +cachetools==7.0.1 +cbor2==5.8.0 +certifi==2026.1.4 +cffi==2.0.0 +charset-normalizer==3.4.4 +click==8.3.1 +cloudpickle==3.1.2 +compressed-tensors==0.13.0 +cryptography==46.0.5 +cuda-bindings==13.1.1 +cuda-pathfinder==1.3.5 +cuda-python==13.1.1 +cupy-cuda12x==14.0.1 +datasets==4.5.0 +depyf==0.20.0 +dill==0.4.0 +diskcache==5.6.3 +distro==1.9.0 +dnspython==2.8.0 +docstring_parser==0.17.0 +einops==0.8.2 +email-validator==2.3.0 +fastapi==0.132.0 +fastapi-cli==0.0.23 +fastapi-cloud-cli==0.13.0 +fastar==0.8.0 +filelock==3.24.3 +flashinfer-python==0.6.1 +frozenlist==1.8.0 +fsspec==2025.10.0 +gguf==0.17.1 +grpcio==1.78.1 +grpcio-reflection==1.78.1 +h11==0.16.0 +hf-xet==1.3.0 +httpcore==1.0.9 +httptools==0.7.1 +httpx==0.28.1 +httpx-sse==0.4.3 +huggingface_hub==0.36.2 +idna==3.11 +ijson==3.5.0 +interegular==0.3.3 +Jinja2==3.1.6 +jiter==0.13.0 +jmespath==1.1.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +lark==1.2.2 +llguidance==1.3.0 +llmsql==0.1.15 +llvmlite==0.44.0 +lm-format-enforcer==0.11.3 +loguru==0.7.3 +markdown-it-py==4.0.0 +MarkupSafe==3.0.3 +mcp==1.26.0 +mdurl==0.1.2 +mistral_common==1.9.1 +model-hosting-container-standards==0.1.13 +mpmath==1.3.0 +msgpack==1.1.2 +msgspec==0.20.0 +multidict==6.7.1 +multiprocess==0.70.18 +networkx==3.6.1 +ninja==1.13.0 +numba==0.61.2 +numpy==2.2.6 +nvidia-cublas-cu12==12.8.4.1 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cudnn-cu12==9.10.2.21 +nvidia-cudnn-frontend==1.18.0 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cufile-cu12==1.13.1.3 +nvidia-curand-cu12==10.3.9.90 +nvidia-cusolver-cu12==11.7.3.90 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cusparselt-cu12==0.7.1 +nvidia-cutlass-dsl==4.4.0 +nvidia-cutlass-dsl-libs-base==4.4.0 +nvidia-ml-py==13.590.48 +nvidia-nccl-cu12==2.27.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nvshmem-cu12==3.3.20 +nvidia-nvtx-cu12==12.8.90 +openai==2.23.0 +openai-harmony==0.0.8 +opencv-python-headless==4.13.0.92 +outlines_core==0.2.11 +packaging==26.0 +pandas==3.0.1 +partial-json-parser==0.2.1.1.post7 +pillow==12.1.1 +prometheus-fastapi-instrumentator==7.1.0 +prometheus_client==0.24.1 +propcache==0.4.1 +protobuf==6.33.5 +psutil==7.2.2 +py-cpuinfo==9.0.0 +pyarrow==23.0.1 +pybase64==1.4.3 +pycountry==26.2.16 +pycparser==3.0 +pydantic==2.12.5 +pydantic-extra-types==2.11.0 +pydantic-settings==2.13.1 +pydantic_core==2.41.5 +Pygments==2.19.2 +PyJWT==2.11.0 +python-dateutil==2.9.0.post0 +python-dotenv==1.2.1 +python-json-logger==4.0.0 +python-multipart==0.0.22 +PyYAML==6.0.3 +pyzmq==27.1.0 +ray==2.54.0 +referencing==0.37.0 +regex==2026.2.19 +requests==2.32.5 +rich==14.3.3 +rich-toolkit==0.19.4 +rignore==0.7.6 +rpds-py==0.30.0 +safetensors==0.7.0 +sentencepiece==0.2.1 +sentry-sdk==2.53.0 +setproctitle==1.3.7 +setuptools==80.10.2 +shellingham==1.5.4 +six==1.17.0 +sniffio==1.3.1 +sse-starlette==3.2.0 +starlette==0.52.1 +supervisor==4.3.0 +sympy==1.14.0 +tabulate==0.9.0 +tiktoken==0.12.0 +tokenizers==0.22.2 +torch==2.9.1 +torchaudio==2.9.1 +torchvision==0.24.1 +tqdm==4.67.3 +transformers==4.57.6 +triton==3.5.1 +typer==0.24.1 +typer-slim==0.24.0 +typing-inspection==0.4.2 +typing_extensions==4.15.0 +urllib3==2.6.3 +uvicorn==0.41.0 +uvloop==0.22.1 +vllm==0.15.1 +watchfiles==1.1.1 +websockets==16.0 +xgrammar==0.1.29 +xxhash==3.6.0 +yarl==1.22.0 diff --git a/leaderboard/Qwen2.5-7B-Instruct/5fewshots/run.yaml b/leaderboard/Qwen2.5-7B-Instruct/5fewshots/run.yaml new file mode 100644 index 0000000..0492a50 --- /dev/null +++ b/leaderboard/Qwen2.5-7B-Instruct/5fewshots/run.yaml @@ -0,0 +1,56 @@ +date: 2026-02-24 +# ===================== +# Model Information +# ===================== +model: + name: Qwen/Qwen2.5-7B-Instruct + revision: main + commit_hash: a09a35458c702b33eeacc393d103063234e8bc28 + parameter_count: 7B + dtype: bfloat16 + thinking: false + +type: open-source # open-source | proprietary + +# ===================== +# Package Information +# ===================== +llmsql: + version: 0.1.15 + commit_hash: 79175212c90b1fc094abd2c9666c23d903060014 + +# ===================== +# Benchmark Information +# ===================== +version: 2.0 + +# ===================== +# Environment Information +# ===================== +os_name: Ubuntu 24.04.3 LTS +python_version: 3.12.12 +pip_freeze: requirements.txt +device: 4xH200 + +# ===================== +# Function Inputs / Inference Backend +# ===================== +inference: + backend: vllm # vllm | transformers + arguments: + batch_size: 20000 + tetensor_parallel_size: 4 + do_sample: false + max_new_tokens: 256 + temperature: 0.0 + num_fewshots: 5 + seed: 42 + llm_kwargs: + dtype: bfloat16 + +# ===================== +# Results +# ===================== +results: + execution_accuracy: 0.7940 + answers_path: https://huggingface.co/datasets/llmsql-bench/benchmark-evaluation-results/blob/main/Qwen2.5-7B-Instruct/5fewshots/Qwen2.5-7B-Instruct_outputs.jsonl diff --git a/leaderboard/Qwen3-0.6B/5fewshots/inference_script.py b/leaderboard/Qwen3-0.6B/5fewshots/inference_script.py new file mode 100644 index 0000000..30b281f --- /dev/null +++ b/leaderboard/Qwen3-0.6B/5fewshots/inference_script.py @@ -0,0 +1,26 @@ +import os + +from dotenv import load_dotenv + +from llmsql import evaluate, inference_vllm + +load_dotenv() + +MODEL_NAME = "Qwen/Qwen3-0.6B" + +results = inference_vllm( + model_name=MODEL_NAME, + output_file=f"{MODEL_NAME}_outputs.jsonl", + batch_size=20000, + tensor_parallel_size=4, + do_sample=True, + hf_token=os.environ["HF_TOKEN"], + max_new_tokens=1024, + temperature=0.6, + sampling_kwargs={"top_p": 0.95, "top_k": 20, "min_p": 0}, + num_fewshots=5, + seed=42, + llm_kwargs={"dtype": "bfloat16"}, +) + +evaluate(results) diff --git a/leaderboard/Qwen3-0.6B/5fewshots/requirements.txt b/leaderboard/Qwen3-0.6B/5fewshots/requirements.txt new file mode 100644 index 0000000..929f583 --- /dev/null +++ b/leaderboard/Qwen3-0.6B/5fewshots/requirements.txt @@ -0,0 +1,172 @@ +accelerate==1.12.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.13.3 +aiosignal==1.4.0 +annotated-doc==0.0.4 +annotated-types==0.7.0 +anthropic==0.83.0 +anyio==4.12.1 +apache-tvm-ffi==0.1.8.post2 +astor==0.8.1 +attrs==25.4.0 +blake3==1.0.8 +cachetools==7.0.1 +cbor2==5.8.0 +certifi==2026.1.4 +cffi==2.0.0 +charset-normalizer==3.4.4 +click==8.3.1 +cloudpickle==3.1.2 +compressed-tensors==0.13.0 +cryptography==46.0.5 +cuda-bindings==13.1.1 +cuda-pathfinder==1.3.5 +cuda-python==13.1.1 +cupy-cuda12x==14.0.1 +datasets==4.5.0 +depyf==0.20.0 +dill==0.4.0 +diskcache==5.6.3 +distro==1.9.0 +dnspython==2.8.0 +docstring_parser==0.17.0 +einops==0.8.2 +email-validator==2.3.0 +fastapi==0.132.0 +fastapi-cli==0.0.23 +fastapi-cloud-cli==0.13.0 +fastar==0.8.0 +filelock==3.24.3 +flashinfer-python==0.6.1 +frozenlist==1.8.0 +fsspec==2025.10.0 +gguf==0.17.1 +grpcio==1.78.1 +grpcio-reflection==1.78.1 +h11==0.16.0 +hf-xet==1.3.0 +httpcore==1.0.9 +httptools==0.7.1 +httpx==0.28.1 +httpx-sse==0.4.3 +huggingface_hub==0.36.2 +idna==3.11 +ijson==3.5.0 +interegular==0.3.3 +Jinja2==3.1.6 +jiter==0.13.0 +jmespath==1.1.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +lark==1.2.2 +llguidance==1.3.0 +llmsql==0.1.15 +llvmlite==0.44.0 +lm-format-enforcer==0.11.3 +loguru==0.7.3 +markdown-it-py==4.0.0 +MarkupSafe==3.0.3 +mcp==1.26.0 +mdurl==0.1.2 +mistral_common==1.9.1 +model-hosting-container-standards==0.1.13 +mpmath==1.3.0 +msgpack==1.1.2 +msgspec==0.20.0 +multidict==6.7.1 +multiprocess==0.70.18 +networkx==3.6.1 +ninja==1.13.0 +numba==0.61.2 +numpy==2.2.6 +nvidia-cublas-cu12==12.8.4.1 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cudnn-cu12==9.10.2.21 +nvidia-cudnn-frontend==1.18.0 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cufile-cu12==1.13.1.3 +nvidia-curand-cu12==10.3.9.90 +nvidia-cusolver-cu12==11.7.3.90 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cusparselt-cu12==0.7.1 +nvidia-cutlass-dsl==4.4.0 +nvidia-cutlass-dsl-libs-base==4.4.0 +nvidia-ml-py==13.590.48 +nvidia-nccl-cu12==2.27.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nvshmem-cu12==3.3.20 +nvidia-nvtx-cu12==12.8.90 +openai==2.23.0 +openai-harmony==0.0.8 +opencv-python-headless==4.13.0.92 +outlines_core==0.2.11 +packaging==26.0 +pandas==3.0.1 +partial-json-parser==0.2.1.1.post7 +pillow==12.1.1 +prometheus-fastapi-instrumentator==7.1.0 +prometheus_client==0.24.1 +propcache==0.4.1 +protobuf==6.33.5 +psutil==7.2.2 +py-cpuinfo==9.0.0 +pyarrow==23.0.1 +pybase64==1.4.3 +pycountry==26.2.16 +pycparser==3.0 +pydantic==2.12.5 +pydantic-extra-types==2.11.0 +pydantic-settings==2.13.1 +pydantic_core==2.41.5 +Pygments==2.19.2 +PyJWT==2.11.0 +python-dateutil==2.9.0.post0 +python-dotenv==1.2.1 +python-json-logger==4.0.0 +python-multipart==0.0.22 +PyYAML==6.0.3 +pyzmq==27.1.0 +ray==2.54.0 +referencing==0.37.0 +regex==2026.2.19 +requests==2.32.5 +rich==14.3.3 +rich-toolkit==0.19.4 +rignore==0.7.6 +rpds-py==0.30.0 +safetensors==0.7.0 +sentencepiece==0.2.1 +sentry-sdk==2.53.0 +setproctitle==1.3.7 +setuptools==80.10.2 +shellingham==1.5.4 +six==1.17.0 +sniffio==1.3.1 +sse-starlette==3.2.0 +starlette==0.52.1 +supervisor==4.3.0 +sympy==1.14.0 +tabulate==0.9.0 +tiktoken==0.12.0 +tokenizers==0.22.2 +torch==2.9.1 +torchaudio==2.9.1 +torchvision==0.24.1 +tqdm==4.67.3 +transformers==4.57.6 +triton==3.5.1 +typer==0.24.1 +typer-slim==0.24.0 +typing-inspection==0.4.2 +typing_extensions==4.15.0 +urllib3==2.6.3 +uvicorn==0.41.0 +uvloop==0.22.1 +vllm==0.15.1 +watchfiles==1.1.1 +websockets==16.0 +xgrammar==0.1.29 +xxhash==3.6.0 +yarl==1.22.0 diff --git a/leaderboard/Qwen3-0.6B/5fewshots/run.yaml b/leaderboard/Qwen3-0.6B/5fewshots/run.yaml new file mode 100644 index 0000000..ba714f8 --- /dev/null +++ b/leaderboard/Qwen3-0.6B/5fewshots/run.yaml @@ -0,0 +1,60 @@ +date: 2026-02-24 +# ===================== +# Model Information +# ===================== +model: + name: Qwen/Qwen3-0.6B + revision: main + commit_hash: c1899de289a04d12100db370d81485cdf75e47ca + parameter_count: 0.6B + dtype: bfloat16 + thinking: true + +type: open-source # open-source | proprietary + +# ===================== +# Package Information +# ===================== +llmsql: + version: 0.1.15 + commit_hash: 79175212c90b1fc094abd2c9666c23d903060014 + +# ===================== +# Benchmark Information +# ===================== +version: 2.0 + +# ===================== +# Environment Information +# ===================== +os_name: Ubuntu 24.04.3 LTS +python_version: 3.12.12 +pip_freeze: requirements.txt +device: 4xH200 + +# ===================== +# Function Inputs / Inference Backend +# ===================== +inference: + backend: vllm # vllm | transformers + arguments: + batch_size: 20000 + tetensor_parallel_size: 4 + do_sample: true + max_new_tokens: 1024 + temperature: 0.6 + sampling_kwargs: + top_p: 0.95 + top_k: 20 + min_p: 0 + num_fewshots: 5 + seed: 42 + llm_kwargs: + dtype: bfloat16 + +# ===================== +# Results +# ===================== +results: + execution_accuracy: 0.4983 + answers_path: https://huggingface.co/datasets/llmsql-bench/benchmark-evaluation-results/blob/main/Qwen3-0.6B/5fewshots/Qwen3-0.6B_outputs.jsonl diff --git a/leaderboard/gpt-oss-120b/5fewshots/inference_script.py b/leaderboard/gpt-oss-120b/5fewshots/inference_script.py new file mode 100644 index 0000000..e99d509 --- /dev/null +++ b/leaderboard/gpt-oss-120b/5fewshots/inference_script.py @@ -0,0 +1,26 @@ +import os + +from dotenv import load_dotenv + +from llmsql import evaluate, inference_vllm + +load_dotenv() + +MODEL_NAME = "openai/gpt-oss-120b" + +results = inference_vllm( + model_name=MODEL_NAME, + output_file=f"{MODEL_NAME}_outputs.jsonl", + batch_size=20000, + tensor_parallel_size=4, + do_sample=True, + hf_token=os.environ["HF_TOKEN"], + max_new_tokens=1024, + temperature=1.0, + sampling_kwargs={"top_p": 0.95, "repetition_penalty": 1.0}, + num_fewshots=5, + seed=42, + llm_kwargs={"dtype": "bfloat16"}, +) + +evaluate(results) diff --git a/leaderboard/gpt-oss-120b/5fewshots/requirements.txt b/leaderboard/gpt-oss-120b/5fewshots/requirements.txt new file mode 100644 index 0000000..929f583 --- /dev/null +++ b/leaderboard/gpt-oss-120b/5fewshots/requirements.txt @@ -0,0 +1,172 @@ +accelerate==1.12.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.13.3 +aiosignal==1.4.0 +annotated-doc==0.0.4 +annotated-types==0.7.0 +anthropic==0.83.0 +anyio==4.12.1 +apache-tvm-ffi==0.1.8.post2 +astor==0.8.1 +attrs==25.4.0 +blake3==1.0.8 +cachetools==7.0.1 +cbor2==5.8.0 +certifi==2026.1.4 +cffi==2.0.0 +charset-normalizer==3.4.4 +click==8.3.1 +cloudpickle==3.1.2 +compressed-tensors==0.13.0 +cryptography==46.0.5 +cuda-bindings==13.1.1 +cuda-pathfinder==1.3.5 +cuda-python==13.1.1 +cupy-cuda12x==14.0.1 +datasets==4.5.0 +depyf==0.20.0 +dill==0.4.0 +diskcache==5.6.3 +distro==1.9.0 +dnspython==2.8.0 +docstring_parser==0.17.0 +einops==0.8.2 +email-validator==2.3.0 +fastapi==0.132.0 +fastapi-cli==0.0.23 +fastapi-cloud-cli==0.13.0 +fastar==0.8.0 +filelock==3.24.3 +flashinfer-python==0.6.1 +frozenlist==1.8.0 +fsspec==2025.10.0 +gguf==0.17.1 +grpcio==1.78.1 +grpcio-reflection==1.78.1 +h11==0.16.0 +hf-xet==1.3.0 +httpcore==1.0.9 +httptools==0.7.1 +httpx==0.28.1 +httpx-sse==0.4.3 +huggingface_hub==0.36.2 +idna==3.11 +ijson==3.5.0 +interegular==0.3.3 +Jinja2==3.1.6 +jiter==0.13.0 +jmespath==1.1.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +lark==1.2.2 +llguidance==1.3.0 +llmsql==0.1.15 +llvmlite==0.44.0 +lm-format-enforcer==0.11.3 +loguru==0.7.3 +markdown-it-py==4.0.0 +MarkupSafe==3.0.3 +mcp==1.26.0 +mdurl==0.1.2 +mistral_common==1.9.1 +model-hosting-container-standards==0.1.13 +mpmath==1.3.0 +msgpack==1.1.2 +msgspec==0.20.0 +multidict==6.7.1 +multiprocess==0.70.18 +networkx==3.6.1 +ninja==1.13.0 +numba==0.61.2 +numpy==2.2.6 +nvidia-cublas-cu12==12.8.4.1 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cudnn-cu12==9.10.2.21 +nvidia-cudnn-frontend==1.18.0 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cufile-cu12==1.13.1.3 +nvidia-curand-cu12==10.3.9.90 +nvidia-cusolver-cu12==11.7.3.90 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cusparselt-cu12==0.7.1 +nvidia-cutlass-dsl==4.4.0 +nvidia-cutlass-dsl-libs-base==4.4.0 +nvidia-ml-py==13.590.48 +nvidia-nccl-cu12==2.27.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nvshmem-cu12==3.3.20 +nvidia-nvtx-cu12==12.8.90 +openai==2.23.0 +openai-harmony==0.0.8 +opencv-python-headless==4.13.0.92 +outlines_core==0.2.11 +packaging==26.0 +pandas==3.0.1 +partial-json-parser==0.2.1.1.post7 +pillow==12.1.1 +prometheus-fastapi-instrumentator==7.1.0 +prometheus_client==0.24.1 +propcache==0.4.1 +protobuf==6.33.5 +psutil==7.2.2 +py-cpuinfo==9.0.0 +pyarrow==23.0.1 +pybase64==1.4.3 +pycountry==26.2.16 +pycparser==3.0 +pydantic==2.12.5 +pydantic-extra-types==2.11.0 +pydantic-settings==2.13.1 +pydantic_core==2.41.5 +Pygments==2.19.2 +PyJWT==2.11.0 +python-dateutil==2.9.0.post0 +python-dotenv==1.2.1 +python-json-logger==4.0.0 +python-multipart==0.0.22 +PyYAML==6.0.3 +pyzmq==27.1.0 +ray==2.54.0 +referencing==0.37.0 +regex==2026.2.19 +requests==2.32.5 +rich==14.3.3 +rich-toolkit==0.19.4 +rignore==0.7.6 +rpds-py==0.30.0 +safetensors==0.7.0 +sentencepiece==0.2.1 +sentry-sdk==2.53.0 +setproctitle==1.3.7 +setuptools==80.10.2 +shellingham==1.5.4 +six==1.17.0 +sniffio==1.3.1 +sse-starlette==3.2.0 +starlette==0.52.1 +supervisor==4.3.0 +sympy==1.14.0 +tabulate==0.9.0 +tiktoken==0.12.0 +tokenizers==0.22.2 +torch==2.9.1 +torchaudio==2.9.1 +torchvision==0.24.1 +tqdm==4.67.3 +transformers==4.57.6 +triton==3.5.1 +typer==0.24.1 +typer-slim==0.24.0 +typing-inspection==0.4.2 +typing_extensions==4.15.0 +urllib3==2.6.3 +uvicorn==0.41.0 +uvloop==0.22.1 +vllm==0.15.1 +watchfiles==1.1.1 +websockets==16.0 +xgrammar==0.1.29 +xxhash==3.6.0 +yarl==1.22.0 diff --git a/leaderboard/gpt-oss-120b/5fewshots/run.yaml b/leaderboard/gpt-oss-120b/5fewshots/run.yaml new file mode 100644 index 0000000..fe878e3 --- /dev/null +++ b/leaderboard/gpt-oss-120b/5fewshots/run.yaml @@ -0,0 +1,60 @@ +date: 2026-02-24 +# ===================== +# Model Information +# ===================== +model: + name: openai/gpt-oss-120b + revision: main + commit_hash: b5c939de8f754692c1647ca79fbf85e8c1e70f8a + parameter_count: 120B + dtype: bfloat16 + thinking: true + +type: open-source # open-source | proprietary + +# ===================== +# Package Information +# ===================== +llmsql: + version: 0.1.15 + commit_hash: 79175212c90b1fc094abd2c9666c23d903060014 + +# ===================== +# Benchmark Information +# ===================== +version: 2.0 + +# ===================== +# Environment Information +# ===================== +os_name: Ubuntu 24.04.3 LTS +python_version: 3.12.12 +pip_freeze: requirements.txt +device: 4xH200 + +# ===================== +# Function Inputs / Inference Backend +# ===================== +inference: + backend: vllm # vllm | transformers + arguments: + batch_size: 20000 + tetensor_parallel_size: 4 + do_sample: true + max_new_tokens: 1024 + temperature: 1.0 + sampling_kwargs: + top_p: 0.95 + repetition_penalty: 1.0 + num_fewshots: 5 + seed: 42 + llm_kwargs: + dtype: bfloat16 + + +# ===================== +# Results +# ===================== +results: + execution_accuracy: 0.9049 + answers_path: https://huggingface.co/datasets/llmsql-bench/benchmark-evaluation-results/blob/main/gpt-oss-120b/5fewshots/gpt-oss-120b_outputs.jsonl diff --git a/leaderboard/gpt-oss-20b/5fewshots/inference_script.py b/leaderboard/gpt-oss-20b/5fewshots/inference_script.py new file mode 100644 index 0000000..3125855 --- /dev/null +++ b/leaderboard/gpt-oss-20b/5fewshots/inference_script.py @@ -0,0 +1,26 @@ +import os + +from dotenv import load_dotenv + +from llmsql import evaluate, inference_vllm + +load_dotenv() + +MODEL_NAME = "openai/gpt-oss-20b" + +results = inference_vllm( + model_name=MODEL_NAME, + output_file=f"{MODEL_NAME}_outputs.jsonl", + batch_size=20000, + tensor_parallel_size=4, + do_sample=True, + hf_token=os.environ["HF_TOKEN"], + max_new_tokens=1024, + temperature=1.0, + sampling_kwargs={"top_p": 0.95, "repetition_penalty": 1.0}, + num_fewshots=5, + seed=42, + llm_kwargs={"dtype": "bfloat16"}, +) + +evaluate(results) diff --git a/leaderboard/gpt-oss-20b/5fewshots/requirements.txt b/leaderboard/gpt-oss-20b/5fewshots/requirements.txt new file mode 100644 index 0000000..929f583 --- /dev/null +++ b/leaderboard/gpt-oss-20b/5fewshots/requirements.txt @@ -0,0 +1,172 @@ +accelerate==1.12.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.13.3 +aiosignal==1.4.0 +annotated-doc==0.0.4 +annotated-types==0.7.0 +anthropic==0.83.0 +anyio==4.12.1 +apache-tvm-ffi==0.1.8.post2 +astor==0.8.1 +attrs==25.4.0 +blake3==1.0.8 +cachetools==7.0.1 +cbor2==5.8.0 +certifi==2026.1.4 +cffi==2.0.0 +charset-normalizer==3.4.4 +click==8.3.1 +cloudpickle==3.1.2 +compressed-tensors==0.13.0 +cryptography==46.0.5 +cuda-bindings==13.1.1 +cuda-pathfinder==1.3.5 +cuda-python==13.1.1 +cupy-cuda12x==14.0.1 +datasets==4.5.0 +depyf==0.20.0 +dill==0.4.0 +diskcache==5.6.3 +distro==1.9.0 +dnspython==2.8.0 +docstring_parser==0.17.0 +einops==0.8.2 +email-validator==2.3.0 +fastapi==0.132.0 +fastapi-cli==0.0.23 +fastapi-cloud-cli==0.13.0 +fastar==0.8.0 +filelock==3.24.3 +flashinfer-python==0.6.1 +frozenlist==1.8.0 +fsspec==2025.10.0 +gguf==0.17.1 +grpcio==1.78.1 +grpcio-reflection==1.78.1 +h11==0.16.0 +hf-xet==1.3.0 +httpcore==1.0.9 +httptools==0.7.1 +httpx==0.28.1 +httpx-sse==0.4.3 +huggingface_hub==0.36.2 +idna==3.11 +ijson==3.5.0 +interegular==0.3.3 +Jinja2==3.1.6 +jiter==0.13.0 +jmespath==1.1.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +lark==1.2.2 +llguidance==1.3.0 +llmsql==0.1.15 +llvmlite==0.44.0 +lm-format-enforcer==0.11.3 +loguru==0.7.3 +markdown-it-py==4.0.0 +MarkupSafe==3.0.3 +mcp==1.26.0 +mdurl==0.1.2 +mistral_common==1.9.1 +model-hosting-container-standards==0.1.13 +mpmath==1.3.0 +msgpack==1.1.2 +msgspec==0.20.0 +multidict==6.7.1 +multiprocess==0.70.18 +networkx==3.6.1 +ninja==1.13.0 +numba==0.61.2 +numpy==2.2.6 +nvidia-cublas-cu12==12.8.4.1 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cudnn-cu12==9.10.2.21 +nvidia-cudnn-frontend==1.18.0 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cufile-cu12==1.13.1.3 +nvidia-curand-cu12==10.3.9.90 +nvidia-cusolver-cu12==11.7.3.90 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cusparselt-cu12==0.7.1 +nvidia-cutlass-dsl==4.4.0 +nvidia-cutlass-dsl-libs-base==4.4.0 +nvidia-ml-py==13.590.48 +nvidia-nccl-cu12==2.27.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nvshmem-cu12==3.3.20 +nvidia-nvtx-cu12==12.8.90 +openai==2.23.0 +openai-harmony==0.0.8 +opencv-python-headless==4.13.0.92 +outlines_core==0.2.11 +packaging==26.0 +pandas==3.0.1 +partial-json-parser==0.2.1.1.post7 +pillow==12.1.1 +prometheus-fastapi-instrumentator==7.1.0 +prometheus_client==0.24.1 +propcache==0.4.1 +protobuf==6.33.5 +psutil==7.2.2 +py-cpuinfo==9.0.0 +pyarrow==23.0.1 +pybase64==1.4.3 +pycountry==26.2.16 +pycparser==3.0 +pydantic==2.12.5 +pydantic-extra-types==2.11.0 +pydantic-settings==2.13.1 +pydantic_core==2.41.5 +Pygments==2.19.2 +PyJWT==2.11.0 +python-dateutil==2.9.0.post0 +python-dotenv==1.2.1 +python-json-logger==4.0.0 +python-multipart==0.0.22 +PyYAML==6.0.3 +pyzmq==27.1.0 +ray==2.54.0 +referencing==0.37.0 +regex==2026.2.19 +requests==2.32.5 +rich==14.3.3 +rich-toolkit==0.19.4 +rignore==0.7.6 +rpds-py==0.30.0 +safetensors==0.7.0 +sentencepiece==0.2.1 +sentry-sdk==2.53.0 +setproctitle==1.3.7 +setuptools==80.10.2 +shellingham==1.5.4 +six==1.17.0 +sniffio==1.3.1 +sse-starlette==3.2.0 +starlette==0.52.1 +supervisor==4.3.0 +sympy==1.14.0 +tabulate==0.9.0 +tiktoken==0.12.0 +tokenizers==0.22.2 +torch==2.9.1 +torchaudio==2.9.1 +torchvision==0.24.1 +tqdm==4.67.3 +transformers==4.57.6 +triton==3.5.1 +typer==0.24.1 +typer-slim==0.24.0 +typing-inspection==0.4.2 +typing_extensions==4.15.0 +urllib3==2.6.3 +uvicorn==0.41.0 +uvloop==0.22.1 +vllm==0.15.1 +watchfiles==1.1.1 +websockets==16.0 +xgrammar==0.1.29 +xxhash==3.6.0 +yarl==1.22.0 diff --git a/leaderboard/gpt-oss-20b/5fewshots/run.yaml b/leaderboard/gpt-oss-20b/5fewshots/run.yaml new file mode 100644 index 0000000..74b6638 --- /dev/null +++ b/leaderboard/gpt-oss-20b/5fewshots/run.yaml @@ -0,0 +1,59 @@ +date: 2026-02-24 +# ===================== +# Model Information +# ===================== +model: + name: openai/gpt-oss-20b + revision: main + commit_hash: 6cee5e81ee83917806bbde320786a8fb61efebee + parameter_count: 20B + dtype: bfloat16 + thinking: true + +type: open-source # open-source | proprietary + +# ===================== +# Package Information +# ===================== +llmsql: + version: 0.1.15 + commit_hash: 79175212c90b1fc094abd2c9666c23d903060014 + +# ===================== +# Benchmark Information +# ===================== +version: 2.0 + +# ===================== +# Environment Information +# ===================== +os_name: Ubuntu 24.04.3 LTS +python_version: 3.12.12 +pip_freeze: requirements.txt +device: 4xH200 + +# ===================== +# Function Inputs / Inference Backend +# ===================== +inference: + backend: vllm # vllm | transformers + arguments: + batch_size: 20000 + tetensor_parallel_size: 4 + do_sample: true + max_new_tokens: 1024 + temperature: 1.0 + sampling_kwargs: + top_p: 0.95 + repetition_penalty: 1.0 + num_fewshots: 5 + seed: 42 + llm_kwargs: + dtype: bfloat16 + +# ===================== +# Results +# ===================== +results: + execution_accuracy: 0.8871 + answers_path: https://huggingface.co/datasets/llmsql-bench/benchmark-evaluation-results/blob/main/gpt-oss-20b/5fewshots/gpt-oss-20b_outputs.jsonl diff --git a/leaderboard/pllum-12b-nc-chat-250715/5fewshots/inference_script.py b/leaderboard/pllum-12b-nc-chat-250715/5fewshots/inference_script.py new file mode 100644 index 0000000..c2e8457 --- /dev/null +++ b/leaderboard/pllum-12b-nc-chat-250715/5fewshots/inference_script.py @@ -0,0 +1,25 @@ +import os + +from dotenv import load_dotenv + +from llmsql import evaluate, inference_vllm + +load_dotenv() + +MODEL_NAME = "CYFRAGOVPL/pllum-12b-nc-chat-250715" + +results = inference_vllm( + model_name=MODEL_NAME, + output_file=f"{MODEL_NAME}_outputs.jsonl", + batch_size=20000, + tensor_parallel_size=4, + do_sample=False, + hf_token=os.environ["HF_TOKEN"], + max_new_tokens=256, + temperature=0.0, + num_fewshots=5, + seed=42, + llm_kwargs={"dtype": "bfloat16"}, +) + +evaluate(results) diff --git a/leaderboard/pllum-12b-nc-chat-250715/5fewshots/requirements.txt b/leaderboard/pllum-12b-nc-chat-250715/5fewshots/requirements.txt new file mode 100644 index 0000000..929f583 --- /dev/null +++ b/leaderboard/pllum-12b-nc-chat-250715/5fewshots/requirements.txt @@ -0,0 +1,172 @@ +accelerate==1.12.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.13.3 +aiosignal==1.4.0 +annotated-doc==0.0.4 +annotated-types==0.7.0 +anthropic==0.83.0 +anyio==4.12.1 +apache-tvm-ffi==0.1.8.post2 +astor==0.8.1 +attrs==25.4.0 +blake3==1.0.8 +cachetools==7.0.1 +cbor2==5.8.0 +certifi==2026.1.4 +cffi==2.0.0 +charset-normalizer==3.4.4 +click==8.3.1 +cloudpickle==3.1.2 +compressed-tensors==0.13.0 +cryptography==46.0.5 +cuda-bindings==13.1.1 +cuda-pathfinder==1.3.5 +cuda-python==13.1.1 +cupy-cuda12x==14.0.1 +datasets==4.5.0 +depyf==0.20.0 +dill==0.4.0 +diskcache==5.6.3 +distro==1.9.0 +dnspython==2.8.0 +docstring_parser==0.17.0 +einops==0.8.2 +email-validator==2.3.0 +fastapi==0.132.0 +fastapi-cli==0.0.23 +fastapi-cloud-cli==0.13.0 +fastar==0.8.0 +filelock==3.24.3 +flashinfer-python==0.6.1 +frozenlist==1.8.0 +fsspec==2025.10.0 +gguf==0.17.1 +grpcio==1.78.1 +grpcio-reflection==1.78.1 +h11==0.16.0 +hf-xet==1.3.0 +httpcore==1.0.9 +httptools==0.7.1 +httpx==0.28.1 +httpx-sse==0.4.3 +huggingface_hub==0.36.2 +idna==3.11 +ijson==3.5.0 +interegular==0.3.3 +Jinja2==3.1.6 +jiter==0.13.0 +jmespath==1.1.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +lark==1.2.2 +llguidance==1.3.0 +llmsql==0.1.15 +llvmlite==0.44.0 +lm-format-enforcer==0.11.3 +loguru==0.7.3 +markdown-it-py==4.0.0 +MarkupSafe==3.0.3 +mcp==1.26.0 +mdurl==0.1.2 +mistral_common==1.9.1 +model-hosting-container-standards==0.1.13 +mpmath==1.3.0 +msgpack==1.1.2 +msgspec==0.20.0 +multidict==6.7.1 +multiprocess==0.70.18 +networkx==3.6.1 +ninja==1.13.0 +numba==0.61.2 +numpy==2.2.6 +nvidia-cublas-cu12==12.8.4.1 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cudnn-cu12==9.10.2.21 +nvidia-cudnn-frontend==1.18.0 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cufile-cu12==1.13.1.3 +nvidia-curand-cu12==10.3.9.90 +nvidia-cusolver-cu12==11.7.3.90 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cusparselt-cu12==0.7.1 +nvidia-cutlass-dsl==4.4.0 +nvidia-cutlass-dsl-libs-base==4.4.0 +nvidia-ml-py==13.590.48 +nvidia-nccl-cu12==2.27.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nvshmem-cu12==3.3.20 +nvidia-nvtx-cu12==12.8.90 +openai==2.23.0 +openai-harmony==0.0.8 +opencv-python-headless==4.13.0.92 +outlines_core==0.2.11 +packaging==26.0 +pandas==3.0.1 +partial-json-parser==0.2.1.1.post7 +pillow==12.1.1 +prometheus-fastapi-instrumentator==7.1.0 +prometheus_client==0.24.1 +propcache==0.4.1 +protobuf==6.33.5 +psutil==7.2.2 +py-cpuinfo==9.0.0 +pyarrow==23.0.1 +pybase64==1.4.3 +pycountry==26.2.16 +pycparser==3.0 +pydantic==2.12.5 +pydantic-extra-types==2.11.0 +pydantic-settings==2.13.1 +pydantic_core==2.41.5 +Pygments==2.19.2 +PyJWT==2.11.0 +python-dateutil==2.9.0.post0 +python-dotenv==1.2.1 +python-json-logger==4.0.0 +python-multipart==0.0.22 +PyYAML==6.0.3 +pyzmq==27.1.0 +ray==2.54.0 +referencing==0.37.0 +regex==2026.2.19 +requests==2.32.5 +rich==14.3.3 +rich-toolkit==0.19.4 +rignore==0.7.6 +rpds-py==0.30.0 +safetensors==0.7.0 +sentencepiece==0.2.1 +sentry-sdk==2.53.0 +setproctitle==1.3.7 +setuptools==80.10.2 +shellingham==1.5.4 +six==1.17.0 +sniffio==1.3.1 +sse-starlette==3.2.0 +starlette==0.52.1 +supervisor==4.3.0 +sympy==1.14.0 +tabulate==0.9.0 +tiktoken==0.12.0 +tokenizers==0.22.2 +torch==2.9.1 +torchaudio==2.9.1 +torchvision==0.24.1 +tqdm==4.67.3 +transformers==4.57.6 +triton==3.5.1 +typer==0.24.1 +typer-slim==0.24.0 +typing-inspection==0.4.2 +typing_extensions==4.15.0 +urllib3==2.6.3 +uvicorn==0.41.0 +uvloop==0.22.1 +vllm==0.15.1 +watchfiles==1.1.1 +websockets==16.0 +xgrammar==0.1.29 +xxhash==3.6.0 +yarl==1.22.0 diff --git a/leaderboard/pllum-12b-nc-chat-250715/5fewshots/run.yaml b/leaderboard/pllum-12b-nc-chat-250715/5fewshots/run.yaml new file mode 100644 index 0000000..6b48d39 --- /dev/null +++ b/leaderboard/pllum-12b-nc-chat-250715/5fewshots/run.yaml @@ -0,0 +1,56 @@ +date: 2026-02-24 +# ===================== +# Model Information +# ===================== +model: + name: CYFRAGOVPL/pllum-12b-nc-chat-250715 + revision: main + commit_hash: 025e26b3fc5ac1fa8714298e671a6cf2418123d7 + parameter_count: 12B + dtype: bfloat16 + thinking: false + +type: open-source # open-source | proprietary + +# ===================== +# Package Information +# ===================== +llmsql: + version: 0.1.15 + commit_hash: 79175212c90b1fc094abd2c9666c23d903060014 + +# ===================== +# Benchmark Information +# ===================== +version: 2.0 + +# ===================== +# Environment Information +# ===================== +os_name: Ubuntu 24.04.3 LTS +python_version: 3.12.12 +pip_freeze: requirements.txt +device: 4xH200 + +# ===================== +# Function Inputs / Inference Backend +# ===================== +inference: + backend: vllm # vllm | transformers + arguments: + batch_size: 20000 + tetensor_parallel_size: 4 + do_sample: false + max_new_tokens: 256 + temperature: 0.0 + num_fewshots: 5 + seed: 42 + llm_kwargs: + dtype: bfloat16 + +# ===================== +# Results +# ===================== +results: + execution_accuracy: 0.3727 + answers_path: https://huggingface.co/datasets/llmsql-bench/benchmark-evaluation-results/blob/main/pllum-12b-nc-chat-250715/5fewshots/pllum-12b-nc-chat-250715_outputs.jsonl From 8901784e4f02bc05517f2803806d1298d74aabe1 Mon Sep 17 00:00:00 2001 From: Viknov Date: Fri, 27 Feb 2026 20:09:16 +0100 Subject: [PATCH 2/3] add table on the main page --- .idea/.gitignore | 8 + .../inspectionProfiles/profiles_settings.xml | 6 + .idea/llmsql-benchmark.iml | 15 ++ .idea/misc.xml | 7 + .idea/modules.xml | 8 + .idea/vcs.xml | 6 + docs/_build/html/.buildinfo | 2 +- .../html/.doctrees/docs/evaluation.doctree | Bin 35496 -> 17486 bytes docs/_build/html/.doctrees/docs/index.doctree | Bin 6806 -> 6793 bytes .../html/.doctrees/docs/inference.doctree | Bin 99019 -> 3054 bytes docs/_build/html/.doctrees/docs/usage.doctree | Bin 8957 -> 8936 bytes docs/_build/html/.doctrees/environment.pickle | Bin 53117 -> 26468 bytes docs/_build/html/.doctrees/index.doctree | Bin 2540 -> 2527 bytes docs/_build/html/_sources/docs/usage.rst.txt | 4 +- docs/_build/html/_static/basic.css | 10 +- .../html/_static/documentation_options.js | 2 +- docs/_build/html/_static/leaderboard.json | 106 ++++++++++ .../_build/html/_static/scripts/front_page.js | 85 ++++++++ docs/_build/html/_static/searchtools.js | 13 +- .../_build/html/_static/styles/front_page.css | 67 +++++++ docs/_build/html/docs/evaluation.html | 69 ++----- docs/_build/html/docs/index.html | 32 ++- docs/_build/html/docs/inference.html | 189 ++---------------- docs/_build/html/docs/usage.html | 36 ++-- docs/_build/html/genindex.html | 104 ++-------- docs/_build/html/index.html | 33 +-- docs/_build/html/leaderboard.json | 106 ++++++++++ docs/_build/html/objects.inv | Bin 405 -> 323 bytes docs/_build/html/search.html | 50 ++--- docs/_build/html/searchindex.js | 2 +- docs/_static/leaderboard.json | 106 ++++++++++ docs/_static/scripts/front_page.js | 83 ++++++++ docs/_static/styles/front_page.css | 67 +++++++ docs/_templates/index.html | 14 +- .../docs/_build/html/_static/leaderboard.json | 106 ++++++++++ leaderboard/docs/_static/leaderboard.json | 106 ++++++++++ leaderboard/generate_leaderboard.py | 34 ++++ leaderboard/leaderboard.json | 106 ++++++++++ 38 files changed, 1150 insertions(+), 432 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/inspectionProfiles/profiles_settings.xml create mode 100644 .idea/llmsql-benchmark.iml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml create mode 100644 docs/_build/html/_static/leaderboard.json create mode 100644 docs/_build/html/leaderboard.json create mode 100644 docs/_static/leaderboard.json create mode 100644 leaderboard/docs/_build/html/_static/leaderboard.json create mode 100644 leaderboard/docs/_static/leaderboard.json create mode 100644 leaderboard/generate_leaderboard.py create mode 100644 leaderboard/leaderboard.json diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/llmsql-benchmark.iml b/.idea/llmsql-benchmark.iml new file mode 100644 index 0000000..d00f9a4 --- /dev/null +++ b/.idea/llmsql-benchmark.iml @@ -0,0 +1,15 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..3ba169c --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..6ad7ce7 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/docs/_build/html/.buildinfo b/docs/_build/html/.buildinfo index f89f358..d7142fb 100644 --- a/docs/_build/html/.buildinfo +++ b/docs/_build/html/.buildinfo @@ -1,4 +1,4 @@ # Sphinx build info version 1 # This file records the configuration used when building these files. When it is not found, a full rebuild will be done. -config: f80c99a6f2b64faef91db5c44c3abb2f +config: 3caef0746bc07fabd8f91030ce7b6533 tags: 645f666f9bcd5a90fca523b33c5a78b7 diff --git a/docs/_build/html/.doctrees/docs/evaluation.doctree b/docs/_build/html/.doctrees/docs/evaluation.doctree index c582b678d8fce8948c18ba58d72135b099893df1..3dff8bedc85d28d10ee6abedc9791edf7cbdc371 100644 GIT binary patch literal 17486 zcmds9X^bSvRi4?IzN(KoJPqqPRzZR7?)6sB?hc+^dJcAX4)1t+dfIcC!?WaMRYp~2 zbyj9=9(~BHJS%&xZAf9sTcG7I$Ow>-j4Xb^ACSc%#A#${1&Bj1AozzZ#33Oe_+CUt zMpkuqZDq~O0v@SrDkEPU?~4~NUc7j(?v4ETg^PQ{zhuVpYjNbb;ezK|JnSUnRM6sH z)Oo8j|9a=O&RjAPnCqb*2Q}VF_JE>hJFXRQuk%XB-jj@lycRjWm(klJ_NYDfN@or) zMQ4~1UI0Bo*);`Ay!9p@tbImYBA-<1zW=nn>x8k;b zsG7CafY*s8PR1g$L9aUYjD6Iew2#^Q?BmG`7Z&f_4tWsX@%%NvapxirS0lfD$8}rb zt8S$V9_^MHtlq&uL;1loC>DYcY@E3ISX>1J6X45i1ZLaLBljn_Y|iecb@<%;T~?1h z0X;Ka*Iy6WrXRD&XIWQ_psYlG<>tqiSPS#(GCz*maTJ!!hUs`=#3Ho0v~-OK8oq0> zS7RPHJY=S4F%&uxXO`KJif#XFXYjHqX3!h@H-v~=BCv2Hf&E_beilMoN zCM+JAj>|0;@U|bIk{L%n=$smcySX6dd(NK4gkubybWQ>2f&_HgK7nZi8V7_78Gt_( z6eXDUNHUejLrTzWQqmaZnteXstZ6?15wIB~u8hC$!ruq+H;3u9pR`ZYDA4z8@|*w9 zG2R7L-k1HS?Bn)_A&mQ#bOVZ#)CGENA5aC!_+@TJae!qbR`6`DKzDz0aP(SwuSv~Mu>|-~tFE6n= z1T|!Sz*aUR8=8q|LMftlMR963QwATbQ4NYaI4EjPN&v@(HtRcq^>hM>xJG>d!G_`@ zx%gs!9Wu*b_gvq!a%uxgAS4Cdr3Nm(#GE?AQV$)ku??EnQ)nJe;~JvR6o+UMTz`5{ zh=_PQMCVshHJ2(j7RD{;ZlxH*xGgnW&G+g~BM!`}%TqPC6F6_D;5=xv8kV0b4$Blc z{fR+gA>!?@tlaEY?6h`Lw%NK9*?L>Z?+B5DEj2P*ZMiu(9yTDc8W&1ZV`K{~x&Dwl zP#Wo>x2tF3SBuZYG{F6pL1%)9)0x;^YbT>pJ9d)26|sYvISn9mX$0a zFVwo3Q|yas7=3BC22gdb6L?`xot+a5&1@H%PllR-*e0JC1IP*$qMs|=l8Hji6D^p_pr|f zl}v23+{LhM(s5!O`!Q_hHGCbZ`*gzCO2%E&Ys3(vPBPjS+i@qEv7LtP;x97Hph5OK z*=hfpO8Jo-`dT7B&?^ySATYn)wZ~_&F(7p&CC$mbd-fC;lw-M}{#&*s|5e^BImbR( zV)*x|vf76Y$0{$f@-yY9SUIKSmVr|RmEOFxbm_t^Mh6mh`Ns8Y?3UP#zVGErH!d++ zjOW>@r^~NBwR4-#_G|OP_1nw0=AN9-wMz}XvXfNiIM_uJzqOfE!ry#9L4#D`xw1I$p3F*9g(zvQ zY8Md}4Ro`C6D_u<1*=+?R4GJ~blLZKDN~^Thew#n+;fdhV?)r!-N%MFd|A0QUJM$-6_wc$JH66!d83eA!klPc&uR#Z;*{9EM}%$EAN^8nVc;ae-(-uz3 z^d#0*rpNq?)N0`|FJd4+F9uRz%z9ZV6FldEYJQdma{7%m&&|0b)8TMO(WubZ(O!2*}ZIU@zDoQczC-{XPYzxOlUL%DzN ziwoKV-h5)H89I*{I#zUs9;sN)bkC6z%-&)xn&*dN{CmOpnZX%0|I?PKXWRP&uZYgXf-SbY(!27|Ne3)21* z{p_}hPA}7NL=&D;?fYTrSwyilOjV6QG86(Y0fDWe2t3lso}@cO-1(6=Od-=%rdj^- zP%K{n%fCE0%fBQ@`%n6cvO#F<`ua2HCC_$^kk)=#3jsI{v6+O%r{dVLUK1x+y^-81 z*f2JN{!RC648J`Tf)|0{SBoO}NQLG=X#4AizI#xDv#neU&)*Nl`X#Xbw}Z3(*MhXa z@H{Q9oalZny-H{62(#eI?Mv56%H&|Tf|%>Ap>Rdb{eKR{{bg|fA4R!;q!T~WGB&0~)JG1Zn?iKXuLAa9VMTd2y@C;l>kJtV#r+BrGRvyXK~z z{EuWaKU7_f!81c)@G>x1D2hRW$?s44yL7TrE|T3zll|n4=^&g2ZWS);RChR9B?puX zCag4^HH0*oJ+UCoyRx?S6iIJ}?mIcG!f-0Mnn?Xs+B25D#qmME?c*IICsZHr*whtq ze|p&uAS6n`brZWfwi@`yFE8p0!X6@$^9lqV@tM<1j^27MbNXsC*ND^HoNVqlr8 zxkwc~?u~2|MSMrsa^uIiwUha89`032Xr2ScM9_2(;GpnwI6{U6AE{Rx>T2 zKC5}=x`@YP*JM1Mn#tZ`y)wR3Xl2lf{@JXCSka&8TG5NNqLuzaCNNS zpd$Jmp?ZO~GQ-1dpOA}gut?%%$gEryKyCT<(h@n9JwFAn7pQM)pdP$!4b#621=Ag{ z{m%tpBKjR+x=ew={b0%>b5mH@x*5JZSY!LM_4TlN(2z|~Ej8 zTfjB5!GpjLLLpjLJVbgsa@|3I_y(t-d&HSanqE12LPJxuB@F=^3W6U3OUDa9K=eC8 zFmKjs2#l<43IMLJQ9P7O7sNFL|530wQv|)-e?)U%pb^dZb3-w1fRR%L7$^E289$vn z!jleON~4alVjW~JwpJM?pI&k6?Pp1&Fl>t&lI5WwxeFFQRsa&B-w~2>76K}S;Zm0N zz_J&;LQvD}4^msR%7tbSCOj0Wu-g4dV*^=zkImz%J>Kw@)c5w|C< znaZ@Slqu+3#J3}suLqCh^22FYHanXv58fP{iF%-L9MVuxmJJp~^SMi=A07(-D)8JW zfIrb|{I$v6dzWtiq5Ve`DX8LM$!t;vaaTy}jJ)fvq=_v#V?TYPVw>@%kGW6Qbsuwo z`i5|5GlV(Aj#iV=Q>tb8ONFo$#(q{q82h}-*nj7xO?ainttCRSA(kT4)WIcvU++fmEy=;&N>9sd_igG`2ty0FzUmP|`MT7gm`ovyt^=?l`X@&5cK~EBnTsSL?!tpqjGQVEZZ~7$wY&DI7=YBr-KLNrpnwRJWK|@A7QAx?rb0t z=42K+xuNjorOK-!Dh%kaLA8mL(B82suH#j9}x-Oa3dd5n#EYWCC+h<1}g-pEU1~a_m!tcqf^N zqk84}O6cJG@$Q%MoNCyJ9CvvD_-UxMsFNI!up7FS5z$H~Innc`r-|mvBZz395og;S zY9H@@(Nj4&NIXFBLK_gdB1>diWrP563uz&b;i4vaAB+t@undd0AvPE!vh#q$sLaVD zL>x;=gFTweQ1&qrDO6GNm;+gxw&GUX$m9{giUQ;~^T0E4g|wU;5G6v=Y4&wdJLIwD z8^~q>)j__|_Cs#uYYHiWNx$ZVj+EB}2Cnlc^c;*(<{&DLsda)iHw9*pcmg&J6hvfG)LWVag|=>a)kIdy_-7*dRGx1A^!}Ehp++JNThwrp?HyOO#>&pfr$Zi|L6(5)W@zKJfH+ z1J9YDUh8R|N_6udRLxUXIf?8l4;CCA)kPoB1}8}kqq7Nw#b6bQ=;-m}1Z1*hWQ#LX zc81I&+DZOo|@8e>(50P20>Sw0%b&oU`b4bYyb*~q%D zVdCX_1IhwI)Luo*_m=pn1euC1)xZ{{$FDhF9Ws#Wg~OyM!noQ)=ClwtkG@Q~80j|& zRyzjWvoFB!bnNe*|F%3nJ`FeAr*Nzt&vBEot)#eOPcgHck5YY5I6ue8V={e49RwP_0Aw zh?5CwE!u1YeWc=x_O|S=Z;7V1L^E5W34EgFmZ-WF+l+`szmxXY>=n_e{f_LBE@@?t zbd4!{q|=`0QCNF%GMyJ9!Pa!mfhfvDj>Kh}bcLdm_~lDTz(py9u2uEjOfE&?k)Fbg zdZMev3Dv8pbhxX1sC`5|VG+;vww1oY)(cHm^JCIe*ymuTxsW{YfIW5WLR$V7fPJHL y1FI6hCxP{iG+fHxkR4_Ut*dx4D;Bae>3B6awn%mxBt|8RbZ)Ai$f}5|3;zRR?-5M^ literal 35496 zcmdsAdypJQdDltz*6H-XJ{!)m$LDfPHd20tX*NIJ=uFiF5!&+g9M%-rtI zdS_O57c5HVVe8DKsDh2bfI}dFsl19JkK&(@;*k_dfHFlw5kirQDpbg$3P=h_iX`Os z^<#QwcCT+{bv6W5X>YfuyTAS({e3-Oe_!{xfnR_2scr0ExZ7{oPJMOOtW@e=#q!(X zaK$UPYF0gHzq>v2c>8R7Ivi=5E56rimaTTU4K>QPQ>`?udi&vaI8N0Ar|Qp&_Cvl^ z4jiu@HMa-sL3`-o_Ov|`4g^k6wPKIL%MYJ1t1XjS7owC%m&NHIdg-VqLp(JzGkRi|#X7g{AV95sWW>6BUluqG^bI2E5T z8Y)*!-$x<757f;X|2AA{HPnxiSzc~hi&Uo-4h7~CJ!;##?MZvgzQW#N?+=g9* zYdChRI(LudF9%*@ZmH6A&RBCNo1P0Q`E%83&4097C;?%+W;U1SFlwKFSP@-jn?7*5 z^62(f3G^5NlIBv=Y}j!!e)+U*8IOoI*7VFH#$v0^_+XgTs<+}BYhKF;JR<^S5M2v_ zS2*>+F{1|FR}HTfG+Kc_YA%^h-4BcaeI7e@oGL7N)r#?G%W687Zs@{@6T66rG8I)}c1=P?t&!CHn6_`%dsu)eH;Wg37 zYy}?bIb{rYZI&^A+MWRSF^0BaFsM7nsk>lb2_AwPyO}^0!3!ZZQfTdga4b#_E>08S zsG?D<+gsz^n)X$Y2m5M>-!=GuE&gAJ|1;pV{c`(NGz!3;2tWP*oW>ha)zYG0u=m>s zA&mV})Io}!Tu-QpzDE{>!$&MLXf-j1*mRzV6)5nhvs0gD*aN&e8^#Eef}N#SwQ2>$ zs)L!LBtYq%5qeUDz}8}q!`p@Kv4s3LnpVYOlg2j|n_kU$?Wq$B$Bab?s&9Br1c~1%emXsUk`(J5eN0MnN10s!hwlCBZeq zQm=YuC8jo@1boug2{mxfU52x0V5<90eW`;qZxN)~n$8u8-j<(4JAw7NtR$k!og}*B zq)>B0u^E1=2Hh`D=8izySaAYdnT5DT2prhgz^pa6ad7G|Kx{V7j&h9=O|bC976XtPY0LXn zocLsZPV544e?KcHsIuV1##%cZcJ|?M__sOv_TeYhSGt@Mz#ZuKPFUQT1!v@EfgtKCq0!1soT82|2&0@$E z%;+dzJQ;_gqUouI8Q8_ZD~h$;tm}LA>eS3ATboVu1~U@p7ZD<-ASm%8Ut^%ck=0st z-fx)nsxb_E>|4z<_&TuY(c8y*I9xUBOD%{|I~;5SVW6E;4Qjf}M;DRIa8wr$HVD3h_F{xCAh-;6Iy4jFG4HSq6E zQ_>C>oyyd_G4+b6gT|DoX;q4NtDw?ThmRe;`?NuCC&rNnPaHQ+Gi&tv`wl;N*r3UH zhjHWeQ)dr$@AC#-pLd^l-NNbV12eIHsiTLxNo5R!4J7gB8%Zhr?#mz-I{9}8P{AFf zk0E@lyTka5B;6gwucUPEkBN+u?i=xfdB&+=xk0aeO#`bq14}e|{gPti8`J)>(=b-N z=5obp&P1}_Q(GnZ2Qb9^l7C$c4JG-{bm})Is2eCbF2xaoMN)-_rq~PbTvRF)BClI1 zB`RUkKsOihMho*aTPaQPT7pEp>Vj9dMk59KGd#jb=57H;wp8Y36tiOMhVd3z_kty@ zyIMW)Cr0dvW(`4xTeF7PcSO21?rbhGH&wKb==Mm-<*8h9iRFZQGHStA6gM*KJ-~z` zX0`b{COi>?!BW#}H98)V+iR%oRsDul?s&N2n;M*KUUkK(1a|xAB`|01GL2@^wehD_ za;x|w4TKoMMm`Jo4C)>|+0h4vH2?>XS?L_qNVucvt#p(#2Oj)u&9#n4v9;E)WQ<&7 zr~<`}cM>+tPsTkt`d3!Dkh&EF2pIvi_1&ka_eu9jR@Hq6{-kK7H04fs=E_Z{!914Y zO)OES#{4nrHTRh3F_7P41IaOFHLc_lJm!IN-%bOWbl;Mp*K@oomm0?Ll2_?u-?aM# zDkh0EttZ@&SYS#nN52)pzCyrr6YMsi_T>zy2`2v)tEwOC?I&Rg$oZ-KIpoeU4Y}x3 z>l5zWF|bg6wB^7ZG$WiD6XC^mqNwtno~Uv=sPgT+RLLYpU*H7HEdXP2vnZt?S*Va6 z-rkES90Jk7Ge_aFr1+bVew zp=cGs)&gujC6*8db{^pWrd5|!nKkaQ7R^>QzzUqd&J_i*rXq2A+lxGq3+FzN9Lj}r z&(10jxO-Jk9J&J>x*{)!UQn?dPx43#X1Z93=Bs-`{MA7G_Uwq?%Bt!`^ZFCa3Yk@d z7Ou&?nh2?w3X4Q3e?x`+Lp@>t8eo4aFZM4e+lRzFl>ClrC*>N1#;&$MV_x#eYWOtwr<4!?Qy<2}FTSbOaw=!p8>})U zr*n1;L!j?EWf(r+69w-91wWIQf-k7h?DlPMrKt8Eq~LUvE8+Rao=`sw)c+wn>fd5j z^}=&K+j64)HL;a$tRMu#l5StxR+38&<8-rSDbtX|apEkJrrA zE__cHE5s*v^s-pJ7l>bumKhhTm$IsQw%>T%Ty<)#no)1nN*3IBY>QQj0F*?<@f%fh zO)>wbi1R(QRW!JvCmP%b8qDRTK@R5YtNuB?SxFbk##O_9^2W3gP6M~fB5zaf@@f_C zmM$2-u!N|cdfQCJg19Dctvp5Co1tKb`(zkS0b3KoU!^=_(NnxWsHYn57&)Od-m$U! z+5YJfuL&WM3NBfwSiGEz#pDsQqZZ5~b`*nAo|FR=siR(`eFOTog`lAl_O0r2*~$hM zxtjAdQR7~VMv=#NbUZeGj9WRGzqPenDWe_T<=;i=?Bhn=PRxPW7K|abb`ZsK#ya$h zsk2fFe%{gK@&P743GvA9?(V03(sq{(mJl~k`GQ&y~Fw~xt1l2|11JY<&73Q%p~b;pj8Q@QR5;-v}oeMP8??^}`SKYAk5 z8-ezBbC8MZcO}yi3JlhfDT>S$WHDAue{*6D_eb+m;Q5 zT}d{JZRVoYYjO0*x#(M&(VoO{Fa^&F9MNQ=0Ke4 zcSZc>*eg8g;A0}{C@NM#_RO^MFnO`Xt<2AHMTK5_R3tgk6GUTx;UbbW`lI38y-si1EmUe|3VZMQ6 zOi!`6J+{p>MP_A+g3eidJ7W1#^azh^O~1U`iEtr%cW@@Efx>Y}Lq%CMSQO2~Hkr=$ zME?@#xt4?eRA13wE!oq1>Gl`xFHxkRjEALVlUxudLSo1HUUxwxw!{|u;)#rH#uLqQ zpH|nJ<^HY*nM2z^m^0SVk{PK|CCeYqMN78Wk6N(BzLi+)|IJ-%@Jh2?ON3%WELo_j zgV%WYt1D`wl+tqym~fd}14Gr#Km7>3^wTm4dY*E!@X{dmU1vv;DTLT~q` zZ2axRzD2%Cgqu=9a`8&N%RL6|Qz>ZLs1Zs=a%p^Eg6y(TvTsYT`VSXXkh$SR9329q6;M6PD zYG;7eM&VuI5JhAm`2?wm(_w$ZbeipO3MQsiwQe&u4kmAhSMy2*O&xo10j>@5lYWn| zQ$(@%i@88?Z}44OooQDahqeu18^f!#&xZqeIdCd%_b2F_fbTXw0dJ zCcj#CA5ahqKwpNRT`Ff)G>19NbySpkxWL-n1PD7AL>KUH_BO6`g~UcR%{wx9TBHPTW8M) zb!Sq&Y_5d~F4MyIE0@{DBCCWXjn$@V%ryuDmZllWOM+R2Qx+LdY($vip4~Ik|Ay zBxEIG`1jF>MYQNa10<_)N2<1vH6>XFBE9Ura?m%Hv@D&#Iz zomw3zqAtCI%*27Xs6J0xDzZWz-!5d5bmN8v6BqgZ!5lasj^JS>b9ug zN%zsH`|zzlJp&vQN?2lOtZm^4qive7!&0;)6;**E*;5ft-8K%U!(k#Br`Iq+*^E@@ z)1-2rq!KyOeQcz3J_BBBtR_Vl$7SJ5KUb3=l6^k^Bdp^XM?v9iuVfn+N>UYEea7ZV zvgSk>{Z`EUxR#k=Uzh#)k8yr|7=Ltb+vI%sP2T7OtdaX6{E=!q>^s1vZCkjPv6?XP z8t`5spg4}VtT#a|6b7H8o;O_Tg?(OUgu!R{VE%*+X5&>zW(-~)kJ?UY#I*9|9Cn|J zQy>}OEw;dtg$mzqYVmdd0FOIDDg?{E&Jf1zza%1bgED{kNihjpN1Kg(^D$cgn-+H1 z*XX46zfcowA4$-WVJq^4)R?&T8B*=wn?m@1HUYZQ3G~Ntph;SfD$^KEwy}m)r@X%6 zD&C!9WDqNgiEsxR#YVJmP$zP6qkvO>ti=k>U~%g0aC>772jQR~SZh%FEi)u-Rzh{j@xa zA#+L#W+Pc}4K$0%!Va(GCMAKx(TIY39d)5&dGdWGgJ#GiBXuppS{dO=!2N7OMh-J$oH^_cX zCzWnGQAC)ybg?4FA8LuA73EK$Db!t35p_3bpVdRrDIx(Si;z99hoTcQ0VT0m=rCYz zS3Q+7Sr@c}h>JqpWTn)UJB40y*(;rr`W8CNXqSGd9lciP2q;N$=x&$#hjeD$)rmt6 zT=waq=)^%l>B{f4T{5&_vNF7sX81*!B|iS77RoZ0v&^!2ExuY25wk3dS-MXTMQ4_B zXpD#SP;??9pgaS<$)!ukc_wsjT23l@SRX}4iN)CFwuTQO$>q4bWwtdh+t>%im+na; z6J@6fOrzh*ten*&t9y4^L(`0A0 zY{#&6O`s#IfRe?m?A1fjnUx&4T&0JS0hjD%rJ#q^gIVE-Cb=A!x6EecO8xdlH`9`c zrx4~PQKn4H5k2BM6LS}uW;8LU^-y#=ML@}-A4_^DI)M>Tw!p*$+Cju6;6*br4bG@6 zBs{Mjz1GACC|OL*(|Ra6amazoNAys1;vk@8GBKajg2~G8tu--x{7L1bH8DA6*%!6= zYDGjq$zo!@p@*U~F*$Jgjvk6mL zk@Aohd|U!@nHUbHM;HFBj7Q1MNv+1oLGIUU@zrV^0VRvZvGh=M8Yc%Xz8*>j(Mf6? zicw*r19yEt`=R9IaXqXaG!92}3pCDU8${J~Q#R>C8icM%7c0Eqr^j2TYMw^ZjH>42 zdMG+oBcNnaHGiy!qLUW^C826|AbLroY1m^$$S-Jt_pE3blpgiW=NW@?=$Ra}zMw@{ zt7in1EPCd9dMG+QlLMFk)kDc3Hwit{cau~y_@)G8Y*cI1d(bc(#hx@wmPX-AYdA&8 zK@wHn!j7N_(&Bf9unoxfVW&0`2$ziR8e@?tA`LzkdV$bL;&w&r^$o z>2Yoyz?ia!NKF$4@sPn7rQmhvS4!b}q1iKzwNgg9TRYq>zZOm0b(9E~+AJED0#jtv z$u*Qchtvjnb&?G7M;em+O7SXaL5T$+I2sP(2ZEe21a`TS3Kynh;@$hxsPITZB~Y%- zHt=y15Uz+qz*jpQrKmwfK_Do&ortRp2eZV!u&qgZ5SJ<-&YDFW@^Oyh*#0sJX5hEo z5{AW!pFpeyd%(jKPGQ(MbUED1$&gPf+bi_pS7YU-1Nmpb+SK zZm{N14Kb-zcF3zl^s7-Q7>6BX9?${2u>*No4j-}))Tz9&dzM(7xx6?75pEt5Bx59r zqDAF?nt&BESI^}l#z#3ZK7>CY!bCX0@^xd~(UZ7@qRzowSSdE8yHcu0W-!E)N(SF{blS+Cp?3E^-dkak!MPPaH$;r8x9foRmE6 zehg5f4c%mDs9;aBP*LLf6j8Mpx`IbvsD_&Edvm7!1skUpB^=4e=5nMbZC=&$4Y{U$^L((=&Hp@vgrpKvWcFg*ff(JA;pq( zKby1U-?xe-xGRFin04WWINCL`mAHqaD;J^=0I9;{fwl_4shvqeWup6)Xw+SW(UTP2 zuSwCp>)_RZ(miZW@kurs?ZC55g2S<$F`{~Oq;8E*1$xXB6b|uT|Ro_H5aV=pXO zbqd+XIo-nLkkD9(%`(A^M47_>Xo`YbyyQfG^Yci%@ub7IEcnt|9}XlPZd^&N z#!r()qH2RKHS9Ve(#|$%ACkSs?8CDuT4f^rGaHhgo}@^B`EhfD@hgI-n{7_;r*jcp z-yfYp^4`tOi~BSgKfdaYXSa4)`MHnts&8h!x^KZBxyll66Yi6-vT*1@eqf(;-bB9@ ziT*l8Sv{k~3&K~ZQS^e~z9$B=V|{s9cvbi0 z*Kt`~;P*VOhhyTQ{Jj@7gg?oTD($X&vP{;ao(P_A1p)Hac#4NBd{dwgJ6=VeBBHzm zngdqe2t49;Ks#Ncqgc_C1}0YX zHspXemz+8-N^i{H(6nl|;q0QS*POrR=9_Q6ZR!r(NB6xCzUz1K28OeU*pY1)p8d#o zKlM(dv=%?X;DP^ZjkGaOF>odu{s9o z;1DZSCk1E+mn3!&7EUqQSeC*;A2%ohjD0Y?Bxss-zsgR^D8lctgfmg;y3uLeq(a*? zbZ8f385Ksw&sMXFU%SWyT{6qd^b}rbx?>ARn4^z6^0Z=4&WF2vxW92OeQ}k91&8^J z!!74Die;}>!_4u~+axcssun9L()1{@q)%W1V4^|Q^nF$^#-*lMwa!?e<@RvYHmf-F zj{h78_gbq?Py}j0%U7iX6|2-*LVx2cX0uMIvFI1m4J9MsMcE>JXng!sJBW-&3AAvl zw}QpOEd}P%zasgNAJ18|abafF!g)%AG*o!|!`+!ls;Pd!O>;rE8B5CefScbvF0@^#fn8~!Wbl-3$+jq zLOJ6qa-xV+k|i&}fI9`Jmt7;nj>1c#}daeRl8=FwLXw;0fkn?OX2a!Ae-%(*cj zgZt)jwg|R(P<@xER$Rmd7b41Xl%99Z;8gqw}S9Le)nyQ%p!IvnUN5+mvPnQIz+?6N%H~d&(**>q{k~e^+m{l&*1wk)LJfD$V{&mal{HKU2y(< zxR-h+j{8=1kt?+IaEDbt(_Uzm!eOpWFnuA3Sl_Vo2XVJ69ahHVW-kT>$qLwEyO6R) zQsv>kL@DjPuwwMK82z9fMW;gL?C7$G+w71Z)5l5*_4$)&cO5g{eS)mzSMBHB<4~pU zYw5$nxyx>eK0ZJn&#-Se9LRl!J|3r5Z>EpaSS+~@;G-3eP`_+7Mb?J(yl#JWoprj- zI$UR+;S+tVvrg7o2iVxYnwqe2jM>lI53oVlU*jX7n=AMT=xzr-0y+YdkAT)_Yy|8G zDTXWmMq8rnz?fFJD=vVHRJ-s7z)d8`#7jue$a0bH8yOFV(uh88S)Ez)tYkyzqYxop8658W-w-woowz5O6& zG&X6W8z35VH6Ln0mPQub9!S3}i{BIBz4u|=jMHQ4%eelF#MKbz z`Du>on=)_=CXBRN^T4|qt@y6orC;&Mx28$d(=LVvgfZ?5``ODx*nj`RkrQMf`69_3 whh#v@AjAUnMz`)ud6(hNh(K&c?sV$qYO6vTV2PCYXoW17Q1Y}kYn5jI55Ya!T>t<8 diff --git a/docs/_build/html/.doctrees/docs/index.doctree b/docs/_build/html/.doctrees/docs/index.doctree index 18a85b95df75a1f08d67983297438cfa09857d2a..eb88b105296d0b27cb563eaa471637c7a36155f4 100644 GIT binary patch delta 2117 zcmb7F-)kII6rS0i`O)1Cs~hPgF-xmS+&GKIBoDQy)E_ZMO0^XfH!ZWflbqSk&a5+c zG)3x0M9RXgu-wsCZC`vURo4e0Q0QOKKR`h$#exW;zE}nE%-o&sWaE}};m*UocfWJK z?|kRn&z4$CwB-IL4r}3fGAATr=LGkdc#&NyJVQ}nwzB;RQoJ38zo(@+wbUd zSv4w9SFE!ojArwXW>nR!qGco8)LO(TgTStfE7JlUw$u$3<5X0P{n&@@d3O%t@B}2+ zUC6Q7NPbp=MIQG!OdWU13OwxH$>aGUT#w+I!8ON>$kC$(Sin4sgKw?74iw;Vn1p#u zBo>gugk<(}~fM4yfr9>9Okt< z$Ve}U+7*KR5IY?ArC_o6;U|4D_@9RHa+C-0>3X`41erm?%W4iX?9KReGeTF5qSl~9_vY!@#0-T@{cp%12YO)Du!*AUszNl=2y_Z|`DRbBkM1vz?NAQw8|9Yl|hM|M%``#(vL?H}6h8-spef zwNP3(B|*O1V}mD9ZAKJ?{gV&c;29Y0hV$9C$1b3pm&U9nHxWp+WyHN#-DzUOwCW0R zX$(kR_kDnKxwh!#!$j;_nzBUZks@G5$8J#<5wb09kkj>RHJsnlVk_JgIjR3w&v};j30*%8%kXz=Nf1W?4{`D5@ zwBT0Dzu0mh?;U9{%27;2*@l~N&4+;74oz*CCNWH+p21hx1EQ&l=E>SQeQ-fcARnh( F{125=*jWGo delta 2003 zcmb7FO=#O@7>@j-#-4xTw74WnnxrB9!LdzA;=@)#8SU1PE_7=Tr6n73{57u>*;4h> zB!`d$I*O?--lqHD7d zeSY+Q)v<5K4zGmo-sx_0er8frK&UL__)@8sE~LyvNuDuuLDF&+xkNOpHr2yKYIB8X z#0*^~3%W(E+EQQb?-Y}&J)ZwE2^_?r2ZGRRp3j3^CV52wL&~2e+MF&c`Pq^x+$8yn zsw|QjUCT>SS-T-+i=;9GWnP`jV?E8i36&CA$f=rc!I8;;QG^Knk@>VQVn$SQov65j zVD*!$){=Ds!f*->Ok0qq(Ln!k4#wOjrr^kwQRJb|UNeC22k{!h>o{H~>F0sdr!z2u zP3K#$nzjtcKo)vo7#j(B46&gcy&w3ANz)s_wPg-2?5E^*FxC&|F)@SLY}Rtwn8D2z z%lg`KNpf(tRgJ8l*Y+aOW)ad5f^LKc!;Td6d1&yIBZj-lUL16bICO2@KkzyO*6!*A zbkL8&eaQP-xcd?ZTC1LRw`9jd@uA1xjs91-=W5!FO;j|Umn2>n zF>Rd9VDln->|+kT**}<413c-6B-t~NkjeenR;=65heWmT~7f_FA`?)WgLHl$6y4a>} z_(bl)T0eM^sXvYW!#J^lH2(jWxy-@C)_P!MeudxQUKQ#lt1Rk(&cp9Z&KKy3#Ci3M zq9+m^iht6`vl5@!nP>%f9zsDdE zk1=g+FzyWMy=Ub1Su*aGbBNyRxQ(jCQj1qOv4DVk`D(J)Wwp%fmKg%QEIB^oTop&{ zM%>YQrV0Mm*e9v3hOuu_UCVnU9HsX*;U|-vCAuzDX0td!RGpMixW?pv`>MbBY6q-; z{^5jb!g3yndwgIWR+9}C&mkD5zjPLu4BhJdib>Yb9efsGhU-J=p97^bh~gzADPFXh zuEQU&1)Jie{p38}3^P*U%S2yP@N~6LsdCj>UWN54Y*n3wRRad>S0cn*6a`UMVG}kS R2zXI1bbZFK=%!K3{SW0ssHgw{ diff --git a/docs/_build/html/.doctrees/docs/inference.doctree b/docs/_build/html/.doctrees/docs/inference.doctree index c2b53f884d712ff9e11583b4de5c30ef0a7a7fff..efab5a5120df428cf6b3bcbcbdafd9550a035166 100644 GIT binary patch delta 663 zcmb7CO)mpc6rJ1A`e;%ts8G^~2(6ifU?CC`R*0S5G)(Jp%S@-{&8Yf_l88v2MB~XU zSz6flM_Bj`h^^n?)o67k@sj%%=iQuh?z!*zBJ>&UE(P!2!_xO*kxeV(I$49N<>*uZ zrE)3;Qu%OPTu+nGQGP9~MX(pc*oQL4*xbU5wnm9fwVbtYZEK5!?mAXpGYpgN8A_Vu zGHB}dt|n-ye`Rx9#1=SFZR&9BUk(t0PzyS!)$=ih1HgR}|yoX#>NwvQvTq@Vs+S KL*iGtY4r<5nB)uq literal 99019 zcmeHw3zQsJd8S6vj5MRSAF|C@a#=E-9($%ovMpneJq8I|# zc2)IA12zbPjg=IMcin&!2M8eq2%LmG)`ToZ-a`-~Au=`~Q3E*1gAv-?@3k&Dm92KT0!ox^r&jT-M*DC(+(?QCPR%d! z``7qZIdI%Ye7iktkJxJtw+r@YI2<@Z&5|yKXI$4M=*XX`)$0Dewdr|~-maV8kr}A9&;B@( zR*IevB5l86WozDE5e_?zigirVA5*P;4v1i%3kur_|GymmpMd`-i8!DU&!IvT!qrXF z@!H``-L150))fZBFAA*qS1It!hQHu?b;|>vD6%`Ib#Jgo4hC(rWnXAd3!NOaw}a;G zbgy$bPaO15vA24o{zzyLa`&$j>S=2SQY8U4Y0yWvotix(zcGViGI5Fl` z+?efL#$?KxiFGP|DcUVImuHFH!Ay-s!(Gs}3U`Xpa1$lmCZ!p6d zP0w9&pm`gn;R8&~G8VWpjeFt>D%C1X-eUx|85m8=BYxnI1<(+u19zHOXwhm|o=KXy z(IO8tYNoen!QaMW%Y+{QYt?*X$uZ#}mTFMadaOpJNLzTp-WpD5*0e1Qd5dgFV2MqX zS4z+*8JEb4cv*J|8Odx7&+MiH3G(M;NZfbs0^hj>TzzX;`bq1si%Kliw-4Fd?3=)S zR!2S#bsY^yyw<$ulx5Yex#pNzZvtWS78f52*d=|n1Q8sEUrS*()SLi(saBe=x#c4g z`W=41kc>m4n^aOzFh3ygHDjg5Jh->L<^G1sL;Va$O4sE?Lhuf%>34kt%VGqC4& z+5Yn1MMHpHzwl=6uak%CR#_p+>7-TCk4#wj20!dsw~Z&7f?TeCG|; z+-2-c-8@!mSVv2N3(d}-yJ*j)?0jI=p>3N%%d_To6<@&%lZIP@#_Z3%k;+LlwIgsn zcxWY9ZdyBLjUDp~HP;MwU&_V0pWVti!73d&n!LZ_mVD|+aQA+*=3DFzxN(xZ)=S3z z|H9aeH-HBo^yj;CXgGSTUYqrsCP_HgLZk96ZwY=5Lm+}vC&IN+plOFA&E>#`Y%|QG6FY#P#LP^GzhcSERy}R;7;&?Wh{sbW({g# zl^q`ljKyw(~(=W%_w`1?l9y#L@bMQ zH@Ji6`NY-70GxL}Xmlq%hGOJ`nfeJMEJtjI2jW^PL9H-p6u?(_ZmHQRm~4kvX%-bd z5(T!6bt_hct0BLYvzMsXxdQsAa0Ox_^wUVRlXDDb)ru8eghcX#Sp#wb<38g}vVlk^ z36kET*$Q0t8=7{B9_@uwHO800{<*10?Y=q@-uopxS^dyNbI&BTnzMFhiU! z(i}3?7*g(-di|*D9jQ1h)0Hs^B)u;rmr$; zjvw4D9045bFbql1VW1V^dfFh#m=hT|iN=}i;jo~~5K+w5zd;cDUw0?=LbwiSVj2`r zYZh)G6{RCW^w^TeR_wO@PQtri1Kxd_Ju>|0i*cl&E1hn?VF+Txig1*gBWa7R>AV(3=E(czJiNI;# zc!|JYBJhZu_h=D#%+oA!iW`~Kz`|E(w63(YCw?}iR+5y_)JJ~wIP&O-eb6#71y$F> z5K?y*3WpkyLQBKtKp==Cz?PM)f+`8#N?|x--8D3_<2{Hx+BLOrfw`q}p3_S;J1{AJ? zGJX^KsS7Zac>>19muJD9;B`kkBzYC&I3gU1Uur)C#vGPs8Nxy05__47=HQ>!e05q@ zQ5QShg-Thy94>b_9ybI>Ia@)wFSho5k}ho=rI3cgeJq14OE`*#Ec+x`V!t>o)vZMP zc^eAb%~7g1!)?M*s<)Cu;z*H~T+v6Dt&MnB^$voBIgS_+ApQL68Bo5%2Nd#AZ;&PC zsb_UHJOj9jyvzm19EN}5wtIo-UL|^$9lc8GTLrc0Z|S%_^1zfU!e}2R6{2rI)84V6 zGXEwMLiPJ}YJD6>!Tg@1=(OP{a1_k%If_E2n>m@reR{6;)W(q@&996wE0y$TG4Lb#TlnI7+&`p;1c2ej19RY432A zJ6}~p^4fhh8Ml%Qe=7>3Y53|+c#0DA7))O+ zT@6M$Os+!>byQx|MHt7z=*5;eL3ioGi&mfR!I-~H58K!Xf2rVa=f%;)>@V-WO%CIYLXZ0B_C!Z z1^2e-%drzA1O8Q6cGVSRr_{(xhAyD?sNM%pjFT095?(05FJYv|QCX@F!R3w&id*UT z;TlZy>N*WVqo&*Ofp#9gl^DOJp6^`{Gc#uLZy}6MI+7~yT z&0pQ6zOzf?n_*fql93+?A@9V6#CYYka4U2vb$DeFN5Oc7qv-HT3rE46gQF_zl)<_1d~CQPva;WKBN%L z2IZZ4)Cv=4MQi*E96bl+Fwy&>oLyNb0;JKgi zIEosu(npFe!BH^($RNu;90emAj*@Ec)x<-jKGGo+Cbz*EOwNQi)fRV0)G&0nRQ(EC z{i=p7Qni$56&{L`TS)_G6{^2Z#X6djbo3xvdmSC=@L3y2$?LDHfX5_Regj3xZwZ1j zNN;6xmB@g^VX5|yqqQHDC7!@hG%TTdRXPhkg`;S+JRPt47LKAJOKSU0j(Yto3Zv^* zgB>?zEU0E@XtzH@^09Wf-{eClE8mKvU|D$vS)C?<~v198Xbxx^dcND zM;^|qp;}t@IgP=_ab$WePydv3acxvSl2e=zlHPzzily7v!mXO=_MJEiMkE|Xhe#H2 z6wFIFilV-GYUl_RFGDyUYL{Y@jLT)^g0c79y}lMNDIW2A+c_44gem{CuPP;=S{ znvh_jW;5KX8ESUmC>XbJ6dj)2g`;4O!%>t_bGjJQkrdB0p3XvvKK?$mLb6w63OjxE z@vlm=T6!Pr)O_QtB*!Hj*`mTiS+CpC`xW6^^=~Z>L@)udp}xx z9UbX-_V3~-m}jGwP}8$dqbRzT_(`N^sRl_;&%TTbF(`w46Gzc7h)#O;FE|QDn_Oc$ z&h-M0q9IFaGf$>xqieB5BDZ(-m7Xyzsev0!&oV@#ov6HMdd5+7(zD$-3YMN_kmZ#) z3g#jlML#{e5y#4rp0O&ZR+jxuBRw;Etu2x^PPRe0DaytYXGJQufD4YLV)JmTW-1oo zC>W=36dg`^D~^IW3`bE?v3xIO+Gr(Y56F~$^^;jV%nYTY!VjahNBM)J=!BTZaTLrQ zGRX1-j)J)ZN6`u~PoXfn2Jc8<`U)|u{wn#PAtpmBdj^#kbrFuD6JnmnQLqq`L6)_n zSbCr%i++gNieu#nF-%a^%Cf&{gqX9`j-uf=3dgL9+9u2fq`yULdq0kpgJGDMeKE|L-PX8E zBbaqk;fP(7b3)iZ#)ZYG<#D)GlUhEDqhQp+QFN%~t2hegA{<4bmSMU$Ak|+wx$pu) zng4`VAUC~moSd&Q{(+h)1Hoj__{y=|fuE!35X@#A1tXXYvTVmuG`vS87+8NYYJ025 zZ{a3WI8qLRVPf`0FlTgGDl?5=M&q)Pgwh!y=QX&T7`N<&TQ#}mRvZQ67LKCBEhdhF zc?U;PxFzcs&^`*En_2q$0@|fE%0Mg`)cp=rUNm#zC_2RQ7>Wn` z6dgW!21mi1f}<#Wl08@X5n6%VjM87O@;o(D21?1G@iW)u&Q&;y4yBxrqhOShL6%E! z6bh=MJJmQD|L6}k|U8+lv$2QIoRE}To{+!4!3G@ ziG!nHT*6Uwxa25~f_Vc+QMe@Akje=ZJU5TfO62VEJU|VUfkiUt`U9xAXkx-qbXer~ za1@M1GRX2LIEsefC@ivaPfQ}@r{It;;5a!rgbCRfhs>ah`%81WVJiXYgCeD+Z0UGL z#5XBdgg(BF>jUGSlW?mh_xu1y!MKN`=y1=<^|_-eM^QA||Frtk&L`8Q*7a!Zb#$cT*>~V57(1etQ0Hwe zpeVYQ=y>fpfNAxq21%c{^#)XkK^f#B97V$*Ixp4Vg`;4!$u*|qT)&BkLzJwr5l5|tOt&NzxrcJ_4~1Gj)J)kM^Tb88rG3_?CM?+b11m%1KHlO6O^1C!po_FGIR(t==ye4T$FM+ ziVo#iI0{BN8D#Ns6b-*oC}$km^DJtOXc^Hu--6dH2SG8l^hHqT_T08DjgvO2cSVXw zo))tIDlR+5Oz(nQHJRzdI10v097Ttj9>-BI5925bGv%9G`dPFRKNE&}f*DFjg@1$A z9wi)(qLT*yGme6}Lk3xXh@)Wcz)`f);E|13Y}Yk-M<=VVG?>+20 zBYD>=Cxxt^z-7fa=MUgkP0slYj)HLxN73P&uiz+{?{E~AbI9jP$at>$&D1}l;Q2YH zk8h^_jv6Qf=VZ|Jk5O?^&fzFJoU>swns{KGlR=ho97V%#6wVpi6E!;cC0g1fj*^2_ zn23F`%Bk~aP_|(w@;6w~*O-^3amtt|7YQey5>oENrNsE;D!5gXPY&TI7@u$y9X@$I zj)M6GN9lu4nke|sgimVJKpFTXgRXx86&K|bj-taSkK!m8pJb5bw{R2!-@#EZH{d9J(8miX_|Jqso}&iJKpz=&efkzOQP9w}4t?yzQ84<*Aj@tXMMD;a zK34K(2>d~QinjJj947~XFd;J%2wprqs}4K%Iwi6XaH(l}X59)b4-WmHO?KzCe`Jr_ zPqSS>?eE&#Zy0LL!@kJt>AuLG6|}sDFK>x_8oBdsu_`r8mpII3>lMPhL-i$96hUPnhdLG;gY6pRcpe@$58X%ywfV~O;6#os||uVV@IZB#nS zbN!;=L{D~FznegG&^c^B+_as>v7&V>C@wkH(Xv~y+To_?XUS}oZDr$odxTElJm>y) z!9EwH*$Mx@9R8nx|0fT(>7Z|5dW#n9o3^V2TRpUDmOoRg)%|;GGfrc{@~lSL62D4; zXEyu=*Q;Be-wua8Yr%mO_StX*epTFZJKP9=eQ`sZNVEPFS(=0P_Jgf?Ne54wq+W$^ zmG3OJ!!h_f?ao)Na?rMv%8s!5wL?CXS&S`=^ZIX zU55T2B!c-aq6d)0-p0(MCa1C)hzbtb+rp2|*lyjLX*zbRHgmP*AAxTyXBI1-vt-R2 z@?2uS$djhyy6d`DvAH};94YXe#-iaa7|bPwoy9N3L|sm|NjhYYSEsp@V6o9~Y{9W= z6_!AC+G{i%rq@$U8Q)61Mcf*W9g5O2KeCa0N@)9rEZVMKM@o-Z_mh9N*;613FXu#M?y<3@7U zU=7xAJ>xa|xi^!7Go%!A*a9S@8*6~(nY9ua)@>|Gu;DlY6DbodBedJdq}>|Rt_}Zy zR=0-3#>wN47*g+mT#=gwl-a2xw5U+>2@>}u2$?mm@v22@gEqoynN|EATC`e$e2r9;US#n3YaH;F=Y}cZHq8Hj`AT1y%KHc*#iN z@v)HRPZN5*-LiRBRCS~yXznZDmXNK&4?B=H!p#tI#pMD*F)(>~clJOY`45B3FK3#i|- z>TY0_NES)>;C9=Be8V#W*Pz!8cHN-Y3l(d@gaL?I<2J8lO>$0G-EvFEVyM7-2e_XrgM#XZ5z!Dq1gGgbbFts6w2Qv9(vW4R#0# zc$6~6W}%DE?Y=bKGQ;$~$Qt}eh;Re{?uT# zKU=YuoU&D_n@wVZtLc%^teIu24r#stX^a_Ii_3;+9Xrfc;OX49bN4~t|aWSOGlQI$A67TbMp9mKps)MMgrE;3^rmHow{$fFxsou$4vpH2`XnFFTk&ri ztm6NaCFyL_BBK1gm5M_&4P96mMQD*m;TwdV_JZHgBzeE@Z5T-31iWo z34_>`!envmAOxUUV}jDt#1#C98yq?6h$r+UG3vya=~;$tHYzo%V)zyr5D%={vH@p| zqfQNaBukd(IS|>F%v#Ix4Tx-3sw=3QS#oz!PYM(KPQr#-F7ex4U4mRmxWu0hNT(-{ zKT5TF^7yYzL6_!Hq1ot6hj!x&M8j~9Q^!JEN%49+?1u*HuyJBcXD=z;7v&P<2=TF~ z{$C}mqh;)`^p*g|PsXcX>Rm2NO7}U@8`Er_aW5&QT^bWj6Pk8o5hlb`tO89vDk zchn$L-Vrz54lfdI6gXi(ZM1V&Gu}=+F9`GfFpv3uK+TfY($lkVk{a|gi@mi|^W56^&FYHWh-eaOj+=!%8 zLYQAl=%;(jkVX!EAyT*6)X8aC*FBs1(9WoG@JrO`1BvJ0b>a>C*J+))7wxBEe-NJC zvth?dQ5tqqC%v8J?L7p^*sy2Soli*1zR^79)LV7dbZJ7u7uBPCYnF?p{UV4>ZZ&~l zNhe%BUw}pIVJ*Z(4`w+orw@oIdrWX+DCdOCFCzBS>!Ng zt%<3z3EF7bh(WV-WTHz4HnrWPESbxhE9Ht%^!pM9(z5qsUF}V-B<%fI*HM9PN`t<= zsjFC|Rw^?dNSINN6R?<{ z${NaQj#0`fJzna`t>3D;LU+4JDd$?aDLvOAS7gtEL*tc>OZGb{i*;$_sP3^ z>;Cq^>V86A&x9!dcGo*Xy1*63gQ>5sw@f)H)$`S5Pq^0AlS0zBC-kJH_Mh*nc5)@5 z_A|OwT~s1@S8sJ6AFR4Bmsf2d>OR=2&%j1*9Q6Ilzkk3#& z%ZFK@KFqO!oDV)hH7jOdGOvS#Hrq`?n(MPjL#Iz%3wLN5h)A8uK)8pz60`;$5`9Y& zeKCXIbRoToeJ_-LM}fVtg-?|lWzu~~e*6Tj9%fCoS*bPfy#<@C?1OOOTJn*veB5xS zokqnvMkY2l-00`V==3zq)AE+#;(C5DI^F^$_%}>`Txe6njm|~o$dd@3Zv_D)J?sPI zFVc2DZm*97lsY#>svC~p84-u`1)!+0ff$0QEg0i*U#P7kor58 z--gf4?AdSv#$#Xx_Z8UJMN9DIQ9C@Jm7GRCL!t%kk%K|ogzlg{9n~*Z#%MUgKeZE| z4DnXiwZg-W56JC@qA%^Lj}c#nS>pXm$?f7JVH~6Lj z(R~S~VEOQ^pbsq#&O`)Z+!Ni$>&--%8K?lj?TF}iEC z$b*a;`Ro(^HWphZ`~Wzs<{L|nNk2NI2AVg^Fg>Eea|$wEp;cmH~X;AzB_h? zQ#u8RC}ML`H0g<-Njwn^()L|oa#9C6CUg5wN}!%}1@q4Eb|WanJPb;+}oe*EL`weXAVon4*QH(YU!C?7*LaX~Sg6 z13Io(_1#8|7mi1o=m&{&J4iY*6Q3-~MWta4UXGnn3dxYcAmX--XuV zBe^|U2G@Sq3H#mQNOL)`Ar-TK<$n6<_uE8s8I>B5(p-nhl=k@hUw$N->I(yCsxJdT z5iZ_KNjZ)R@bVvK#lUyt3Wf6_a|kQuG7M2Iz?c07ZQrD%{Mk8l!zt+-l@^^Y$I6l! zH9oX#()w%k+ZQ^4Ys#Qa1H#D^op=w8$MWFgiyDM2tL*sDTH~(4i@ln{M5AW{A?tpj zDpUBl;uC`T3*mN_I8Ku|4VgJ#S7ACbTASu|RDxhlj{3O69F&Fmsx*4yfT)NrzS=L zoQJ10Oc&x)eesJHo@|H39z_y&w6V1NFQb^od)oGYi%#6Jcjbx|iHeFqHA}wknldh! zVjn}wA4|>}^R8QixlOmhr;u>n>g4xmazho5S)wVy8%W=a~g@Wm#id8KGlYoa!4Xo~7VB9uHkYu2FO3qn0$EAJ^IeyN;z zSTu_$V;!Xk&ZM3*kgc7j#OopVX|1@F}B@^HM22@9FTOCC=da&(VWvxl>oy7uQ3 z9b${hTyoK*F@2TMeaS5Q>ycN59Vt`7TVqD-T~Kh+Whf^V>~GuuU0E!JFXe834Jhf7 z_}=i57ejwBp%1z^2XM{5#2VbFHp(dRO`aCoxi*$CZS|6Z{lD$!Q?>JqqMhn~@))*o z+K;2k+;J9=#7nH85Z+6y0BcbLb;iqj4nuD#a+q*a!ds~O7OKh;mSdt!PoRAlX2+!$}%Cx(z9PxH)J#}nD!`ve2jFQPHRZMK|@ml)) zs^Ymr7f@%csw;r9lb}(!EFAOWs6MOM3ZNfL5luFLMoqVOdj0yj zo8VT>xNqVp80~Nr9olhm6wG%xib6Z9GVMn478E=;>Cj5#{6&K|k zj-takAHh*D&dDIlCvX%Ezfm}6Xm`~3;FoA=pTSXbuniNjFSZ#+2946VMyy`La~g3% zNcs(2QjAl+4!3G@$}>0$#wi>{hf|)%Q83@&C<>?K+ZTHM%dh~SpIcxI5Z$>R$!FF= zmB3IsDm2jAqg26B)Xet2xOv?tHLoI$g1JKmS+2xUFn8c6=^5-nw_!=bT#v%&8a&6= zojonCvJINSW@u#Eca~q~_aKz}G+H6qF|u`~DNN3^_RDEjOW%9rJ81RM zb^19o0uva-i;6f7&tAj?N^6wF09ihfr12^=d& zR>rEJT3PltjjU`tKA(dp1xos`rQ?=}uTrjvX7VLmd@Lb*5^mK@$i9K2V9df%beQEC z90hY5j-qHZ>zj-pq42T?WJ|y33zM9T^E|athEc@~>OS*qG#AlOw+`c+kE39WlR=hC za1;$$RK~&HWbDJSaxe}Pv@gavtLqH0G_pyqjm9&hGA{-V`c0n>#R~{!o`zN+H@$G2oJq`Ps8puu z7#TEPLSg`3=gBjq3%R)M| z?b5p%{{2oG;cQAgA>!4J^MZW-9oGa#K0kz8HOXgeJDPD|W&qghwzAC#9MbWjyPvTWsszK6Ul|76KF(`w)7e~=Bh|a6BkKrg7ZE}t2 zIM*NJC>pY)HuL1Gvd^M0xxK5eS7l5~YSczwm1T%VCsBFP+>E2>i-!xv86|;R-oR*wrSpE0qxC*eO>|D6D zBfO`7_;WFig0Txn(P5XpI11)E97RdWXjmtnlRngx$SrO~!DS!F^s=m<+~Ou`pbXu? z47#qP;-ZwpQFJKhjW`NMIT>U*fum^njY2u&$RYqyHwi5xx=ioJYnFqcm|FTGsB`0W zhq|p}kj6>L1+O_JCr=C6KaR_eG1JH3R!wI5FE|RuOdLgrnV!T^Fc0G>3Nz)K$Nvqq z5)8t%Pj*pIZ9+ z0cI#275-GDS9ybaM7_Ka~0gG8DtLOC>X176di_qJ&uC83`bFdOt!I=CJHXw{-+jPYScg( z##S=u`WH}fQL5o6I#ly0j)GB523dX!N73*bg=#jFu@!l-!)QRI^MYuHe~8y12L&+& z^hH6Zr!LZ%$~kh$NM3bsA0g`(a9J_V`5fG;$vJrIT>_)`VKVl(9pFG=j_B$FwV&!%WfP+Ll%W|hW12_4t|N2 z_DUQj2dgj<`(l+-#Uk#}qOO;vaf-YR70+ViQ$or+a49i9xeadBo0vxz$Y1W{p+Z>Xb!_sbok_V za1@MBGRX2N97V%#R6b!Xl6-i5nZzfb$5C?d2@|m|KG{fiV2hX}UIX~uMQ_ii^?* zN712=%W)KpJ~GI1Esmn$Hwt~M^);AOHlOcZ5Ep*!Pev#LarLRZe85OtOu8spRsoi?Eg{jyW zgl(H4#H!A~eI{8#zEC{}3ReyIqims3JrB_5A(g34kYBsO0`%~~ zc6E|GsB^!~JZ_&*~K16h*>lV;`|4*7jRk4Z2%x$F7HDQ>=Ic zmR`j~2Js5iW8_vvstvpPy?A$8AzsO)fk6AJPCfSCo_Oq#>6x(iujR%2oXMwz+74t< zTa|B8d>y$7+oZT!N~v_E1$HY=ODoEc1;ojRwd=HjHnYfzAA* zP_0Xk4CU=PqH!VdR4J=k%p`LSFu)>ZxCyqv*%}TTCyzg3NRJ~|OHh7>exrZ!K-Cf>%+8Y6;=I~1Fl&i$}a3QZX|n*vRw}o+Fip-MdOL_ zv5@59gd*@7a=nJMhNMGa6eOjR)3Q=i^5yDRW$Ulum5=Ke9gF&XF!4;heor$E{sygI z_oDKMSGcmd%m&C|_qLi-b^@}yGYJo{5p$aoEiKNYMfF4Q?CP_C^HBAB@RxD!_uwTGOB&|u4E3)@NndxkziK-=^)!R~E z9IT~oNZaLr80%oGJ`c8n4dkyq^jhD59f|oydBCx3BUISiTC&>ibd#lTl)$Z)eo<)W zM+r@LZ)n6nx#6DgYB+KwVYrL)h$$+Vd}43&JvUhM6{CgbJL00*g0|KOU^h>)=`OT6 z*a|weQHiDIxmCmoA?U=*Rn=)(Vkg{`9u>)zge7(;Q9Zr(qCUw7^j5q zESgMIzqe9xh@N5VRM?jrwoz>atj&-uK#P#H7AA{hWM|n~gX~&w)h4El32GDAOxU9b zDZ5ZkOpQ%+FzEy!ScYvjDmAMDyBw0{7+AGsx`XRcr&b$tmn_eNO~;KTv(|!rUx9P2 zRMX2VKSxQCkV^_EhZ6R})yNy-fv)B!R}$v`{R|pBdHhkT!;{BAT z_AorVXLc4VMaj;RI_d2!UJpSsW@oeN3W9{cZZwZM^;VsQ3D|$yHF($CGMbPD_!fK` z1$#)RvPg0*LCz$})54ga&SXr^CZD7_NXy5_71_t2OhH>7QLE%*dYk3(!J6e_Y1uVm zlUq&LtebSbZf>EUi_=4I!No zLT@@>)ti1g*3w$x0%S(sqyKS|Rucy`aEDOQS-G$nP?+f(l9 zAxOrg=?ZD}3c^B%V5FJkNk=^ghOpR0-zbm{m@yA0pbtha!SG6#C@noN@+@U}3*lQI z%w$JS5$~tsr{zoJitJucW-603qGriQ^|s1;2Wyr6>L>fMVd5*rs_!;xMghhY-06yY zv{8d0ZQpPfjJo6d5W3S1l>C4{N^9fi(nSKKKFSvpdhOnZq-5&zUCl(UB+PV4WTvQIdHvYLUlg;u#Bh$!QfNH49wKLSN(8|w;c7PklL6~ zwWwv3v*D)n?3Y|g7-gr97@|VSC-gSMS+q8I#WQ5Rm&+f<5HpBT>ILiQv~RnCVI3pg zLm1sF?3yxmPm!0Od(w$a{h}r^h72K**mVi*;WF`9b4^!ckShseOzI0GOp<&~Zqi9D3K;C&>K=)hx|bTH-~&mNbM!Gr)7*rS7VSX-Hp*p5K*n|`EMWc_bzMj42zDwh!)t#&;nf=k4$DevNE4rT+?LA#(%`F!6KpXK~ zmH1dF>ZydDakKG>Dt}3C!X~OnGf=KN0CR82%TO}y(}y?KjlgsoWNd|dkvRD86VljsM^mN34fjZR_r(1M>7?|W?`TBQCie<{XIhZxSC_)ob;6cGl*Ia6{RO8>xFtF%=Q|uehcc5objKWG{YU7 zcugh;N71XVWNjgu(q(T;PUTCP=Tp6q-4xBIOTVeWUTLgdXf?{j3ES%YzoTbBU!1)3 zs2(G~b`{{ya{TKJ@ashNjr8w*^zZ%j@0;k~H`Bkz$zSNo*k7a+H*T+wDk3$URnybc zk(;hMdAu!|=@NjH$q{!?n3QKv+1qc}WD}sX?NyNdjfSf{^Jx2#cHBPPD=W?tB&y3b zYi>ti9Sf$-nzPuLEyD|0%e%sWksg>jHXXRl*^A&E;1$qFYpyrD^U_N%y<*2zV}|kU z`~Ks{;B5%JJU!|_2mj=Ie)E6)(JveGkpJ420q=2dz4?G~n`PF=uABj+t0K{U>bQ*i zz;&+%E&B7_br+66V62<)ZSHD2sMp%jA`<(E_L)kz2$c$FE!bCvXT%d!r_H7_4Re*? zO)|U&0#|~mrKEvKMGPnc0XQmR+7nP@@^G7Y$QU53Vz#D6T!}TdaKv|89-!D$;c0u5g46*t8GYn+}GfoaS8* zo-_*mAj=ORABTf=p6dosRcpaH)((eeV9FP(LJ4q>K!GuS*z#&{v>|ZIrFpY_gj|KH z3`3ngm?w710z9Tc-cgH-4}}|i=#7!g{TMN7`=M|n>^1K+;XQVpPC0-FjjFW}CJ6a5&sz9dkg70gM9sQK8{*xMIz> z7U6jtkD4BdgHXbQW&vkL!5n3a)SA@KX*63w37m$$Bx#4EtzcpLvT5I0Y=>v77fUc} z!SzaPW`hS7AXP1c<8I zsHg@qwZjcHv$5DR7un2#cDT}Nw8K-Oi|l%pQpIY5=KzdDH9Z1J!UBj18gVL-L)atX zMlucs*+O(GT8+c>MX% zTl0xxPM!EJ8UKVPODaYv30wxI`cL)7$b%OVp~PcK8MBPFYDI<=BnsSw_!wcbLW zOf2qOwFSm$C&E=$W2t?xH6N~JY+?}~gKq*Vboi*mrnFMO*#eaU&kBHU;TbUJldySs zigK1r=BB6NX$$cI`3l0Rkn`2YAoGCLi{A(Tn0~m16k%5i)jOfZS6@eB+o$YLR?jA1 zESv>@THz=uMW)))@IlL+us?Nz7J)40T!<;`zV-Ge?W^o7tM7%UM|lPNM8S@~^0<hdj4ROvFD0 znVtvUpzoouZ$sj?4>R|nMa+WL2}eRN1bQayo04K2vc<2Gd|w-Ks38&X?I-Q0sfMZ_ zg-TP+vR~v2k}CPaq!n(E-JKd{1I%7#F&T`)&Jb3M$}vmJXGl||XQ6r2=iOS|g6PT9 z5+~!*qskLUBUF0RDa!L9KzTtM=;rj~v-lOr6M%EeZi_G~1Z7D4YRr=%>aKY;KmQ20 z*8}ZafInbb9WW6%q7O}m0mf>SPeT&A(O>p~grc)?%3u$3zlpzW;m2FU1Fr#3^honF zSmQ8ufcL<+LDPsC$y~x6hT|KM7l=%&5_Nqb8GIfbA4>*FZbbS!&p|mf1smq+z)-lF zPEQY4Uw3f-O(Z6>)U5j3;4PFtEKoY}`TxZp8Lo@9L`h(s(7%g5Z<*DJC19U9~`J1r>2z3wb>9t%Yi^8;##3GEom!M)Y=MbXOrjk+H3YF zCP>BQfRNE1FmL&5BtU?KkU$9eQXwIg3!q+5#hD8yI3Us9k(jsZ^*%R07$WV+*`597 zo0)Is?cdqovr|`x-}&`0WBnQ7HMyi~Wl5cF-*>`;n08^BW!kSK!F-x#$J#$P&V^AF z1yKxn(X_FUc_RP3Dky5cCU43c`BQ>g(d2qwlB()U5?2&zB~;~=N*-WU+vqC4A5kVDJ@t$B{H491?z6WlQybQookzW_MQ+ur1vh!HqlTNf0dD41^CX9thF z4DpsAR>YXY*FBFmU63UjL>P#?55O}2mg&>oOo{VJ5(A%a{YyN-M_TAfa*=X zxU;{z4SNWjf1rv0(%xk?I0HkxdsV>cKpF<+Vju+qSPjJSM}ZOO#O=YEI-MY5_>&^B zF9+h><*+lwrjsLj7rlqx2bq3=u94+N)J*H`09_zsOxvoLOT5H2WTmn$$qkct%Fuh3 zpq^1%GXTKunA{)F5hd?3C*O_6`G^r(7Z3|=^>fjHTF!y2G* z2+QGxU>cgS5j?sU6=84x2S}$ElN2;SGDb)$Yl6Z{t3^pJRZMI;Q~JUmo!i0dtf$hz z*T(8z&^Hc_-x#sSa@U7HCZghTV5RPP65%`Jxp+#PC)DlJDALb|!%7&px*8s4{G^Z5fStj{`SX8!>i=v|xu delta 2628 zcmbVOO>7%Q6!tpKChIs(oTRR$tsRFFn%1#hO4|bxMIv!!;-pI3O9cwm+TLWmUVF{1 z*GUhR!vR%+1u314_-QK;2P9NgLYNZ=Ku8Fo9+3DsqDmknE^z3*5^r|B_PB|S!O}k2 zp10q7-<$8fnLksHQirz(_Z~jsvOdO`RzZdGRzk?6WIM~^OZ5gBOf|~22j{YCy`PTla&I}<>aEM6{|`mxsg{3H8HuWDw3G1 zYDrlxX|KxhtXR&WlAsoo5JR(#t_d4rLe;7!ie%h+7IF9nv%&=N9p=>1>*j0bb#o5F zCGBt*P~?K170@*KcV^8*N8y=;=NLT4*GwJ7P#lHPJb-zeEG^KB?k5=z*W52pB#zsl zgXlaOS_2#o|L&e!2^!ps<#kbo97VGoc%}cq83>q$U_s*~zU6+p-Udn2AYxzHvjDdE zZ&Y7sQYFsMlNb^ z9v?hchwt>O{Sg}ih+@CQh;x(JestE}b^O!d$+;MM4qTg}w6>pxG)J6TAlBaMaU5f# zAg*PWzs#c#=;p}j-4*k$ni=t9u`I&a@ zU9%q{q5FDv|Hd8Et)t-l9YyuTfxYbro_M#oVm}uHp+C!=fSnb&S*Z2H{V!G^e;`d-mqFA05K9cG7r?o{F%( z1<2iZWX&|#bww=+RnzdGYDuOj6J9lvm%UY&I)b(P(HPANS|Jw~w}etf7VUPblj}lr zCE8x;+DwrQ=$yjzTNjqNQWMo$L8N=sUWrmGkAV)5IwqS*!&RsI^fXva(N(yIpm{vS zpJie=!(U~_8h80mn2~RQx#1U8RZ;Owc(S2}Lku(Dcr$X({n8$+NjHfJ5aTE2En?mP yqZ?dVD2Xs;;93V4D2aIQSe`qU*N%>6$o~WY`K-6mZR=HOs&R8>m0_*ZCGijGdQ)%! diff --git a/docs/_build/html/.doctrees/environment.pickle b/docs/_build/html/.doctrees/environment.pickle index 2bd04f5ea5b1176b06e0e17b602f35c5a0e29152..09b1883e635ff5d34b8359dfd5b8eeed9eb5f8a3 100644 GIT binary patch literal 26468 zcmcJ23z!|pRi>Wvl14LnSeD-{$w)J@=8k2@N!E^S$&&oY$YVv4ariuPyXW4X>F&9= z@73+TBaLN+1hB2~b~XlIhd3l5MRESNmLhBwOt@8tut!om;7w)28oN!e%?$qAO{oVYTEoyfmnW)9tL! zJ)Sm?BG!JMB@f;{)y~#FocPTI-?%5D(H?wVm~sbmK`oZ68A&Zqh8XBSF7zCRi7&4+et`!H{I=Dc9qf5UmaB zj+%JSL6wjXZ@@p3`f(E0JrkyW;h!FwgNqv$MgX`?cmm|x9L}XFF|K}ZZ7RB z?Yj-r3KV$@OzfACJ5x*1{5D{})%0C6Ja;Cn`fLx#Jlm|-({rV>Chgo=%)z;K=FIwO zilFONtM$|FOzVj!7TmPy#YxpmWr7y7YpuA84$n6IByr1CFG>8Qy_gNuyd?FTBv$k- zj4S>b5`n^8(wdnG&w#44HKk@kvPbE(*(#^#YK2X=lWf>OQ?9luKB*ctDY6KLv;Gr7 zt2UjKDxmZJ(xKxZ^?K9C6b^^+vI5zvJ$o=^>nr|@*Q%y&6O=)$c$Esq6_+_2G|

=RLhZA$B9w?55z#I&c)P@p#~hq_4sjHwE_ek%2v7lNFwLz$X8 z1AJqeeYfe)_-GSElr+4u4}7hcf{HMRzYUFrS+WFBQo4XXXT7z0T%Ys8cD5-1O;kx2 z`K+R)#sX*xEp04P>kTzq!1IRGuQf1xej-eQ6g}YPFcP(ixYNzBf@Lt74bJ&~!#(Xa zV-Pv}v@vNl8Zys05cO=5o@yGVOjtwX;4rbrMdDV&7(aeIi_T1DL+C-^qC?YIa#*#TECKnafOgQGbP=0}$;Y9c zFQSQ$ZD3^y?WWbl1)~XP7MRXh83fu)6HZkNJBSP9Vy8k~38o}H)dD@s60xGLr_r$b z)FBLVi=Cubbs`AsH>^kT{8f<$X7NPe9-LI)6OWKJ%O1qNi~GWAOfdL{FbN;1QAF;7 zyaO)@6R?XaIvmui6_&ul@*Ayb4oP_Def#_L_B$YOta`G-HH?6i3z%yyo*@B;6^uKa z34Ji`-c~#p*H5D(>m)5HOFhAhS)asPVWo|TYNK37>+Mk;n?;(B*apdnRUsT%_88Sp zqfW{PmloItZoF<8yfp`@UK;M0g9oKTup~{Pt99Q9X*DpLd4>c#A32Im!tiq;xx* zhSVh@*OUlcQwJK2Ltg4l9id<`b(CACS4*aj3g6bBTld{EbtIggO2X71!_F|r*_c`; zuS}Une2oI|z@}$9kzJuihYi?mH7a2G*y}_XxN3Om8gG2?2438-`0!^#0KRrO#y z#H`d~uZjjp9Ov4>UD<%lafuaTuih*bEAGtM56i+6)ciDja37G!9NEMc$ytD0h4fXw z^b#1lq$Q1Nm}(MQ)G&%_t`Z*dI)nrjjFkhvRh1VRgCJMS8IUqkh!nb)95LuWPJFLf z4k$3GcDayDiAfE!YnjyLrCoMNiDrx0Hq|UQepM%GQDP}pfR!fVJ~@)WxBw{VP_*A# zh@zCGYK;_oGkd)LuDkAXo}4Q2Kc}5tvpipzs6C4de7lugkE;tJK$;3M6q%E&tV~V6 z2HvD|aCYnln2&5VU69R&$-AtwtPc%wweitgHcK9SVZjJF8(0 zq%AqNSUH64xXJu1i*B|O63Dq_@`!9>C7dtivW)}A2?^H9srEP#>e_FAa$v;@Bi=b+ zKVHK>=Fc8J(-?u62@W&roE>oRujykmjU5FnHGSDTM#dj4?YZgh$8J1(Ze;YyN2g9s zO`RN{-Rtb0n!0h;e^C*}%Dhf$=?i2An<4 zF~M@;XpUWRIJeFW=GpO_NwB&Saxvq6$c^pbr;glrG0ggc4_km1eQ|0lbuRI1(cAQxd z+*ZSR-$74hy22S}M32QZ!dXNOs*k1z6*Y7z!wgb?=~T;aE*#hRIZ&^F&lJ7l)Uor4g)ropvA5q^xxC3!mPi0On0 zx&cAduaule!fMr-_MHY#OP~kPkTjhJUh13%9(2wDj126iRh2xkgB@d9atj4{aRuWR zl64xP50=YGmn5l3m5g_%Yzkbj;)JnNZ>5b^YE7evh6mA6HI*36L2{-z-7DQ!y~l0^ zM!+0z|4ajOI^$Inf3HJc!x~hsBV;t3aLn&sJuxb!fD$o>$yz!ylc%tC!%_}{ zqoAf8JN7!{A&2VaL*yU=G6XP>EF!fx)I2dROYx2YnK7X7P~AE0J6N2U8BDdqW-z#- z9i(a2l{m!ej{6TEJ3I!C)siFFj=QBM)(kYV%j_h2v3>i6dTeLDUv*pZ+C#|uR&M&( zp!Ep2R>7+%;Df;t<o-4sjguw(UU10*v#r|n99$|UA9~;^2+;qX& z1bFx8NNLaL?op{0yaY3tL&#(50j@x~z6JYG59GK@lR6NwRqKWHEbcHc1jAMHaJ&ILBB9C6Px)c2C8-M{P_C zjiDCST9U|!LcYQzk;h3=*`9pTu{*A!x~q1jEY&%DRcDFE;Sn}f)XuyeLn<{|Nibp+ z@65~ID4wP;kT|<`*^Mpj2kW;cN!2dHdGwZ(omDR*8FnP9T5DIUJLftNP&klk6TQ)63sRH=RtfGi zBdZS^Q|v@7tXA+n2dA;w5XS&jPHHqL&X-jwsn`6Gd2XWkL04s$w|mR|NiQv|9vdrB zJ&e^lQ!5%06ej`<{FrRT*idoG34x4RvNs}y4AYyhhZTn^;<08c7B(r{FE&>mCi8w; z4MXRyX==w*yrW}TTe;9ZD^^Odd;23d?{&uZDeCW#2&|kC2B1nPIHIdX7HE4)>~vhG zPDdSt)c7q`@>)?PQ^_aie(V7dsUd(T(7>G6Co-jpqN>Lna8~hjplZ}d5TjVZ`Bojy zG~7B8t0v5mr3{Uj$mRMJMeTP*AO6v6p0Cuw1!r%cx;04Z*XmB<$Vj!7oD(oQt+EV0 ztw#Z-5#W>p9Wa~F2ktkxJnbWn0}(SsP&Lqg(a!D&(zKD>K0aQq#HA?Vp<%vRiv4sv zZq&wg-u<_Z`^jzNK>Jv&Rvjy|C&&z)fT_$H=>`MSstW3nR!xmS)ju@~1yf$65`x)7 ziLTNgW%@~^=f$x}P;sX9ZRqi>JOOEtQJO>F$vFeQh7E|e1JMfthX&T7j%YF4Wa1=X zZEZQ9U7bUV!?%*MMJaTpNrt)|M@6lfuj7<-)x8fNIpRznc<`|EmctJoc;LwX1BVVB zJAC~3T{zAcrIe`yhfJ>qx;b)S^1g=;+=n7Zj_j|7rFwG~+1F8Be8$x(mma9yTB3+i zV68hDRxy~SFuii>hwk(m1>`|ng=$c}qx1DodTcxF3fwI8_&c*uDJ?qs(d^nHPZKl;HUcGq zYr1sb{^r+}#nj3tXmMMyGMI!<_*_dnt$O@My_wgMxLyLgMtu&JJt(?B%F0e7gtVAl zO+d|?gZf92BCUQ1)nQr03dJf2~bL77FyqC63Wdwl)$;+i0EgXsp&J1Frd`lJ$H`1W5-R8Lt`nadZR?jnpO<^OSW0S zv~lIzi0mw;E^S2U*jWb7KnD$tOPbOuFjRKKiVLHh=x;$vAiqszFmJH+7El7MuY)` z3z8Zm2qU!pDrkMC{lo3-hB7Adm2D+gmKzPFUR-XP+9+vD2e*aL3k`2O6E4->Ff(C? zY|Ub<^*N~GVN)t#LJzd(13VIW^PW3pxgZ##4ey$f@RVhKYT z>^Gr%R)$5O4$H9W2Y$7I-7>B~XN!Fl`$hq-`190>iXCpPACI*_FWZV_UCLO5vX%Q_ zx+s!G9}WF)0hycDwnVE99;CpjC~%#_4by_5)C#NJ)*?~LL7-C- z(9n1*X^u~a@pur@v!iV{K{FxkfI9ESJg-TB0l%mi$fSjl%Mdnn;t{J)3_O&W1dW}F zZ;>JePHt)>Od#YGHZ~w#OyXj+h9IY5T1J3pN$kx-ps>OA4!My+j`#?O=9qTGY^}sm zh-p+3*L<;Jf?Gg1%+`BV81|-p$PZ)_SDIX^mfRkSdTQYr>;gp&WAeA;z-vrtw4;=# z#Y({9D7rvlHmq?A5$-1aX5UDJaM}Sh>$lpz(bSu#W5MS}FIP!!?QAu6Eo-363}bBw zd0KY$ZOd!H9tqJEgOjxYA;x6U=62XSA(b@I_p41x7~_7oeP!^mNM7jXeYEae^M4Cb{7h?gZJ}SH3E@a>? z&7*P_K#pte9BKx@fD}ovuh(^?iy7beY7n_#aK`(`ovqhrLHIcykhm>R#4ZQxhMjhO zo@U%gpmIxL_r%!+`N$rZ<$znlJO!3l%#<|za)^azWx6r1i759f@Q)A^o5)1|8IRi? zM_aUbfqZl%VWwTLRGB3&z$Ks~C%VqV*QO4jSRjC@PH2-<0qB;Fyn0XXKsx9PY6PgF zlbm5+kr(P9csSwY31z4l>hNw*(idF~ifN>r;d(Jft9_3kfEZscy++CYa}Wqo_#F-U zi3sLu0%WV+ zZ$a%FpO!O+u`w!0@P^=4lj*kLjVAb}0{Z4j1KbttHo=i#)CB)gaJvcKv6%Jh^B~)Q z%c4oRGq_6!vp)KV3GwH!;O=0*$$3+-*8~p+hfMIk;C>T)Yu=rS^9FV#4?UQNCWD7e z^3l8>Z_7i+g5yL;a3XlvWc%LW5fglS@O_f{=%jq6@?s~0$4nkKZ|d>DlWfy@*UEXQ zlIQY+8B=s@GPo6O&jx{f!#sDiD1f=3D&JZV%eS7l-u0)X#WqYZ^ZvE7?%z|pS#2aFJPeU z;05`Pf1YeWyF9u(i?=J`cOuqkIW&g&a#C|3CRr&r}@ayvZjq~#PP4)S$;9pAg zXI1pKRhfUKV!xw4|2p_LlIP!Q&h@*h{(q-F|6UdOJ(c%AsMvoDz9~iiBKWp^|5By? zx8T1^^siLWzYhLJqW=RGb6x&VmFs`$cK$c!i>%`RBw3Dj@Z=uX13O>ouYcfQA2%V4q1%6Sk-;N9*vS?D)Ns8j^(RxIp zLH=mK28M_DqbnL_cw+&`)@wypjKePGzpqh&Pb7}&(7uA8hE%Gd9m=XB^vzNJ+%Ov9 z%$$22U(ne&+Kk51w#VwCEeH#{i>_c7w&F+NS0W(X&NRq{0G&t$>JiDXEiZT#Lgxi` zHA2xf_>tJP2&lZ*5q`aa-@uq-V%r(pVPcyYWB;Uv?_q40iS1@=M8~oXIUdwu0v<{n z&nPqPF-2}-Y_ExpF;+6MamH>ov3-o)Vq&)2&yr^38CoC z_!0Q+2#iYT4uo(xsNQsEtISwUlm5d*Ji?q8C=JSdrcE5Ns@ziX7`aLjx&pL%Wmgnh0Il=T`+nA+r$F|L z8dBZMm~TW*ArdwDqX7xSsRd{uJgHN0(#y3mP?LZvDsbY@33g0hPB3-NTScSOcH|5q zizeHGm166Aa7btbPJnSn3A z(b42NvVO%BcqEmz_5G+(_N?fgCX@|n1x#KFE1LV;yG)Mg2M~&W5I+L{A;z9Ev8Ne( zhCh1Eoo9H_0zAv`KehlF!|%2L{{-QjlHbFG=d9EpX81=e!1D~ZEx-#5zt;l%D8uiw z052k(xA1;D@?(hP3+DrN$_EjNKExkAvL9#o!}t+2`~(7dl^N;yqKl+^(XBg5oMd>K8Agf>(ai`8a907$1|c=T!isB4 zMWNkRdcSnFNZW(77W%FG%dU%Z1pQXMxrK$L+1nVBw%){$)OtHZtRuP;Kb-db3LHm1 ztY=Y~+LBVaJ+V65wiFoPS|3!Nuxzf(u3VZJrs}wbG;`fBuML+iC;IdIl?JS42KldA zQdeP(VK>ZmL{p6Z2knVf*%eDlDT_yYV$HJlc`XhFmUb&iD*y*Kq6a;yuJ*(MIFo2= z282EySd_y)m3tj<_$9z=dNJz*KIj@lt@8+MAJF8|q6gg_Z)jz!U{Jwa^Ck!^R-U@z zxQ5mMn6s&zhv|@a(r}~)UKgPPL#{cvio;rK^N#p*d5w%I-2{ah<_~NeO|#SM?07X>-Ur5O}yqD6R`MjBge5Nr>wN zO?ok)9qJz|82)Da+LIDa#F`*FzhL#H?*tASeG29j7h;0lxFyItSFX8q)yn#y_2&9g zQ)x4rXL|>?-u%MvO!R2>qu9u^y;mGpP|aj=fULu3d#~U5n}3EvvI3v&efDvQwRnM& z9v)cx^c~37!5@Nt<`^(z+L!lH|uT-7OlBw-M4 zK{N$>)zjWWLYAo*06D{R*;beL_vnm-anexA8)<=0JHs~RL&SvTiG4NL(Mc}4xh87E zy=K)%X}3paTTmCxF_pZT_O_%XuJho;k^=+Kcd+}L-f6|`xwrI{16+;;!6%iV8LHEql%6d}|c5>cq(<5k2M^!WuE#gOwCCU(l>iFIGp!?6^lWlmI zdyVrDxCV9Nm^St^dI4$C$MAzbM(@R!4E%lgqTKt{=L72VA@%vNd|J^*@Qpr-A2eTM z_?4R<*hH2pag{cf6;9p;sr-gY?O5S;@@AArZ9*`IA3e66!1UC)m~%z1?v z+3Q}pcUL}OHr`k0+;4YwF8T)In4n)(pI=v>-%y|5l8>Gs)sGg`Ms-{#rFODd5<;4a zbVvYd7C9ek+a^NPT4n1kT+7s~NY;~D6YHj1;1$Y;F<(iY)HkAX_S4PLg}uLak$kv% zuii|+E~8iIM(u6~$)f4uenf+^&jl%57HEIIHC!y4EQD9HeATk}8;tw4(I)wzvc+SM z=s|y4&9JTb&wz3qkd+(Us-`Eix~!V~OQO1qm6%-sSE|PU+HH;2jH#ObNNS4y6rT%C z!btS(i=;$mnE!xhOij#pe14a03I9gOYx24)m+C_Z`bEZxAVtc!we5WYt9hYsYHt(E4)A=_1j2I{Ik?0c`W%Nn@dmxXwa#95*ki$qN$(dUTqjAw4YJ#ib3`l4XM)rci; z3LH{TI87;?aPffC6w#u73N4VL#}`<2H_}6g9H;71gyFx2$<*YzbXj&$#V>QF z?DP8yffvKw>o5BZpICrC0k_zAxnTzHTXk%#Yt~*S>}4Bw!u^d6kL3+7p+;iD~%mV`R6zWcIaeZx-ZAH(vP7HXCYCpUigDBs>~ z6-!axCI3-eP%We`swjH#&2DK&b3sw@Tx=yrFQRsRCR{EVL?1-zYuHDkk1@YA_;G|z zVw<;>pHH{F=+gp^%9tH5`V1m({SN*`pG9ag`aFIYqc7n1eDp;Le+j>E{CY_uFXI<_ zwCF1Yu8h8lPxi*wwD&IEcE7hhxi!F(`YlxwOjZ!paX0 z?!tS`Uvb#7%cn~acNic?a>MX~t$?=^k*;&BH&=8$x7yZ2r$tY!T>XH0{7 z@CE*!|L*bkT(?#HNj?SdR#R{>dhK>^(E^d)TRPGG<;#Zdp`CK@Fq;c*bt}bgf%KQU z#aKbKU+A`q%Z_xMeANS*&VlBvzPxJdlD>J_5YLmZYI+QhB-|e=F z%Z_}g@L<(+?o}6rY_xypvZ0+SJTy8+)f8?eEkpRA zs2^Z&wNbvK=Q8bs+0f3qG&=@krZl?@={KNl`^@kS@Y+{p_9>8lZMRiicBBs;lY89G zaXik5=K*Y-AHHlj->?&xm~i!PEK#j4L-{@3;;TUU-Q8BP6y=4$|Eeec5A)qv2}E5~ zQS>72cEPe2(46dMXD)hXc)jI>S7k^&_c~D)hjuBc_K9xkpdh8w-BwXdV%lvLY7$%7 z5R|{XIF~<;O%DO8YbVx<@fde+cvi#tilY8RuT1;I%2u|41+9k&;1wYtxbm5McmNG+ z-7;?9hs!RA{w-px zhB5a4xmGqvoiAz=PZav9;8vwv800;93yRYyA8ix=6Ev8uEVJEgom{C)7HZRc17VrB zTFEs@J;zm^+?dTn5Zt1Y5SOXq*#pzKHH$;9iQnT>lQjv~K{_rY0G}KuMvhKYM!x_M{9XAAUrD0vq2@#U3kw^}Q|L$e94B2z*Bb}~UBsiWOVNKS zzf=)SM!$scV%A?^0|fXhfI*2c9nm?)!iV;}Dn=Sng~2GP5e2L2LyAL;D+V8O6V%qq zR^gQ+X8tu)54_-Acs^C%f$=rsZ)jy2#M{^|o`n%|=*Y^K)QgPKx24j2>Q>VY(MmQ` zx|)87fM@IALpxjCus5CRsp85$o*7`4K3w!xZj;ffUb-CRe+xo8?1S)56n|U*uDE9w zlo2U+apYB1ULwbmS^I$eDS>C&)6c*I$~-a;AGF58SRR90(^@nIZBx{XZ`6mMZ2g0R zD95g#N4LRSad{@#IZ6wYYPF-Or447BkIIA2&XU&PUc=>5;6;$3F#`YQ z_3Y=M^a3$hFOl<5o6&ufCuEa2%dG zq#WRYoA0Es5yN;St8N1u*p39re6c2zX}L%xZG;eeT|-Xas+D! zE8y&VP(cqFpscz{5<)M1bl@h51vLbo-{hrui^&hk8x`^wKwQl>)$0SOzvBWpnE`!P z30L>Mc-z`QKm=G0bk7?s8ws3aUoNczu{0+D38b}*S@0d4dyutwJ)?MgSUc8$wJ`OeuBLP=ag3?REvOY zqEjSzJ!_`V3*Odg-=|)$hv@V)W^Wx>q5UYs7NC`nTI=U+a$m0O%FYB%xpbWKctvL# zo^WVT)<()TohecspLj6o+d9+fidr_mSa`R-PM+tB*V5D*4N9N?S>_akdyYPFLH_#zNfa z^s>!Qb*uHt-XQ3My|K^ho$nR0Ys;-xZ#o;PG-o^QIIdLNz3dhPquZ(0i=|e%lhms- zy=-0SXs3M`U%d|^4gajZ|aR=?9;IlD1!cH6UY3CL_lk{lGtK$NCn%~ z`2=BCp$4bpHqp~d9?90W<9b|9QQ0*Q7IzePJdBxrGjjzXRCsw&Rj* zUTLmck69j&d9K}TcFq^i*|_uP&-FZD(p>kYmljK%cDa$(%N^;U zi`nag?-D9J-;UF?G+QsHX`J>hW+Mybv=g^W(rwpQ8Z`XJ_l(N=zyTBqfXFrfmJOq`NSa7Av6p^~F6$LHf;hj4s?< zZ46|{uGzjFUACbT&y~CNPN@ybAXdti3fk3}WphwK8)GqvS@R3oX2vZvD_!(H5yh~Z zI?dTqbB4nO-QH zje)NXl2HkR_}kQ4oF|6>C3O~2=WOjlv(Y?RuJ*E<63|4Qbm4jxCAAhoQz&VRk8QTJ zw}9tOop_;z-iuRV8l>ovk_#hI+mF&ryIR38n9eqyjN?}6Ou5|vk+V))(r&9I{pqHsXE^v z$)GN`Cg@+5J~%8%_9Jv%OI62*+NFG(Qy4l?e<+IfV z6b=s4D07gM>eU8*;>J8GGo9Ut8YCrDXa++Lqn4c|LGdY|9dsvEoX&2r>DaUj{xgj+ zGhAInxt)4i0;8$UEi#^8vmnqmW(jmvHzoLaPF#SMqDwlbx}awnBEe^=Y|wD+sXb`q zEn$=y)u|wCy=6Vh^RKCu!N^`pO1r1^dF*+zW|@M({Ln9$716=q8^R=Fphk(j1oBSG zX*C7AsH4J3vtD5i46L}-one!NcRu~(Ix`!ONH|74S>bj304e*hQZRUKl;COw?XJ#M zV=(Tu-NwmA^9(9-gT$q?0w?)Uwobmg)k+T^TE=V>rT50n*F554d~K417!|^iWscEu zT1`?uxU__A;=Eg$1#fjhT1(3vUGPQ$2$p0h^jcFlhICu#O_w0W%twx5m|`!}f-yWx z7-kqdK-(<>4RohLr8CtE$^I%+ZyEC#VWjSAgTx3PBUJ^Bl$70rl?sqboX)nZ!i$I$ z4TKzkp^oVn^B@AZNXj$qDv6%l7w|?fM^;}#x-tZh5dm(x>-rgVHN=4%eV}7&Hw9rq zA8LN(rPJAZNd%VD1R-ybkGb;cDk_CZ85|eW-0Y+YDYiJpZb&Xh5bz*-d! zW=C{Vvr(?20Mg3&Uh+gXB3)i2Kup~?3t(l|B!=p&un6t0Rt#nYiaANHaf_t%K_%^j zYkuMr;O`Qbw(8Z6p{R={jFMYPWeEpEC{n`+9O1WVei1nsR5D@*gpLj*3{y+C7{j5F z#^v^GLitI{<+!prrcGD~W!mJIa#nkNT1vp zBx=VCU{xjsdnYV_>Bwa}i!%2x`u-p<;h_L<$VOL$#L{e1Zetz*>jO^j4w5DieJB@V zDy>%+K-!YxmLP?2p;CH!o>^}WB2tLGvt$|BmP++>(TO?{_$UP0Af--uBIEVm@$$%3 zS6OK<0yiqR;v@0=-m|R&q)srLvFO}Lguix-xwR2#V6h#`R8p8cUflk`6Ysq5-1)-T z%g0|h@xluyCg;baZ7;lV-~HRh9*ZU?qxJqgYTV!lLFJ78u~f%xg;4%-ZHOI=AQ0!J~qW^U*O^qZhT<7o#)P zPNFqcDycB{yH)WE?}7#n*`;G7Q$MgdkEjkrw=JSM$88SRE7HUq8=iijKNT#{#` zE6%8zq8gA(<4Q4lzFMzGGjY`7;tBKs8j_+rNx2i90Uk`s5wr~KrdyXZjcC4Io{`i- zLFGmT?G}=aTGbdVmz^$A0!VW@yvxJG;ij7M1Bws#h` zMGp+^O$2WnD-^enZ5tD?;3eo$!yGRGFUMBdq5Y&XHUM2)4a|ctC?OJ=B#YxGeD7(? z_cgod2}5H(a=zIKIzR|87wHim#A%5{-qX#F(qd31NuuDB*g?cT#xN*~JW|;9LSx%l zh-slQKq1tkd=wPs{b6DK^&tG9lN7o%=lDY>@}LLEhI#onkbkfV%2zdcl*5w2ZQQ7& z&to!@=Ce~m?l2bJAC5^OVhf86mfkUHqO=f>K$FvULM|sEpuapV-Rr+xLeprLa_+vP&n(e1d-vlI>lY-%QYOhHOHV3puLb29oc zF*TxE7o!z?FT&z%zKUgl<`ak(#co&?Q!IfBr#Xq@2UV3>KCCV0Co{A#dP1z!n$?Dm zGab>8px6;$;74RC#)OJhP8G(=R4ht}R_XgMy)v zH#9Z+LSr;HtbqYFT-xxUhc3zp(BRc(;eUvDajB_p+1vQ9wjj0Q9LF4GDy zjRdO{sDxRBS${uL;^H6QSP(Hm4Vnjg#tAHFN{IopFq6Jj!A-gG^2h)jUVO;$OKs(I@F)+GvI5p4WcPQ)P%sH4eL^6bTPZhev^QW z?d3xDIv45}-zsH`j_7t94edLYibhf2#472|ea{^@5KZqoxHo$1-h;cIJ@DkNJ$sJq zJ$m$UEa!bLwR~WaSzZ9$9N0De^mDtOMwSByo~&1k&GtN!-$3p0*^<$~tbrD;D|#6X zHma1(`US%kx>r{HP^I3bfm~?o+X6K4Sm)Qf3k@{+P7Q={sve(?>(sLnIT)+J>@6BO zs7laqS$wM+>Y7FqOZp&~+6gE>MVYR_ko84?qcY<*4GOdwPLZ}nqeMe%XqObQZl>8h zi9it5dQ`sIZa2@QAO};XASTor8eC|&kTP(wnPW#p&p-)%mk%_&fy-kRHmQpa;>99^ zfe156wGBjxVrG-iPXhw8-06ebF;+UDizF_KN)(OX62@X8fhGV7?I5&MU5LSpdi(DQ zL(eqZC#h83e|;EQhRv{v&$UhmViL33au zP!g1Em7adG{VBDl2I&MXrEWv*C)Jl?j-{;uoVe9&I~ZxHoWjh}JPAV|6kQ-?HQNXw zUCds`fQ9l&sDG3cX#~Vu6NW~NP>h0VW3E}s6ZvKvoFg(DI6%K!Y)nAG>#TVQW79@A zn$VJTFw22(yD0q*8!_8%LMQC(NJKwjc5I!wgaM_l-99z(uF(fxgaT7y%|wZq>$(k? zW3rnim?kebjmXSmVHul3Ze|%d3zal9E}agI1UJr>s+AIKbfV4$DS`ZMng#O)i|-5G zJd4SXs>ze_B8aKNv{0Es8wj%oYb$TfIn)k=jb>|d&pJ(q>!}lhxIts@I$Ce2si8$t zbhYSEm?o$p>m*n~RZ3#5#o&CyEP8Tz+sW;-t(NK+2TfI5B(A@8w=i^A`|f4LrOMmP zM3^GG^QdL>BvkXTG5Ihd2-^D*T&R^#msNRwNw9ZP{nH98MASUMV4EfYbgwFtOXR`b z#wl=a8Zs(ktusc>i`ji9(WM$NYSNkMkz)sA3tHN^4W+Oe0MR6jk9wTM^%kbFMg_`L zOmJZ6KDZK}rm|A(W!J}zi7qJM4j*lDS<6j!^$wUAe6%Qvp_ug{CufY!FDQd69Iy}y zDAU>PKCPN2E$hN0rC1$F0GA#q1^)pGHD6q_<*gOc*Fn@LkvY3L1SJcE#jSolg(Mt4+uGh83#xg)3q3v zAz_&?GkIU%fcpM4X8JHVoVvD9`Vd>bsB~%X%dVGiWMUk(w+pdY8!?O_wPhR1br|4g zVodB53vRcm6pFK30?tD9EZDP8VNLUafOeP7jdE10>FF9A$5(+;W_5wQUD%G)n{^`* z!q*0&+4`XD`z%}+kAdinQzaFtdT_jf1=W|r#4xH>A?V6J|b$^|WsW!0u#JuL3NVa>YjRki_7AWHlc1Q^~Gt$@+v|q$tfH4z%$a^WN-}ab;0)CxddWTw|)~uAX;NR zaZ2;e<~)eowH1+fxJXPfFfD{}_qaIg(gA%}2R2A79FUF-Z6F0)1Wr?0ST<4Gif5}B zTtTAy93(Qwas|E$V&4*hBtBc_{6p;f5{IOtQwI~>zZ|e&vW9s+ml9Ry%IBtBps4?W zseEXg{ybA&)3QwbP{bClm@VTZq77Y*;R7O^O~)rua*$M3rn6%cG;BU$%K8~lkByx z&m_;<*ZrWc_u4EM5(oT) z_RaWWPY^ z;#-w8lH3{YWfjQ|xOArEM|2qcbA9}1@?%nik0#$F&mT|zg*-o%{7ZTMRPxjE{In+j znTzuEv&qlN^XHR)CC^{DAdg?v$7hm%BcI=`pTDGeep$c%tv>!;@+*?&R}JU-_X_{l z^zrMO{Uk?Egsqr+ogIX8m8upUdZ805HepUuvrVZOZv8 z^cPvhU%TP)Hwyi4lm8XyDS>f{5X4c4*F33#aQZ|xQo*6EcAu68Tlh2zz3V-;@4 zkA&ZW7h!m|K#m3IBq~r(NP;_E#@FHPf`r|Lx7zFRBVTvpMbqBH@Hg1-h+p^GuTg$& zwO=>$i}e!-_w#F;{VMQl%zR~=93FJd01qLyXFC%;V6%+#Yr=jN`88?39^}^!`}GjN z9=2bP;AQ{S0`O73-XpJX;_LPD`ewXpo8N-B+GpWM!XLxSn7q9eZ%&OQAyWuZd^~R7 zvh`F0sf!b|6zIn_+K*5=;R>M@{JC`aQ*`l@) zg;GD=*z&;YAFB1Px$3Iy4ToUkpkNdQ%N80aA?MLI8C5%2O=sgem_PJ)DSY&mly;#e zwWshSt=-M9J@#uaUKC&Z82+>ke+Dm#lV@Fu&t{7K{4qo30N)P=0n>Q5EWLJ!5r>1= zw}r7s7;!X+J;wLvf`GU4{rMo^MZ8;fQY$lJCWxKodnE{n`92o}%;VkVPw=4)th%>a znBXLXZAoe6;XUJH;Op(B^v@1a$B| zZDO&ebE*~SJka+Qc+NYroG?2AEUa#jwRSrEID?OiHrd%ACRE;FHPBLB9mH6zUZ`{t zMH|Es7k8ttoJuO$4Q9AJ1GPlYg1E~kv-%Jyi0EMprJ`$Tpbm1W)G7Rgg;ZNa#cMC| z#~|+<-(L;_&g0#-GusTclPE|6;ft=Y5{RCG1HDzaxD?rX9r|2*$_RQFAj-^C`&|2$ z-DtEaE*Hiv=d^d*6t&O8TkZ4lBjI1buP?M;@8Q=g{4pczy?nnA1YG3%`+|Tk;(HbZ zd@3F-e=rE>@%_t#fUn^DR|WxJg?CrNSBD=T!iO6xU&Ab4 z%OBI25A*%&@FOk%db~I&AK?d)n%842gsPofJHCOizLBsvseCkq^-bZ&H-{hJf)7`{ zZw)^_7JhtN`0?%G$H&8u?+8D>6CbWM-xYp*clhx=VV>{BhvP%v7k+$ynCAx=BhCCl zyeY5x1QSTuCmCiiEbC)RTfe137}m-B!djURte5%Vnwd|ooB8D0nNO~t`NSHU&)N?G zuts&eGucM9X^M|ArW6>8;E_oyrwLJ&KE_412KGDZ+`>&yo6N>6H$$yGZ00Jr^Xn3O z$gs?P>OD|o!2zBdN7TjF>N}ICvUM1hqU`LwAI1dkmP;pfWsq$$A7--+Bs2+#PnYZJ zqza9lNvip}DU61_4`iD~X$u`JSKPwD&(lyrb%N}-TYtEjR?i;BCpO%{Ho@&pvXkjO z*c4a0?^;mkcKjq;pm}hGNrG3HV5?71djRhec&`u4HbNkP@e&(^e5TPLe!W!G$Gs78 zUDVptmK7J(2WqXgnP6rq_CdZ$Sr73|pgzhs!l=CkKkRh3*^KSpZ2C~GSZQ uGg zj{d*|8|a|4gwgQo?Dqc1YDX7Sh_f5}95n25+`r!KY_!4JFGzo9A1rSZVKwZ=KbwuF z{oej-vRnIdsm-Fd|GI(lxd9Cs`m2?8DhN*Ah#IU>SiSvE!cW84BOuRlEuy~gRNdvm zVe*GV^2KZ&@IkK=>WvGqUO-VxV;MAa+zXXm1Ct2)ntLu_3o7eo!t1EdhozY6by(%N z`wI(p;PoODTu5~iUf3`QLRDGeUM%PxFJv29*r~;dM=Tw5L06Xv1gAx9dJ$%P>@YUn-@C;%CObJMYUb5#+^9<00Nuj{76X5mxHva+JG~u_g+aRWCV&B zP&!K;IT1z8V%nX_hiPxNebAH$K6dOyS=HKt%FXR<4e0TWHawud+W|$ zczEyLH3R(SeQR(0XVD=Sl5J8THbh;^VfeuwnNguOk0DLYc()j!`gWE;4Gi)G ziS)9g#o|hKpNM-2`x2f2lia9NPbW#&lO5XAt1rnoFEK=*P5`P3PhiT1r};N^Rx!{d z{Ua-@MFk3X?31A*tYq7tGpxbVc=7?)*r}C4JPm|QrnD?D zx2X9!N_*MmapLU541#gg$2!)nJZP4o_t6p^oh zFM86vYLZvc`s{8GUNOfy`j&kFe8QNlD!tkQkWy>mhXXO&2yZvKCY0-E%8Mz4mF%_> zcc#*>4t#|sf9|zRVw&ylP+3`VY7fJ+G3vnis1BCug!`GAzQM!2i6s@fc5DG4e5 zWfutK9!UWvVGbKu-!7j~(TbAXy4eJ~{vm)sxl#L&fIkMyG}+AS&1%kUM}s$gs4RU? zk7pNPH!{6HI=$M5kvE+wXCr}s2&ckptK$3t2M@Dj98M$Gs}zT~b=a`zKj_GSd4?Rl zo2~7X;VU9X8)yrobs1>e# z42f!=#1AT4`!+l&Iei>Y0QgRQe78QnS0CRmk8bS;@vMCUKd48-_vP!c={ZW6(IH3+ zv~+<*6JJ{VPa*rODZYB?;z!;JKVQMyiYeV5YklAVZUqVu?uQ~M`A<;W>%?-4aa8cm+_=y;)adXk;VmwD>g1}av7S|d>DO?8VdMAo4l}rZ0 zN`|kJ zEjw>D5Q835NPj7iYJY>ruudq{u0rL9#ni4}DndHs#;0|O5FLg43HIjZ#^nQ7st>Zw zTWN8I{4tT5at$xL4Ml2;3wT=%?rL{C5N{L^wZB*rX-lDVwbzR=wg{+Fy4 zbsqa)Bp{aLPqn`TrJ|Btx2jCs2}hKjiel|%VV4Y@g z!k#BFIM?2SI<00v2e{v|S`dB&Z4wZ#WhcqvZLQfo;is4(I*dCUnsRE>LSBG+;q@(p%8Q@1?fI&_(FJil6u`cA2= z7Q~et$B&~@Lzn;3TR5B2XdFw#RNt8Wz-9N3p|S(WsM5g~Up7uw+QpO04OcNKzuv{5S||YbhVO&k<3K(yp&L}aKLU(^!k%4g}rRcRvLRz z@PsSa3jpPI#Zx0Aqp-?heao6FM6=nZt#Hez&Q2KJ z(;#!)grZ-iTzkW6LG+`16)cSWN4cO5q*Z|MwfOXEX~%FuvD_@l1-}DTH8b>}0;l#p zY{<$jf3;6Cy%hLEcsl`YOz0ExO0Qh|l!Vs~qIb%*pTNgw|2_U{KZ&>L+E3&6V(ruT zy-@oZdH-4bVz!N8JOAqO_tI)X{I%XA6YHNJkfr5wIHrI(qXO8G8N2{@ohu>(k7 zkFrB)b`|sDn$?0>G16DP?A>@o&XJ0a;;eWaPZlD5@`@pS(^j10fD=w8QZ01==i{qI zm?!ix0r3wgi`-$fmw{8*LAi1zhtxw?45@coy=7qdhd(dwd>LdU2VOmXpIt2oPt*If zi(W%nWqFO|h`|hwyw3GW_p3yKHzjyRu!OM5KGxftBPzME}gMwpB2i8OF>mLTkd3xYNq zR}madA>I<+-4s3@{eBoy1Qw{GAi(Y|aoFJ2tu_vaUlo@*Ya!P!Z;TSDE_OEZgd&>4NsZ^?yPFzhEBS%mFEHV-`Sxxr0u z^I`|p9+R%_;#g0&hm9y$Yd*sy>t^jvHg=9?L_QNpXxJ}lzPLGxA>Hg+9PfvhLhUnP z2ifcTwx%J{p|kc&NRdZn?N<;qUHeu3Vyhw1cBp-bHv?|6cXl1-MG@S0xxt2D4TSI7 zuSwQlmtU%qrfa`}_lwzjkBt+ku^?!-clif?j5B_K9kW>}Xt#4%Vi*Szh?(Zz6TLzq zyRKE{g@xQAgjw~mNbUaKr0d^uHs4-W)EpBl&5%4Y-9c}y(IXam*#U2k3cdJnR(1gw z9w_jY)@H1iJ-)(vEY%*ZUhNv(JC$u)0WxS&>t**}8VQonKwm@XZ}kmV8{-Na%*1GD zahZlPNzg!FbS&%={Z6if2iaxV{iA&?9x^ZYvU~H59hRoQlXuuzeQ381fq(|CNH&my z;%+y)2B$}pu74W?2BR9+?~#sk{Ng&SEWi9$skY0Tr@s%XCIa)xB(qjjq`Zhvd4#z1$`Sw-Vh4Si6pdvPRp|yV$|*lUGgP zwFdyxfL}-O>#RD&6!3;++9H$`Lbw4CYUB9P%3Ur|hM+bLtIX?6WrmPQ&tw+m_N~mE zN7RbwCxmh5kP38b_aKyAUBI(8#w0f!6xtfnRoI39h#;c&FjMcrkHIwY#6Ko?sfKL=BV#lZgxg^b^~|Y`GYtu@vo`lWKWz%$|UP>SfuXFu01Dpa96vUqd+h{ zi?5mb;A6M{S<)X2FK!<-PrKtmln^Ygf0kS4d++VdycgTPK$i*{6Q=8EbsIJ^0<+^ZS&m4-b< z6e>Jk1Q`Y+I6|SsdcG*Nz!$4y>V&TvzfPJs+d0^fhcjo1V(^|dvLzgTxOP)Ix9%0q z4S?C^eddhM=Ni>hdvw5V>xrArWbCq7dM#G{JBX7 z3~oI|hmsKkA^zdoNbGOtG0=>?1+p*iM>7W{yS-G|?fD3nPVtQm$!^WZ;Y3nyHfGFS z`4|Dm6H9IU9r<`}z4k@+WfqDFe8VyFd%0lp@984){OUa!=2^ut9 z|1x-T;T`whf7Ml^qoX4$^SvCw%@`~A8yPv4;Mypy5|4a34Pg*B6h$ieM3(JFFM2^B zoGS`rFXFZ+Tq%X?_NX*XN7%a)xF`d+664NaIU5#IIBxeqH$@Q2=W*JUxfYqlf(76V zPh7i^#E}efvn$%}6f`_lVI(qtQtJ|qme&{dHov%`d1~}+@KT@T&qKvWCw3GceQJWc zhjY+#9RF`-|Qn4SLpt9>YT#^5ju)9@KH#G<7sqfG&%$Km2zjtn}TMa_h1Y~#if%t1$huw zrCE|a@PxEaZVq#086aaLBhx%scjR6CGvx269H-vteS6Pz=W!h0{61X!I+QHs(yWE@ zB8Lo4%8EyDb<2FxsV_#jR!)e_UOz>6<%ZC=xK)~sKiJ2Fi*IxT6G$JmjzM^^J)|R* zu3S1}0rAo`EHXgCdewvof>J)Z4aYsx?FY0oTjgc67%t=yBQgxk7JEzmhe4nkL8l#z zMl&jQH5N?QO*Neb_Sg?A^EPxdX>a&mTInf6uc=N{4qHdq&GNxo}+w zF6T&#e4=&cR7aZ~p}Ew$aorDf`L>BMZ7k04Xlb44#v^e5PM5B|ab*aUTwdi9xW1x^ zlT$z)xM5B2T#4IC#)4KE+4}9y$1Vkd(96gAEjQkz<0By4phF7yd>gOeAa9)l(am;_ zgUtY~jl~lq$K(R&f#8vmC#3^}RTqaPmOIhbh~sZ(1kDrfK^Va?Aj`5r9nC{o;7;C7 z8NDq1JJ9Pg8qJ{|IPo3;$UR+_#v(FK>t&97D1x@@V`~KdPnjvMM+fv^8;D#`QpAvQ zQbsEDHV&JHnncd%=QOc1;=K<*(lOo^C~1V*?8MDZ;5Bj`F+?!?Kb&WsNq^K!X%}%E5N_^_>Ul5uh!1gJs#)+Yx0L59X1ogPuhYWHITe?a@ojw3diRpILzndFhm8jyP^r#OM*w0(o=H7 z2}y`^cy1PUlemZxOJX;e(U(j^PfT1c69AfuhA~e?XE^mc1Zk9?{cVI10>}n!&?66l zHANGT;|S5TbKDOPFo6>%cFJhugv6QtSu1@(yquMJKn1|45DI={xa(79OC#sbSX5(b zBNS>O6)wM)Wkkv=R4I$A<)UZynF7glCOJFe!dde-gemy}mjmHaMgbTHXKS8A`Ij*H+7P0MFn#d(@x=AW4@+_c^xCWij;XlX11twxeOxKS6`3}M_55N`uJW`1fM;w*Jc5h``3S5Jm-z9L4w;PvKfbzoX1B{-HWopC zQ@<9uFqPtf^j9Dx6e-2Xr9ilouMt-YL0xRW$&(V@&S;w;V;gQ!#>tz8Z}V#a^Abo# zxy^N34jj&zi6FD#ei>Y~9Jgplh#}eW+SW6?R+lq&8|$Qqr;k87Lm%RTWe00(x}wd2-|0{wG3L&H><)x zfP&m~;+x+$i5!w`!Vq8?j;j!La^ZmBQLebyI0k7`8urJ4~c`=YEmF_ z|Fd00aM6km2Jhveh_3Rv`RAxPQcEglRyI)qvk~SboFNrg76YPm%a5H=Ft)L%pnNYU zb4909lMUU4agN>#3f9?cx;tBsyQ$AZ3KS_PpA-|?v1V)H>0CRos=3w9TxtaHndNY0Y(L>V* zc(pn2!p3nCVm9( zMV7pvW;tw3OSA%l%8p05)Nh-B6q_{wVsDNRF zX)2vMCbx#;#y!l(&6DzJ0ETlFpyob-sB?N9tKmjizo-b0lmkIN^}qlm<8b1!1)=Gs zRf+{p97dF|D;wN|i#0xvg#o)NXyt}3!Nxh8kNIIC>qe?eQ1r6O7nwlw<8;)|rO>JP zkG@m`lL-lq!K)ae1xVjuFUpa?3!-VULMP0Zk z&?vHfmT&$*S^dK=Xz)N98=BMK!ev+W*?c zs(5iJg+f`;CD=s&;swK2*r)^G(Z1w}9u37+@(0&R%10r2bPD-I<3^Aj^8<$kZQtIX z*T?(I-d!5tD5q^47}sn#7jxrwtTKR@FheFB6-Bf{^*e*(!j%{#3MPkQjEk!!jrIhl z0Xc|oAY1URqcbCQ^7Incj-8r#CC zOJUV4UV4)}I+g={$}k`r?H~lHO;uef$Sp%iRJa8fJXjRfB>BWSsBFImAkn$b@lp98 zrca=MMGEWC8elRJn!%%DO_XAnt;xos^(^4RA(u|@ryi)o~j>bv*7l^lLL$SS<1-p`F8Xv%nO0kueEaJ z&W%2g>o6%+Tj>+>{J9{#n-sMoR#7jA8KS^1gi`oGS=z{RVVd(Qo112e2WH&Hyqqtt zSQ742D!J;I>h2t>RYUELpOBH#!KI1RY{Ne8X)ZU4yv_~x0(+^NxR)~;g_dBwDZp+s zA9Lv-FvnqjwWhzZak&|2YV>I%wKnEQ1L0BlM(fkd_E9>(pr>k8`<~AA1TCQY!f;?oc46&jAy4F|uONwlR=L#T%p1Asa`t&*w2o5C$8L z8x%z^lV>A?$YD-fu%8JSYX2<2sX%#eo z#vChlnkuaqb)glMuS|>Gs1v3^Oq{R9_U&5Fd|PD-Hj;*q34lu`b2qh63KX{ecM^PJoeOJ<5>ddD2-6qFIsk79{J4X?^8|;SbSHFj%G
7k3?_Y7n1CX%NCuEYhffs<7ud1~ zf{}F5Ii2k$Cxht@bKj2hnC&^5Y-#5w&&dqyruqQh(@rg9HI#AmJMZr4l-v);=rEL@ zSovmzj&QemuZGZ@2v>N(LeSg4>tJ;Cdwc0J3B|QIUAez7?uomkKW^b2AHOT-zqZ5~ zW0&w-TaCXO{efRoKef=02TIT@;h)Ajt6by^RQ( zlDR;;K5`W@ixaBbjgOj!3?vLig5|n9IwWQV zxJ<(NOI@@?<0vymXvZG0j$SKq=Q2DAE0ysTd3Kdbb6xt_l}grSDY!v>#CmHr(~*&8 zT7>^rwcTtSAALJa>ZRw7?k&B2$LI;nh@%gV=Hhoh_tb%1N5#YH=;2+v_m*}an%?*9 z)0%eYBM*Mon|8h>pF<$-o!<4-0Rk-@goDF zy+&NVJDRw+(d*0t^B!m%8#WZ}9mmFMHN72{4Nd>#X@(OyFW1wo&mYXQAJ2Nk9Yu$g z^Vfp^ao!su^p-)SFpL2!f%EI}3dUiB*gBw?v22Y9OS;9-183zfR|BhuA+_2Xv4O#) zEfl8SXts5E&7H0D@~%$KYsqQrh7@iM%9qh5t*6)G2S*eMZ1(YO$}2JHwe%YNkL+PS<}P&gWw^hY3{DFVSucp?}- z7D1@G4j4_ed`R{4^ZsxbgdzSxOSm664o+5dvjZZd;Bu`VPu5L`p1Jj74XlREDO*2!gnU z1U_LfEn2-ahFCB>oR|z+>?Gbj#D%L=92GENnz)`qUCyC#nTr;xVx0_t1(LOg6bjcX z0~2~m?2PtQv3mgC&x>puuI7frm8}Mrx7=gm9YaEieP`s^#%kP9d~Sye@yH68tgNkl z>92`f3Vl>lEU+6LI2F=1@S-gqYKj+SjGG#u^)GsD#EZk6Rq~bz&($DZ2OnVc#&@J| z*dSSdh)=tm>x*T&0N`G!Ky%W*Dqx=3S2}iR|K4e%EO^ZwX4L3J%a`ixgj$@z$*DF* z55sk+3~!Bjzmyb28`3S(?89+udqNkQXxu-%T`yM+b|ZJe{9~K-K&Tz#qF}v&NL~yc zwAgchd|Zd}OrC1kc8t=yMl6|N$jm94IBKl(6ROw9jSdJX%+l{Q;M2n=#>wO3)Wbdp zi$rAnsW9f?na10mur9_CaPe9APT~X%5k_DvFZ2A5yeqD>&saho*?G?(S;7lCqHV@A zh@#o~ZBe1qnmRX3C^|n@6tWb^s4PfQa5ol$@U1EB?0RGqG%ZD<>9@ACGV zfdvO3t=JYTitKRzj#OF)C$@%R=-KH?9U(0Z*%`|$2tzHYj>iC%s`pT=W3uBM2()p^ z7-~?ph0CTAWB8OIZ~2|=Oj{Q<7+j_WR@xb3!yUF%V8c}400H#TNf$_Oz&X;bq}1dW zq*EoEt(1`WLONtj1zAAZLyPp(Egl>yu5gs)_DlLy+qb)7mGCvnP_c`n1TG#TM{|k{ zP-0(_&|^i>&&|hR%x2RoSXp2K=Kg1+TYNQUW||&UQ+~zl$su^Ggw-%T#I`KNK=y5H zaUcO>CuiV4mTlS9ij}}6R$_=j)$;V7H{&ts6)-R_&w*Gd|E%Gb=v&TWY}*2>ajuPo zF;{Jrx!FlZe_%6Rh4~m9MwfNQR2(gB79OaEx7x5$c{e+T@UO3`wpIBy8H%z-u`o|y zb+Ri0I|s<-_qbp(!}ewdu;DU6P9^FU4o9gqhTSPRd1H3W8~jWimNoPlD<+Z_O>+1~ z6d9VsV6IZO>A4iF$v?b3&~s3h&%f8!4j6F8mD&PxSeYBm83sd!wWN74D09r42pw2M zD#3K@wF%BvSShNhF$c?920?r@Rt}l2i9lmT404Jkag&=n(~JHr3n0O7 z>Np0nx93%;glNXZ$kO`|kH*-O1n@WzPPVY=#y8F|ykiooLEgt*fhQd=rw=ydV=XPW z%3J${{f6ltcg?><7u_W{?wif@-EOsFY#4BTNQz)Ca+SRW35)OS$=AZjp&R7=kw$tU zy)8!PD=ebnq1oZiR%k?YTw2vZ$)Q-($r@?9!<^E2HeFV`E%Rx{qk)*Sjv^C1mHn`C*wu(WwY5H0ufKe9T^dWhpnu( z)iPEYlg7l6*Q3C-!|&c+ddF>kjk~11qB3I?Mmey4Q@B)G>$@ZjYz#W%5ErLI%m|cC zEoPZOw_@W65ZTj9EU7NcZqeDRLy`VHF=N3}xVZ;y;w__5c^(oc99%g#uv%i2BZHx8 za)Z^K(O%q;ixps5cS-3og$gq&$;;vf$t^{qyYc# zVW$+rWZy)fV(agogXKn-Cu|tqpKFfSRrGJ{ay{3V71*HO&_mesly6F|JKeU(1udMB zQiEdj?3^Bu@H?-3nKsesssdR3-F}!OhU!EaLFlJ5IF~VYS1E^k5c>(@VEJsA5v)^& zpXtnjyLtEJbcJ>lj~bZIxr8sN-~6n=H47_2a(BXQcS1Bn7zl&cNwBLnr?<>zv$F_s#VnwBIx zDmZ!e$8w*rcgK~A+pPkmWCpaWe5=dM{015nN8& zVWZ9pIn-3DDLqvoi+I0A7R0~Ny$v~-#NryrFp&D&VKqKWY{4NgIT@~W3It-oO?odh zd$@v708{rhVk*inxIjMz12Dg40RBvPv6{+6*5|t?F2qaBdBb)gn9BN6jA(~~`UvAS z?L-Uv%JiOvzT$4H%;ft`1&ESS2Uli1?O4!D_gH1dBa>b2x~wv@sy-X2%ebX5UykWC z)v_F*`rC7LM{16@_>fSY9LgYl>|#8ETr!pOsT?fP%)mTKB*NZSohr%rFm;==aDa-X zxTU8q!$gH=qF21YWW8>+*)RbQhbBcX3C=*5ftX)_>o&e);Q2##@!3<++`c`(fg4Kx+8dw|jh0izZ zL&HJ8C7e$I?{c3(7s!4p4m5sr1hacEC^tbwf&UZ~yIe?E5)k6heIRgO=EnJ{QQO@7 zb`aB8?9Y$}o`@cpigtKq3Ye&o;%#MM#F%T2N7S!df!nMO3akp;hRrcoaQ9{>7?cxX zt}Kfnl(2+JdAm6Y)jlqQ9}t6~l@Dt=Ke>m-Aii359>*3JaQLn_D`^U_7_%^3$iouIX^E zopCX)dxpdXJuoCTK=_bYhhFZoktXHF<5OK6=nmBemvkKWID-)`lWt&}h;US*;W}1* zVWZ{Q*)E2{A_mi7gQ7_vXbGPEt5EzP5Gf%y)Ll0duKNherA7cM^@SD^SUo|p(L7VI zssgN*XU8zZHI?}l0)Kk5M}q=1a(ii7!5agEPF~y?J7L!l7FqVhRUG!M_7f}c<))Tk z$ucNhhLKaz+^8Ab;2hTKP&!9lMv*^{6vobr8-$VBB{Ri-vmh@7(8f(AO;?j86ld!= qY|jX0&LLh_^7MyaXZectfWK@1SE&Na$H>{)>9T4mTnE*iDgHm#QhY%G diff --git a/docs/_build/html/.doctrees/index.doctree b/docs/_build/html/.doctrees/index.doctree index 90c54d1c9b8406b23d4bf9a3c8a334f34947858f..a9d0e116120116caf4b852fb756338315cff1e50 100644 GIT binary patch delta 140 zcmaDOd|z0sfpzK?P6jaO;Y-O+E-lH-Db|}PZpxIwFfmqIsE5C}AR{xcLN75TB`-fE zwRp>SlpORWUWumV% zQwGDvDrLqXXZ?))+*JL7%#6|;eV5eY?2`Nf{q&Tg%(7JdfTH}Y)Z~(4{hXZK;=&x= zq}06RjNHVcY<-}4#rm0fDXA5DMa3mkdRTHY^HQgHGiE4c@JxQhXw1m9S&Zo}j BH9Y_T diff --git a/docs/_build/html/_sources/docs/usage.rst.txt b/docs/_build/html/_sources/docs/usage.rst.txt index 4468249..806a4e8 100644 --- a/docs/_build/html/_sources/docs/usage.rst.txt +++ b/docs/_build/html/_sources/docs/usage.rst.txt @@ -33,11 +33,11 @@ Using transformers backend. batch_size=8, max_new_tokens=256, temperature=0.7, - model_args={ + model_kwargs={ "attn_implementation": "flash_attention_2", "torch_dtype": "bfloat16", }, - generate_kwargs={ + generation_kwargs={ "do_sample": False, }, ) diff --git a/docs/_build/html/_static/basic.css b/docs/_build/html/_static/basic.css index 93a4776..4738b2e 100644 --- a/docs/_build/html/_static/basic.css +++ b/docs/_build/html/_static/basic.css @@ -741,14 +741,6 @@ abbr, acronym { cursor: help; } -.translated { - background-color: rgba(207, 255, 207, 0.2) -} - -.untranslated { - background-color: rgba(255, 207, 207, 0.2) -} - /* -- code displays --------------------------------------------------------- */ pre { @@ -911,4 +903,4 @@ div.math:hover a.headerlink { #top-link { display: none; } -} +} \ No newline at end of file diff --git a/docs/_build/html/_static/documentation_options.js b/docs/_build/html/_static/documentation_options.js index 35c998a..9da1c65 100644 --- a/docs/_build/html/_static/documentation_options.js +++ b/docs/_build/html/_static/documentation_options.js @@ -1,5 +1,5 @@ const DOCUMENTATION_OPTIONS = { - VERSION: '0.1.13', + VERSION: '0.1.15', LANGUAGE: 'en', COLLAPSE_INDEX: false, BUILDER: 'html', diff --git a/docs/_build/html/_static/leaderboard.json b/docs/_build/html/_static/leaderboard.json new file mode 100644 index 0000000..c842948 --- /dev/null +++ b/docs/_build/html/_static/leaderboard.json @@ -0,0 +1,106 @@ +[ + { + "model": "openai/gpt-oss-120b", + "type": "open-source", + "fewshots": 5, + "backend": "vllm", + "accuracy": 0.9049, + "date": "2026-02-24" + }, + { + "model": "openai/gpt-oss-20b", + "type": "open-source", + "fewshots": 5, + "backend": "vllm", + "accuracy": 0.8871, + "date": "2026-02-24" + }, + { + "model": "meta-llama/Llama-3.3-70B-Instruct", + "type": "open-source", + "fewshots": 5, + "backend": "vllm", + "accuracy": 0.8607, + "date": "2026-02-24" + }, + { + "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", + "type": "open-source", + "fewshots": 5, + "backend": "vllm", + "accuracy": 0.8519, + "date": "2026-02-24" + }, + { + "model": "Qwen/Qwen2.5-7B-Instruct", + "type": "open-source", + "fewshots": 5, + "backend": "vllm", + "accuracy": 0.794, + "date": "2026-02-24" + }, + { + "model": "mistralai/Mistral-Nemo-Instruct-2407", + "type": "open-source", + "fewshots": 5, + "backend": "vllm", + "accuracy": 0.7599, + "date": "2026-02-24" + }, + { + "model": "Qwen/Qwen2.5-1.5B-Instruct", + "type": "open-source", + "fewshots": 5, + "backend": "vllm", + "accuracy": 0.6401, + "date": "2026-02-24" + }, + { + "model": "meta-llama/Llama-3.2-3B-Instruct", + "type": "open-source", + "fewshots": 5, + "backend": "vllm", + "accuracy": 0.5415, + "date": "2026-02-24" + }, + { + "model": "CYFRAGOVPL/PLLuM-12B-chat", + "type": "open-source", + "fewshots": 5, + "backend": "vllm", + "accuracy": 0.5224, + "date": "2026-02-24" + }, + { + "model": "Qwen/Qwen3-0.6B", + "type": "open-source", + "fewshots": 5, + "backend": "vllm", + "accuracy": 0.4983, + "date": "2026-02-24" + }, + { + "model": "CYFRAGOVPL/PLLuM-12B-nc-chat", + "type": "open-source", + "fewshots": 5, + "backend": "vllm", + "accuracy": 0.4044, + "date": "2026-02-24" + }, + { + "model": "CYFRAGOVPL/pllum-12b-nc-chat-250715", + "type": "open-source", + "fewshots": 5, + "backend": "vllm", + "accuracy": 0.3727, + "date": "2026-02-24" + }, + { + "model": "meta-llama/Llama-3.2-1B-Instruct", + "type": "open-source", + "fewshots": 5, + "backend": "vllm", + "accuracy": 0.2678, + "date": "2026-02-24" + } +] \ No newline at end of file diff --git a/docs/_build/html/_static/scripts/front_page.js b/docs/_build/html/_static/scripts/front_page.js index 03fd423..1e0c0dc 100644 --- a/docs/_build/html/_static/scripts/front_page.js +++ b/docs/_build/html/_static/scripts/front_page.js @@ -48,3 +48,88 @@ if (searchInput) { } }); } + +document.addEventListener("DOMContentLoaded", async () => { + const container = document.getElementById('leaderboard-container'); + if (!container) return; + + try { + const response = await fetch('_static/leaderboard.json'); + const rows = await response.json(); + renderLeaderboard(rows); + } catch (e) { + container.innerHTML = '

Error loading leaderboard 😢

'; + console.error(e); + } +}); + +function renderLeaderboard(rows) { + const container = document.getElementById('leaderboard-container'); + container.innerHTML = ''; + + const table = document.createElement('table'); + table.className = 'leaderboard-table'; + + const thead = document.createElement('thead'); + thead.innerHTML = ` + + Rank + Model + Type + Fewshots + Backend + Accuracy + Date + `; + table.appendChild(thead); + + const tbody = document.createElement('tbody'); + rows.forEach((row, i) => { + const tr = document.createElement('tr'); + + // Берём только вторую часть после слеша + const modelName = row.model.includes('/') ? row.model.split('/')[1] : row.model; + + // Модель с ссылкой + const modelCell = document.createElement('td'); + if (row.url) { + const a = document.createElement('a'); + a.href = row.url; + a.target = "_blank"; + a.rel = "noopener"; + a.textContent = modelName; // <-- здесь только вторая часть + modelCell.appendChild(a); + } else { + modelCell.textContent = modelName; + } + + // Accuracy + const accuracyCell = document.createElement('td'); + const barContainer = document.createElement('div'); + barContainer.className = 'accuracy-bar'; + const fill = document.createElement('div'); + fill.className = 'fill'; + fill.style.width = `${(row.accuracy*100).toFixed(2)}%`; + const text = document.createElement('span'); + text.textContent = `${(row.accuracy*100).toFixed(2)}%`; + barContainer.appendChild(fill); + barContainer.appendChild(text); + accuracyCell.appendChild(barContainer); + + // Вставка остальных ячеек + tr.innerHTML += `${i+1}`; + tr.appendChild(modelCell); + tr.innerHTML += ` + ${row.type} + ${row.fewshots} + ${row.backend} + `; + tr.appendChild(accuracyCell); + tr.innerHTML += `${row.date}`; + + tbody.appendChild(tr); + }); + + table.appendChild(tbody); + container.appendChild(table); +} \ No newline at end of file diff --git a/docs/_build/html/_static/searchtools.js b/docs/_build/html/_static/searchtools.js index 2c774d1..91f4be5 100644 --- a/docs/_build/html/_static/searchtools.js +++ b/docs/_build/html/_static/searchtools.js @@ -513,9 +513,11 @@ const Search = { // perform the search on the required terms searchTerms.forEach((word) => { const files = []; + // find documents, if any, containing the query word in their text/title term indices + // use Object.hasOwnProperty to avoid mismatching against prototype properties const arr = [ - { files: terms[word], score: Scorer.term }, - { files: titleTerms[word], score: Scorer.title }, + { files: terms.hasOwnProperty(word) ? terms[word] : undefined, score: Scorer.term }, + { files: titleTerms.hasOwnProperty(word) ? titleTerms[word] : undefined, score: Scorer.title }, ]; // add support for partial matches if (word.length > 2) { @@ -547,8 +549,9 @@ const Search = { // set score for the word in each file recordFiles.forEach((file) => { - if (!scoreMap.has(file)) scoreMap.set(file, {}); - scoreMap.get(file)[word] = record.score; + if (!scoreMap.has(file)) scoreMap.set(file, new Map()); + const fileScores = scoreMap.get(file); + fileScores.set(word, record.score); }); }); @@ -587,7 +590,7 @@ const Search = { break; // select one (max) score for the file. - const score = Math.max(...wordList.map((w) => scoreMap.get(file)[w])); + const score = Math.max(...wordList.map((w) => scoreMap.get(file).get(w))); // add result to the result list results.push([ docNames[file], diff --git a/docs/_build/html/_static/styles/front_page.css b/docs/_build/html/_static/styles/front_page.css index 55bce1c..1d3bcfb 100644 --- a/docs/_build/html/_static/styles/front_page.css +++ b/docs/_build/html/_static/styles/front_page.css @@ -248,3 +248,70 @@ pre span { background: none !important; color: inherit !important; } + +.leaderboard-box { + padding: 1rem; + background: #fff; + border-radius: 12px; + box-shadow: 0 6px 20px rgba(0,0,0,0.08); + overflow-x: auto; +} + +.leaderboard-table { + width: 100%; + border-collapse: collapse; + font-family: 'Inter', 'Roboto', sans-serif; + font-size: 0.95rem; + text-align: center; +} + +.leaderboard-table th { + background: linear-gradient(180deg, #f6f6f6 0%, #e9e9e9 100%); + color: #111827; + font-weight: 600; + font-size: 0.95rem; + padding: 14px 10px; + text-transform: uppercase; + letter-spacing: 0.5px; + border-bottom: 2px solid #ddd; + text-align: center; + box-shadow: inset 0 -1px 0 rgba(0,0,0,0.05); +} + +.leaderboard-table td { + padding: 10px; + border-bottom: 1px solid #e0e0e0; +} + +.leaderboard-table tbody tr:nth-child(even) { + background-color: #f9f9f9; +} + +/* Accuracy bar */ +.accuracy-bar { + position: relative; + width: 100%; + height: 20px; + background: #e0e0e0; + border-radius: 10px; + overflow: hidden; +} + +.accuracy-bar .fill { + height: 100%; + background: linear-gradient(90deg,#4caf50,#81c784); + border-radius: 10px 0 0 10px; +} + +.accuracy-bar span { + position: absolute; + width: 100%; + text-align: center; + top: 0; + left: 0; + font-size: 0.8rem; + font-weight: 600; + line-height: 20px; + color: #000; +} + diff --git a/docs/_build/html/docs/evaluation.html b/docs/_build/html/docs/evaluation.html index e7d2eec..5335a85 100644 --- a/docs/_build/html/docs/evaluation.html +++ b/docs/_build/html/docs/evaluation.html @@ -5,21 +5,21 @@ - Evaluation API Reference — LLMSQL 0.1.13 documentation + Evaluation API Reference — LLMSQL 0.1.15 documentation - + - - + + - + - + +
- +

Evaluation API Reference

The evaluate() function allows you to benchmark Text-to-SQL model outputs @@ -153,37 +150,6 @@

Report Saving -

LLMSQL Evaluation Module

-

Provides the evaluate() function to benchmark Text-to-SQL model outputs -on the LLMSQL benchmark.

-

See the documentation for full usage details.

-

-
-
-llmsql.evaluation.evaluate.evaluate(outputs, *, workdir_path: str | None = 'llmsql_workdir', questions_path: str | None = None, db_path: str | None = None, save_report: str | None = None, show_mismatches: bool = True, max_mismatches: int = 5) → dict[source]
-

Evaluate predicted SQL queries against the LLMSQL benchmark.

-
-
Parameters:
-
    -

  • outputs – Either a JSONL file path or a list of dicts.

    • -

  • workdir_path – Directory for auto-downloads (ignored if all paths provided).

    • -

  • questions_path – Manual path to benchmark questions JSONL.

    • -

  • db_path – Manual path to SQLite benchmark DB.

    • -

  • save_report – Optional manual save path. If None → auto-generated.

    • -

  • show_mismatches – Print mismatches while evaluating.

    • -

  • max_mismatches – Max mismatches to print.

    • -
-
-
Returns:
-

Metrics and mismatches.

-
-
Return type:
-

dict

-
-
-
-

—

- + \ No newline at end of file diff --git a/docs/_build/html/docs/index.html b/docs/_build/html/docs/index.html index d78ae06..009a960 100644 --- a/docs/_build/html/docs/index.html +++ b/docs/_build/html/docs/index.html @@ -5,22 +5,22 @@ - LLMSQL package Documentation — LLMSQL 0.1.13 documentation + LLMSQL package Documentation — LLMSQL 0.1.15 documentation - + - - + + - + - + +
- +

LLMSQL package Documentation

← Back to main page

@@ -161,20 +158,17 @@

Navigation

  • index
    • -
  • - modules |
  • next |
  • previous |
    • -
  • LLMSQL 0.1.13 documentation »
    • -
  • LLMSQL package Documentation
    • +
  • LLMSQL 0.1.15 documentation »
    • +
  • LLMSQL package Documentation
    • - + \ No newline at end of file diff --git a/docs/_build/html/docs/inference.html b/docs/_build/html/docs/inference.html index caad396..138e1e2 100644 --- a/docs/_build/html/docs/inference.html +++ b/docs/_build/html/docs/inference.html @@ -5,22 +5,22 @@ - Inference API Reference — LLMSQL 0.1.13 documentation + Inference API Reference — LLMSQL 0.1.15 documentation - + - - + + - + - + +
    - -
    -

    Inference API Reference

    -
    -

    LLMSQL Transformers Inference Function

    -

    This module provides a single function inference_transformers() that performs -text-to-SQL generation using large language models via the Transformers backend.

    -

    Example

    -
    from llmsql.inference import inference_transformers
-
-results = inference_transformers(
-    model_or_model_name_or_path="Qwen/Qwen2.5-1.5B-Instruct",
-    output_file="outputs/preds_transformers.jsonl",
-    questions_path="data/questions.jsonl",
-    tables_path="data/tables.jsonl",
-    num_fewshots=5,
-    batch_size=8,
-    max_new_tokens=256,
-    temperature=0.7,
-    model_args={
-        "torch_dtype": "bfloat16",
-    },
-    generate_kwargs={
-        "do_sample": False,
-    },
-)
-
    -
    -

    Notes

    -

    This function uses the HuggingFace Transformers backend and may produce -slightly different outputs than the vLLM backend even with the same inputs -due to differences in implementation and numerical precision.

    -
    -
    -
    -llmsql.inference.inference_transformers.inference_transformers(model_or_model_name_or_path: str | AutoModelForCausalLM, tokenizer_or_name: str | Any | None = None, *, trust_remote_code: bool = True, dtype: dtype = torch.float16, device_map: str | dict[str, int] | None = 'auto', hf_token: str | None = None, model_kwargs: dict[str, Any] | None = None, tokenizer_kwargs: dict[str, Any] | None = None, chat_template: str | None = None, max_new_tokens: int = 256, temperature: float = 0.0, do_sample: bool = False, top_p: float = 1.0, top_k: int = 50, generation_kwargs: dict[str, Any] | None = None, output_file: str = 'llm_sql_predictions.jsonl', questions_path: str | None = None, tables_path: str | None = None, workdir_path: str = 'llmsql_workdir', num_fewshots: int = 5, batch_size: int = 8, seed: int = 42) → list[dict[str, str]][source]
    -

    Inference a causal model (Transformers) on the LLMSQL benchmark.

    -
    -
    Parameters:
    -
      -

    • model_or_model_name_or_path – Model object or HF model name/path.

      • -

    • tokenizer_or_name – Tokenizer object or HF tokenizer name/path.

      • -

    • Loading (# Tokenizer)

      • -

    • trust_remote_code – Whether to trust remote code (default: True).

      • -

    • dtype – Torch dtype for model (default: float16).

      • -

    • device_map – Device placement strategy (default: “auto”).

      • -

    • hf_token – Hugging Face authentication token.

      • -

    • model_kwargs – Additional arguments for AutoModelForCausalLM.from_pretrained(). -Note: ‘dtype’, ‘device_map’, ‘trust_remote_code’, ‘token’ -are handled separately and will override values here.

      • -

    • Loading

      • -

    • tokenizer_kwargs – Additional arguments for AutoTokenizer.from_pretrained(). ‘padding_side’ defaults to “left”. -Note: ‘trust_remote_code’, ‘token’ are handled separately and will override values here.

      • -

    • Chat (# Prompt &)

      • -

    • chat_template – Optional chat template to apply before tokenization.

      • -

    • Generation (#)

      • -

    • max_new_tokens – Maximum tokens to generate per sequence.

      • -

    • temperature – Sampling temperature (0.0 = greedy).

      • -

    • do_sample – Whether to use sampling vs greedy decoding.

      • -

    • top_p – Nucleus sampling parameter.

      • -

    • top_k – Top-k sampling parameter.

      • -

    • generation_kwargs – Additional arguments for model.generate(). -Note: ‘max_new_tokens’, ‘temperature’, ‘do_sample’, -‘top_p’, ‘top_k’ are handled separately.

      • -

    • Benchmark (#)

      • -

    • output_file – Output JSONL file path for completions.

      • -

    • questions_path – Path to benchmark questions JSONL.

      • -

    • tables_path – Path to benchmark tables JSONL.

      • -

    • workdir_path – Working directory path.

      • -

    • num_fewshots – Number of few-shot examples (0, 1, or 5).

      • -

    • batch_size – Batch size for inference.

      • -

    • seed – Random seed for reproducibility.

      • -
    -
    -
    Returns:
    -

    List of generated SQL results with metadata.

    -
    -
    -
    - + +
    +

    Inference API Reference

    —

    -
    -

    LLMSQL vLLM Inference Function

    -

    This module provides a single function inference_vllm() that performs -text-to-SQL generation using large language models via the vLLM backend.

    -

    Example

    -
    from llmsql.inference import inference_vllm
-
-results = inference_vllm(
-    model_name="Qwen/Qwen2.5-1.5B-Instruct",
-    output_file="outputs/predictions.jsonl",
-    questions_path="data/questions.jsonl",
-    tables_path="data/tables.jsonl",
-    num_fewshots=5,
-    batch_size=8,
-    max_new_tokens=256,
-    temperature=0.7,
-    tensor_parallel_size=1,
-)
-
    -
    -

    Notes

    -

    This function uses the vLLM backend. Outputs may differ from the Transformers -backend due to differences in implementation, batching, and numerical precision.

    -
    -
    -
    -llmsql.inference.inference_vllm.inference_vllm(model_name: str, *, trust_remote_code: bool = True, tensor_parallel_size: int = 1, hf_token: str | None = None, llm_kwargs: dict[str, Any] | None = None, use_chat_template: bool = True, max_new_tokens: int = 256, temperature: float = 1.0, do_sample: bool = True, sampling_kwargs: dict[str, Any] | None = None, output_file: str = 'llm_sql_predictions.jsonl', questions_path: str | None = None, tables_path: str | None = None, workdir_path: str = 'llmsql_workdir', num_fewshots: int = 5, batch_size: int = 8, seed: int = 42) → list[dict[str, str]][source]
    -

    Run SQL generation using vLLM.

    -
    -
    Parameters:
    -
      -

    • model_name – Hugging Face model name or path.

      • -

    • Loading (# Model)

      • -

    • trust_remote_code – Whether to trust remote code (default: True).

      • -

    • tensor_parallel_size – Number of GPUs for tensor parallelism (default: 1).

      • -

    • hf_token – Hugging Face authentication token.

      • -

    • llm_kwargs – Additional arguments for vllm.LLM(). -Note: ‘model’, ‘tokenizer’, ‘tensor_parallel_size’, -‘trust_remote_code’ are handled separately and will -override values here.

      • -

    • Generation (#)

      • -

    • max_new_tokens – Maximum tokens to generate per sequence.

      • -

    • temperature – Sampling temperature (0.0 = greedy).

      • -

    • do_sample – Whether to use sampling vs greedy decoding.

      • -

    • sampling_kwargs – Additional arguments for vllm.SamplingParams(). -Note: ‘temperature’, ‘max_tokens’ are handled -separately and will override values here.

      • -

    • Benchmark (#)

      • -

    • output_file – Path to write outputs (will be overwritten).

      • -

    • questions_path – Path to questions.jsonl (auto-downloads if missing).

      • -

    • tables_path – Path to tables.jsonl (auto-downloads if missing).

      • -

    • workdir_path – Directory to store downloaded data.

      • -

    • num_fewshots – Number of few-shot examples (0, 1, or 5).

      • -

    • batch_size – Number of questions per generation batch.

      • -

    • seed – Random seed for reproducibility.

      • -
    -
    -
    Returns:
    -

    List of dicts containing question_id and generated completion.

    -
    -
    -
    -

    —

    - + \ No newline at end of file diff --git a/docs/_build/html/docs/usage.html b/docs/_build/html/docs/usage.html index e53712a..dde17b7 100644 --- a/docs/_build/html/docs/usage.html +++ b/docs/_build/html/docs/usage.html @@ -5,22 +5,22 @@ - Usage Overview — LLMSQL 0.1.13 documentation + Usage Overview — LLMSQL 0.1.15 documentation - + - - + + - + - + +
    - +

    Usage Overview

    LLMSQL package provides two primary components:

    @@ -79,11 +76,11 @@

    Basic Examplebatch_size=8, max_new_tokens=256, temperature=0.7, - model_args={ + model_kwargs={ "attn_implementation": "flash_attention_2", "torch_dtype": "bfloat16", }, - generate_kwargs={ + generation_kwargs={ "do_sample": False, }, ) @@ -182,21 +179,18 @@

    Navigation

  • index
    • -
  • - modules |
  • next |
  • previous |
    • -
  • LLMSQL 0.1.13 documentation »
    • +
  • LLMSQL 0.1.15 documentation »
  • LLMSQL package Documentation »
    • -
  • Usage Overview
    • +
  • Usage Overview

    • - + \ No newline at end of file diff --git a/docs/_build/html/genindex.html b/docs/_build/html/genindex.html index 1be9b96..62acac1 100644 --- a/docs/_build/html/genindex.html +++ b/docs/_build/html/genindex.html @@ -4,20 +4,20 @@ - Index — LLMSQL 0.1.13 documentation + Index — LLMSQL 0.1.15 documentation - + - - + + - + - + +
    - +

    Index

    - E - | I - | L - | M - +
    -

    E

    - - -
    - -

    I

    - - - -
    - -

    L

    - - - -
      -
    • - llmsql.evaluation.evaluate - -
      • -
    • - llmsql.inference.inference_transformers - -
      • -
      -
    • - llmsql.inference.inference_vllm - -
      • -
    - -

    M

    - - -
    -
    @@ -142,14 +69,11 @@

    Navigation

  • index
    • -
  • - modules |
    • -
  • LLMSQL 0.1.13 documentation »
    • -
  • Index
    • +
  • LLMSQL 0.1.15 documentation »
    • +
  • Index
    • - + \ No newline at end of file diff --git a/docs/_build/html/index.html b/docs/_build/html/index.html index c3b94b6..bd4199d 100644 --- a/docs/_build/html/index.html +++ b/docs/_build/html/index.html @@ -10,7 +10,7 @@ LLMSQL Project — Text-to-SQL Benchmark - + @@ -126,13 +126,20 @@

    2️⃣ Inference from CLI

    --output-file outputs/preds.jsonl \ --batch-size 8 \ --temperature 0.9 \ ---generate-kwargs '{"do_sample": false, "top_p": 0.95}' +--generation-kwargs '{"do_sample": false, "top_p": 0.95}'

    3️⃣ Evaluation API (Python)

    -
    from llmsql import LLMSQLEvaluator
+
    from llmsql import evaluate
+
+report =evaluate(outputs="path_to_your_outputs.jsonl")
+print(report)
+
    
+
    Or with ther results from the infernece:
    
+
    from llmsql import evaluate
+
+# results = inference_transformers(...) or infernce_vllm(...)
 
-evaluator = LLMSQLEvaluator()
-report = evaluator.evaluate(outputs_path="path_to_your_outputs.jsonl")
+report =evaluate(outputs=results)
 print(report)
 
    
 
@@ -146,15 +153,9 @@ 
       
     
 
-    
    📊 Leaderboard [in progress]
    
-    
    
-      
    
-        The official Leaderboard is currently empty and in progress.
-        
-          Submit
-        
-        your model results to be the first on the ranking!
-      
    
+    
    📊 Leaderboard — Execution Accuracy (EX)
    
+    
    
+      
    Loading leaderboard...
    
     
    
 
 
@@ -162,7 +163,7 @@ 
    📄 Citation
    
     
    @inproceedings{llmsql_bench,
   title={LLMSQL: Upgrading WikiSQL for the LLM Era of Text-to-SQL},
   author={Pihulski, Dzmitry and Charchut, Karol and Novogrodskaia, Viktoria and Koco{'n}, Jan},
-  booktitle={2025 IEEE ICDMW},
+  booktitle={2025 IEEE ICувцDMW},
   year={2025},
   organization={IEEE}
 }
@@ -202,4 +203,4 @@ 
    📄 Citation
    
 
 
 
-
+
\ No newline at end of file
diff --git a/docs/_build/html/leaderboard.json b/docs/_build/html/leaderboard.json
new file mode 100644
index 0000000..c842948
--- /dev/null
+++ b/docs/_build/html/leaderboard.json
@@ -0,0 +1,106 @@
+[
+  {
+    "model": "openai/gpt-oss-120b",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.9049,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "openai/gpt-oss-20b",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.8871,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "meta-llama/Llama-3.3-70B-Instruct",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.8607,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.8519,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "Qwen/Qwen2.5-7B-Instruct",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.794,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "mistralai/Mistral-Nemo-Instruct-2407",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.7599,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "Qwen/Qwen2.5-1.5B-Instruct",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.6401,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "meta-llama/Llama-3.2-3B-Instruct",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.5415,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "CYFRAGOVPL/PLLuM-12B-chat",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.5224,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "Qwen/Qwen3-0.6B",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.4983,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "CYFRAGOVPL/PLLuM-12B-nc-chat",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.4044,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "CYFRAGOVPL/pllum-12b-nc-chat-250715",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.3727,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "meta-llama/Llama-3.2-1B-Instruct",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.2678,
+    "date": "2026-02-24"
+  }
+]
\ No newline at end of file
diff --git a/docs/_build/html/objects.inv b/docs/_build/html/objects.inv
index 201680cb037070ec8153a07d39141277ed4dc841..ef93dc279cb0908985244f5ac23dd6dec053de78 100644
GIT binary patch
delta 215
zcmV;|04V>J1H%H4bbp1B(GG$j6o&743SPikx2?6c)rFfb%k=;RCv_n>0n4{fK~vMM
zJ9hs6`#20HsaiutxFI@|0xQk7Oj7J^Fq=lJROcL@hUy{m@f99Wf}ntGI};4vGl!iz
zJP5-065xW+CrJPwB{ew+ZH2v}GcIVl#HsvmabXdoTK%uVPEZ|vvxRd7WtG7TB_M{~
zIk^LGp?=z5*c{R9iKE6&NUSr>VUKdj7KBCZ^?@To$qa{1Jxc#zQLWSF2PU)vQqpt^
Rx|w5Qmq*RJ(FeR^vFk$dWC8#H

delta 298
zcmV+_0oDG)0+j=hbbqapPfvs}5XJBP6qD!`|6Di5#6%C9uo2yOHnj{3v9#D0ly7gL
zixlXJ2XB*k^M3Q%;k?uz`~n-wYibH9h7zG_H>FH#j&KF3Wctz@%1z17oEn`&MEFU}
zgqPKVbt+tki5{vD87dSE$LiH6Ds(1QiApcyjgAxP){wt!ntunM?OIvVSnZ$kcBlaP
z5BX*v4%(y}+k+^43wRhu?~*M%>?3sU@AF#4{u$MJ6e6hlSXJ3S!WOIDXCOb;Z}
zi~)?F9=1P&fgrW+@N+`nb?%pb;ITo~6!=LFD8}rZz9H{SLw8tcOR{*PH*q5*n8I+p
wCH8IsVAV#0;3FJ6cYBBS)D>IOn%rJs;?_tN%cn-ybDq7^6Z>xR4V?pLrr9cy7XSbN

diff --git a/docs/_build/html/search.html b/docs/_build/html/search.html
index 212e3b0..cf12260 100644
--- a/docs/_build/html/search.html
+++ b/docs/_build/html/search.html
@@ -4,18 +4,18 @@
   
     
     
-    Search — LLMSQL 0.1.13 documentation
+    Search — LLMSQL 0.1.15 documentation
     
-    
+    
     
-    
-
-    
+    
+    
+    
     
     
     
     
-    
+    
     
     
     
@@ -23,7 +23,7 @@
     
     
     
-
+     
 
   
     
+    
      
 
     
    
       
    
         
    
           
    
-
+            
   
    Search
    
-
+  
   
-
-
+  
+  
   
    
     Searching for multiple words only shows matches that contain
     all words.
   
    
-
-
+  
+  
   
    
     
     
     
   
    
-
-
+  
+  
   
    
-
+  
 
             
    
           
    
@@ -89,14 +86,11 @@ 
    Navigation
    
         
  • 
           index
    • 
-        
  • 
-          modules |
    • 
-        
  • LLMSQL 0.1.13 documentation »
    • 
-        
  • Search
    • 
+        
  • LLMSQL 0.1.15 documentation »
    • 
+        
  • Search
    •  
       
     
    
     
   
-
+
\ No newline at end of file
diff --git a/docs/_build/html/searchindex.js b/docs/_build/html/searchindex.js
index 970540c..f5962dd 100644
--- a/docs/_build/html/searchindex.js
+++ b/docs/_build/html/searchindex.js
@@ -1 +1 @@
-Search.setIndex({"alltitles": {"Basic Example": [[3, "basic-example"]], "Contents": [[1, null]], "Evaluation API Reference": [[0, null]], "Example: Running your first evaluation (with transformers backend)": [[1, "example-running-your-first-evaluation-with-transformers-backend"]], "Features": [[0, "features"]], "Full Documentation": [[1, "full-documentation"]], "Function Arguments": [[0, "function-arguments"]], "Getting Started": [[1, "getting-started"]], "Inference API Reference": [[2, null]], "Input Format": [[0, "input-format"]], "Installation": [[1, "installation"]], "LLMSQL Evaluation Module": [[0, "llmsql-evaluation-module"]], "LLMSQL Transformers Inference Function": [[2, "llmsql-transformers-inference-function"]], "LLMSQL package Documentation": [[1, null]], "LLMSQL vLLM Inference Function": [[2, "llmsql-vllm-inference-function"]], "Output Metrics": [[0, "output-metrics"]], "Report Saving": [[0, "report-saving"]], "Typical workflow": [[3, "typical-workflow"]], "Usage Examples": [[0, "usage-examples"]], "Usage Overview": [[3, null]]}, "docnames": ["docs/evaluation", "docs/index", "docs/inference", "docs/usage", "index"], "envversion": {"sphinx": 64, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1}, "filenames": ["docs/evaluation.rst", "docs/index.rst", "docs/inference.rst", "docs/usage.rst", "index.rst"], "indexentries": {"evaluate() (in module llmsql.evaluation.evaluate)": [[0, "llmsql.evaluation.evaluate.evaluate", false]], "llmsql.evaluation.evaluate": [[0, "module-llmsql.evaluation.evaluate", false]], "module": [[0, "module-llmsql.evaluation.evaluate", false]]}, "objects": {"llmsql.evaluation": [[0, 0, 0, "-", "evaluate"]], "llmsql.evaluation.evaluate": [[0, 1, 1, "", "evaluate"]], "llmsql.inference": [[2, 0, 0, "-", "inference_transformers"], [2, 0, 0, "-", "inference_vllm"]], "llmsql.inference.inference_transformers": [[2, 1, 1, "", "inference_transformers"]], "llmsql.inference.inference_vllm": [[2, 1, 1, "", "inference_vllm"]]}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "function", "Python function"]}, "objtypes": {"0": "py:module", "1": "py:function"}, "terms": {"0": [1, 2, 3], "1": [0, 1, 2, 3], "2": 0, "256": [1, 2, 3], "3": 0, "30": 0, "4096": 3, "42": 2, "5": [0, 1, 2, 3], "50": 2, "5b": [1, 2, 3], "7": [1, 2, 3], "8": [1, 2, 3], "9": 3, "By": 0, "If": 0, "It": 0, "The": 0, "accuraci": [0, 3], "activ": 0, "addit": 2, "ag": 0, "against": 0, "all": 0, "allow": 0, "ani": 2, "api": 1, "appli": 2, "ar": [0, 2], "argument": 2, "attn_implement": 3, "authent": 2, "auto": [0, 2], "automat": 0, "automodelforcausallm": 2, "autotoken": 2, "back": 1, "backend": [2, 3], "batch": 2, "batch_siz": [1, 2, 3], "befor": 2, "bench": 0, "benchmark": [0, 2], "bfloat16": [1, 2, 3], "bool": [0, 2], "both": 0, "can": 0, "causal": 2, "chat": 2, "chat_templ": 2, "code": 2, "complet": 2, "compon": 3, "comput": 3, "configur": 0, "contain": [0, 2], "count": 0, "cover": 1, "current": 0, "data": [1, 2, 3], "databas": 0, "dataset": 3, "db": 0, "db_path": 0, "decod": 2, "default": [0, 2], "descript": 0, "detail": 0, "devic": 2, "device_map": 2, "dict": [0, 2], "dict_list": 0, "dictionari": 0, "differ": 2, "directori": [0, 2], "displai": 0, "do_sampl": [1, 2, 3], "document": 0, "download": [0, 2], "dtype": 2, "due": 2, "either": 0, "error": 0, "evalu": 3, "evaluation_results_": 0, "even": 2, "everyth": 1, "exact": 0, "exampl": 2, "execut": 0, "face": 2, "fals": [1, 2, 3], "few": 2, "file": [0, 2], "flash_attention_2": 3, "float": 2, "float16": 2, "follow": 0, "from": [0, 1, 2, 3], "from_pretrain": 2, "full": 0, "gener": [0, 2, 3], "generate_kwarg": [1, 2, 3], "generation_kwarg": 2, "gold": 0, "gold_non": 0, "gpu": 2, "gpu_memory_util": 3, "greedi": 2, "guid": 1, "handl": 2, "here": 2, "hf": 2, "hf_token": 2, "how": 0, "hug": 2, "huggingfac": 2, "i": 0, "ignor": 0, "implement": 2, "import": [0, 1, 2, 3], "infer": [1, 3], "inference_transform": [1, 2, 3], "inference_vllm": [2, 3], "input": 2, "input_mod": 0, "inspect": 3, "instruct": [1, 2, 3], "int": [0, 2], "invalid": 0, "json": 0, "jsonl": [0, 1, 2, 3], "jsonl_path": 0, "k": 2, "kei": 0, "languag": 2, "larg": 2, "left": 2, "level": 3, "list": [0, 2], "llm": [2, 3], "llm_kwarg": [2, 3], "llm_sql_predict": 2, "llmsql": 3, "llmsql_workdir": [0, 2], "llmsqlevalu": 3, "load": 2, "log": 0, "made": [0, 1, 2, 3], "mai": 2, "main": 1, "manual": 0, "match": 0, "max": 0, "max_mismatch": 0, "max_model_len": 3, "max_new_token": [1, 2, 3], "max_token": 2, "maximum": [0, 2], "metadata": 2, "metric": 3, "mismatch": 0, "miss": [0, 2], "mode": 0, "model": [0, 1, 2, 3], "model_arg": [1, 2, 3], "model_kwarg": 2, "model_nam": [2, 3], "model_or_model_name_or_path": [1, 2, 3], "modul": 2, "name": [0, 2], "need": 1, "none": [0, 2], "note": 2, "nucleu": 2, "null": 0, "num_fewshot": [1, 2, 3], "number": [0, 2], "numer": 2, "object": 2, "option": [0, 2], "output": [1, 2, 3], "output_fil": [1, 2, 3], "outputs_path": 3, "overal": 0, "overrid": [0, 2], "overview": 1, "overwritten": 2, "own": 0, "packag": 3, "padding_sid": 2, "page": 1, "parallel": 2, "paramet": [0, 2], "pass": 3, "path": [0, 2], "path_to_output": 0, "per": 2, "perform": [2, 3], "pip": 1, "placement": 2, "precis": 2, "pred_non": 0, "predict": [0, 2, 3], "predicted_sql": 0, "preds_transform": [1, 2, 3], "preds_vllm": 3, "primari": 3, "print": [0, 1, 3], "produc": 2, "project": 1, "prompt": 2, "provid": [0, 2, 3], "python": 0, "queri": [0, 3], "question": [0, 1, 2, 3], "question_id": [0, 2], "questions_path": [0, 1, 2, 3], "qwen": [1, 2, 3], "qwen2": [1, 2, 3], "random": 2, "refer": 1, "remot": 2, "report": 3, "reproduc": 2, "requir": 0, "result": [0, 1, 2, 3], "return": [0, 2], "run": [2, 3], "same": 2, "sampl": 2, "sampling_kwarg": 2, "samplingparam": 2, "save_report": 0, "see": 0, "seed": 2, "select": 0, "separ": 2, "sequenc": 2, "shot": 2, "should": 0, "show_mismatch": 0, "singl": 2, "size": 2, "skip": 0, "slightli": 2, "some": 3, "sourc": [0, 1, 2], "sql": [0, 1, 2, 3], "sql_error": 0, "sqlite": 0, "sqlite_t": 0, "store": 2, "str": [0, 2], "strategi": 2, "summari": 0, "support": 0, "tabl": [0, 1, 2, 3], "tables_path": [1, 2, 3], "take": 3, "task": 3, "team": [0, 1, 2, 3], "temperatur": [1, 2, 3], "templat": 2, "tensor": 2, "tensor_parallel_s": [2, 3], "text": [0, 1, 2], "than": 2, "thi": [0, 1, 2], "time": 3, "timestamp": 0, "token": 2, "tokenizer_kwarg": 2, "tokenizer_or_nam": 2, "top": 2, "top_k": 2, "top_p": 2, "torch": 2, "torch_dtyp": [1, 2, 3], "total": 0, "transform": 3, "true": [0, 2], "trust": 2, "trust_remote_cod": 2, "two": 3, "type": 0, "us": [0, 1, 2, 3], "usag": 1, "use_chat_templ": 2, "uuid": 0, "v": 2, "valu": 2, "via": 2, "vllm": 3, "wa": 0, "welcom": 1, "were": 0, "where": 0, "whether": 2, "while": 0, "work": 2, "workdir": 0, "workdir_path": [0, 2], "write": 2, "you": [0, 1], "your": 0}, "titles": ["Evaluation API Reference", "LLMSQL package Documentation", "Inference API Reference", "Usage Overview", "<no title>"], "titleterms": {"api": [0, 2], "argument": 0, "backend": 1, "basic": 3, "content": 1, "document": 1, "evalu": [0, 1], "exampl": [0, 1, 3], "featur": 0, "first": 1, "format": 0, "full": 1, "function": [0, 2], "get": 1, "infer": 2, "input": 0, "instal": 1, "llmsql": [0, 1, 2], "metric": 0, "modul": 0, "output": 0, "overview": 3, "packag": 1, "refer": [0, 2], "report": 0, "run": 1, "save": 0, "start": 1, "transform": [1, 2], "typic": 3, "usag": [0, 3], "vllm": 2, "workflow": 3, "your": 1}})
+Search.setIndex({"alltitles":{"Basic Example":[[3,"basic-example"]],"Contents":[[1,null]],"Evaluation API Reference":[[0,null]],"Example: Running your first evaluation (with transformers backend)":[[1,"example-running-your-first-evaluation-with-transformers-backend"]],"Features":[[0,"features"]],"Full Documentation":[[1,"full-documentation"]],"Function Arguments":[[0,"function-arguments"]],"Getting Started":[[1,"getting-started"]],"Inference API Reference":[[2,null]],"Input Format":[[0,"input-format"]],"Installation":[[1,"installation"]],"LLMSQL package Documentation":[[1,null]],"Output Metrics":[[0,"output-metrics"]],"Report Saving":[[0,"report-saving"]],"Typical workflow":[[3,"typical-workflow"]],"Usage Examples":[[0,"usage-examples"]],"Usage Overview":[[3,null]]},"docnames":["docs/evaluation","docs/index","docs/inference","docs/usage","index"],"envversion":{"sphinx":65,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.viewcode":1},"filenames":["docs\\evaluation.rst","docs\\index.rst","docs\\inference.rst","docs\\usage.rst","index.rst"],"indexentries":{},"objects":{},"objnames":{},"objtypes":{},"terms":{"0":[1,3],"1":[0,1,3],"2":0,"256":[1,3],"3":0,"30":0,"4096":3,"5":[0,1,3],"5b":[1,3],"7":[1,3],"8":[1,3],"9":3,"By":0,"It":0,"The":0,"accuraci":[0,3],"activ":0,"ag":0,"against":0,"allow":0,"api":1,"ar":0,"attn_implement":3,"automat":0,"back":1,"backend":3,"batch_siz":[1,3],"bench":0,"benchmark":0,"bfloat16":[1,3],"both":0,"can":0,"compon":3,"comput":3,"configur":0,"contain":0,"count":0,"cover":1,"current":0,"data":[1,3],"databas":0,"dataset":3,"db":0,"db_path":0,"default":0,"descript":0,"detail":0,"dict":0,"dict_list":0,"dictionari":0,"directori":0,"displai":0,"do_sampl":[1,3],"download":0,"error":0,"evalu":3,"evaluation_results_":0,"everyth":1,"exact":0,"execut":0,"fals":[1,3],"file":0,"flash_attention_2":3,"follow":0,"from":[0,1,3],"gener":3,"generate_kwarg":1,"generation_kwarg":3,"gold":0,"gold_non":0,"gpu_memory_util":3,"guid":1,"how":0,"i":0,"ignor":0,"import":[0,1,3],"infer":[1,3],"inference_transform":[1,3],"inference_vllm":3,"input_mod":0,"inspect":3,"instruct":[1,3],"invalid":0,"json":0,"jsonl":[0,1,3],"jsonl_path":0,"kei":0,"level":3,"list":0,"llm":3,"llm_kwarg":3,"llmsql":[0,2,3],"llmsql_workdir":0,"llmsqlevalu":3,"log":0,"made":[0,1,2,3],"main":1,"match":0,"max_mismatch":0,"max_model_len":3,"max_new_token":[1,3],"maximum":0,"metric":3,"mismatch":0,"miss":0,"mode":0,"model":[0,1,3],"model_arg":1,"model_kwarg":3,"model_nam":3,"model_or_model_name_or_path":[1,3],"name":0,"need":1,"none":0,"null":0,"num_fewshot":[1,3],"number":0,"option":0,"output":[1,3],"output_fil":[1,3],"outputs_path":3,"overal":0,"overrid":0,"overview":1,"own":0,"packag":3,"page":1,"pass":3,"path":0,"path_to_output":0,"perform":3,"pip":1,"pred_non":0,"predict":[0,3],"predicted_sql":0,"preds_transform":[1,3],"preds_vllm":3,"primari":3,"print":[0,1,3],"project":1,"provid":[0,3],"python":0,"queri":[0,3],"question":[0,1,3],"question_id":0,"questions_path":[0,1,3],"qwen":[1,3],"qwen2":[1,3],"refer":1,"report":3,"requir":0,"result":[0,1,3],"return":0,"run":3,"save_report":0,"select":0,"should":0,"show_mismatch":0,"skip":0,"some":3,"sourc":1,"sql":[0,1,3],"sql_error":0,"sqlite":0,"sqlite_t":0,"summari":0,"support":0,"tabl":[0,1,3],"tables_path":[1,3],"take":3,"task":3,"team":[0,1,2,3],"temperatur":[1,3],"tensor_parallel_s":3,"text":[0,1],"thi":[0,1],"time":3,"timestamp":0,"torch_dtyp":[1,3],"total":0,"transform":3,"true":0,"two":3,"us":[0,1,3],"usag":1,"uuid":0,"vllm":3,"wa":0,"welcom":1,"were":0,"where":0,"while":0,"workdir":0,"workdir_path":0,"you":[0,1],"your":0},"titles":["Evaluation API Reference","LLMSQL package Documentation","Inference API Reference","Usage Overview","<no title>"],"titleterms":{"api":[0,2],"argument":0,"backend":1,"basic":3,"content":1,"document":1,"evalu":[0,1],"exampl":[0,1,3],"featur":0,"first":1,"format":0,"full":1,"function":0,"get":1,"infer":2,"input":0,"instal":1,"llmsql":1,"metric":0,"output":0,"overview":3,"packag":1,"refer":[0,2],"report":0,"run":1,"save":0,"start":1,"transform":1,"typic":3,"usag":[0,3],"workflow":3,"your":1}})
\ No newline at end of file
diff --git a/docs/_static/leaderboard.json b/docs/_static/leaderboard.json
new file mode 100644
index 0000000..c842948
--- /dev/null
+++ b/docs/_static/leaderboard.json
@@ -0,0 +1,106 @@
+[
+  {
+    "model": "openai/gpt-oss-120b",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.9049,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "openai/gpt-oss-20b",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.8871,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "meta-llama/Llama-3.3-70B-Instruct",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.8607,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.8519,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "Qwen/Qwen2.5-7B-Instruct",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.794,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "mistralai/Mistral-Nemo-Instruct-2407",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.7599,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "Qwen/Qwen2.5-1.5B-Instruct",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.6401,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "meta-llama/Llama-3.2-3B-Instruct",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.5415,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "CYFRAGOVPL/PLLuM-12B-chat",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.5224,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "Qwen/Qwen3-0.6B",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.4983,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "CYFRAGOVPL/PLLuM-12B-nc-chat",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.4044,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "CYFRAGOVPL/pllum-12b-nc-chat-250715",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.3727,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "meta-llama/Llama-3.2-1B-Instruct",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.2678,
+    "date": "2026-02-24"
+  }
+]
\ No newline at end of file
diff --git a/docs/_static/scripts/front_page.js b/docs/_static/scripts/front_page.js
index 03fd423..4f2cdc5 100644
--- a/docs/_static/scripts/front_page.js
+++ b/docs/_static/scripts/front_page.js
@@ -48,3 +48,86 @@ if (searchInput) {
     }
   });
 }
+
+document.addEventListener("DOMContentLoaded", async () => {
+  const container = document.getElementById('leaderboard-container');
+  if (!container) return;
+
+  try {
+    const response = await fetch('_static/leaderboard.json');
+    const rows = await response.json();
+    renderLeaderboard(rows);
+  } catch (e) {
+    container.innerHTML = '
    Error loading leaderboard 😢
    ';
+    console.error(e);
+  }
+});
+
+function renderLeaderboard(rows) {
+  const container = document.getElementById('leaderboard-container');
+  container.innerHTML = '';
+
+  const table = document.createElement('table');
+  table.className = 'leaderboard-table';
+
+  const thead = document.createElement('thead');
+  thead.innerHTML = `
+    
+      Rank
+      Model
+      Type
+      Fewshots
+      Backend
+      Accuracy
+      Date
+    `;
+  table.appendChild(thead);
+
+  const tbody = document.createElement('tbody');
+  rows.forEach((row, i) => {
+    const tr = document.createElement('tr');
+
+    const modelName = row.model.includes('/') ? row.model.split('/')[1] : row.model;
+
+    const modelCell = document.createElement('td');
+    if (row.url) {
+      const a = document.createElement('a');
+      a.href = row.url;
+      a.target = "_blank";
+      a.rel = "noopener";
+      a.textContent = modelName;   // <-- здесь только вторая часть
+      modelCell.appendChild(a);
+    } else {
+      modelCell.textContent = modelName;
+    }
+
+    // Accuracy
+    const accuracyCell = document.createElement('td');
+    const barContainer = document.createElement('div');
+    barContainer.className = 'accuracy-bar';
+    const fill = document.createElement('div');
+    fill.className = 'fill';
+    fill.style.width = `${(row.accuracy*100).toFixed(2)}%`;
+    const text = document.createElement('span');
+    text.textContent = `${(row.accuracy*100).toFixed(2)}%`;
+    barContainer.appendChild(fill);
+    barContainer.appendChild(text);
+    accuracyCell.appendChild(barContainer);
+
+
+    tr.innerHTML += `${i+1}`;
+    tr.appendChild(modelCell);
+    tr.innerHTML += `
+      ${row.type}
+      ${row.fewshots}
+      ${row.backend}
+    `;
+    tr.appendChild(accuracyCell);
+    tr.innerHTML += `${row.date}`;
+
+    tbody.appendChild(tr);
+  });
+
+  table.appendChild(tbody);
+  container.appendChild(table);
+}
\ No newline at end of file
diff --git a/docs/_static/styles/front_page.css b/docs/_static/styles/front_page.css
index 55bce1c..1d3bcfb 100644
--- a/docs/_static/styles/front_page.css
+++ b/docs/_static/styles/front_page.css
@@ -248,3 +248,70 @@ pre span {
     background: none !important;
     color: inherit !important;
 }
+
+.leaderboard-box {
+  padding: 1rem;
+  background: #fff;
+  border-radius: 12px;
+  box-shadow: 0 6px 20px rgba(0,0,0,0.08);
+  overflow-x: auto;
+}
+
+.leaderboard-table {
+  width: 100%;
+  border-collapse: collapse;
+  font-family: 'Inter', 'Roboto', sans-serif;
+  font-size: 0.95rem;
+  text-align: center;
+}
+
+.leaderboard-table th {
+  background: linear-gradient(180deg, #f6f6f6 0%, #e9e9e9 100%);
+  color: #111827;
+  font-weight: 600;
+  font-size: 0.95rem;
+  padding: 14px 10px;
+  text-transform: uppercase;
+  letter-spacing: 0.5px;
+  border-bottom: 2px solid #ddd;
+  text-align: center;
+  box-shadow: inset 0 -1px 0 rgba(0,0,0,0.05);
+}
+
+.leaderboard-table td {
+  padding: 10px;
+  border-bottom: 1px solid #e0e0e0;
+}
+
+.leaderboard-table tbody tr:nth-child(even) {
+  background-color: #f9f9f9;
+}
+
+/* Accuracy bar */
+.accuracy-bar {
+  position: relative;
+  width: 100%;
+  height: 20px;
+  background: #e0e0e0;
+  border-radius: 10px;
+  overflow: hidden;
+}
+
+.accuracy-bar .fill {
+  height: 100%;
+  background: linear-gradient(90deg,#4caf50,#81c784);
+  border-radius: 10px 0 0 10px;
+}
+
+.accuracy-bar span {
+  position: absolute;
+  width: 100%;
+  text-align: center;
+  top: 0;
+  left: 0;
+  font-size: 0.8rem;
+  font-weight: 600;
+  line-height: 20px;
+  color: #000;
+}
+
diff --git a/docs/_templates/index.html b/docs/_templates/index.html
index 65bcb22..a93b256 100644
--- a/docs/_templates/index.html
+++ b/docs/_templates/index.html
@@ -153,15 +153,9 @@ 
       
     
 
-    
    📊 Leaderboard [in progress]
    
-    
    
-      
    
-        The official Leaderboard is currently empty and in progress.
-        
-          Submit
-        
-        your model results to be the first on the ranking!
-      
    
+    
    📊 Leaderboard — Execution Accuracy (EX)
    
+    
    
+      
    Loading leaderboard...
    
     
    
 
 
@@ -169,7 +163,7 @@ 
    📄 Citation
    
     
    @inproceedings{llmsql_bench,
   title={LLMSQL: Upgrading WikiSQL for the LLM Era of Text-to-SQL},
   author={Pihulski, Dzmitry and Charchut, Karol and Novogrodskaia, Viktoria and Koco{'n}, Jan},
-  booktitle={2025 IEEE ICDMW},
+  booktitle={2025 IEEE ICувцDMW},
   year={2025},
   organization={IEEE}
 }
diff --git a/leaderboard/docs/_build/html/_static/leaderboard.json b/leaderboard/docs/_build/html/_static/leaderboard.json
new file mode 100644
index 0000000..c842948
--- /dev/null
+++ b/leaderboard/docs/_build/html/_static/leaderboard.json
@@ -0,0 +1,106 @@
+[
+  {
+    "model": "openai/gpt-oss-120b",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.9049,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "openai/gpt-oss-20b",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.8871,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "meta-llama/Llama-3.3-70B-Instruct",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.8607,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.8519,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "Qwen/Qwen2.5-7B-Instruct",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.794,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "mistralai/Mistral-Nemo-Instruct-2407",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.7599,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "Qwen/Qwen2.5-1.5B-Instruct",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.6401,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "meta-llama/Llama-3.2-3B-Instruct",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.5415,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "CYFRAGOVPL/PLLuM-12B-chat",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.5224,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "Qwen/Qwen3-0.6B",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.4983,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "CYFRAGOVPL/PLLuM-12B-nc-chat",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.4044,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "CYFRAGOVPL/pllum-12b-nc-chat-250715",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.3727,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "meta-llama/Llama-3.2-1B-Instruct",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.2678,
+    "date": "2026-02-24"
+  }
+]
\ No newline at end of file
diff --git a/leaderboard/docs/_static/leaderboard.json b/leaderboard/docs/_static/leaderboard.json
new file mode 100644
index 0000000..c842948
--- /dev/null
+++ b/leaderboard/docs/_static/leaderboard.json
@@ -0,0 +1,106 @@
+[
+  {
+    "model": "openai/gpt-oss-120b",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.9049,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "openai/gpt-oss-20b",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.8871,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "meta-llama/Llama-3.3-70B-Instruct",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.8607,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.8519,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "Qwen/Qwen2.5-7B-Instruct",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.794,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "mistralai/Mistral-Nemo-Instruct-2407",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.7599,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "Qwen/Qwen2.5-1.5B-Instruct",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.6401,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "meta-llama/Llama-3.2-3B-Instruct",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.5415,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "CYFRAGOVPL/PLLuM-12B-chat",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.5224,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "Qwen/Qwen3-0.6B",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.4983,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "CYFRAGOVPL/PLLuM-12B-nc-chat",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.4044,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "CYFRAGOVPL/pllum-12b-nc-chat-250715",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.3727,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "meta-llama/Llama-3.2-1B-Instruct",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.2678,
+    "date": "2026-02-24"
+  }
+]
\ No newline at end of file
diff --git a/leaderboard/generate_leaderboard.py b/leaderboard/generate_leaderboard.py
new file mode 100644
index 0000000..b795556
--- /dev/null
+++ b/leaderboard/generate_leaderboard.py
@@ -0,0 +1,34 @@
+import yaml
+import json
+from pathlib import Path
+import shutil
+
+BASE_DIR = Path(__file__).parent
+DOCS_DIR = Path(__file__).parent.parent / "docs/_static"
+BUILD_DIR = DOCS_DIR / "_build/html"
+
+rows = []
+
+for path in BASE_DIR.rglob("run.yaml"):
+    with open(path) as f:
+        data = yaml.safe_load(f)
+
+    rows.append({
+        "model": data["model"]["name"],
+        "type": data.get("type", ""),
+        "fewshots": data["inference"]["arguments"]["num_fewshots"],
+        "backend": data["inference"]["backend"],
+        "accuracy": data["results"]["execution_accuracy"],
+        "date": str(data["date"]),
+    })
+
+rows.sort(key=lambda x: x["accuracy"], reverse=True)
+
+json_file = DOCS_DIR / "leaderboard.json"
+with open(json_file, "w") as f:
+    json.dump(rows, f, indent=2)
+
+if BUILD_DIR.exists():
+    shutil.copy(json_file, BUILD_DIR / "leaderboard.json")
+
+print(f"✅ leaderboard.json in {json_file}")
\ No newline at end of file
diff --git a/leaderboard/leaderboard.json b/leaderboard/leaderboard.json
new file mode 100644
index 0000000..c842948
--- /dev/null
+++ b/leaderboard/leaderboard.json
@@ -0,0 +1,106 @@
+[
+  {
+    "model": "openai/gpt-oss-120b",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.9049,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "openai/gpt-oss-20b",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.8871,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "meta-llama/Llama-3.3-70B-Instruct",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.8607,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.8519,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "Qwen/Qwen2.5-7B-Instruct",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.794,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "mistralai/Mistral-Nemo-Instruct-2407",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.7599,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "Qwen/Qwen2.5-1.5B-Instruct",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.6401,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "meta-llama/Llama-3.2-3B-Instruct",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.5415,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "CYFRAGOVPL/PLLuM-12B-chat",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.5224,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "Qwen/Qwen3-0.6B",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.4983,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "CYFRAGOVPL/PLLuM-12B-nc-chat",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.4044,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "CYFRAGOVPL/pllum-12b-nc-chat-250715",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.3727,
+    "date": "2026-02-24"
+  },
+  {
+    "model": "meta-llama/Llama-3.2-1B-Instruct",
+    "type": "open-source",
+    "fewshots": 5,
+    "backend": "vllm",
+    "accuracy": 0.2678,
+    "date": "2026-02-24"
+  }
+]
\ No newline at end of file

From e1b7bf2e357d436294818d3fa2d0695b01769ec1 Mon Sep 17 00:00:00 2001
From: Dzmitry Pihulski 
Date: Fri, 27 Feb 2026 20:27:41 +0100
Subject: [PATCH 3/3] .idea deleted

---
 .gitignore                                     |  2 ++
 .idea/.gitignore                               |  8 --------
 .idea/inspectionProfiles/profiles_settings.xml |  6 ------
 .idea/llmsql-benchmark.iml                     | 15 ---------------
 .idea/misc.xml                                 |  7 -------
 .idea/modules.xml                              |  8 --------
 .idea/vcs.xml                                  |  6 ------
 7 files changed, 2 insertions(+), 50 deletions(-)
 delete mode 100644 .idea/.gitignore
 delete mode 100644 .idea/inspectionProfiles/profiles_settings.xml
 delete mode 100644 .idea/llmsql-benchmark.iml
 delete mode 100644 .idea/misc.xml
 delete mode 100644 .idea/modules.xml
 delete mode 100644 .idea/vcs.xml

diff --git a/.gitignore b/.gitignore
index a3c3066..1574e58 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,3 +13,5 @@ llmsql_workdir
 
 evaluation_*
 coverage.xml
+
+.idea
diff --git a/.idea/.gitignore b/.idea/.gitignore
deleted file mode 100644
index 13566b8..0000000
--- a/.idea/.gitignore
+++ /dev/null
@@ -1,8 +0,0 @@
-# Default ignored files
-/shelf/
-/workspace.xml
-# Editor-based HTTP Client requests
-/httpRequests/
-# Datasource local storage ignored files
-/dataSources/
-/dataSources.local.xml
diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml
deleted file mode 100644
index 105ce2d..0000000
--- a/.idea/inspectionProfiles/profiles_settings.xml
+++ /dev/null
@@ -1,6 +0,0 @@
-
-  
-    
-
\ No newline at end of file
diff --git a/.idea/llmsql-benchmark.iml b/.idea/llmsql-benchmark.iml
deleted file mode 100644
index d00f9a4..0000000
--- a/.idea/llmsql-benchmark.iml
+++ /dev/null
@@ -1,15 +0,0 @@
-
-
-  
-    
-    
-    
-  
-  
-    
-  
-    
-
\ No newline at end of file
diff --git a/.idea/misc.xml b/.idea/misc.xml
deleted file mode 100644
index 3ba169c..0000000
--- a/.idea/misc.xml
+++ /dev/null
@@ -1,7 +0,0 @@
-
-
-  
-    
-  
-
\ No newline at end of file
diff --git a/.idea/modules.xml b/.idea/modules.xml
deleted file mode 100644
index 6ad7ce7..0000000
--- a/.idea/modules.xml
+++ /dev/null
@@ -1,8 +0,0 @@
-
-
-  
-    
-      
-    
-  
-
\ No newline at end of file
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
deleted file mode 100644
index 35eb1dd..0000000
--- a/.idea/vcs.xml
+++ /dev/null
@@ -1,6 +0,0 @@
-
-
-  
-    
-  
-
\ No newline at end of file