integrate routellm

[sammcj]

https://github.com/lm-sys/RouteLLM

e.g.

pip install "routellm[serve,eval]"

import os

from routellm.controller import Controller

client = Controller(
  routers=["mf"],
  strong_model="llama3.1-8b-Q6_K",
  weak_model="llama3.1-70b-IQ3_XS"
)

response = client.chat.completions.create(
  # This tells RouteLLM to use the MF router with a cost threshold of 0.11593
  model="router-mf-0.11593",
  messages=[
    {"role": "user", "content": "Write the game snake in python"}
  ]
)

message_content = response['choices'][0]['message']['content']
model_name = response['model']

print(f"Message content: {message_content}")
print(f"Model name: {model_name}")