config.json

{
    "__comment::log_level": "Log level for the app, see https://docs.python.org/3/library/logging.html#logging-levels",
    "__comment::idle_polling_interval": "The interval in seconds to check for new messages when the app has no tasks",
    "__comment::tokenizer_file": "The tokenizer file name inside the model directory (loader.model_path)",
    "__comment::loader": "CTranslate2 loader options, see https://opennmt.net/CTranslate2/python/ctranslate2.Translator.html#ctranslate2.Translator.__init__. Use 'model_path' key for local paths or 'model_name' key for models hosted on Hugging Face. Both can't be used at the same time.",
    "__comment::inference": "CTranslate2 inference options, see the kwargs in https://opennmt.net/CTranslate2/python/ctranslate2.Translator.html#ctranslate2.Translator.translate_batch",
    "__comment::changes_to_the_config": "the program needs to be restarted if you change this file since it is stored in memory on startup",
    "log_level": 20,
    "idle_polling_interval": 5,
    "idle_polling_interval_with_trigger": 300,
    "tokenizer_file": "spiece.model",
    "loader": {
        "model_name": "Nextcloud-AI/madlad400-3b-mt-ct2-int8_float32",
        "inter_threads": 4,
        "intra_threads": 0
    },
    "inference": {
        "max_batch_size": 8192,
        "sampling_temperature": 0.0001,
        "disable_unk": true,
        "max_decoding_length": 10000,
        "repetition_penalty": 1.3,
        "patience": 1.5
    }
}