config.sample.json

{
    "id": "mac",
    "host": "localhost",
    "port": "8080",
    "log": "booster.log",
    "deadline": 180,
    "swap": "/home/sessions",
    "debug": "",

    "pods": {

        "cpu": {
            "model": "default",
            "prompt": "default",
            "sampling": "janus",
            "threads": 8,
            "gpus": [ 0 ],
            "batch": 512
        }
    },

    "models": {

        "default": {
            "name": "Hermes 8B",
            "path": "~/models/Hermes-2-Pro-Llama-3-8B-Q4_K_M.gguf",
            "context": "2K",
            "predict": "1K"
        }
    },

    "prompts": {

        "default": {
            "locale": "en_US",
            "prompt": "Today is {DATE}. You are a helpful virtual assistant.",
            "system": "<|im_start|>system\n{PROMPT}<|im_end|>",
            "user": "\n<|im_start|>user\n{USER}<|im_end|>",
            "assistant": "\n<|im_start|>assistant\n{ASSISTANT}<|im_end|>"
        }
    },

    "samplings": {

        "janus": {
            "janus": 1,
            "depth": 200,
            "scale": 0.97,
            "hi": 0.99,
            "lo": 0.96
        },

        "mirostat": {
            "mirostat": 0,
            "mirostatENT": 0,
            "mirostatLR": 0
        },

        "basic": {
            "temperature": 0.1,
            "top_k": 8,
            "top_p": 0.4,
            "typicalP": 1.0,
            "repetition_penalty": 1.1,
            "penaltyLastN": -1
        }
    }
}