Local models: a complete overview

# Download a model and ask it a question — all on your own server
ollama pull qwen3:8b
ollama run qwen3:8b "Summarise the delivery terms in two sentences"

# The same model over HTTP: an OpenAI-compatible API on port 11434
curl http://localhost:11434/v1/chat/completions \
  -H 'Content-Type: application/json' \
  -d '{"model": "qwen3:8b", "messages": [{"role": "user", "content": "Hello"}]}'

// One client for a cloud and a local model: only the address, the key and the model change
const LLM_URL = process.env.LLM_URL ?? 'http://localhost:11434/v1';
const LLM_MODEL = process.env.LLM_MODEL ?? 'qwen3:8b';

export async function complete(prompt: string): Promise<string> {
  const res = await fetch(`${LLM_URL}/chat/completions`, {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      Authorization: `Bearer ${process.env.LLM_KEY ?? 'local'}`,
    },
    body: JSON.stringify({
      model: LLM_MODEL,
      messages: [{ role: 'user', content: prompt }],
      temperature: 0.2, // fewer surprises in business answers
    }),
  });
  if (!res.ok) throw new Error(`LLM: HTTP ${res.status}`);
  const data = (await res.json()) as { choices: { message: { content: string } }[] };
  return data.choices[0].message.content;
}

# /etc/systemd/system/ollama.service.d/override.conf — Ollama on a production server
[Service]
# Listen only for applications on this server, not for the internet
Environment="OLLAMA_HOST=127.0.0.1:11434"
# Keep the model in memory between requests — no cold start
Environment="OLLAMA_KEEP_ALIVE=24h"
# How many requests one model serves at the same time
Environment="OLLAMA_NUM_PARALLEL=4"
Environment="OLLAMA_MAX_LOADED_MODELS=1"
# A ceiling: the model cannot take the memory of the whole server
MemoryMax=24G