localai

# Run LocalAI with Docker (CPU-only, no GPU needed)
docker run -p 8080:8080 \
  -v ./models:/build/models \
  localai/localai:latest-cpu

# With GPU support (NVIDIA CUDA)
docker run -p 8080:8080 --gpus all \
  -v ./models:/build/models \
  localai/localai:latest-gpu-nvidia-cuda-12

# Docker Compose for production

# docker-compose.yml — Production LocalAI setup
version: "3.8"
services:
  localai:
    image: localai/localai:latest-cpu
    ports:
      - "8080:8080"
    volumes:
      - ./models:/build/models
    environment:
      - THREADS=4                    # CPU threads for inference
      - CONTEXT_SIZE=4096            # Default context window
      - GALLERIES=[{"name":"model-gallery","url":"github:mudler/LocalAI/gallery/index.yaml@master"}]
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080/readyz"]
      interval: 30s
      timeout: 10s

# Install models from the gallery (via API)
curl -X POST http://localhost:8080/models/apply \
  -H "Content-Type: application/json" \
  -d '{"id": "huggingface://TheBloke/Mistral-7B-Instruct-v0.2-GGUF/mistral-7b-instruct-v0.2.Q5_K_M.gguf"}'

# Or download GGUF files directly into the models directory
wget -P ./models/ \
  https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q5_K_M.gguf

# Create a model configuration
cat > ./models/mistral.yaml << 'EOF'
name: mistral
backend: llama-cpp
parameters:
  model: mistral-7b-instruct-v0.2.Q5_K_M.gguf
  temperature: 0.7
  top_p: 0.9
  top_k: 40
  context_size: 8192
template:
  chat_message: |
    {{.RoleName}}: {{.Content}}
  chat: |
    [INST] {{.Input}} [/INST]
EOF

# List available models
curl http://localhost:8080/v1/models | jq '.data[].id'

// src/local-ai.ts — Use LocalAI with OpenAI SDK
import OpenAI from "openai";

const ai = new OpenAI({
  apiKey: "not-needed",
  baseURL: "http://localhost:8080/v1",
});

// Chat completions
async function chat(prompt: string) {
  const response = await ai.chat.completions.create({
    model: "mistral",                      // Model name from config
    messages: [
      { role: "system", content: "You are a helpful assistant." },
      { role: "user", content: prompt },
    ],
    temperature: 0.7,
  });
  return response.choices[0].message.content;
}

// Embeddings
async function embed(texts: string[]) {
  const response = await ai.embeddings.create({
    model: "text-embedding-ada-002",       // Mapped to local embedding model
    input: texts,
  });
  return response.data.map(d => d.embedding);
}

// Image generation (Stable Diffusion backend)
async function generateImage(prompt: string) {
  const response = await ai.images.generate({
    model: "stablediffusion",
    prompt,
    n: 1,
    size: "512x512",
  });
  return response.data[0].url;
}

// Audio transcription (Whisper backend)
async function transcribe(audioPath: string) {
  const response = await ai.audio.transcriptions.create({
    model: "whisper-1",
    file: fs.createReadStream(audioPath),
  });
  return response.text;
}

// Text-to-speech
async function textToSpeech(text: string) {
  const response = await ai.audio.speech.create({
    model: "tts-1",
    voice: "alloy",
    input: text,
  });
  const buffer = Buffer.from(await response.arrayBuffer());
  fs.writeFileSync("output.mp3", buffer);
}

# models/chat-model.yaml — Chat model
name: chat
backend: llama-cpp
parameters:
  model: llama-3.1-8b-instruct.Q5_K_M.gguf
  context_size: 8192
  threads: 4
  gpu_layers: 0                            # 0 = CPU only, increase for GPU offloading

---
# models/code-model.yaml — Code completion model
name: code
backend: llama-cpp
parameters:
  model: codellama-7b-instruct.Q5_K_M.gguf
  context_size: 16384
  threads: 4

---
# models/embedding-model.yaml — Embedding model
name: embedding
backend: sentencetransformers
parameters:
  model: all-MiniLM-L6-v2

---
# models/whisper-model.yaml — Audio transcription
name: whisper-1
backend: whisper
parameters:
  model: whisper-base.bin
  language: en

// LocalAI supports function calling with compatible models
async function chatWithFunctions(prompt: string) {
  const response = await ai.chat.completions.create({
    model: "mistral",
    messages: [{ role: "user", content: prompt }],
    tools: [
      {
        type: "function",
        function: {
          name: "get_current_weather",
          description: "Get the weather for a location",
          parameters: {
            type: "object",
            properties: {
              location: { type: "string" },
              unit: { type: "string", enum: ["celsius", "fahrenheit"] },
            },
            required: ["location"],
          },
        },
      },
    ],
    tool_choice: "auto",
  });
  return response;
}

# Docker (recommended)
docker pull localai/localai:latest-cpu

# Binary (Linux/macOS)
curl -Lo local-ai https://github.com/mudler/LocalAI/releases/latest/download/local-ai-$(uname -s)-$(uname -m)
chmod +x local-ai
./local-ai --models-path ./models

# Homebrew (macOS)
brew install localai

Add Localai to my Next.js app for the AI chat feature. I want streaming responses.

My Localai calls are slow and expensive. Help me optimize the setup.