Run a Local LLM Stack for Private AI Applications

npx terminal-skills install localai llamafile

# docker-compose.yml — Production LocalAI with multiple models
version: "3.8"
services:
  localai:
    image: localai/localai:latest-gpu-nvidia-cuda-12
    ports:
      - "8080:8080"
    volumes:
      - ./models:/build/models
      - ./config:/build/config
    environment:
      - THREADS=16                         # Match server's physical CPU cores
      - CONTEXT_SIZE=8192                  # Default context window
      - DEBUG=false
      - CORS=true
      - CORS_ALLOW_ORIGINS=*
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: 1
              capabilities: [gpu]
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080/readyz"]
      interval: 30s
      timeout: 10s
      retries: 3

# config/chat.yaml — Chat model (Llama 3.1 8B for general use)
name: llama-chat
backend: llama-cpp
parameters:
  model: Meta-Llama-3.1-8B-Instruct.Q5_K_M.gguf
  context_size: 8192
  threads: 8
  gpu_layers: 33                           # Offload all layers to GPU
  temperature: 0.7
  top_p: 0.9
  repeat_penalty: 1.1
template:
  chat_message: |
    <|start_header_id|>{{.RoleName}}<|end_header_id|>

    {{.Content}}<|eot_id|>
  chat: |
    <|begin_of_text|><|start_header_id|>system<|end_header_id|>

    {{.Input}}<|eot_id|>

---
# config/embedding.yaml — Embedding model for RAG and search
name: embedding
backend: sentencetransformers
parameters:
  model: all-MiniLM-L6-v2

---
# config/whisper.yaml — Audio transcription
name: whisper-1
backend: whisper
parameters:
  model: ggml-base.en.bin
  language: en

# Create the models directory
mkdir -p models

# Download Llama 3.1 8B (quantized — 5.3 GB, runs on 8GB GPU)
wget -P models/ \
  "https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF/resolve/main/Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf"

# Download embedding model (auto-downloaded by sentencetransformers)
# Just reference it in the config — LocalAI handles the rest

# Download Whisper base model
wget -P models/ \
  "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin"

# Start the server
docker compose up -d

# Verify all models are loaded
curl http://localhost:8080/v1/models | python3 -m json.tool

// src/lib/ai.ts — AI client that works with both OpenAI and LocalAI
import OpenAI from "openai";

// Switch between providers with an environment variable
const ai = new OpenAI({
  apiKey: process.env.AI_API_KEY || "not-needed",
  baseURL: process.env.AI_BASE_URL || "http://localhost:8080/v1",
});

// Chat — identical API whether it's OpenAI or LocalAI
export async function summarizeTicket(ticketContent: string): Promise<string> {
  const response = await ai.chat.completions.create({
    model: process.env.AI_CHAT_MODEL || "llama-chat",
    messages: [
      {
        role: "system",
        content: "You are a support ticket analyst. Summarize the customer's issue in 2-3 sentences, identify the severity (low/medium/high), and suggest the best team to handle it.",
      },
      { role: "user", content: ticketContent },
    ],
    temperature: 0.3,
    max_tokens: 300,
  });
  return response.choices[0].message.content ?? "";
}

// Embeddings — for semantic search over internal documents
export async function embedDocuments(texts: string[]): Promise<number[][]> {
  const response = await ai.embeddings.create({
    model: process.env.AI_EMBED_MODEL || "embedding",
    input: texts,
  });
  return response.data.map((d) => d.embedding);
}

// Transcription — for converting voice messages and call recordings
export async function transcribeAudio(audioBuffer: Buffer): Promise<string> {
  const file = new File([audioBuffer], "audio.wav", { type: "audio/wav" });
  const response = await ai.audio.transcriptions.create({
    model: "whisper-1",
    file,
  });
  return response.text;
}

# Create a company-standard llamafile for code review
# This is a single executable that any developer can download and double-click

# Download the llamafile runtime
wget https://github.com/Mozilla-Ocho/llamafile/releases/latest/download/llamafile
chmod +x llamafile

# Create the code review assistant llamafile
./llamafile \
  --model models/Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf \
  --system-prompt "You are a senior code reviewer at a fintech company. Review code for:
1. Security vulnerabilities (SQL injection, XSS, auth bypass)
2. Financial calculation accuracy (rounding, overflow, currency handling)
3. Error handling and edge cases
4. Compliance with SOC2 and PCI-DSS requirements
Be specific. Reference line numbers. Suggest fixes." \
  --create fintech-code-reviewer.llamafile

# Distribute to the team via internal file share
# Each developer runs it locally — no server, no Docker, no internet

# Quick code review from the terminal
cat pull_request.diff | ./fintech-code-reviewer.llamafile --cli

# Analyze an error log
./fintech-code-reviewer.llamafile --cli \
  --system "Analyze this error log. Identify the root cause and suggest a fix." \
  < production_error.log

# Interactive mode — opens web UI at localhost:8080
./fintech-code-reviewer.llamafile

// src/services/knowledge-search.ts — RAG with local embeddings
import { ai, embedDocuments } from "../lib/ai";

interface SearchResult {
  content: string;
  source: string;
  score: number;
}

async function searchAndAnswer(
  question: string,
  vectorStore: VectorStore
): Promise<{ answer: string; sources: SearchResult[] }> {
  // 1. Embed the question (runs on LocalAI)
  const [queryEmbedding] = await embedDocuments([question]);

  // 2. Find relevant documents
  const results = await vectorStore.search(queryEmbedding, { topK: 5 });

  // 3. Generate answer with context (runs on LocalAI)
  const context = results
    .map((r) => `[Source: ${r.source}]\n${r.content}`)
    .join("\n\n---\n\n");

  const response = await ai.chat.completions.create({
    model: "llama-chat",
    messages: [
      {
        role: "system",
        content: `Answer the question based ONLY on the provided context. If the context doesn't contain the answer, say "I don't have information about that in our documents." Always cite which source document you're referencing.`,
      },
      {
        role: "user",
        content: `Context:\n${context}\n\nQuestion: ${question}`,
      },
    ],
    temperature: 0.2,
    max_tokens: 500,
  });

  return {
    answer: response.choices[0].message.content ?? "",
    sources: results.map((r) => ({
      content: r.content.slice(0, 200) + "...",
      source: r.source,
      score: r.score,
    })),
  };
}