llamafile

# Download a pre-built llamafile (model + runtime in one file)
wget https://huggingface.co/Mozilla/Meta-Llama-3.1-8B-Instruct-llamafile/resolve/main/Meta-Llama-3.1-8B-Instruct.Q5_K_M.llamafile

# Make it executable and run
chmod +x Meta-Llama-3.1-8B-Instruct.Q5_K_M.llamafile
./Meta-Llama-3.1-8B-Instruct.Q5_K_M.llamafile

# Opens web UI at http://localhost:8080
# Also serves OpenAI-compatible API at http://localhost:8080/v1

# Run with specific settings
./Meta-Llama-3.1-8B-Instruct.Q5_K_M.llamafile \
  --host 0.0.0.0 \
  --port 8080 \
  --ctx-size 8192 \
  --threads 8 \
  --gpu auto                  # Use GPU if available (CUDA, Metal, ROCm)

# Build a llamafile from any GGUF model

# 1. Download the llamafile runtime
wget https://github.com/Mozilla-Ocho/llamafile/releases/latest/download/llamafile
chmod +x llamafile

# 2. Download a GGUF model from HuggingFace
wget https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/resolve/main/mistral-7b-instruct-v0.2.Q5_K_M.gguf

# 3. Create the llamafile (packages model + runtime)
./llamafile \
  --model mistral-7b-instruct-v0.2.Q5_K_M.gguf \
  --create my-assistant.llamafile

# 4. The resulting file runs anywhere — no dependencies
./my-assistant.llamafile --port 8080

# With a custom system prompt baked in
./llamafile \
  --model mistral-7b-instruct-v0.2.Q5_K_M.gguf \
  --system-prompt "You are a customer support assistant for Acme Inc. Be helpful and concise." \
  --create acme-support-bot.llamafile

// src/local-llm.ts — Use llamafile with any OpenAI-compatible SDK
import OpenAI from "openai";

// Point to the local llamafile server
const llm = new OpenAI({
  apiKey: "not-needed",                    // llamafile doesn't require auth
  baseURL: "http://localhost:8080/v1",
});

async function chat(prompt: string) {
  const response = await llm.chat.completions.create({
    model: "local",                        // Model name doesn't matter for llamafile
    messages: [
      { role: "system", content: "You are a helpful assistant." },
      { role: "user", content: prompt },
    ],
    temperature: 0.7,
    max_tokens: 512,
  });
  return response.choices[0].message.content;
}

// Streaming works too
async function streamChat(prompt: string) {
  const stream = await llm.chat.completions.create({
    model: "local",
    messages: [{ role: "user", content: prompt }],
    stream: true,
  });
  for await (const chunk of stream) {
    process.stdout.write(chunk.choices[0]?.delta?.content ?? "");
  }
}

// Embeddings
async function embed(text: string) {
  const response = await llm.embeddings.create({
    model: "local",
    input: text,
  });
  return response.data[0].embedding;
}

# One-shot completion (no server, just stdin → stdout)
echo "Explain Docker in one sentence:" | ./llamafile --cli

# With a system prompt
./llamafile --cli \
  --system "You are a Linux expert. Answer in one line." \
  --prompt "How do I find files larger than 100MB?"

# Process a file
cat error_log.txt | ./llamafile --cli \
  --system "Analyze this error log. Identify the root cause."

# Batch processing with a script
for file in reports/*.txt; do
  echo "=== $file ==="
  cat "$file" | ./llamafile --cli \
    --system "Summarize this report in 3 bullet points." \
    --temp 0
done

# Run llamafile as a subprocess from your application
import subprocess
import requests

class LlamafileServer:
    """Manage a llamafile server as a subprocess."""

    def __init__(self, model_path: str, port: int = 8080):
        self.model_path = model_path
        self.port = port
        self.process = None
        self.base_url = f"http://localhost:{port}"

    def start(self):
        self.process = subprocess.Popen(
            [self.model_path, "--port", str(self.port), "--host", "127.0.0.1"],
            stdout=subprocess.DEVNULL,
            stderr=subprocess.DEVNULL,
        )
        # Wait for server to be ready
        import time
        for _ in range(30):
            try:
                requests.get(f"{self.base_url}/health", timeout=1)
                return
            except requests.ConnectionError:
                time.sleep(1)
        raise RuntimeError("llamafile failed to start")

    def chat(self, prompt: str, system: str = "") -> str:
        response = requests.post(
            f"{self.base_url}/v1/chat/completions",
            json={
                "messages": [
                    {"role": "system", "content": system},
                    {"role": "user", "content": prompt},
                ],
                "temperature": 0.7,
            },
        )
        return response.json()["choices"][0]["message"]["content"]

    def stop(self):
        if self.process:
            self.process.terminate()
            self.process.wait()

# Usage
server = LlamafileServer("./my-model.llamafile", port=8081)
server.start()
answer = server.chat("What is the capital of France?")
server.stop()

# No installation needed — llamafile IS the installation
# Download and run. That's it.

# For creating llamafiles:
wget https://github.com/Mozilla-Ocho/llamafile/releases/latest/download/llamafile
chmod +x llamafile

Add Llamafile to my Next.js app for the AI chat feature. I want streaming responses.

My Llamafile calls are slow and expensive. Help me optimize the setup.