Build a Cost-Aware Kubernetes Autoscaler

// src/collector/usage-collector.ts
// Scrapes Kubernetes metrics and stores usage history

import { Redis } from 'ioredis';
import { Pool } from 'pg';

const redis = new Redis(process.env.REDIS_URL!);
const db = new Pool({ connectionString: process.env.DATABASE_URL });

interface PodMetrics {
  namespace: string;
  deployment: string;
  pod: string;
  containers: Array<{
    name: string;
    cpuUsageMillicores: number;
    cpuRequestMillicores: number;
    cpuLimitMillicores: number;
    memoryUsageMb: number;
    memoryRequestMb: number;
    memoryLimitMb: number;
  }>;
  timestamp: number;
}

export async function collectMetrics(): Promise<PodMetrics[]> {
  // Fetch from Kubernetes Metrics Server API
  const metricsResponse = await fetch(
    `${process.env.K8S_API}/apis/metrics.k8s.io/v1beta1/pods`,
    { headers: { Authorization: `Bearer ${process.env.K8S_TOKEN}` } }
  );
  const metrics = await metricsResponse.json() as any;

  // Fetch pod specs for requests/limits
  const podsResponse = await fetch(
    `${process.env.K8S_API}/api/v1/pods`,
    { headers: { Authorization: `Bearer ${process.env.K8S_TOKEN}` } }
  );
  const pods = await podsResponse.json() as any;

  const podSpecs = new Map<string, any>();
  for (const pod of pods.items) {
    podSpecs.set(pod.metadata.name, pod.spec);
  }

  const results: PodMetrics[] = [];

  for (const item of metrics.items) {
    const spec = podSpecs.get(item.metadata.name);
    if (!spec) continue;

    const deployment = item.metadata.labels?.['app'] ??
      item.metadata.ownerReferences?.[0]?.name ?? 'unknown';

    results.push({
      namespace: item.metadata.namespace,
      deployment,
      pod: item.metadata.name,
      containers: item.containers.map((c: any, i: number) => {
        const specContainer = spec.containers[i];
        return {
          name: c.name,
          cpuUsageMillicores: parseCpu(c.usage.cpu),
          cpuRequestMillicores: parseCpu(specContainer?.resources?.requests?.cpu ?? '0'),
          cpuLimitMillicores: parseCpu(specContainer?.resources?.limits?.cpu ?? '0'),
          memoryUsageMb: parseMemory(c.usage.memory),
          memoryRequestMb: parseMemory(specContainer?.resources?.requests?.memory ?? '0'),
          memoryLimitMb: parseMemory(specContainer?.resources?.limits?.memory ?? '0'),
        };
      }),
      timestamp: Date.now(),
    });
  }

  // Store in Redis for real-time access + PostgreSQL for history
  for (const pod of results) {
    const key = `metrics:${pod.namespace}:${pod.deployment}`;
    await redis.lpush(key, JSON.stringify(pod));
    await redis.ltrim(key, 0, 1439);  // 24h at 1-minute intervals
    await redis.expire(key, 172800);
  }

  // Batch insert to PostgreSQL for historical analysis
  if (results.length > 0) {
    const values = results.map(p =>
      `('${p.namespace}', '${p.deployment}', ${JSON.stringify(p.containers).replace(/'/g, "''")}, ${p.timestamp})`
    ).join(',');

    await db.query(`
      INSERT INTO pod_metrics (namespace, deployment, containers, collected_at)
      VALUES ${values}
    `);
  }

  return results;
}

function parseCpu(value: string): number {
  if (value.endsWith('n')) return parseInt(value) / 1_000_000;
  if (value.endsWith('m')) return parseInt(value);
  return parseFloat(value) * 1000;
}

function parseMemory(value: string): number {
  if (value.endsWith('Ki')) return parseInt(value) / 1024;
  if (value.endsWith('Mi')) return parseInt(value);
  if (value.endsWith('Gi')) return parseInt(value) * 1024;
  return parseInt(value) / (1024 * 1024);
}

// src/analyzer/right-sizer.ts
// Analyzes usage patterns and recommends optimal resource requests

import { Pool } from 'pg';

const db = new Pool({ connectionString: process.env.DATABASE_URL });

interface RightSizingRecommendation {
  namespace: string;
  deployment: string;
  currentCpuRequest: number;      // millicores
  recommendedCpuRequest: number;
  currentMemoryRequest: number;   // MB
  recommendedMemoryRequest: number;
  potentialSavingsPercent: number;
  monthlySavingsUsd: number;
  confidence: 'high' | 'medium' | 'low';
  dataPointsAnalyzed: number;
}

export async function analyzeRightSizing(
  namespace: string,
  deployment: string,
  daysToAnalyze: number = 7
): Promise<RightSizingRecommendation> {
  // Get historical usage data
  const result = await db.query(`
    SELECT containers
    FROM pod_metrics
    WHERE namespace = $1 AND deployment = $2
      AND collected_at > NOW() - INTERVAL '${daysToAnalyze} days'
    ORDER BY collected_at
  `, [namespace, deployment]);

  if (result.rows.length < 100) {
    throw new Error('Insufficient data for recommendation');
  }

  // Extract CPU and memory usage across all samples
  const cpuSamples: number[] = [];
  const memorySamples: number[] = [];
  let currentCpuRequest = 0;
  let currentMemoryRequest = 0;

  for (const row of result.rows) {
    const containers = row.containers as any[];
    for (const c of containers) {
      cpuSamples.push(c.cpuUsageMillicores);
      memorySamples.push(c.memoryUsageMb);
      currentCpuRequest = c.cpuRequestMillicores;
      currentMemoryRequest = c.memoryRequestMb;
    }
  }

  // Calculate p95 usage (we want headroom, not average)
  cpuSamples.sort((a, b) => a - b);
  memorySamples.sort((a, b) => a - b);

  const cpuP95 = cpuSamples[Math.floor(cpuSamples.length * 0.95)];
  const memP95 = memorySamples[Math.floor(memorySamples.length * 0.95)];

  // Add 20% buffer above p95 for safety
  const recommendedCpu = Math.ceil(cpuP95 * 1.2);
  const recommendedMemory = Math.ceil(memP95 * 1.2);

  // Don't recommend below minimum thresholds
  const finalCpu = Math.max(50, recommendedCpu);    // at least 50m
  const finalMemory = Math.max(64, recommendedMemory); // at least 64MB

  // Calculate savings
  const cpuSavingsPercent = currentCpuRequest > 0
    ? ((currentCpuRequest - finalCpu) / currentCpuRequest) * 100
    : 0;
  const memorySavingsPercent = currentMemoryRequest > 0
    ? ((currentMemoryRequest - finalMemory) / currentMemoryRequest) * 100
    : 0;

  // Rough cost estimate: $0.048/vCPU/hour, $0.006/GB/hour (on-demand)
  const cpuSavingsMonthly = Math.max(0, (currentCpuRequest - finalCpu) / 1000) * 0.048 * 730;
  const memSavingsMonthly = Math.max(0, (currentMemoryRequest - finalMemory) / 1024) * 0.006 * 730;

  return {
    namespace,
    deployment,
    currentCpuRequest,
    recommendedCpuRequest: finalCpu,
    currentMemoryRequest,
    recommendedMemoryRequest: finalMemory,
    potentialSavingsPercent: Math.max(cpuSavingsPercent, memorySavingsPercent),
    monthlySavingsUsd: cpuSavingsMonthly + memSavingsMonthly,
    confidence: result.rows.length > 1000 ? 'high' : result.rows.length > 500 ? 'medium' : 'low',
    dataPointsAnalyzed: result.rows.length,
  };
}

// src/scaler/predictive.ts
// Pre-scales based on historical traffic patterns

import { Redis } from 'ioredis';

const redis = new Redis(process.env.REDIS_URL!);

interface ScaleDecision {
  deployment: string;
  namespace: string;
  currentReplicas: number;
  targetReplicas: number;
  reason: string;
  estimatedCostChange: number;  // $/hour
}

export async function predictAndScale(
  namespace: string,
  deployment: string,
  currentReplicas: number,
  currentCpuPercent: number,
  config: {
    minReplicas: number;
    maxReplicas: number;
    targetCpuPercent: number;
    costPerReplicaHour: number;
    budgetMaxReplicaHour: number;
  }
): Promise<ScaleDecision> {
  // 1. Check scheduled events (demos, deployments, marketing campaigns)
  const scheduled = await getScheduledScaleEvents(deployment);
  if (scheduled) {
    return {
      deployment, namespace, currentReplicas,
      targetReplicas: Math.min(scheduled.replicas, config.maxReplicas),
      reason: `Scheduled: ${scheduled.reason}`,
      estimatedCostChange: (scheduled.replicas - currentReplicas) * config.costPerReplicaHour,
    };
  }

  // 2. Check historical pattern (same hour, same day of week)
  const historicalTarget = await getHistoricalReplicas(deployment);

  // 3. Reactive component (current CPU)
  const reactiveTarget = Math.ceil(
    currentReplicas * (currentCpuPercent / config.targetCpuPercent)
  );

  // 4. Take the max of historical and reactive (conservative)
  let targetReplicas = Math.max(historicalTarget, reactiveTarget);

  // 5. Apply bounds
  targetReplicas = Math.max(config.minReplicas, Math.min(config.maxReplicas, targetReplicas));

  // 6. Budget guardrail
  const hourlyCost = targetReplicas * config.costPerReplicaHour;
  if (hourlyCost > config.budgetMaxReplicaHour) {
    const budgetCappedReplicas = Math.floor(config.budgetMaxReplicaHour / config.costPerReplicaHour);
    targetReplicas = Math.max(config.minReplicas, budgetCappedReplicas);
  }

  // 7. Scale-down dampening: don't scale down more than 1 pod per cycle
  if (targetReplicas < currentReplicas) {
    targetReplicas = Math.max(targetReplicas, currentReplicas - 1);
  }

  const reason = targetReplicas > currentReplicas
    ? `Scale up: CPU at ${currentCpuPercent}% (target ${config.targetCpuPercent}%), historical suggests ${historicalTarget}`
    : targetReplicas < currentReplicas
    ? `Scale down: CPU at ${currentCpuPercent}%, safe to reduce`
    : 'No change needed';

  return {
    deployment, namespace, currentReplicas, targetReplicas, reason,
    estimatedCostChange: (targetReplicas - currentReplicas) * config.costPerReplicaHour,
  };
}

async function getScheduledScaleEvents(deployment: string): Promise<{
  replicas: number; reason: string;
} | null> {
  const event = await redis.get(`scale:scheduled:${deployment}`);
  if (!event) return null;
  const parsed = JSON.parse(event);

  // Check if event is within 15-minute window
  const now = Date.now();
  if (parsed.startAt <= now && parsed.endAt >= now) {
    return { replicas: parsed.replicas, reason: parsed.reason };
  }
  return null;
}

async function getHistoricalReplicas(deployment: string): Promise<number> {
  const now = new Date();
  const hour = now.getHours();
  const dayOfWeek = now.getDay();

  // Look up what replica count was needed at this time last week
  const key = `scale:history:${deployment}:${dayOfWeek}:${hour}`;
  const historical = await redis.get(key);

  return historical ? parseInt(historical) : 1;
}

// src/costs/attribution.ts
// Calculates per-team, per-service cloud cost

import { Pool } from 'pg';

const db = new Pool({ connectionString: process.env.DATABASE_URL });

// Pricing: on-demand rates per hour
const PRICING = {
  cpuPerCoreHour: 0.048,     // $/vCPU/hour
  memoryPerGbHour: 0.006,    // $/GB/hour
  storagePerGbMonth: 0.10,   // $/GB/month for PVCs
};

interface CostBreakdown {
  team: string;
  namespace: string;
  deployment: string;
  dailyCpuCost: number;
  dailyMemoryCost: number;
  dailyTotalCost: number;
  monthlyProjection: number;
  utilizationPercent: number;
  wastedSpendPercent: number;
}

export async function calculateCostAttribution(
  date: string  // YYYY-MM-DD
): Promise<CostBreakdown[]> {
  // Get average resource usage and requests per deployment
  const result = await db.query(`
    SELECT
      namespace,
      deployment,
      AVG((c->>'cpuUsageMillicores')::float) as avg_cpu_usage,
      AVG((c->>'cpuRequestMillicores')::float) as avg_cpu_request,
      AVG((c->>'memoryUsageMb')::float) as avg_mem_usage,
      AVG((c->>'memoryRequestMb')::float) as avg_mem_request,
      COUNT(DISTINCT pod) as replica_count
    FROM pod_metrics,
      jsonb_array_elements(containers::jsonb) as c
    WHERE collected_at::date = $1::date
    GROUP BY namespace, deployment
  `, [date]);

  // Map namespaces to teams
  const teamMap: Record<string, string> = {
    'api': 'Backend',
    'frontend': 'Frontend',
    'ml': 'Data Science',
    'payments': 'Payments',
    'default': 'Platform',
  };

  return result.rows.map(row => {
    const cpuRequestCores = row.avg_cpu_request / 1000;
    const memRequestGb = row.avg_mem_request / 1024;

    const dailyCpuCost = cpuRequestCores * PRICING.cpuPerCoreHour * 24;
    const dailyMemoryCost = memRequestGb * PRICING.memoryPerGbHour * 24;
    const dailyTotalCost = (dailyCpuCost + dailyMemoryCost) * row.replica_count;

    const cpuUtil = row.avg_cpu_request > 0 ? (row.avg_cpu_usage / row.avg_cpu_request) * 100 : 0;
    const memUtil = row.avg_mem_request > 0 ? (row.avg_mem_usage / row.avg_mem_request) * 100 : 0;
    const avgUtil = (cpuUtil + memUtil) / 2;

    return {
      team: teamMap[row.namespace] ?? row.namespace,
      namespace: row.namespace,
      deployment: row.deployment,
      dailyCpuCost: Math.round(dailyCpuCost * 100) / 100,
      dailyMemoryCost: Math.round(dailyMemoryCost * 100) / 100,
      dailyTotalCost: Math.round(dailyTotalCost * 100) / 100,
      monthlyProjection: Math.round(dailyTotalCost * 30 * 100) / 100,
      utilizationPercent: Math.round(avgUtil),
      wastedSpendPercent: Math.round(Math.max(0, 100 - avgUtil)),
    };
  });
}

// src/scaler/executor.ts
// Applies scaling decisions with graceful pod draining

export async function applyScaleDecision(decision: {
  namespace: string;
  deployment: string;
  currentReplicas: number;
  targetReplicas: number;
}): Promise<{ success: boolean; message: string }> {
  if (decision.targetReplicas === decision.currentReplicas) {
    return { success: true, message: 'No scaling needed' };
  }

  // Scale via Kubernetes API
  try {
    const response = await fetch(
      `${process.env.K8S_API}/apis/apps/v1/namespaces/${decision.namespace}/deployments/${decision.deployment}/scale`,
      {
        method: 'PATCH',
        headers: {
          'Authorization': `Bearer ${process.env.K8S_TOKEN}`,
          'Content-Type': 'application/strategic-merge-patch+json',
        },
        body: JSON.stringify({ spec: { replicas: decision.targetReplicas } }),
      }
    );

    if (!response.ok) {
      const error = await response.text();
      return { success: false, message: `K8s API error: ${error}` };
    }

    // Record for historical pattern learning
    const { Redis } = await import('ioredis');
    const redis = new Redis(process.env.REDIS_URL!);
    const now = new Date();
    const key = `scale:history:${decision.deployment}:${now.getDay()}:${now.getHours()}`;
    await redis.setex(key, 604800, String(decision.targetReplicas));  // 7-day TTL
    redis.disconnect();

    return {
      success: true,
      message: `Scaled ${decision.deployment}: ${decision.currentReplicas} → ${decision.targetReplicas}`,
    };
  } catch (err: any) {
    return { success: false, message: err.message };
  }
}