mnemo_cards/tools/deploy/fix-nginx-oom-protection.sh

#!/bin/bash

# Script to protect nginx from OOM killer and enable auto-recovery
# This script configures systemd override for nginx service
# Usage: ./fix-nginx-oom-protection.sh

set -e

SERVER_IP="147.45.152.129"
SERVER_USER="root"

echo "🔧 Configuring nginx OOM protection and auto-recovery..."
echo "Server: $SERVER_IP"
echo ""

ssh "$SERVER_USER@$SERVER_IP" << 'ENDSSH'
    set -e
    
    echo "📋 Backing up current nginx service configuration..."
    if [ -f /etc/systemd/system/nginx.service ]; then
        cp /etc/systemd/system/nginx.service /etc/systemd/system/nginx.service.backup.$(date +%Y%m%d_%H%M%S) || true
    fi
    
    echo ""
    echo "📁 Creating systemd override directory..."
    mkdir -p /etc/systemd/system/nginx.service.d
    
    echo ""
    echo "📝 Creating override configuration..."
    cat > /etc/systemd/system/nginx.service.d/override.conf << 'OVERRIDE'
[Service]
# Protect nginx from OOM killer
# OOMScoreAdjust ranges from -1000 to 1000
# -1000 means "never kill this process" (highest protection)
# 1000 means "kill this process first" (lowest protection)
OOMScoreAdjust=-1000

# Automatic restart configuration
Restart=always
RestartSec=5
StartLimitIntervalSec=0
StartLimitBurst=0

# Additional resource limits (optional, uncomment if needed)
# MemoryLimit=512M
# CPUQuota=50%
OVERRIDE
    
    echo "✅ Override configuration created:"
    cat /etc/systemd/system/nginx.service.d/override.conf
    echo ""
    
    echo "🔄 Reloading systemd daemon..."
    systemctl daemon-reload
    
    echo ""
    echo "🧪 Testing nginx configuration..."
    nginx -t
    
    if [ $? -eq 0 ]; then
        echo "✅ Nginx configuration is valid"
    else
        echo "❌ Nginx configuration test failed!"
        exit 1
    fi
    
    echo ""
    echo "🚀 Restarting nginx to apply new settings..."
    systemctl restart nginx
    
    echo ""
    echo "⏳ Waiting for nginx to start..."
    sleep 3
    
    echo ""
    echo "🔍 Verifying nginx status..."
    if systemctl is-active --quiet nginx; then
        echo "✅ Nginx is running"
    else
        echo "❌ Nginx failed to start!"
        systemctl status nginx --no-pager || true
        exit 1
    fi
    
    echo ""
    echo "📊 Checking nginx service configuration..."
    systemctl show nginx | grep -E "(OOMScoreAdjust|Restart|RestartSec)" || true
    
    echo ""
    echo "✅ Nginx OOM protection and auto-recovery configured successfully!"
    echo ""
    echo "📋 Configuration summary:"
    echo "   - OOMScoreAdjust: -1000 (maximum protection from OOM killer)"
    echo "   - Restart: always (automatic restart on failure)"
    echo "   - RestartSec: 5 (5 seconds delay before restart)"
    echo "   - StartLimitIntervalSec: 0 (no restart limit)"
    echo ""
    echo "💡 To verify OOM score, run: cat /proc/\$(pgrep -f 'nginx: master')/oom_score_adj"
ENDSSH

echo ""
echo "✅ Nginx OOM protection and auto-recovery setup completed!"
echo ""
echo "📝 What was configured:"
echo "   1. OOMScoreAdjust=-1000 - nginx will be the last process killed by OOM killer"
echo "   2. Restart=always - nginx will automatically restart if it crashes"
echo "   3. RestartSec=5 - 5 second delay before restart"
echo "   4. StartLimitIntervalSec=0 - no limit on restart attempts"
echo ""
echo "🔍 To check if nginx is protected, SSH to server and run:"
echo "   cat /proc/\$(pgrep -f 'nginx: master')/oom_score_adj"
echo "   (should output: -1000)"