From d6a63562542bba1f1bb3602993b5f0b8180d61cf Mon Sep 17 00:00:00 2001
From: Dmitry <cinnabarflower@gmail.com>
Date: Sat, 3 Jan 2026 16:48:08 +0300
Subject: [PATCH] tools

---
 tools/deploy/NGINX_OOM_PROTECTION.md     | 161 +++++++++++++++++++++++
 tools/deploy/check-and-restore-nginx.sh  | 130 ++++++++++++++++++
 tools/deploy/fix-nginx-oom-protection.sh | 113 ++++++++++++++++
 tools/deploy/web-app/deploy.sh           |  38 ++++++
 4 files changed, 442 insertions(+)
 create mode 100644 tools/deploy/NGINX_OOM_PROTECTION.md
 create mode 100755 tools/deploy/check-and-restore-nginx.sh
 create mode 100755 tools/deploy/fix-nginx-oom-protection.sh

diff --git a/tools/deploy/NGINX_OOM_PROTECTION.md b/tools/deploy/NGINX_OOM_PROTECTION.md
new file mode 100644
index 0000000..e6e96c1
--- /dev/null
+++ b/tools/deploy/NGINX_OOM_PROTECTION.md
@@ -0,0 +1,161 @@
+# Защита nginx от OOM killer и автоматическое восстановление
+
+## Проблема
+
+При сборке Flutter web приложения на сервере процесс сборки может потреблять много памяти, что приводит к падению nginx из-за OOM (Out of Memory) killer.
+
+## Решение
+
+Реализованы два механизма защиты:
+
+1. **Защита от OOM killer** - настройка systemd для защиты nginx от убийства при нехватке памяти
+2. **Автоматическое восстановление** - настройка systemd для автоматического перезапуска nginx при падении
+
+## Использование
+
+### 1. Настройка защиты nginx от OOM killer
+
+Запустите скрипт для настройки systemd override:
+
+```bash
+./tools/deploy/fix-nginx-oom-protection.sh
+```
+
+Этот скрипт:
+- Создает systemd override для nginx с `OOMScoreAdjust=-1000` (максимальная защита)
+- Настраивает автоматический перезапуск при падении (`Restart=always`)
+- Настраивает задержку перед перезапуском (`RestartSec=5`)
+- Отключает лимит на количество перезапусков
+
+### 2. Проверка и восстановление nginx
+
+Для проверки статуса nginx и автоматического восстановления (если нужно):
+
+```bash
+./tools/deploy/check-and-restore-nginx.sh
+```
+
+Этот скрипт:
+- Проверяет, запущен ли nginx
+- Проверяет, отвечает ли nginx на запросы
+- Автоматически восстанавливает nginx, если он упал
+- Показывает информацию о памяти и OOM событиях
+
+### 3. Автоматическая проверка после деплоя
+
+Скрипт деплоя (`tools/deploy/web-app/deploy.sh`) автоматически проверяет и восстанавливает nginx после перезапуска.
+
+## Что было настроено
+
+### Systemd Override для nginx
+
+Создается файл `/etc/systemd/system/nginx.service.d/override.conf`:
+
+```ini
+[Service]
+# Защита от OOM killer (от -1000 до 1000)
+# -1000 = никогда не убивать этот процесс
+OOMScoreAdjust=-1000
+
+# Автоматический перезапуск
+Restart=always
+RestartSec=5
+StartLimitIntervalSec=0
+StartLimitBurst=0
+```
+
+### Проверка защиты
+
+Чтобы проверить, что nginx защищен от OOM killer:
+
+```bash
+# SSH на сервер
+ssh root@147.45.152.129
+
+# Проверить OOM score
+cat /proc/$(pgrep -f 'nginx: master')/oom_score_adj
+# Должно вывести: -1000
+```
+
+### Мониторинг
+
+Для проверки OOM событий в системных логах:
+
+```bash
+# Проверить последние OOM убийства
+dmesg | grep -i "oom\|killed process" | tail -20
+
+# Проверить логи nginx
+journalctl -u nginx --since "1hour ago" -n 50
+```
+
+## Автоматический мониторинг (опционально)
+
+Можно настроить cron для периодической проверки nginx:
+
+```bash
+# Добавить в crontab (каждые 5 минут)
+*/5 * * * * /root/mnemo_cards/tools/deploy/check-and-restore-nginx.sh >> /var/log/nginx-check.log 2>&1
+```
+
+## Дополнительные рекомендации
+
+1. **Мониторинг памяти**: Регулярно проверяйте использование памяти:
+   ```bash
+   free -h
+   ```
+
+2. **Swap**: Убедитесь, что на сервере настроен swap (функция `setup_swap` в `config.sh`)
+
+3. **Ограничение памяти для сборки**: Рассмотрите возможность ограничения памяти для процесса сборки Flutter через переменные окружения:
+   - `DART_VM_OPTIONS="--old-gen-heap-size=1024"`
+   - `NODE_OPTIONS="--max-old-space-size=256"`
+
+4. **Сборка на отдельном сервере**: Для больших проектов рассмотрите возможность сборки на отдельном сервере или в CI/CD пайплайне.
+
+## Устранение неполадок
+
+### Nginx не запускается после настройки
+
+1. Проверьте конфигурацию nginx:
+   ```bash
+   nginx -t
+   ```
+
+2. Проверьте логи:
+   ```bash
+   journalctl -u nginx -n 50
+   ```
+
+3. Проверьте, что systemd override создан:
+   ```bash
+   cat /etc/systemd/system/nginx.service.d/override.conf
+   ```
+
+4. Перезагрузите systemd:
+   ```bash
+   systemctl daemon-reload
+   systemctl restart nginx
+   ```
+
+### Nginx все еще падает
+
+1. Проверьте, что OOM protection активна:
+   ```bash
+   cat /proc/$(pgrep -f 'nginx: master')/oom_score_adj
+   ```
+
+2. Проверьте использование памяти:
+   ```bash
+   free -h
+   ps aux --sort=-%mem | head -10
+   ```
+
+3. Рассмотрите увеличение swap или ограничение памяти для других процессов
+
+## Файлы
+
+- `tools/deploy/fix-nginx-oom-protection.sh` - настройка защиты от OOM killer
+- `tools/deploy/check-and-restore-nginx.sh` - проверка и восстановление nginx
+- `tools/deploy/web-app/deploy.sh` - обновлен для проверки nginx после деплоя
+
diff --git a/tools/deploy/check-and-restore-nginx.sh b/tools/deploy/check-and-restore-nginx.sh
new file mode 100755
index 0000000..4099647
--- /dev/null
+++ b/tools/deploy/check-and-restore-nginx.sh
@@ -0,0 +1,130 @@
+#!/bin/bash
+
+# Script to check nginx status and restore it if needed
+# This script can be run manually or via cron
+# Usage: ./check-and-restore-nginx.sh
+
+set -e
+
+SERVER_IP="147.45.152.129"
+SERVER_USER="root"
+
+echo "🔍 Checking nginx status and restoring if needed..."
+echo "Server: $SERVER_IP"
+echo ""
+
+ssh "$SERVER_USER@$SERVER_IP" << 'ENDSSH'
+    set -e
+    
+    NGINX_STATUS=0
+    
+    # Check if nginx is running
+    if systemctl is-active --quiet nginx; then
+        echo "✅ Nginx is running"
+        
+        # Additional check: can nginx respond?
+        if curl -f -s http://localhost/health > /dev/null 2>&1 || \
+           curl -f -s http://localhost > /dev/null 2>&1; then
+            echo "✅ Nginx is responding to requests"
+        else
+            echo "⚠️  Nginx process is running but not responding to requests"
+            NGINX_STATUS=1
+        fi
+    else
+        echo "❌ Nginx is not running!"
+        NGINX_STATUS=1
+    fi
+    
+    # If nginx is not working, try to restore it
+    if [ $NGINX_STATUS -ne 0 ]; then
+        echo ""
+        echo "🔧 Attempting to restore nginx..."
+        
+        # Check nginx configuration
+        echo "🧪 Testing nginx configuration..."
+        if nginx -t 2>&1; then
+            echo "✅ Nginx configuration is valid"
+        else
+            echo "❌ Nginx configuration has errors!"
+            echo "📋 Recent nginx errors:"
+            journalctl -u nginx --since "10min ago" --no-pager -n 20 || true
+            exit 1
+        fi
+        
+        # Try to start nginx
+        echo ""
+        echo "🚀 Starting nginx..."
+        if systemctl start nginx; then
+            sleep 2
+            
+            if systemctl is-active --quiet nginx; then
+                echo "✅ Nginx started successfully"
+                
+                # Verify it's responding
+                sleep 1
+                if curl -f -s http://localhost > /dev/null 2>&1; then
+                    echo "✅ Nginx is responding to requests"
+                else
+                    echo "⚠️  Nginx started but not responding yet (may need a moment)"
+                fi
+            else
+                echo "❌ Failed to start nginx!"
+                echo "📋 Recent nginx errors:"
+                journalctl -u nginx --since "10min ago" --no-pager -n 30 || true
+                exit 1
+            fi
+        else
+            echo "❌ Failed to start nginx service!"
+            echo "📋 Recent nginx errors:"
+            journalctl -u nginx --since "10min ago" --no-pager -n 30 || true
+            exit 1
+        fi
+    fi
+    
+    # Check for OOM kills in system logs
+    echo ""
+    echo "🔍 Checking for OOM kills in system logs..."
+    if dmesg | grep -i "oom\|killed process.*nginx" | tail -5; then
+        echo "⚠️  Found OOM kill events related to nginx!"
+        echo "💡 Consider checking system memory usage: free -h"
+    else
+        echo "✅ No recent OOM kill events for nginx"
+    fi
+    
+    # Show current memory usage
+    echo ""
+    echo "📊 Current system memory usage:"
+    free -h || true
+    
+    # Show nginx process info
+    echo ""
+    echo "📊 Nginx process information:"
+    if pgrep -f 'nginx: master' > /dev/null; then
+        MASTER_PID=$(pgrep -f 'nginx: master' | head -1)
+        echo "   Master PID: $MASTER_PID"
+        
+        # Check OOM score
+        if [ -f "/proc/$MASTER_PID/oom_score_adj" ]; then
+            OOM_SCORE=$(cat "/proc/$MASTER_PID/oom_score_adj")
+            echo "   OOM Score Adjust: $OOM_SCORE"
+            if [ "$OOM_SCORE" = "-1000" ]; then
+                echo "   ✅ Nginx is protected from OOM killer"
+            else
+                echo "   ⚠️  Nginx OOM protection may not be configured correctly"
+            fi
+        fi
+        
+        # Show memory usage of nginx processes
+        echo "   Memory usage:"
+        ps aux | grep -E 'nginx: (master|worker)' | grep -v grep || true
+    else
+        echo "   ⚠️  Nginx master process not found"
+    fi
+    
+    echo ""
+    echo "✅ Nginx check completed"
+ENDSSH
+
+echo ""
+echo "✅ Nginx status check completed!"
+
diff --git a/tools/deploy/fix-nginx-oom-protection.sh b/tools/deploy/fix-nginx-oom-protection.sh
new file mode 100755
index 0000000..196ab34
--- /dev/null
+++ b/tools/deploy/fix-nginx-oom-protection.sh
@@ -0,0 +1,113 @@
+#!/bin/bash
+
+# Script to protect nginx from OOM killer and enable auto-recovery
+# This script configures systemd override for nginx service
+# Usage: ./fix-nginx-oom-protection.sh
+
+set -e
+
+SERVER_IP="147.45.152.129"
+SERVER_USER="root"
+
+echo "🔧 Configuring nginx OOM protection and auto-recovery..."
+echo "Server: $SERVER_IP"
+echo ""
+
+ssh "$SERVER_USER@$SERVER_IP" << 'ENDSSH'
+    set -e
+    
+    echo "📋 Backing up current nginx service configuration..."
+    if [ -f /etc/systemd/system/nginx.service ]; then
+        cp /etc/systemd/system/nginx.service /etc/systemd/system/nginx.service.backup.$(date +%Y%m%d_%H%M%S) || true
+    fi
+    
+    echo ""
+    echo "📁 Creating systemd override directory..."
+    mkdir -p /etc/systemd/system/nginx.service.d
+    
+    echo ""
+    echo "📝 Creating override configuration..."
+    cat > /etc/systemd/system/nginx.service.d/override.conf << 'OVERRIDE'
+[Service]
+# Protect nginx from OOM killer
+# OOMScoreAdjust ranges from -1000 to 1000
+# -1000 means "never kill this process" (highest protection)
+# 1000 means "kill this process first" (lowest protection)
+OOMScoreAdjust=-1000
+
+# Automatic restart configuration
+Restart=always
+RestartSec=5
+StartLimitIntervalSec=0
+StartLimitBurst=0
+
+# Additional resource limits (optional, uncomment if needed)
+# MemoryLimit=512M
+# CPUQuota=50%
+OVERRIDE
+    
+    echo "✅ Override configuration created:"
+    cat /etc/systemd/system/nginx.service.d/override.conf
+    echo ""
+    
+    echo "🔄 Reloading systemd daemon..."
+    systemctl daemon-reload
+    
+    echo ""
+    echo "🧪 Testing nginx configuration..."
+    nginx -t
+    
+    if [ $? -eq 0 ]; then
+        echo "✅ Nginx configuration is valid"
+    else
+        echo "❌ Nginx configuration test failed!"
+        exit 1
+    fi
+    
+    echo ""
+    echo "🚀 Restarting nginx to apply new settings..."
+    systemctl restart nginx
+    
+    echo ""
+    echo "⏳ Waiting for nginx to start..."
+    sleep 3
+    
+    echo ""
+    echo "🔍 Verifying nginx status..."
+    if systemctl is-active --quiet nginx; then
+        echo "✅ Nginx is running"
+    else
+        echo "❌ Nginx failed to start!"
+        systemctl status nginx --no-pager || true
+        exit 1
+    fi
+    
+    echo ""
+    echo "📊 Checking nginx service configuration..."
+    systemctl show nginx | grep -E "(OOMScoreAdjust|Restart|RestartSec)" || true
+    
+    echo ""
+    echo "✅ Nginx OOM protection and auto-recovery configured successfully!"
+    echo ""
+    echo "📋 Configuration summary:"
+    echo "   - OOMScoreAdjust: -1000 (maximum protection from OOM killer)"
+    echo "   - Restart: always (automatic restart on failure)"
+    echo "   - RestartSec: 5 (5 seconds delay before restart)"
+    echo "   - StartLimitIntervalSec: 0 (no restart limit)"
+    echo ""
+    echo "💡 To verify OOM score, run: cat /proc/\$(pgrep -f 'nginx: master')/oom_score_adj"
+ENDSSH
+
+echo ""
+echo "✅ Nginx OOM protection and auto-recovery setup completed!"
+echo ""
+echo "📝 What was configured:"
+echo "   1. OOMScoreAdjust=-1000 - nginx will be the last process killed by OOM killer"
+echo "   2. Restart=always - nginx will automatically restart if it crashes"
+echo "   3. RestartSec=5 - 5 second delay before restart"
+echo "   4. StartLimitIntervalSec=0 - no limit on restart attempts"
+echo ""
+echo "🔍 To check if nginx is protected, SSH to server and run:"
+echo "   cat /proc/\$(pgrep -f 'nginx: master')/oom_score_adj"
+echo "   (should output: -1000)"
+
diff --git a/tools/deploy/web-app/deploy.sh b/tools/deploy/web-app/deploy.sh
index 3bb5ef5..40c1a6f 100755
--- a/tools/deploy/web-app/deploy.sh
+++ b/tools/deploy/web-app/deploy.sh
@@ -103,6 +103,44 @@ ssh "$SERVER_USER@$SERVER_IP" << EOF
     systemctl restart nginx
     systemctl enable nginx
     
+    # Wait a moment for nginx to start
+    sleep 2
+    
+    # Check and restore nginx if needed
+    echo "🔍 Verifying nginx status..."
+    if ! systemctl is-active --quiet nginx; then
+        echo "⚠️  Nginx is not running after restart, attempting to restore..."
+        
+        # Check configuration
+        if nginx -t; then
+            echo "✅ Configuration is valid, trying to start nginx again..."
+            systemctl start nginx
+            sleep 2
+            
+            if systemctl is-active --quiet nginx; then
+                echo "✅ Nginx restored successfully"
+            else
+                echo "❌ Failed to restore nginx!"
+                echo "📋 Recent nginx errors:"
+                journalctl -u nginx --since "5min ago" --no-pager -n 20 || true
+                exit 1
+            fi
+        else
+            echo "❌ Nginx configuration has errors!"
+            nginx -t
+            exit 1
+        fi
+    else
+        echo "✅ Nginx is running"
+        
+        # Verify nginx is responding
+        if curl -f -s http://localhost > /dev/null 2>&1; then
+            echo "✅ Nginx is responding to requests"
+        else
+            echo "⚠️  Nginx is running but not responding (may need a moment)"
+        fi
+    fi
+    
     # Configure firewall
     ufw allow '$FIREWALL_ALLOW_NGINX'
     ufw allow $FIREWALL_ALLOW_SSH