diff --git a/.forgejo/workflows/deploy.yaml b/.forgejo/workflows/deploy.yaml index 1e6a7c0..b3e7ed4 100644 --- a/.forgejo/workflows/deploy.yaml +++ b/.forgejo/workflows/deploy.yaml @@ -426,258 +426,33 @@ jobs: - name: Fix Permissions and Reload Nginx run: | ssh -o ServerAliveInterval=60 -o ServerAliveCountMax=3 -o ConnectTimeout=30 -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} << 'ENDSSH' - # Set a global timeout for the entire step (20 minutes max) - timeout 1200 bash -c ' set -e echo "🔧 Setting correct permissions for web files..." - chown -R www-data:www-data /var/www/mnemo_cards - chmod -R 755 /var/www/mnemo_cards + sudo chown -R www-data:www-data /var/www/mnemo_cards + sudo chmod -R 755 /var/www/mnemo_cards echo "✅ Permissions fixed" - # Pre-reload diagnostics - echo "🔍 Pre-reload diagnostics..." - - # Check running processes BEFORE reload - echo "📊 Processes BEFORE reload:" - echo "Forgejo processes:" - pgrep -f forgejo || echo "No forgejo processes" - echo "Backend processes:" - pgrep -f mnemo_cards_server || echo "No backend processes" - echo "Nginx processes:" - pgrep -f nginx || echo "No nginx processes" - - echo "📊 Current nginx status:" - sudo systemctl status nginx --no-pager || true - - # Check system load and memory - echo "📊 System BEFORE reload:" - uptime || true - free -h || true - - echo "🌐 Testing upstream connectivity before reload:" - if timeout 10 nc -z 127.0.0.1 3000 2>/dev/null; then - echo "✅ Forgejo (port 3000): reachable" - else - echo "❌ Forgejo (port 3000): NOT reachable before reload!" - fi - - if timeout 10 nc -z 127.0.0.1 8081 2>/dev/null; then - echo "✅ Backend (port 8081): reachable" - else - echo "❌ Backend (port 8081): NOT reachable before reload!" - fi - - echo "🌐 Testing site accessibility before reload:" - for site in "https://mnemo-cards.online/" "https://code.mnemo-cards.online/" "https://vscode.mnemo-cards.online/"; do - if timeout 15 curl -I --max-time 10 "$site" 2>/dev/null | grep -q "200\|301\|302"; then - echo "✅ $site accessible before reload" - else - echo "⚠️ $site not accessible before reload" - timeout 10 curl -I --max-time 5 "$site" || echo "Pre-reload check failed" - fi - done - echo "🔄 Reloading nginx to serve new web app..." - echo "📋 Current nginx configuration test:" - if timeout 30 sudo nginx -t 2>&1; then + if sudo nginx -t 2>&1; then echo "✅ nginx configuration is valid" - echo "🚀 Executing nginx reload..." sudo systemctl reload nginx - RELOAD_RESULT=$? - if [ $RELOAD_RESULT -eq 0 ]; then echo "✅ nginx reloaded successfully" - # Only proceed with post-reload checks if nginx reload was successful - # Wait for services to stabilize after reload - echo "⏳ Waiting for services to stabilize after reload..." - sleep 5 - - # IMMEDIATE upstream check after reload - echo "🔍 IMMEDIATE upstream check after reload (0 seconds wait):" - if timeout 5 nc -z 127.0.0.1 3000 2>/dev/null; then - echo "✅ Forgejo (port 3000): reachable immediately after reload" + # Basic verification + sleep 2 + if sudo systemctl is-active --quiet nginx; then + echo "✅ nginx is running" else - echo "❌ CRITICAL: Forgejo (port 3000): NOT reachable immediately after reload!" - echo "🔍 Checking forgejo process:" - pgrep -f forgejo || echo "No forgejo process found" - ps aux | grep forgejo | grep -v grep || echo "No forgejo in ps" - echo "🔍 Checking forgejo service status:" - timeout 10 sudo systemctl status forgejo --no-pager || echo "Could not get forgejo status" - echo "🔍 Checking if port 3000 is bound:" - ss -tlnp | grep :3000 || echo "Port 3000 not bound" + echo "❌ nginx failed to reload" + sudo systemctl status nginx --no-pager || true + exit 1 fi - if timeout 5 nc -z 127.0.0.1 8081 2>/dev/null; then - echo "✅ Backend (port 8081): reachable immediately after reload" - else - echo "❌ CRITICAL: Backend (port 8081): NOT reachable immediately after reload!" - echo "🔍 Checking backend process:" - pgrep -f mnemo_cards_server || echo "No backend process found" - ps aux | grep mnemo_cards_server | grep -v grep || echo "No backend in ps" - echo "🔍 Checking backend service status:" - timeout 10 sudo systemctl status mnemo_cards_server --no-pager --lines=3 2>/dev/null || echo "Could not get backend status" - echo "🔍 Checking if port 8081 is bound:" - ss -tlnp | grep :8081 || echo "Port 8081 not bound" - fi - - # Check for OOM killer activity - echo "🚨 Checking for OOM killer activity:" - timeout 10 sudo dmesg | grep -i "oom\|kill" | tail -5 || echo "No OOM killer activity detected" - - # Check systemd logs for service events around reload time - echo "📋 Checking systemd journal for recent service events:" - timeout 10 sudo journalctl --since "5 minutes ago" --no-pager -u nginx -u mnemo_cards_server --grep "(start|stop|reload|fail)" | tail -10 || echo "No recent systemd events" - - # Check nginx logs for errors after reload - echo "📋 Checking nginx error logs after reload:" - timeout 10 sudo tail -30 /var/log/nginx/error.log | grep -E "(connect|upstream|error)" | tail -10 || echo "No recent upstream errors in nginx logs" - - # Check running processes AFTER reload - echo "📊 Processes AFTER reload:" - echo "Forgejo processes:" - pgrep -f forgejo || echo "No forgejo processes" - echo "Backend processes:" - pgrep -f mnemo_cards_server || echo "No backend processes" - echo "Nginx processes:" - pgrep -f nginx || echo "No nginx processes" - - # Check system load and memory after reload - echo "📊 System AFTER reload:" - uptime || true - free -h || true - - # Check nginx status again - echo "📊 Nginx status after reload:" - timeout 15 sudo systemctl status nginx --no-pager || echo "Could not get nginx status" - - # Check all services status - echo "📊 Checking service statuses after nginx reload..." - - # Check nginx - echo "🔍 Checking nginx status..." - if timeout 30 sudo systemctl is-active --quiet nginx; then - echo "✅ nginx: running" - # Check loaded configuration - echo "📋 Loaded nginx sites:" - timeout 10 ls -la /etc/nginx/sites-enabled/ || echo "Could not list sites" - else - echo "❌ nginx: not running or status check timed out" - timeout 15 sudo systemctl status nginx --no-pager || echo "Could not get nginx status" - fi - - # Check backend - echo "🔍 Checking backend status..." - if timeout 30 sudo systemctl is-active --quiet mnemo_cards_server; then - echo "✅ backend: running" - else - echo "❌ backend: not running or status check timed out" - timeout 15 sudo systemctl status mnemo_cards_server --no-pager --lines=3 2>/dev/null || echo "Could not get backend status" - - # Check backend logs for errors - echo "📋 Checking backend logs for recent errors:" - timeout 10 sudo journalctl -u mnemo_cards_server --since "10 minutes ago" --no-pager | grep -i "error\|fail\|crash\|oom\|kill" | tail -5 || echo "No recent backend errors" - - # Try to start it if it's stopped - echo "🚀 Attempting to start backend..." - timeout 30 sudo systemctl start mnemo_cards_server 2>&1 - START_EXIT_CODE=$? - if [ $START_EXIT_CODE -eq 0 ]; then - sleep 5 # Give more time to start - if timeout 30 sudo systemctl is-active --quiet mnemo_cards_server; then - echo "✅ backend: started successfully" - else - echo "❌ backend: failed to start" - timeout 15 sudo systemctl status mnemo_cards_server --no-pager --lines=3 2>/dev/null || echo "Could not get status after start attempt" - fi - else - echo "❌ backend start command timed out or failed (exit code: $START_EXIT_CODE)" - fi - fi - - # Check forgejo (if exists) - if sudo systemctl is-active --quiet forgejo 2>/dev/null; then - echo "✅ forgejo: running" - elif pgrep -f forgejo >/dev/null 2>&1; then - echo "✅ forgejo: running (not as service)" - else - echo "⚠️ forgejo: not found or not running" - # Check if forgejo process exists but not responding - if pgrep -f "forgejo" >/dev/null 2>&1; then - echo "🔍 Found forgejo process, but may be unresponsive" - ps aux | grep forgejo | grep -v grep || true - fi - fi - - # Check for orphaned processes - echo "🔍 Checking for orphaned processes..." - # Kill any remaining flutter/dart processes that might interfere - pkill -f "flutter\|dart" || true - # Kill any remaining act processes - pkill -f "act" || true - sleep 1 - - # Check web app files - echo "📁 Checking web app files:" - if [ -f "/var/www/mnemo_cards/index.html" ]; then - echo "✅ index.html exists" - ls -la /var/www/mnemo_cards/ | head -10 || true - else - echo "❌ index.html not found!" - ls -la /var/www/ || true - fi - - # Check system resources - echo "📊 System resources after cleanup:" - free -h || true - df -h / || true - - # Check network connectivity to upstream services - echo "🌐 Checking upstream connectivity..." - if timeout 10 nc -z 127.0.0.1 3000 2>/dev/null; then - echo "✅ Forgejo (port 3000): reachable" - else - echo "⚠️ Forgejo (port 3000): not reachable or check timed out" - fi - - if timeout 10 nc -z 127.0.0.1 8081 2>/dev/null; then - echo "✅ Backend (port 8081): reachable" - else - echo "⚠️ Backend (port 8081): not reachable or check timed out" - fi - - # Wait a bit more before testing sites - sleep 3 - - # Test site accessibility with retries and timeouts - echo "🌐 Testing site accessibility..." - for site in "https://mnemo-cards.online/" "https://code.mnemo-cards.online/" "https://vscode.mnemo-cards.online/"; do - for attempt in {1..3}; do - echo "🔍 Testing $site (attempt $attempt)..." - if timeout 15 curl -I --max-time 10 "$site" 2>/dev/null | grep -q "200\|301\|302"; then - echo "✅ $site accessible (attempt $attempt)" - break - else - CURL_EXIT=$? - if [ $CURL_EXIT -eq 124 ]; then - echo "⏰ Timeout testing $site (attempt $attempt)" - else - echo "❌ $site returned error (attempt $attempt)" - fi - - if [ $attempt -eq 3 ]; then - echo "❌ $site not accessible after 3 attempts" - timeout 10 curl -I --max-time 5 "$site" || echo "Final check also failed" - else - echo "⏳ $site attempt $attempt failed, retrying in 3 seconds..." - sleep 3 - fi - fi - done - done - echo "✅ Web app deployment completed" else - echo "❌ nginx reload failed, skipping post-reload checks" + echo "❌ nginx configuration test failed!" + sudo nginx -t || true + exit 1 fi final-verification: