cinnabarflower/mnemo_cards
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

deploy

Browse source
This commit is contained in:
Dmitry 2025-11-27 21:26:56 +03:00
parent 514bc68254
commit c2a55e22aa
1 changed files with 13 additions and 238 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

251
.forgejo/workflows/deploy.yaml
View file

@ -426,258 +426,33 @@ jobs:
- name: Fix Permissions and Reload Nginx
run: |
ssh -o ServerAliveInterval=60 -o ServerAliveCountMax=3 -o ConnectTimeout=30 -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} << 'ENDSSH'
# Set a global timeout for the entire step (20 minutes max)
timeout 1200 bash -c '
set -e
echo "🔧 Setting correct permissions for web files..."
chown -R www-data:www-data /var/www/mnemo_cards
chmod -R 755 /var/www/mnemo_cards
sudo chown -R www-data:www-data /var/www/mnemo_cards
sudo chmod -R 755 /var/www/mnemo_cards
echo "✅ Permissions fixed"
# Pre-reload diagnostics
echo "🔍 Pre-reload diagnostics..."
# Check running processes BEFORE reload
echo "📊 Processes BEFORE reload:"
echo "Forgejo processes:"
pgrep -f forgejo || echo "No forgejo processes"
echo "Backend processes:"
pgrep -f mnemo_cards_server || echo "No backend processes"
echo "Nginx processes:"
pgrep -f nginx || echo "No nginx processes"
echo "📊 Current nginx status:"
sudo systemctl status nginx --no-pager || true
# Check system load and memory
echo "📊 System BEFORE reload:"
uptime || true
free -h || true
echo "🌐 Testing upstream connectivity before reload:"
if timeout 10 nc -z 127.0.0.1 3000 2>/dev/null; then
echo "✅ Forgejo (port 3000): reachable"
else
echo "❌ Forgejo (port 3000): NOT reachable before reload!"
fi
if timeout 10 nc -z 127.0.0.1 8081 2>/dev/null; then
echo "✅ Backend (port 8081): reachable"
else
echo "❌ Backend (port 8081): NOT reachable before reload!"
fi
echo "🌐 Testing site accessibility before reload:"
for site in "https://mnemo-cards.online/" "https://code.mnemo-cards.online/" "https://vscode.mnemo-cards.online/"; do
if timeout 15 curl -I --max-time 10 "$site" 2>/dev/null | grep -q "200\|301\|302"; then
echo "✅ $site accessible before reload"
else
echo "⚠️ $site not accessible before reload"
timeout 10 curl -I --max-time 5 "$site" || echo "Pre-reload check failed"
fi
done
echo "🔄 Reloading nginx to serve new web app..."
echo "📋 Current nginx configuration test:"
if timeout 30 sudo nginx -t 2>&1; then
if sudo nginx -t 2>&1; then
echo "✅ nginx configuration is valid"
echo "🚀 Executing nginx reload..."
sudo systemctl reload nginx
RELOAD_RESULT=$?
if [ $RELOAD_RESULT -eq 0 ]; then
echo "✅ nginx reloaded successfully"
# Only proceed with post-reload checks if nginx reload was successful
# Wait for services to stabilize after reload
echo "⏳ Waiting for services to stabilize after reload..."
sleep 5
# IMMEDIATE upstream check after reload
echo "🔍 IMMEDIATE upstream check after reload (0 seconds wait):"
if timeout 5 nc -z 127.0.0.1 3000 2>/dev/null; then
echo "✅ Forgejo (port 3000): reachable immediately after reload"
# Basic verification
sleep 2
if sudo systemctl is-active --quiet nginx; then
echo "✅ nginx is running"
else
echo "❌ CRITICAL: Forgejo (port 3000): NOT reachable immediately after reload!"
echo "🔍 Checking forgejo process:"
pgrep -f forgejo || echo "No forgejo process found"
ps aux | grep forgejo | grep -v grep || echo "No forgejo in ps"
echo "🔍 Checking forgejo service status:"
timeout 10 sudo systemctl status forgejo --no-pager || echo "Could not get forgejo status"
echo "🔍 Checking if port 3000 is bound:"
ss -tlnp | grep :3000 || echo "Port 3000 not bound"
echo "❌ nginx failed to reload"
sudo systemctl status nginx --no-pager || true
exit 1
fi
if timeout 5 nc -z 127.0.0.1 8081 2>/dev/null; then
echo "✅ Backend (port 8081): reachable immediately after reload"
else
echo "❌ CRITICAL: Backend (port 8081): NOT reachable immediately after reload!"
echo "🔍 Checking backend process:"
pgrep -f mnemo_cards_server || echo "No backend process found"
ps aux | grep mnemo_cards_server | grep -v grep || echo "No backend in ps"
echo "🔍 Checking backend service status:"
timeout 10 sudo systemctl status mnemo_cards_server --no-pager --lines=3 2>/dev/null || echo "Could not get backend status"
echo "🔍 Checking if port 8081 is bound:"
ss -tlnp | grep :8081 || echo "Port 8081 not bound"
fi
# Check for OOM killer activity
echo "🚨 Checking for OOM killer activity:"
timeout 10 sudo dmesg | grep -i "oom\|kill" | tail -5 || echo "No OOM killer activity detected"
# Check systemd logs for service events around reload time
echo "📋 Checking systemd journal for recent service events:"
timeout 10 sudo journalctl --since "5 minutes ago" --no-pager -u nginx -u mnemo_cards_server --grep "(start|stop|reload|fail)" | tail -10 || echo "No recent systemd events"
# Check nginx logs for errors after reload
echo "📋 Checking nginx error logs after reload:"
timeout 10 sudo tail -30 /var/log/nginx/error.log | grep -E "(connect|upstream|error)" | tail -10 || echo "No recent upstream errors in nginx logs"
# Check running processes AFTER reload
echo "📊 Processes AFTER reload:"
echo "Forgejo processes:"
pgrep -f forgejo || echo "No forgejo processes"
echo "Backend processes:"
pgrep -f mnemo_cards_server || echo "No backend processes"
echo "Nginx processes:"
pgrep -f nginx || echo "No nginx processes"
# Check system load and memory after reload
echo "📊 System AFTER reload:"
uptime || true
free -h || true
# Check nginx status again
echo "📊 Nginx status after reload:"
timeout 15 sudo systemctl status nginx --no-pager || echo "Could not get nginx status"
# Check all services status
echo "📊 Checking service statuses after nginx reload..."
# Check nginx
echo "🔍 Checking nginx status..."
if timeout 30 sudo systemctl is-active --quiet nginx; then
echo "✅ nginx: running"
# Check loaded configuration
echo "📋 Loaded nginx sites:"
timeout 10 ls -la /etc/nginx/sites-enabled/ || echo "Could not list sites"
else
echo "❌ nginx: not running or status check timed out"
timeout 15 sudo systemctl status nginx --no-pager || echo "Could not get nginx status"
fi
# Check backend
echo "🔍 Checking backend status..."
if timeout 30 sudo systemctl is-active --quiet mnemo_cards_server; then
echo "✅ backend: running"
else
echo "❌ backend: not running or status check timed out"
timeout 15 sudo systemctl status mnemo_cards_server --no-pager --lines=3 2>/dev/null || echo "Could not get backend status"
# Check backend logs for errors
echo "📋 Checking backend logs for recent errors:"
timeout 10 sudo journalctl -u mnemo_cards_server --since "10 minutes ago" --no-pager | grep -i "error\|fail\|crash\|oom\|kill" | tail -5 || echo "No recent backend errors"
# Try to start it if it's stopped
echo "🚀 Attempting to start backend..."
timeout 30 sudo systemctl start mnemo_cards_server 2>&1
START_EXIT_CODE=$?
if [ $START_EXIT_CODE -eq 0 ]; then
sleep 5 # Give more time to start
if timeout 30 sudo systemctl is-active --quiet mnemo_cards_server; then
echo "✅ backend: started successfully"
else
echo "❌ backend: failed to start"
timeout 15 sudo systemctl status mnemo_cards_server --no-pager --lines=3 2>/dev/null || echo "Could not get status after start attempt"
fi
else
echo "❌ backend start command timed out or failed (exit code: $START_EXIT_CODE)"
fi
fi
# Check forgejo (if exists)
if sudo systemctl is-active --quiet forgejo 2>/dev/null; then
echo "✅ forgejo: running"
elif pgrep -f forgejo >/dev/null 2>&1; then
echo "✅ forgejo: running (not as service)"
else
echo "⚠️ forgejo: not found or not running"
# Check if forgejo process exists but not responding
if pgrep -f "forgejo" >/dev/null 2>&1; then
echo "🔍 Found forgejo process, but may be unresponsive"
ps aux | grep forgejo | grep -v grep || true
fi
fi
# Check for orphaned processes
echo "🔍 Checking for orphaned processes..."
# Kill any remaining flutter/dart processes that might interfere
pkill -f "flutter\|dart" || true
# Kill any remaining act processes
pkill -f "act" || true
sleep 1
# Check web app files
echo "📁 Checking web app files:"
if [ -f "/var/www/mnemo_cards/index.html" ]; then
echo "✅ index.html exists"
ls -la /var/www/mnemo_cards/ | head -10 || true
else
echo "❌ index.html not found!"
ls -la /var/www/ || true
fi
# Check system resources
echo "📊 System resources after cleanup:"
free -h || true
df -h / || true
# Check network connectivity to upstream services
echo "🌐 Checking upstream connectivity..."
if timeout 10 nc -z 127.0.0.1 3000 2>/dev/null; then
echo "✅ Forgejo (port 3000): reachable"
else
echo "⚠️ Forgejo (port 3000): not reachable or check timed out"
fi
if timeout 10 nc -z 127.0.0.1 8081 2>/dev/null; then
echo "✅ Backend (port 8081): reachable"
else
echo "⚠️ Backend (port 8081): not reachable or check timed out"
fi
# Wait a bit more before testing sites
sleep 3
# Test site accessibility with retries and timeouts
echo "🌐 Testing site accessibility..."
for site in "https://mnemo-cards.online/" "https://code.mnemo-cards.online/" "https://vscode.mnemo-cards.online/"; do
for attempt in {1..3}; do
echo "🔍 Testing $site (attempt $attempt)..."
if timeout 15 curl -I --max-time 10 "$site" 2>/dev/null | grep -q "200\|301\|302"; then
echo "✅ $site accessible (attempt $attempt)"
break
else
CURL_EXIT=$?
if [ $CURL_EXIT -eq 124 ]; then
echo "⏰ Timeout testing $site (attempt $attempt)"
else
echo "❌ $site returned error (attempt $attempt)"
fi
if [ $attempt -eq 3 ]; then
echo "❌ $site not accessible after 3 attempts"
timeout 10 curl -I --max-time 5 "$site" || echo "Final check also failed"
else
echo "⏳ $site attempt $attempt failed, retrying in 3 seconds..."
sleep 3
fi
fi
done
done
echo "✅ Web app deployment completed"
else
echo "❌ nginx reload failed, skipping post-reload checks"
echo "❌ nginx configuration test failed!"
sudo nginx -t || true
exit 1
fi
final-verification: