743 lines
34 KiB
YAML
743 lines
34 KiB
YAML
name: Deploy Mnemo Cards
|
||
|
||
on:
|
||
workflow_dispatch:
|
||
push:
|
||
branches:
|
||
- master
|
||
paths:
|
||
- 'mnemo_cards_web_v2/**'
|
||
- 'mnemo_cards_backend/**'
|
||
- 'mnemo_cards_common/**'
|
||
- 'mnemo_cards_common_backend/**'
|
||
jobs:
|
||
deploy-backend:
|
||
name: Deploy Backend
|
||
runs-on: ubuntu-latest
|
||
steps:
|
||
- name: Setup Node.js
|
||
uses: actions/setup-node@v4
|
||
with:
|
||
node-version: '20'
|
||
|
||
- name: Checkout code
|
||
uses: actions/checkout@v3
|
||
|
||
- name: Install Dependencies
|
||
run: |
|
||
apt-get update && apt-get install -y rsync openssh-client
|
||
|
||
- name: Setup SSH
|
||
run: |
|
||
echo "🔍 Debugging SSH connection..."
|
||
echo "Target host: ${{ secrets.SSH_HOST }}"
|
||
ping -c 3 ${{ secrets.SSH_HOST }} || echo "⚠️ Ping failed"
|
||
nslookup ${{ secrets.SSH_HOST }} || echo "⚠️ DNS lookup failed"
|
||
nc -zv ${{ secrets.SSH_HOST }} 22 || echo "⚠️ Port 22 not reachable"
|
||
|
||
mkdir -p ~/.ssh
|
||
echo "${{ secrets.SSH_KEY }}" > ~/.ssh/id_rsa
|
||
chmod 600 ~/.ssh/id_rsa
|
||
ssh-keyscan -H ${{ secrets.SSH_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || echo "⚠️ SSH keyscan failed"
|
||
|
||
- name: Sync Source Code
|
||
run: |
|
||
# Sync common packages
|
||
rsync -avz --delete --exclude '.*' --exclude 'build' --exclude 'doc' \
|
||
mnemo_cards_common \
|
||
mnemo_cards_common_backend \
|
||
${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }}:~/
|
||
|
||
# Sync backend project
|
||
rsync -avz --delete --exclude '.*' --exclude 'build' --exclude 'server.exe' --exclude 'isar' \
|
||
mnemo_cards_backend \
|
||
${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }}:~/
|
||
|
||
# Sync deploy tools
|
||
rsync -avz --delete tools/deploy \
|
||
${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }}:~/mnemo_cards/tools/
|
||
|
||
- name: Ensure SSL Certificates
|
||
run: |
|
||
ssh -o ServerAliveInterval=60 -o ServerAliveCountMax=3 -o ConnectTimeout=30 -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} << 'ENDSSH'
|
||
set -e
|
||
echo "🔐 Ensuring SSL certificates are available..."
|
||
|
||
if [ -d "/etc/letsencrypt/live/mnemo-cards.online" ] && [ -f "/etc/letsencrypt/live/mnemo-cards.online/fullchain.pem" ]; then
|
||
echo "✅ SSL certificates are present and valid"
|
||
else
|
||
echo "⚠️ SSL certificates missing - they should be renewed automatically by cron"
|
||
echo "Manual certificate renewal can be done with:"
|
||
echo "sudo certbot renew"
|
||
fi
|
||
ENDSSH
|
||
|
||
- name: Get Dependencies
|
||
run: |
|
||
ssh -o ServerAliveInterval=60 -o ServerAliveCountMax=3 -o ConnectTimeout=30 -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} << 'ENDSSH'
|
||
set -e
|
||
cd ~/mnemo_cards_backend
|
||
echo "📦 Getting dependencies..."
|
||
dart pub get
|
||
ENDSSH
|
||
|
||
- name: Compile Server
|
||
run: |
|
||
ssh -o ServerAliveInterval=60 -o ServerAliveCountMax=3 -o ConnectTimeout=30 -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} << 'ENDSSH'
|
||
set -e
|
||
cd ~/mnemo_cards_backend
|
||
echo "🔨 Compiling server..."
|
||
dart compile exe --verbose lib/main.dart -o server_build.exe
|
||
ENDSSH
|
||
|
||
- name: Stop Service
|
||
run: |
|
||
ssh -o ServerAliveInterval=60 -o ServerAliveCountMax=3 -o ConnectTimeout=30 -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} << 'ENDSSH'
|
||
set -e
|
||
echo "🛑 Stopping backend service..."
|
||
sudo systemctl stop mnemo_cards_server || true
|
||
echo "✅ Backend service stopped"
|
||
echo "🔍 Checking nginx status after stopping backend..."
|
||
if sudo systemctl is-active --quiet nginx; then
|
||
echo "✅ nginx still running"
|
||
else
|
||
echo "🚨 nginx also stopped! This is the problem!"
|
||
echo "🚀 Starting nginx..."
|
||
sudo systemctl start nginx
|
||
sleep 2
|
||
if sudo systemctl is-active --quiet nginx; then
|
||
echo "✅ nginx restarted"
|
||
else
|
||
echo "❌ Failed to restart nginx!"
|
||
exit 1
|
||
fi
|
||
fi
|
||
ENDSSH
|
||
|
||
- name: Update Service Configuration
|
||
run: |
|
||
ssh -o ServerAliveInterval=60 -o ServerAliveCountMax=3 -o ConnectTimeout=30 -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} << 'ENDSSH'
|
||
set -e
|
||
cd ~/mnemo_cards_backend
|
||
echo "⚙️ Updating service configuration..."
|
||
sudo chmod 777 mnemo_cards_server.service
|
||
sudo cp mnemo_cards_server.service /etc/systemd/system/
|
||
ENDSSH
|
||
|
||
- name: Update Executable
|
||
run: |
|
||
ssh -o ServerAliveInterval=60 -o ServerAliveCountMax=3 -o ConnectTimeout=30 -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} << 'ENDSSH'
|
||
set -e
|
||
cd ~/mnemo_cards_backend
|
||
echo "🔄 Replacing executable..."
|
||
sudo cp server_build.exe server.exe
|
||
rm server_build.exe
|
||
ENDSSH
|
||
|
||
- name: Configure Certificates
|
||
run: |
|
||
ssh -o ServerAliveInterval=60 -o ServerAliveCountMax=3 -o ConnectTimeout=30 -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} << 'ENDSSH'
|
||
set -e
|
||
# Check certificates and configure mode
|
||
# Используем multi-domain сертификат mnemo-cards.online (если есть)
|
||
if [ -d "/etc/letsencrypt/live/mnemo-cards.online" ] && [ -f "/etc/letsencrypt/live/mnemo-cards.online/fullchain.pem" ]; then
|
||
echo "✅ Let's Encrypt multi-domain сертификат найден. Настраиваем HTTPS режим..."
|
||
sudo sed -i 's|ExecStart=/root/mnemo_cards_backend/server.exe.*|ExecStart=/root/mnemo_cards_backend/server.exe --certs /etc/letsencrypt/live/mnemo-cards.online -a 0.0.0.0 -p 8081|' /etc/systemd/system/mnemo_cards_server.service
|
||
elif [ -d "/etc/letsencrypt/live/api.mnemo-cards.online" ]; then
|
||
echo "✅ Let's Encrypt сертификат для API найден. Настраиваем dual mode..."
|
||
sudo sed -i 's|ExecStart=/root/mnemo_cards_backend/server.exe.*|ExecStart=/root/mnemo_cards_backend/server.exe --dual --certs /etc/letsencrypt/live/api.mnemo-cards.online -a 0.0.0.0 -p 8081|' /etc/systemd/system/mnemo_cards_server.service
|
||
else
|
||
echo "⚠️ No certificates found. Configuring HTTP mode..."
|
||
sudo sed -i 's|ExecStart=/root/mnemo_cards_backend/server.exe.*|ExecStart=/root/mnemo_cards_backend/server.exe -a 0.0.0.0 -p 8081|' /etc/systemd/system/mnemo_cards_server.service
|
||
fi
|
||
ENDSSH
|
||
|
||
- name: Restart Service
|
||
run: |
|
||
ssh -o ServerAliveInterval=60 -o ServerAliveCountMax=3 -o ConnectTimeout=30 -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} << 'ENDSSH'
|
||
set -e
|
||
echo "🚀 Restarting service..."
|
||
sudo systemctl daemon-reload
|
||
sudo systemctl start mnemo_cards_server
|
||
sleep 2
|
||
sudo systemctl status mnemo_cards_server --no-pager --lines=5 || echo "Could not get status"
|
||
ENDSSH
|
||
|
||
- name: Generate and Deploy Nginx Configs
|
||
run: |
|
||
ssh -o ServerAliveInterval=60 -o ServerAliveCountMax=3 -o ConnectTimeout=30 -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} << 'ENDSSH'
|
||
set -e
|
||
echo "🔧 Generating nginx configurations..."
|
||
cd ~/mnemo_cards/tools/deploy
|
||
./generate-nginx-configs.sh
|
||
|
||
echo "🧹 Checking for potential conflicts..."
|
||
|
||
# Check if nginx.conf includes generated_configs directly (this would cause duplication)
|
||
if grep -q "generated_configs" /etc/nginx/nginx.conf; then
|
||
echo "⚠️ WARNING: nginx.conf includes generated_configs directly!"
|
||
echo "This may cause server_name conflicts. Please remove the include line:"
|
||
grep -n "generated_configs" /etc/nginx/nginx.conf
|
||
echo "For now, proceeding with sites-enabled approach..."
|
||
fi
|
||
|
||
# Check for existing conflicting server blocks
|
||
echo "📊 Checking for existing server_name conflicts..."
|
||
for domain in "mnemo-cards.online" "code.mnemo-cards.online" "vscode.mnemo-cards.online"; do
|
||
if grep -r "server_name $domain" /etc/nginx/sites-enabled/ 2>/dev/null | grep -v "$domain.conf:"; then
|
||
echo "⚠️ Found conflicting server_name $domain in other files:"
|
||
grep -r "server_name $domain" /etc/nginx/sites-enabled/ 2>/dev/null
|
||
fi
|
||
done
|
||
|
||
echo "🧹 Cleaning up old configuration files..."
|
||
# Remove old files without .conf extension that might cause conflicts
|
||
rm -f /etc/nginx/sites-available/forgejo
|
||
rm -f /etc/nginx/sites-available/vscode
|
||
rm -f /etc/nginx/sites-available/vscode.mnemo-cards.online
|
||
rm -f /etc/nginx/sites-available/mnemo_cards
|
||
rm -f /etc/nginx/sites-available/mnemo_cards_main
|
||
|
||
# Remove any existing .conf files that we're about to replace
|
||
rm -f /etc/nginx/sites-enabled/vscode.mnemo-cards.online.conf
|
||
rm -f /etc/nginx/sites-enabled/code.mnemo-cards.online.conf
|
||
rm -f /etc/nginx/sites-enabled/mnemo-cards.online.conf
|
||
rm -f /etc/nginx/sites-available/vscode.mnemo-cards.online.conf
|
||
rm -f /etc/nginx/sites-available/code.mnemo-cards.online.conf
|
||
rm -f /etc/nginx/sites-available/mnemo-cards.online.conf
|
||
|
||
echo "📋 Deploying new configurations..."
|
||
cp generated_configs/*.conf /etc/nginx/sites-available/
|
||
|
||
echo "🔗 Creating symbolic links..."
|
||
ln -sf /etc/nginx/sites-available/mnemo-cards.online.conf /etc/nginx/sites-enabled/
|
||
ln -sf /etc/nginx/sites-available/code.mnemo-cards.online.conf /etc/nginx/sites-enabled/
|
||
ln -sf /etc/nginx/sites-available/vscode.mnemo-cards.online.conf /etc/nginx/sites-enabled/
|
||
|
||
echo "📊 Testing nginx configuration..."
|
||
if /usr/sbin/nginx -t; then
|
||
echo "✅ Configuration is valid"
|
||
echo "🔄 Reloading nginx..."
|
||
systemctl reload nginx
|
||
echo "✅ nginx reloaded successfully"
|
||
else
|
||
echo "❌ Configuration test failed!"
|
||
echo "🔍 Debugging configuration errors..."
|
||
/usr/sbin/nginx -t 2>&1 || true
|
||
|
||
# Try to identify the problematic file
|
||
echo "🔍 Checking individual site configs..."
|
||
for config in /etc/nginx/sites-enabled/*.conf; do
|
||
echo "Testing $(basename "$config")..."
|
||
if ! /usr/sbin/nginx -t -c /etc/nginx/nginx.conf -g "include $config;" 2>/dev/null; then
|
||
echo "❌ Error in $(basename "$config")"
|
||
head -20 "$config"
|
||
fi
|
||
done
|
||
|
||
exit 1
|
||
fi
|
||
|
||
echo "🔍 Running final checks..."
|
||
|
||
# Check nginx service status
|
||
if systemctl is-active --quiet nginx; then
|
||
echo "✅ nginx service is running"
|
||
else
|
||
echo "❌ nginx service is not running!"
|
||
systemctl status nginx --no-pager || true
|
||
exit 1
|
||
fi
|
||
|
||
# Test site accessibility
|
||
echo "🌐 Testing site accessibility..."
|
||
if curl -I --max-time 5 https://mnemo-cards.online/ 2>/dev/null | grep -q "200\|301\|302"; then
|
||
echo "✅ mnemo-cards.online is accessible"
|
||
else
|
||
echo "⚠️ mnemo-cards.online not responding"
|
||
fi
|
||
|
||
if curl -I --max-time 5 https://code.mnemo-cards.online/ 2>/dev/null | grep -q "200\|301\|302"; then
|
||
echo "✅ code.mnemo-cards.online is accessible"
|
||
else
|
||
echo "⚠️ code.mnemo-cards.online not responding"
|
||
fi
|
||
|
||
if curl -I --max-time 5 https://vscode.mnemo-cards.online/ 2>/dev/null | grep -q "200\|301\|302"; then
|
||
echo "✅ vscode.mnemo-cards.online is accessible"
|
||
else
|
||
echo "⚠️ vscode.mnemo-cards.online not responding"
|
||
fi
|
||
|
||
echo "✅ Nginx configuration deployment completed successfully!"
|
||
ENDSSH
|
||
|
||
|
||
|
||
deploy-web:
|
||
name: Deploy Web App
|
||
runs-on: ubuntu-latest
|
||
container:
|
||
image: ghcr.io/cirruslabs/flutter:3.35.5
|
||
options: --memory=2048m --memory-swap=4096m
|
||
needs: deploy-backend
|
||
if: always() && needs.deploy-backend.result == 'success'
|
||
steps:
|
||
- name: Install System Dependencies
|
||
run: |
|
||
apt-get update && apt-get install -y rsync openssh-client jq git curl wget gnupg2 software-properties-common ca-certificates
|
||
|
||
- name: Setup Node.js
|
||
run: |
|
||
# Install Node.js manually in container
|
||
curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
|
||
apt-get install -y nodejs
|
||
echo "Node.js version: $(node --version)"
|
||
echo "NPM version: $(npm --version)"
|
||
# Set Node.js memory limits for increased container memory (512MB)
|
||
echo "NODE_OPTIONS=--max-old-space-size=512" >> $GITHUB_ENV || echo "NODE_OPTIONS=--max-old-space-size=512" >> ~/.bashrc || true
|
||
|
||
- name: Setup SSH
|
||
run: |
|
||
mkdir -p ~/.ssh
|
||
echo "${{ secrets.SSH_KEY }}" > ~/.ssh/id_rsa
|
||
chmod 600 ~/.ssh/id_rsa
|
||
ssh-keyscan -H ${{ secrets.SSH_HOST }} >> ~/.ssh/known_hosts
|
||
|
||
- name: Checkout code
|
||
uses: actions/checkout@v3
|
||
with:
|
||
fetch-depth: 0
|
||
lfs: false
|
||
|
||
- name: Configure Git
|
||
run: |
|
||
git config --global http.postBuffer 52428800
|
||
git config --global http.maxRequestBuffer 100M
|
||
git config --global core.compression 0
|
||
|
||
- name: Build Web App
|
||
working-directory: mnemo_cards_web_v2
|
||
env:
|
||
# Memory optimization for increased container limits
|
||
DART_VM_OPTIONS: "--old-gen-heap-size=1024 --max-old-space-size=1024 --gc-on-threshold --low-memory-mode --enable-isolate-groups --use-slow-path"
|
||
# Minimal Node.js memory
|
||
NODE_OPTIONS: "--max-old-space-size=256"
|
||
# Limit parallel compilation to reduce memory pressure
|
||
FLUTTER_BUILD_PARALLEL: "1"
|
||
# Disable analytics to save memory
|
||
FLUTTER_ANALYTICS_DISABLED: "true"
|
||
# Reduce GC pressure
|
||
DART_FLAGS: "--disable-dart-dev --no-background-compilation"
|
||
# Additional memory optimization
|
||
FLUTTER_WEB_USE_SKIA: "false"
|
||
# Disable web security for faster builds
|
||
FLUTTER_WEB_DISABLE_SECURITY: "true"
|
||
run: |
|
||
echo "🔍 Проверяем Flutter..."
|
||
which flutter || echo "Flutter not found in PATH"
|
||
flutter --version || echo "Flutter command failed"
|
||
|
||
echo "📊 Текущее состояние памяти:"
|
||
free -h || true
|
||
|
||
echo "🧹 Очищаем кэш и временные файлы для экономии памяти..."
|
||
flutter clean || true
|
||
# Clear additional caches
|
||
rm -rf ~/.pub-cache/hosted/pub.dev/.cache || true
|
||
rm -rf ~/.dartServer || true
|
||
rm -rf ~/.flutter-devtools || true
|
||
rm -rf ~/.flutter/bin/cache/dart-sdk/bin/snapshots/*.snapshot || true
|
||
rm -rf /tmp/dart* || true
|
||
rm -rf /tmp/flutter* || true
|
||
rm -rf .dart_tool/cache || true
|
||
|
||
# Drop caches to free memory (if available)
|
||
echo 3 > /proc/sys/vm/drop_caches 2>/dev/null || true
|
||
|
||
# Kill any lingering Dart/Flutter processes
|
||
pkill -f dart || true
|
||
pkill -f flutter || true
|
||
sleep 2
|
||
|
||
# Sync memory to disk to free up RAM
|
||
sync || true
|
||
|
||
echo "📦 Получаем зависимости..."
|
||
timeout 300 flutter pub get || (echo "❌ Flutter pub get failed" && exit 1)
|
||
|
||
# Free memory after pub get
|
||
echo 3 > /proc/sys/vm/drop_caches 2>/dev/null || true
|
||
sync || true
|
||
|
||
# Final memory cleanup before build
|
||
echo "🧹 Финальная очистка памяти перед сборкой..."
|
||
echo 3 > /proc/sys/vm/drop_caches 2>/dev/null || true
|
||
sync || true
|
||
sleep 2
|
||
|
||
echo "📊 Память перед сборкой:"
|
||
free -h || true
|
||
|
||
# Use HTML renderer instead of CanvasKit for better memory efficiency
|
||
export FLUTTER_WEB_USE_SKIA=false
|
||
|
||
echo "🔨 Собираем веб-приложение (оптимизировано для 2GB памяти)..."
|
||
|
||
# Try multiple build modes from least to most memory intensive
|
||
echo "🔄 Пробуем разные режимы сборки по возрастанию сложности..."
|
||
|
||
# First try debug mode (least memory intensive)
|
||
echo "🐛 Попытка 1: Debug mode..."
|
||
if timeout 1800 flutter build web --debug \
|
||
--dart-define=API_BASE_URL=https://api.mnemo-cards.online \
|
||
--no-tree-shake-icons \
|
||
--no-pub \
|
||
--suppress-analytics; then
|
||
echo "✅ Debug сборка успешна!"
|
||
else
|
||
echo "⚠️ Debug провалился, попытка 2: Profile mode..."
|
||
|
||
# Try profile mode
|
||
if timeout 2400 flutter build web --profile \
|
||
--dart-define=API_BASE_URL=https://api.mnemo-cards.online \
|
||
--no-tree-shake-icons \
|
||
--no-pub \
|
||
--suppress-analytics; then
|
||
echo "✅ Profile сборка успешна!"
|
||
else
|
||
echo "❌ Все режимы сборки провалились!"
|
||
exit 1
|
||
fi
|
||
fi
|
||
|
||
echo "✅ Сборка завершена"
|
||
|
||
echo "📊 Финальное состояние памяти:"
|
||
free -h || true
|
||
|
||
- name: Deploy Web App
|
||
run: |
|
||
echo "📦 Копируем веб-приложение..."
|
||
timeout 120 rsync -avz --delete mnemo_cards_web_v2/build/web/ \
|
||
${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }}:/var/www/mnemo_cards/ || (echo "❌ Rsync failed" && exit 1)
|
||
echo "✅ Веб-приложение скопировано"
|
||
|
||
- name: Fix Permissions and Reload Nginx
|
||
run: |
|
||
ssh -o ServerAliveInterval=60 -o ServerAliveCountMax=3 -o ConnectTimeout=30 -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} << 'ENDSSH'
|
||
# Set a global timeout for the entire step (20 minutes max)
|
||
timeout 1200 bash -c '
|
||
set -e
|
||
echo "🔧 Setting correct permissions for web files..."
|
||
chown -R www-data:www-data /var/www/mnemo_cards
|
||
chmod -R 755 /var/www/mnemo_cards
|
||
echo "✅ Permissions fixed"
|
||
|
||
# Pre-reload diagnostics
|
||
echo "🔍 Pre-reload diagnostics..."
|
||
|
||
# Check running processes BEFORE reload
|
||
echo "📊 Processes BEFORE reload:"
|
||
echo "Forgejo processes:"
|
||
pgrep -f forgejo || echo "No forgejo processes"
|
||
echo "Backend processes:"
|
||
pgrep -f mnemo_cards_server || echo "No backend processes"
|
||
echo "Nginx processes:"
|
||
pgrep -f nginx || echo "No nginx processes"
|
||
|
||
echo "📊 Current nginx status:"
|
||
sudo systemctl status nginx --no-pager || true
|
||
|
||
# Check system load and memory
|
||
echo "📊 System BEFORE reload:"
|
||
uptime || true
|
||
free -h || true
|
||
|
||
echo "🌐 Testing upstream connectivity before reload:"
|
||
if timeout 10 nc -z 127.0.0.1 3000 2>/dev/null; then
|
||
echo "✅ Forgejo (port 3000): reachable"
|
||
else
|
||
echo "❌ Forgejo (port 3000): NOT reachable before reload!"
|
||
fi
|
||
|
||
if timeout 10 nc -z 127.0.0.1 8081 2>/dev/null; then
|
||
echo "✅ Backend (port 8081): reachable"
|
||
else
|
||
echo "❌ Backend (port 8081): NOT reachable before reload!"
|
||
fi
|
||
|
||
echo "🌐 Testing site accessibility before reload:"
|
||
for site in "https://mnemo-cards.online/" "https://code.mnemo-cards.online/" "https://vscode.mnemo-cards.online/"; do
|
||
if timeout 15 curl -I --max-time 10 "$site" 2>/dev/null | grep -q "200\|301\|302"; then
|
||
echo "✅ $site accessible before reload"
|
||
else
|
||
echo "⚠️ $site not accessible before reload"
|
||
timeout 10 curl -I --max-time 5 "$site" || echo "Pre-reload check failed"
|
||
fi
|
||
done
|
||
|
||
echo "🔄 Reloading nginx to serve new web app..."
|
||
echo "📋 Current nginx configuration test:"
|
||
if timeout 30 sudo nginx -t 2>&1; then
|
||
echo "✅ nginx configuration is valid"
|
||
echo "🚀 Executing nginx reload..."
|
||
sudo systemctl reload nginx
|
||
RELOAD_RESULT=$?
|
||
if [ $RELOAD_RESULT -eq 0 ]; then
|
||
echo "✅ nginx reloaded successfully"
|
||
else
|
||
echo "❌ nginx reload failed with exit code $RELOAD_RESULT"
|
||
sudo systemctl status nginx --no-pager || true
|
||
fi
|
||
else
|
||
echo "❌ nginx configuration test failed!"
|
||
sudo nginx -t || true
|
||
exit 1
|
||
fi
|
||
|
||
# Wait for services to stabilize after reload
|
||
echo "⏳ Waiting for services to stabilize after reload..."
|
||
sleep 5
|
||
|
||
# IMMEDIATE upstream check after reload
|
||
echo "🔍 IMMEDIATE upstream check after reload (0 seconds wait):"
|
||
if timeout 5 nc -z 127.0.0.1 3000 2>/dev/null; then
|
||
echo "✅ Forgejo (port 3000): reachable immediately after reload"
|
||
else
|
||
echo "❌ CRITICAL: Forgejo (port 3000): NOT reachable immediately after reload!"
|
||
echo "🔍 Checking forgejo process:"
|
||
pgrep -f forgejo || echo "No forgejo process found"
|
||
ps aux | grep forgejo | grep -v grep || echo "No forgejo in ps"
|
||
echo "🔍 Checking forgejo service status:"
|
||
timeout 10 sudo systemctl status forgejo --no-pager || echo "Could not get forgejo status"
|
||
echo "🔍 Checking if port 3000 is bound:"
|
||
ss -tlnp | grep :3000 || echo "Port 3000 not bound"
|
||
fi
|
||
|
||
if timeout 5 nc -z 127.0.0.1 8081 2>/dev/null; then
|
||
echo "✅ Backend (port 8081): reachable immediately after reload"
|
||
else
|
||
echo "❌ CRITICAL: Backend (port 8081): NOT reachable immediately after reload!"
|
||
echo "🔍 Checking backend process:"
|
||
pgrep -f mnemo_cards_server || echo "No backend process found"
|
||
ps aux | grep mnemo_cards_server | grep -v grep || echo "No backend in ps"
|
||
echo "🔍 Checking backend service status:"
|
||
timeout 10 sudo systemctl status mnemo_cards_server --no-pager --lines=3 2>/dev/null || echo "Could not get backend status"
|
||
echo "🔍 Checking if port 8081 is bound:"
|
||
ss -tlnp | grep :8081 || echo "Port 8081 not bound"
|
||
fi
|
||
|
||
# Check for OOM killer activity
|
||
echo "🚨 Checking for OOM killer activity:"
|
||
timeout 10 sudo dmesg | grep -i "oom\|kill" | tail -5 || echo "No OOM killer activity detected"
|
||
|
||
# Check systemd logs for service events around reload time
|
||
echo "📋 Checking systemd journal for recent service events:"
|
||
timeout 10 sudo journalctl --since "5 minutes ago" --no-pager -u nginx -u mnemo_cards_server --grep "(start|stop|reload|fail)" | tail -10 || echo "No recent systemd events"
|
||
|
||
# Check nginx logs for errors after reload
|
||
echo "📋 Checking nginx error logs after reload:"
|
||
timeout 10 sudo tail -30 /var/log/nginx/error.log | grep -E "(connect|upstream|error)" | tail -10 || echo "No recent upstream errors in nginx logs"
|
||
|
||
# Check running processes AFTER reload
|
||
echo "📊 Processes AFTER reload:"
|
||
echo "Forgejo processes:"
|
||
pgrep -f forgejo || echo "No forgejo processes"
|
||
echo "Backend processes:"
|
||
pgrep -f mnemo_cards_server || echo "No backend processes"
|
||
echo "Nginx processes:"
|
||
pgrep -f nginx || echo "No nginx processes"
|
||
|
||
# Check system load and memory after reload
|
||
echo "📊 System AFTER reload:"
|
||
uptime || true
|
||
free -h || true
|
||
|
||
# Check nginx status again
|
||
echo "📊 Nginx status after reload:"
|
||
timeout 15 sudo systemctl status nginx --no-pager || echo "Could not get nginx status"
|
||
|
||
# Check all services status
|
||
echo "📊 Checking service statuses after nginx reload..."
|
||
|
||
# Check nginx
|
||
echo "🔍 Checking nginx status..."
|
||
if timeout 30 sudo systemctl is-active --quiet nginx; then
|
||
echo "✅ nginx: running"
|
||
# Check loaded configuration
|
||
echo "📋 Loaded nginx sites:"
|
||
timeout 10 ls -la /etc/nginx/sites-enabled/ || echo "Could not list sites"
|
||
else
|
||
echo "❌ nginx: not running or status check timed out"
|
||
timeout 15 sudo systemctl status nginx --no-pager || echo "Could not get nginx status"
|
||
fi
|
||
|
||
# Check backend
|
||
echo "🔍 Checking backend status..."
|
||
if timeout 30 sudo systemctl is-active --quiet mnemo_cards_server; then
|
||
echo "✅ backend: running"
|
||
else
|
||
echo "❌ backend: not running or status check timed out"
|
||
timeout 15 sudo systemctl status mnemo_cards_server --no-pager --lines=3 2>/dev/null || echo "Could not get backend status"
|
||
|
||
# Check backend logs for errors
|
||
echo "📋 Checking backend logs for recent errors:"
|
||
timeout 10 sudo journalctl -u mnemo_cards_server --since "10 minutes ago" --no-pager | grep -i "error\|fail\|crash\|oom\|kill" | tail -5 || echo "No recent backend errors"
|
||
|
||
# Try to start it if it's stopped
|
||
echo "🚀 Attempting to start backend..."
|
||
if timeout 30 sudo systemctl start mnemo_cards_server 2>&1; then
|
||
sleep 5 # Give more time to start
|
||
if timeout 30 sudo systemctl is-active --quiet mnemo_cards_server; then
|
||
echo "✅ backend: started successfully"
|
||
else
|
||
echo "❌ backend: failed to start"
|
||
timeout 15 sudo systemctl status mnemo_cards_server --no-pager --lines=3 2>/dev/null || echo "Could not get status after start attempt"
|
||
fi
|
||
else
|
||
echo "❌ backend start command timed out or failed"
|
||
sudo systemctl start mnemo_cards_server || echo "Start command failed"
|
||
fi
|
||
fi
|
||
|
||
# Check forgejo (if exists)
|
||
if sudo systemctl is-active --quiet forgejo 2>/dev/null; then
|
||
echo "✅ forgejo: running"
|
||
elif pgrep -f forgejo >/dev/null 2>&1; then
|
||
echo "✅ forgejo: running (not as service)"
|
||
else
|
||
echo "⚠️ forgejo: not found or not running"
|
||
# Check if forgejo process exists but not responding
|
||
if pgrep -f "forgejo" >/dev/null 2>&1; then
|
||
echo "🔍 Found forgejo process, but may be unresponsive"
|
||
ps aux | grep forgejo | grep -v grep || true
|
||
fi
|
||
fi
|
||
|
||
# Check for orphaned processes
|
||
echo "🔍 Checking for orphaned processes..."
|
||
# Kill any remaining flutter/dart processes that might interfere
|
||
pkill -f "flutter\|dart" || true
|
||
# Kill any remaining act processes
|
||
pkill -f "act" || true
|
||
sleep 1
|
||
|
||
# Check web app files
|
||
echo "📁 Checking web app files:"
|
||
if [ -f "/var/www/mnemo_cards/index.html" ]; then
|
||
echo "✅ index.html exists"
|
||
ls -la /var/www/mnemo_cards/ | head -10 || true
|
||
else
|
||
echo "❌ index.html not found!"
|
||
ls -la /var/www/ || true
|
||
fi
|
||
|
||
# Check system resources
|
||
echo "📊 System resources after cleanup:"
|
||
free -h || true
|
||
df -h / || true
|
||
|
||
# Check network connectivity to upstream services
|
||
echo "🌐 Checking upstream connectivity..."
|
||
if timeout 10 nc -z 127.0.0.1 3000 2>/dev/null; then
|
||
echo "✅ Forgejo (port 3000): reachable"
|
||
else
|
||
echo "⚠️ Forgejo (port 3000): not reachable or check timed out"
|
||
fi
|
||
|
||
if timeout 10 nc -z 127.0.0.1 8081 2>/dev/null; then
|
||
echo "✅ Backend (port 8081): reachable"
|
||
else
|
||
echo "⚠️ Backend (port 8081): not reachable or check timed out"
|
||
fi
|
||
|
||
# Wait a bit more before testing sites
|
||
sleep 3
|
||
|
||
# Test site accessibility with retries and timeouts
|
||
echo "🌐 Testing site accessibility..."
|
||
for site in "https://mnemo-cards.online/" "https://code.mnemo-cards.online/" "https://vscode.mnemo-cards.online/"; do
|
||
for attempt in {1..3}; do
|
||
echo "🔍 Testing $site (attempt $attempt)..."
|
||
if timeout 15 curl -I --max-time 10 "$site" 2>/dev/null | grep -q "200\|301\|302"; then
|
||
echo "✅ $site accessible (attempt $attempt)"
|
||
break
|
||
else
|
||
CURL_EXIT=$?
|
||
if [ $CURL_EXIT -eq 124 ]; then
|
||
echo "⏰ Timeout testing $site (attempt $attempt)"
|
||
else
|
||
echo "❌ $site returned error (attempt $attempt)"
|
||
fi
|
||
|
||
if [ $attempt -eq 3 ]; then
|
||
echo "❌ $site not accessible after 3 attempts"
|
||
timeout 10 curl -I --max-time 5 "$site" || echo "Final check also failed"
|
||
else
|
||
echo "⏳ $site attempt $attempt failed, retrying in 3 seconds..."
|
||
sleep 3
|
||
fi
|
||
fi
|
||
done
|
||
done
|
||
|
||
echo "✅ Web app deployment completed"
|
||
else
|
||
echo "❌ nginx configuration test failed!"
|
||
sudo nginx -t || true
|
||
exit 1
|
||
fi
|
||
' || echo "Step timed out after 20 minutes"
|
||
ENDSSH
|
||
|
||
final-verification:
|
||
name: Final Verification
|
||
runs-on: ubuntu-latest
|
||
needs: [deploy-backend, deploy-web]
|
||
if: always()
|
||
steps:
|
||
- name: Wait and Verify All Services
|
||
run: |
|
||
echo "⏳ Ждем завершения всех операций..."
|
||
sleep 10
|
||
|
||
echo "🔍 Финальная проверка доступности всех сайтов..."
|
||
ssh -o ServerAliveInterval=60 -o ServerAliveCountMax=3 -o ConnectTimeout=30 -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} << 'ENDSSH'
|
||
set -e
|
||
echo "📊 Статус всех сервисов:"
|
||
|
||
# Проверяем nginx
|
||
if sudo systemctl is-active --quiet nginx; then
|
||
echo "✅ nginx: работает"
|
||
else
|
||
echo "❌ nginx: НЕ работает!"
|
||
sudo systemctl status nginx --no-pager || true
|
||
fi
|
||
|
||
# Проверяем backend
|
||
if sudo systemctl is-active --quiet mnemo_cards_server; then
|
||
echo "✅ backend: работает"
|
||
else
|
||
echo "❌ backend: НЕ работает!"
|
||
sudo systemctl status mnemo_cards_server --no-pager --lines=3 2>/dev/null || echo "Could not get backend status"
|
||
fi
|
||
|
||
echo ""
|
||
echo "🌐 Тестируем доступность сайтов:"
|
||
|
||
# Тестируем все сайты
|
||
sites=("https://mnemo-cards.online/" "https://api.mnemo-cards.online/" "https://code.mnemo-cards.online/" "https://vscode.mnemo-cards.online/")
|
||
for site in "${sites[@]}"; do
|
||
if curl -I --max-time 10 "$site" 2>/dev/null | grep -q "200\|301\|302"; then
|
||
echo "✅ $site доступен"
|
||
else
|
||
echo "❌ $site НЕДОСТУПЕН!"
|
||
curl -I --max-time 5 "$site" || true
|
||
fi
|
||
done
|
||
|
||
echo ""
|
||
echo "🎉 Проверка завершена!"
|
||
ENDSSH
|
||
|