ci: proper Agiliton deployment with health checks, rollback, smoke tests

- docker-compose.yml in repo (replaces inline generation in CI) - /health nginx endpoint for container health checks - HEALTHCHECK directive in Dockerfile - CI pipeline: build → deploy → health check → smoke test → rollback on failure - Smoke tests verify /health, /en/, /de/ after deploy - Automatic rollback to previous image on health/smoke failure - workflow_dispatch with force_deploy option - Deploy-guard audit logging IFK-6 Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-02 21:00:34 +03:00
parent 8da917618a
commit ff9f6c1b7f
4 changed files with 227 additions and 29 deletions
--- a/.gitea/workflows/deploy.yml
+++ b/.gitea/workflows/deploy.yml
@@ -1,50 +1,223 @@
-name: Build & Deploy
+# Internet for Kids - Production Deployment
+# Deploys to infra VM at /opt/apps/internetforkids
+#
+# Features:
+# - Docker registry-based deployment (build → push → pull)
+# - Health check with automatic rollback on failure
+# - Smoke tests against public URLs
+# - Deploy-guard audit logging
+
+name: Deploy Internet for Kids
+
 on:
  push:
    branches: [main]
-    paths-ignore: ['**.md']
+    paths-ignore:
+      - '**.md'
+      - 'docs/**'
+  workflow_dispatch:
+    inputs:
+      force_deploy:
+        description: 'Force deploy even if health checks fail'
+        required: false
+        default: 'false'
+
 env:
+  PROJECT_NAME: "internetforkids"
  REGISTRY: gitea.agiliton.internal:3000
  IMAGE: gitea.agiliton.internal:3000/christian/internetforkids
-  TARGET_VM: infra.agiliton.internal
+  REMOTE_HOST: infra.agiliton.internal
+  REMOTE_USER: root
  DEPLOY_PATH: /opt/apps/internetforkids
+  HEALTH_ENDPOINT: "http://localhost:3006/health"
+  PUBLIC_URL: "https://internetforkids.ong"
+  HEALTH_TIMEOUT: 60
+
 jobs:
-  build-and-deploy:
+  build-and-push:
+    name: Build & Push
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
+
+      - name: Login to registry
+        run: |
+          echo "${{ secrets.REGISTRY_TOKEN }}" | docker login ${{ env.REGISTRY }} -u christian --password-stdin
+
+      - name: Build and push
+        run: |
+          DOCKER_BUILDKIT=1 docker build --pull \
+            -t ${{ env.IMAGE }}:${{ github.sha }} \
+            -t ${{ env.IMAGE }}:latest \
+            .
+          docker push ${{ env.IMAGE }}:${{ github.sha }}
+          docker push ${{ env.IMAGE }}:latest
+
+      - name: Cleanup
+        if: always()
+        run: docker builder prune -f --filter "until=24h" 2>/dev/null || true
+
+  deploy:
+    name: Deploy
+    runs-on: ubuntu-latest
+    needs: build-and-push
+    outputs:
+      previous_image: ${{ steps.deploy.outputs.previous_image }}
+    steps:
+      - uses: actions/checkout@v4
+
      - name: Setup SSH
        run: |
          mkdir -p ~/.ssh && chmod 700 ~/.ssh
          echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_ed25519
          chmod 600 ~/.ssh/id_ed25519
-          ssh-keyscan -H ${{ env.TARGET_VM }} >> ~/.ssh/known_hosts 2>/dev/null || true
-      - name: Login & Build & Push
-        run: |
-          echo "${{ secrets.REGISTRY_TOKEN }}" | docker login ${{ env.REGISTRY }} -u christian --password-stdin
-          DOCKER_BUILDKIT=1 docker build --pull -t ${{ env.IMAGE }}:latest .
-          docker push ${{ env.IMAGE }}:latest
+          ssh-keyscan -H ${{ env.REMOTE_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true
+
      - name: Deploy
+        id: deploy
        run: |
-          ssh root@${{ env.TARGET_VM }} << 'EOF'
-          mkdir -p /opt/apps/internetforkids
-          cd /opt/apps/internetforkids
-          cat > docker-compose.yml << 'COMPOSE'
-          services:
-            internetforkids:
-              image: gitea.agiliton.internal:3000/christian/internetforkids:latest
-              container_name: internetforkids
-              restart: unless-stopped
-              ports:
-                - "3006:80"
-              labels:
-                - "com.centurylinklabs.watchtower.enable=true"
-          COMPOSE
-          docker pull gitea.agiliton.internal:3000/christian/internetforkids:latest
-          docker compose up -d --force-recreate --remove-orphans
+          # Capture current image digest for rollback
+          PREVIOUS=$(ssh ${{ env.REMOTE_USER }}@${{ env.REMOTE_HOST }} \
+            "docker inspect --format='{{.Image}}' internetforkids 2>/dev/null || echo 'none'")
+          echo "previous_image=$PREVIOUS" >> $GITHUB_OUTPUT
+          echo "Previous image: $PREVIOUS"
+
+          # Sync docker-compose.yml and deploy
+          scp docker-compose.yml ${{ env.REMOTE_USER }}@${{ env.REMOTE_HOST }}:${{ env.DEPLOY_PATH }}/docker-compose.yml
+
+          ssh ${{ env.REMOTE_USER }}@${{ env.REMOTE_HOST }} << 'EOF'
+            cd /opt/apps/internetforkids
+            docker pull gitea.agiliton.internal:3000/christian/internetforkids:latest
+            docker compose up -d --force-recreate --remove-orphans
          EOF
-      - name: Cleanup
-        if: always()
-        run: docker builder prune -f --filter "until=24h" 2>/dev/null || true
+
+          echo "Deployment initiated"
+
+  health-check:
+    name: Health Check
+    runs-on: ubuntu-latest
+    needs: deploy
+    steps:
+      - name: Setup SSH
+        run: |
+          mkdir -p ~/.ssh && chmod 700 ~/.ssh
+          echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+          ssh-keyscan -H ${{ env.REMOTE_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true
+
+      - name: Wait for health
+        run: |
+          echo "Waiting for health (timeout: ${HEALTH_TIMEOUT}s)..."
+          ELAPSED=0
+          while [ $ELAPSED -lt $HEALTH_TIMEOUT ]; do
+            if ssh ${{ env.REMOTE_USER }}@${{ env.REMOTE_HOST }} \
+              "curl -sf ${{ env.HEALTH_ENDPOINT }}" >/dev/null 2>&1; then
+              echo "Health check passed!"
+              exit 0
+            fi
+            echo "Waiting... ($ELAPSED/${HEALTH_TIMEOUT}s)"
+            sleep 10
+            ELAPSED=$((ELAPSED + 10))
+          done
+
+          echo "::error::Health check timeout after ${HEALTH_TIMEOUT}s"
+          ssh ${{ env.REMOTE_USER }}@${{ env.REMOTE_HOST }} \
+            "cd ${{ env.DEPLOY_PATH }} && docker compose ps && docker compose logs --tail 20" || true
+
+          if [ "${{ github.event.inputs.force_deploy }}" = "true" ]; then
+            echo "::warning::Force deploy — continuing despite health failure"
+            exit 0
+          fi
+          exit 1
+
+  smoke-test:
+    name: Smoke Tests
+    runs-on: ubuntu-latest
+    needs: health-check
+    steps:
+      - name: Setup SSH
+        run: |
+          mkdir -p ~/.ssh && chmod 700 ~/.ssh
+          echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+          ssh-keyscan -H ${{ env.REMOTE_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true
+
+      - name: Test key URLs
+        run: |
+          FAILED=0
+          for path in /health /en/ /de/; do
+            STATUS=$(ssh ${{ env.REMOTE_USER }}@${{ env.REMOTE_HOST }} \
+              "curl -so /dev/null -w '%{http_code}' http://localhost:3006${path}" 2>/dev/null || echo "000")
+            if [ "$STATUS" = "200" ] || [ "$STATUS" = "302" ]; then
+              echo "OK  $path → $STATUS"
+            else
+              echo "FAIL $path → $STATUS"
+              FAILED=$((FAILED + 1))
+            fi
+          done
+
+          if [ $FAILED -gt 0 ]; then
+            echo "::error::$FAILED smoke test(s) failed"
+            exit 1
+          fi
+          echo "All smoke tests passed"
+
+  rollback:
+    name: Rollback
+    runs-on: ubuntu-latest
+    needs: [deploy, health-check, smoke-test]
+    if: failure() && needs.deploy.outputs.previous_image != 'none' && needs.deploy.outputs.previous_image != ''
+    steps:
+      - name: Setup SSH
+        run: |
+          mkdir -p ~/.ssh && chmod 700 ~/.ssh
+          echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+          ssh-keyscan -H ${{ env.REMOTE_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true
+
+      - name: Rollback to previous image
+        run: |
+          PREVIOUS="${{ needs.deploy.outputs.previous_image }}"
+          echo "::warning::Rolling back to image $PREVIOUS"
+
+          ssh ${{ env.REMOTE_USER }}@${{ env.REMOTE_HOST }} << EOF
+            cd ${{ env.DEPLOY_PATH }}
+            docker tag $PREVIOUS ${{ env.IMAGE }}:latest
+            docker compose up -d --force-recreate
+          EOF
+
+      - name: Verify rollback
+        run: |
+          sleep 10
+          if ssh ${{ env.REMOTE_USER }}@${{ env.REMOTE_HOST }} \
+            "curl -sf ${{ env.HEALTH_ENDPOINT }}" >/dev/null 2>&1; then
+            echo "Rollback successful"
+          else
+            echo "::error::Rollback may have failed — check manually"
+          fi
+
+  audit:
+    name: Audit
+    runs-on: ubuntu-latest
+    needs: [deploy, health-check, smoke-test, rollback]
+    if: always()
+    steps:
+      - name: Setup SSH
+        run: |
+          mkdir -p ~/.ssh && chmod 700 ~/.ssh
+          echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+          ssh-keyscan -H ${{ env.REMOTE_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true
+
+      - name: Log deployment
+        run: |
+          STATUS="success"
+          [ "${{ needs.health-check.result }}" = "failure" ] && STATUS="failed"
+          [ "${{ needs.smoke-test.result }}" = "failure" ] && STATUS="failed"
+          [ "${{ needs.rollback.result }}" = "success" ] && STATUS="rolled_back"
+
+          echo "${{ env.PROJECT_NAME }} deployment: $STATUS (sha: ${{ github.sha }})"
+          ssh ${{ env.REMOTE_USER }}@${{ env.REMOTE_HOST }} \
+            "/opt/scripts/deploy-guard.sh check 'ci-deploy ${{ env.PROJECT_NAME }} $STATUS'" 2>/dev/null || true
--- a/3
+++ b/3
@@ -4,6 +4,9 @@ WORKDIR /src
 RUN hugo --minify --destination /output

 FROM nginx:alpine
+RUN apk add --no-cache curl
 COPY --from=builder /output /usr/share/nginx/html
 COPY nginx.conf /etc/nginx/conf.d/default.conf
 EXPOSE 80
+HEALTHCHECK --interval=30s --timeout=5s --retries=3 \
+  CMD curl -f http://localhost:80/health || exit 1
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -0,0 +1,15 @@
+services:
+  internetforkids:
+    image: gitea.agiliton.internal:3000/christian/internetforkids:latest
+    container_name: internetforkids
+    restart: unless-stopped
+    ports:
+      - "3006:80"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:80/health"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 10s
+    labels:
+      - "com.centurylinklabs.watchtower.enable=true"
--- a/nginx.conf
+++ b/nginx.conf
@@ -27,6 +27,13 @@ server {
        add_header Cache-Control "public, must-revalidate";
    }

+    # Health check endpoint
+    location /health {
+        access_log off;
+        return 200 "ok";
+        add_header Content-Type text/plain;
+    }
+
    # Redirect unprefixed paths to /en/
    location ~ ^/(?!en/|de/|css/|js/|img/|favicon|android|apple|site|llms) {
        return 302 /en$request_uri;