From ff9f6c1b7fcf202cefc54d8e341c06e0f816f472 Mon Sep 17 00:00:00 2001 From: Christian Gick Date: Thu, 2 Apr 2026 21:00:34 +0300 Subject: [PATCH] ci: proper Agiliton deployment with health checks, rollback, smoke tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - docker-compose.yml in repo (replaces inline generation in CI) - /health nginx endpoint for container health checks - HEALTHCHECK directive in Dockerfile - CI pipeline: build → deploy → health check → smoke test → rollback on failure - Smoke tests verify /health, /en/, /de/ after deploy - Automatic rollback to previous image on health/smoke failure - workflow_dispatch with force_deploy option - Deploy-guard audit logging IFK-6 Co-Authored-By: Claude Opus 4.6 (1M context) --- .gitea/workflows/deploy.yml | 231 +++++++++++++++++++++++++++++++----- Dockerfile | 3 + docker-compose.yml | 15 +++ nginx.conf | 7 ++ 4 files changed, 227 insertions(+), 29 deletions(-) create mode 100644 docker-compose.yml diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 72d6a27..ea3deb0 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -1,50 +1,223 @@ -name: Build & Deploy +# Internet for Kids - Production Deployment +# Deploys to infra VM at /opt/apps/internetforkids +# +# Features: +# - Docker registry-based deployment (build → push → pull) +# - Health check with automatic rollback on failure +# - Smoke tests against public URLs +# - Deploy-guard audit logging + +name: Deploy Internet for Kids + on: push: branches: [main] - paths-ignore: ['**.md'] + paths-ignore: + - '**.md' + - 'docs/**' + workflow_dispatch: + inputs: + force_deploy: + description: 'Force deploy even if health checks fail' + required: false + default: 'false' + env: + PROJECT_NAME: "internetforkids" REGISTRY: gitea.agiliton.internal:3000 IMAGE: gitea.agiliton.internal:3000/christian/internetforkids - TARGET_VM: infra.agiliton.internal + REMOTE_HOST: infra.agiliton.internal + REMOTE_USER: root DEPLOY_PATH: /opt/apps/internetforkids + HEALTH_ENDPOINT: "http://localhost:3006/health" + PUBLIC_URL: "https://internetforkids.ong" + HEALTH_TIMEOUT: 60 + jobs: - build-and-deploy: + build-and-push: + name: Build & Push runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: submodules: true + + - name: Login to registry + run: | + echo "${{ secrets.REGISTRY_TOKEN }}" | docker login ${{ env.REGISTRY }} -u christian --password-stdin + + - name: Build and push + run: | + DOCKER_BUILDKIT=1 docker build --pull \ + -t ${{ env.IMAGE }}:${{ github.sha }} \ + -t ${{ env.IMAGE }}:latest \ + . + docker push ${{ env.IMAGE }}:${{ github.sha }} + docker push ${{ env.IMAGE }}:latest + + - name: Cleanup + if: always() + run: docker builder prune -f --filter "until=24h" 2>/dev/null || true + + deploy: + name: Deploy + runs-on: ubuntu-latest + needs: build-and-push + outputs: + previous_image: ${{ steps.deploy.outputs.previous_image }} + steps: + - uses: actions/checkout@v4 + - name: Setup SSH run: | mkdir -p ~/.ssh && chmod 700 ~/.ssh echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_ed25519 chmod 600 ~/.ssh/id_ed25519 - ssh-keyscan -H ${{ env.TARGET_VM }} >> ~/.ssh/known_hosts 2>/dev/null || true - - name: Login & Build & Push - run: | - echo "${{ secrets.REGISTRY_TOKEN }}" | docker login ${{ env.REGISTRY }} -u christian --password-stdin - DOCKER_BUILDKIT=1 docker build --pull -t ${{ env.IMAGE }}:latest . - docker push ${{ env.IMAGE }}:latest + ssh-keyscan -H ${{ env.REMOTE_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true + - name: Deploy + id: deploy run: | - ssh root@${{ env.TARGET_VM }} << 'EOF' - mkdir -p /opt/apps/internetforkids - cd /opt/apps/internetforkids - cat > docker-compose.yml << 'COMPOSE' - services: - internetforkids: - image: gitea.agiliton.internal:3000/christian/internetforkids:latest - container_name: internetforkids - restart: unless-stopped - ports: - - "3006:80" - labels: - - "com.centurylinklabs.watchtower.enable=true" - COMPOSE - docker pull gitea.agiliton.internal:3000/christian/internetforkids:latest - docker compose up -d --force-recreate --remove-orphans + # Capture current image digest for rollback + PREVIOUS=$(ssh ${{ env.REMOTE_USER }}@${{ env.REMOTE_HOST }} \ + "docker inspect --format='{{.Image}}' internetforkids 2>/dev/null || echo 'none'") + echo "previous_image=$PREVIOUS" >> $GITHUB_OUTPUT + echo "Previous image: $PREVIOUS" + + # Sync docker-compose.yml and deploy + scp docker-compose.yml ${{ env.REMOTE_USER }}@${{ env.REMOTE_HOST }}:${{ env.DEPLOY_PATH }}/docker-compose.yml + + ssh ${{ env.REMOTE_USER }}@${{ env.REMOTE_HOST }} << 'EOF' + cd /opt/apps/internetforkids + docker pull gitea.agiliton.internal:3000/christian/internetforkids:latest + docker compose up -d --force-recreate --remove-orphans EOF - - name: Cleanup - if: always() - run: docker builder prune -f --filter "until=24h" 2>/dev/null || true + + echo "Deployment initiated" + + health-check: + name: Health Check + runs-on: ubuntu-latest + needs: deploy + steps: + - name: Setup SSH + run: | + mkdir -p ~/.ssh && chmod 700 ~/.ssh + echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + ssh-keyscan -H ${{ env.REMOTE_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true + + - name: Wait for health + run: | + echo "Waiting for health (timeout: ${HEALTH_TIMEOUT}s)..." + ELAPSED=0 + while [ $ELAPSED -lt $HEALTH_TIMEOUT ]; do + if ssh ${{ env.REMOTE_USER }}@${{ env.REMOTE_HOST }} \ + "curl -sf ${{ env.HEALTH_ENDPOINT }}" >/dev/null 2>&1; then + echo "Health check passed!" + exit 0 + fi + echo "Waiting... ($ELAPSED/${HEALTH_TIMEOUT}s)" + sleep 10 + ELAPSED=$((ELAPSED + 10)) + done + + echo "::error::Health check timeout after ${HEALTH_TIMEOUT}s" + ssh ${{ env.REMOTE_USER }}@${{ env.REMOTE_HOST }} \ + "cd ${{ env.DEPLOY_PATH }} && docker compose ps && docker compose logs --tail 20" || true + + if [ "${{ github.event.inputs.force_deploy }}" = "true" ]; then + echo "::warning::Force deploy — continuing despite health failure" + exit 0 + fi + exit 1 + + smoke-test: + name: Smoke Tests + runs-on: ubuntu-latest + needs: health-check + steps: + - name: Setup SSH + run: | + mkdir -p ~/.ssh && chmod 700 ~/.ssh + echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + ssh-keyscan -H ${{ env.REMOTE_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true + + - name: Test key URLs + run: | + FAILED=0 + for path in /health /en/ /de/; do + STATUS=$(ssh ${{ env.REMOTE_USER }}@${{ env.REMOTE_HOST }} \ + "curl -so /dev/null -w '%{http_code}' http://localhost:3006${path}" 2>/dev/null || echo "000") + if [ "$STATUS" = "200" ] || [ "$STATUS" = "302" ]; then + echo "OK $path → $STATUS" + else + echo "FAIL $path → $STATUS" + FAILED=$((FAILED + 1)) + fi + done + + if [ $FAILED -gt 0 ]; then + echo "::error::$FAILED smoke test(s) failed" + exit 1 + fi + echo "All smoke tests passed" + + rollback: + name: Rollback + runs-on: ubuntu-latest + needs: [deploy, health-check, smoke-test] + if: failure() && needs.deploy.outputs.previous_image != 'none' && needs.deploy.outputs.previous_image != '' + steps: + - name: Setup SSH + run: | + mkdir -p ~/.ssh && chmod 700 ~/.ssh + echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + ssh-keyscan -H ${{ env.REMOTE_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true + + - name: Rollback to previous image + run: | + PREVIOUS="${{ needs.deploy.outputs.previous_image }}" + echo "::warning::Rolling back to image $PREVIOUS" + + ssh ${{ env.REMOTE_USER }}@${{ env.REMOTE_HOST }} << EOF + cd ${{ env.DEPLOY_PATH }} + docker tag $PREVIOUS ${{ env.IMAGE }}:latest + docker compose up -d --force-recreate + EOF + + - name: Verify rollback + run: | + sleep 10 + if ssh ${{ env.REMOTE_USER }}@${{ env.REMOTE_HOST }} \ + "curl -sf ${{ env.HEALTH_ENDPOINT }}" >/dev/null 2>&1; then + echo "Rollback successful" + else + echo "::error::Rollback may have failed — check manually" + fi + + audit: + name: Audit + runs-on: ubuntu-latest + needs: [deploy, health-check, smoke-test, rollback] + if: always() + steps: + - name: Setup SSH + run: | + mkdir -p ~/.ssh && chmod 700 ~/.ssh + echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + ssh-keyscan -H ${{ env.REMOTE_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true + + - name: Log deployment + run: | + STATUS="success" + [ "${{ needs.health-check.result }}" = "failure" ] && STATUS="failed" + [ "${{ needs.smoke-test.result }}" = "failure" ] && STATUS="failed" + [ "${{ needs.rollback.result }}" = "success" ] && STATUS="rolled_back" + + echo "${{ env.PROJECT_NAME }} deployment: $STATUS (sha: ${{ github.sha }})" + ssh ${{ env.REMOTE_USER }}@${{ env.REMOTE_HOST }} \ + "/opt/scripts/deploy-guard.sh check 'ci-deploy ${{ env.PROJECT_NAME }} $STATUS'" 2>/dev/null || true diff --git a/Dockerfile b/Dockerfile index dbb8305..3755109 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,9 @@ WORKDIR /src RUN hugo --minify --destination /output FROM nginx:alpine +RUN apk add --no-cache curl COPY --from=builder /output /usr/share/nginx/html COPY nginx.conf /etc/nginx/conf.d/default.conf EXPOSE 80 +HEALTHCHECK --interval=30s --timeout=5s --retries=3 \ + CMD curl -f http://localhost:80/health || exit 1 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..15906d5 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,15 @@ +services: + internetforkids: + image: gitea.agiliton.internal:3000/christian/internetforkids:latest + container_name: internetforkids + restart: unless-stopped + ports: + - "3006:80" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:80/health"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s + labels: + - "com.centurylinklabs.watchtower.enable=true" diff --git a/nginx.conf b/nginx.conf index 4b311f3..1a6892e 100644 --- a/nginx.conf +++ b/nginx.conf @@ -27,6 +27,13 @@ server { add_header Cache-Control "public, must-revalidate"; } + # Health check endpoint + location /health { + access_log off; + return 200 "ok"; + add_header Content-Type text/plain; + } + # Redirect unprefixed paths to /en/ location ~ ^/(?!en/|de/|css/|js/|img/|favicon|android|apple|site|llms) { return 302 /en$request_uri;