Repository navigation
Health Checks #1030
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Health Checks | |
| on: | |
| schedule: | |
| # 3시간마다 quick 체크 실행 | |
| - cron: '10 0,3,6,9,12,18,21 * * *' | |
| # 한국시간 00:10 = UTC 15:10, 하루 1회 full fresh 체크 실행 | |
| - cron: '10 15 * * *' | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: health-checks-${{ github.ref }} | |
| cancel-in-progress: false | |
| jobs: | |
| health-checks: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| name: Service health checks | |
| steps: | |
| - name: Run health checks | |
| env: | |
| HEALTH_CHECK_SECRET: ${{ secrets.HEALTH_CHECK_SECRET }} | |
| HEALTH_CHECK_URL: ${{ vars.HEALTH_CHECK_URL || 'https://mcp.aka.page' }} | |
| HEALTH_CHECK_SCHEDULE: ${{ github.event.schedule }} | |
| run: | | |
| if [ -z "${HEALTH_CHECK_SECRET}" ]; then | |
| echo "HEALTH_CHECK_SECRET is not set" | |
| exit 1 | |
| fi | |
| if [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ] || [ "${HEALTH_CHECK_SCHEDULE:-}" = "10 15 * * *" ]; then | |
| HEALTH_CHECK_ENDPOINT="${HEALTH_CHECK_URL}/api/health/checks?mode=full&fresh=true&includeSamples=true&timeoutMs=20000&slowThresholdMs=9000" | |
| HEALTH_CHECK_FORCE_FRESH="true" | |
| else | |
| HEALTH_CHECK_ENDPOINT="${HEALTH_CHECK_URL}/api/health/checks?mode=quick&fresh=true&timeoutMs=5000&slowThresholdMs=3000" | |
| HEALTH_CHECK_FORCE_FRESH="true" | |
| fi | |
| HEALTH_CHECK_ATTEMPT=0 | |
| run_health_check() { | |
| HEALTH_CHECK_ATTEMPT=$((HEALTH_CHECK_ATTEMPT + 1)) | |
| rm -f health-checks.json | |
| if ! curl -fsS --connect-timeout 10 --max-time 240 \ | |
| -H "Authorization: Bearer ${HEALTH_CHECK_SECRET}" \ | |
| -H "x-health-check-force-fresh: ${HEALTH_CHECK_FORCE_FRESH}" \ | |
| "${HEALTH_CHECK_ENDPOINT}" \ | |
| -o health-checks.json; then | |
| echo "Health check HTTP request failed on attempt ${HEALTH_CHECK_ATTEMPT}" > health-check-summary.txt | |
| cp health-check-summary.txt "health-checks-attempt-${HEALTH_CHECK_ATTEMPT}.txt" | |
| return 1 | |
| fi | |
| cp health-checks.json "health-checks-attempt-${HEALTH_CHECK_ATTEMPT}.json" || return 1 | |
| node <<'NODE' | |
| const fs = require('node:fs'); | |
| let payload; | |
| try { payload = JSON.parse(fs.readFileSync('health-checks.json', 'utf8')); } catch {} | |
| const statuses = ['ok', 'degraded', 'fail']; | |
| const valid = payload && statuses.includes(payload.status) && | |
| Array.isArray(payload.checks) && payload.checks.length > 0 && | |
| payload.checks.every(check => check && typeof check.id === 'string' && check.id.length > 0 && | |
| [...statuses, 'skipped'].includes(check.status) && typeof check.message === 'string') && | |
| new Set(payload.checks.map(check => check.id)).size === payload.checks.length && | |
| payload.status === (payload.checks.some(check => check.status === 'fail') ? 'fail' : | |
| payload.checks.some(check => check.status === 'degraded') ? 'degraded' : 'ok'); | |
| if (!valid) { | |
| const summary = 'Health check returned an invalid or inconsistent payload'; | |
| fs.writeFileSync('health-check-summary.txt', summary); | |
| fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary + '\n'); | |
| console.log(summary); | |
| process.exitCode = 1; | |
| } else { | |
| const failedChecks = (payload.checks || []).filter((check) => check.status === 'fail'); | |
| const degradedChecks = (payload.checks || []).filter((check) => check.status === 'degraded'); | |
| const lines = [...failedChecks, ...degradedChecks].map((check) => { | |
| const sample = check.sample?.first ? ` sample=${check.sample.first}` : ''; | |
| return `${check.id}:${check.status}:${check.message}${sample}`; | |
| }); | |
| const runUrl = `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`; | |
| const summary = [ | |
| `status=${payload.status || 'unknown'}`, | |
| `failed=${failedChecks.length}`, | |
| `degraded=${degradedChecks.length}`, | |
| `failedIds=${failedChecks.map((check) => check.id).join(',') || 'none'}`, | |
| `degradedIds=${degradedChecks.map((check) => check.id).join(',') || 'none'}`, | |
| `run=${runUrl}`, | |
| lines.join(' | '), | |
| ].filter(Boolean).join(' '); | |
| fs.writeFileSync('health-check-summary.txt', summary); | |
| fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, `## Service health checks\n\n${summary}\n\n`); | |
| console.log(summary); | |
| if (failedChecks.length > 0 || payload.status === 'fail') { | |
| process.exitCode = 1; | |
| } | |
| } | |
| NODE | |
| } | |
| if ! run_health_check; then | |
| echo "Health check failed on attempt 1; retrying once" | |
| sleep 10 | |
| HEALTH_CHECK_ENDPOINT="${HEALTH_CHECK_ENDPOINT}&cacheBust=true" | |
| run_health_check | |
| fi | |
| - name: Preserve health check attempts | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: health-checks-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: | | |
| health-checks-attempt-*.json | |
| health-checks-attempt-*.txt | |
| health-check-summary.txt | |
| retention-days: 7 | |
| if-no-files-found: warn | |
| - name: Notify health failure | |
| if: failure() && github.event_name == 'schedule' && github.event.schedule == '10 15 * * *' | |
| env: | |
| MOSHI_WEBHOOK_TOKEN: ${{ secrets.MOSHI_WEBHOOK_TOKEN }} | |
| run: | | |
| if [ -z "${MOSHI_WEBHOOK_TOKEN}" ]; then | |
| echo "MOSHI_WEBHOOK_TOKEN is not set; skipping failure notification" | |
| exit 0 | |
| fi | |
| SUMMARY="$(cat health-check-summary.txt 2>/dev/null || echo 'health check command failed')" node <<'NODE' | |
| const fs = require('node:fs'); | |
| fs.writeFileSync( | |
| 'moshi-payload.json', | |
| JSON.stringify({ | |
| token: process.env.MOSHI_WEBHOOK_TOKEN, | |
| title: 'Health Checks Failed', | |
| message: `${process.env.GITHUB_REPOSITORY}: ${process.env.SUMMARY || ''}`.slice(0, 3500), | |
| }), | |
| ); | |
| NODE | |
| curl --fail-with-body --silent --show-error \ | |
| -X POST https://api.getmoshi.app/api/webhook \ | |
| -H "Content-Type: application/json" \ | |
| --data-binary @moshi-payload.json |