Repository navigation
Health Checks #1219
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Health Checks | |
| on: | |
| schedule: | |
| # 3시간마다 quick 체크 실행 | |
| - cron: '10 0,3,6,9,12,18,21 * * *' | |
| # 한국시간 00:10 = UTC 15:10, 하루 1회 full fresh 체크 실행 | |
| - cron: '10 15 * * *' | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: health-checks-${{ github.ref }} | |
| cancel-in-progress: false | |
| jobs: | |
| health-checks: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| name: Service health checks | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - uses: actions/setup-node@v7 | |
| with: | |
| node-version: '24' | |
| - name: Run health checks | |
| env: | |
| HEALTH_CHECK_SECRET: ${{ secrets.HEALTH_CHECK_SECRET }} | |
| HEALTH_CHECK_URL: ${{ vars.HEALTH_CHECK_URL || 'https://mcp.aka.page' }} | |
| HEALTH_CHECK_SCHEDULE: ${{ github.event.schedule }} | |
| run: | | |
| if [ -z "${HEALTH_CHECK_SECRET}" ]; then | |
| echo "HEALTH_CHECK_SECRET is not set" | |
| exit 1 | |
| fi | |
| if [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ] || [ "${HEALTH_CHECK_SCHEDULE:-}" = "10 15 * * *" ]; then | |
| HEALTH_CHECK_ENDPOINT="${HEALTH_CHECK_URL}/api/health/checks?mode=full&fresh=true&includeSamples=true&timeoutMs=20000&slowThresholdMs=9000" | |
| HEALTH_CHECK_FORCE_FRESH="true" | |
| else | |
| HEALTH_CHECK_ENDPOINT="${HEALTH_CHECK_URL}/api/health/checks?mode=quick&fresh=true&timeoutMs=5000&slowThresholdMs=3000" | |
| HEALTH_CHECK_FORCE_FRESH="true" | |
| fi | |
| HEALTH_CHECK_ATTEMPT=0 | |
| run_health_check() { | |
| HEALTH_CHECK_ATTEMPT=$((HEALTH_CHECK_ATTEMPT + 1)) | |
| rm -f health-checks.json | |
| if ! curl -fsS --connect-timeout 10 --max-time 240 \ | |
| -H "Authorization: Bearer ${HEALTH_CHECK_SECRET}" \ | |
| -H "x-health-check-force-fresh: ${HEALTH_CHECK_FORCE_FRESH}" \ | |
| "${HEALTH_CHECK_ENDPOINT}" \ | |
| -o health-checks.json; then | |
| echo "Health check HTTP request failed on attempt ${HEALTH_CHECK_ATTEMPT}" > health-check-summary.txt | |
| cp health-check-summary.txt "health-checks-attempt-${HEALTH_CHECK_ATTEMPT}.txt" | |
| return 1 | |
| fi | |
| cp health-checks.json "health-checks-attempt-${HEALTH_CHECK_ATTEMPT}.json" || return 1 | |
| HEALTH_CHECK_ENDPOINT="${HEALTH_CHECK_ENDPOINT}" node <<'NODE' | |
| const fs = require('node:fs'); | |
| (async () => { | |
| const { isValidHealthPayload, retryMinuteChecks } = await import('./scripts/ops/health-check-retry.ts'); | |
| let payload; | |
| try { payload = JSON.parse(fs.readFileSync('health-checks.json', 'utf8')); } catch {} | |
| const valid = isValidHealthPayload(payload); | |
| if (!valid) { | |
| const summary = 'Health check returned an invalid or inconsistent payload'; | |
| fs.writeFileSync('health-check-summary.txt', summary); | |
| fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary + '\n'); | |
| console.log(summary); | |
| process.exitCode = 1; | |
| } else { | |
| const firstFailures = payload.checks.filter(check => check.status === 'fail'); | |
| fs.writeFileSync('health-check-summary-first.txt', JSON.stringify({ status: payload.status, failedChecks: firstFailures }, null, 2)); | |
| const recovery = await retryMinuteChecks(payload, { endpoint: process.env.HEALTH_CHECK_ENDPOINT, secret: process.env.HEALTH_CHECK_SECRET }); | |
| for (const attempt of recovery.attempts) { | |
| fs.writeFileSync(`health-checks-recheck-${attempt.id}.json`, JSON.stringify(attempt, null, 2)); | |
| } | |
| payload = recovery.payload; | |
| fs.writeFileSync('health-checks.json', JSON.stringify(payload, null, 2)); | |
| const failedChecks = (payload.checks || []).filter((check) => check.status === 'fail'); | |
| const degradedChecks = (payload.checks || []).filter((check) => check.status === 'degraded'); | |
| const lines = [...failedChecks, ...degradedChecks].map((check) => { | |
| const sample = check.sample?.first ? ` sample=${check.sample.first}` : ''; | |
| const diagnostics = [check.errorCode, check.upstreamStatus === undefined ? '' : `upstreamStatus=${check.upstreamStatus}`, | |
| check.quotaReason ? `quotaReason=${check.quotaReason}` : '', check.retryAfter ? `retryAfter=${check.retryAfter}s` : ''].filter(Boolean).join(' '); | |
| return `${check.id}:${check.status}:${check.message}${sample}${diagnostics ? ' ' + diagnostics : ''}`; | |
| }); | |
| const runUrl = `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`; | |
| const summary = [ | |
| `status=${payload.status || 'unknown'}`, | |
| `failed=${failedChecks.length}`, | |
| `degraded=${degradedChecks.length}`, | |
| `failedIds=${failedChecks.map((check) => check.id).join(',') || 'none'}`, | |
| `degradedIds=${degradedChecks.map((check) => check.id).join(',') || 'none'}`, | |
| `firstFailedIds=${firstFailures.map(check => check.id).join(',') || 'none'}`, | |
| `retryCount=${recovery.attempts.length}`, | |
| `recheckedIds=${recovery.attempts.map(attempt => attempt.id).join(',') || 'none'}`, | |
| `run=${runUrl}`, | |
| lines.join(' | '), | |
| ].filter(Boolean).join(' '); | |
| fs.writeFileSync('health-check-summary.txt', summary); | |
| fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, `## Service health checks\n\n${summary}\n\n`); | |
| console.log(summary); | |
| if (failedChecks.length > 0 || payload.status === 'fail') { | |
| process.exitCode = 1; | |
| } | |
| } | |
| })().catch(() => { | |
| const summary = 'Health check retry runner failed; original attempts remain available'; | |
| fs.writeFileSync('health-check-summary.txt', summary); | |
| fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary + '\n'); | |
| process.exitCode = 1; | |
| }); | |
| NODE | |
| } | |
| # 실패 직후 전체 검사를 반복하면 정상 경로까지 중계 한도를 다시 소비합니다. | |
| # 검증된 자체 분당 제한과 명시적 올리브영 원본 429만 대기 후 해당 ID를 한 번 확인합니다. | |
| run_health_check | |
| - name: Preserve health check attempts | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: health-checks-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: | | |
| health-checks-attempt-*.json | |
| health-checks-attempt-*.txt | |
| health-check-summary.txt | |
| health-check-summary-first.txt | |
| health-checks-recheck-*.json | |
| health-checks.json | |
| retention-days: 7 | |
| if-no-files-found: warn | |
| - name: Notify health failure | |
| if: failure() && github.event_name == 'schedule' && github.event.schedule == '10 15 * * *' | |
| env: | |
| MOSHI_WEBHOOK_TOKEN: ${{ secrets.MOSHI_WEBHOOK_TOKEN }} | |
| run: | | |
| if [ -z "${MOSHI_WEBHOOK_TOKEN}" ]; then | |
| echo "MOSHI_WEBHOOK_TOKEN is not set; skipping failure notification" | |
| exit 0 | |
| fi | |
| SUMMARY="$(cat health-check-summary.txt 2>/dev/null || echo 'health check command failed')" node <<'NODE' | |
| const fs = require('node:fs'); | |
| fs.writeFileSync( | |
| 'moshi-payload.json', | |
| JSON.stringify({ | |
| token: process.env.MOSHI_WEBHOOK_TOKEN, | |
| title: 'Health Checks Failed', | |
| message: `${process.env.GITHUB_REPOSITORY}: ${process.env.SUMMARY || ''}`.slice(0, 3500), | |
| }), | |
| ); | |
| NODE | |
| curl --fail-with-body --silent --show-error \ | |
| -X POST https://api.getmoshi.app/api/webhook \ | |
| -H "Content-Type: application/json" \ | |
| --data-binary @moshi-payload.json |