Repository navigation
Expand file tree
/
Copy pathbackup.ts
More file actions
287 lines (264 loc) · 11.4 KB
/
Copy pathbackup.ts
File metadata and controls
287 lines (264 loc) · 11.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
<
8000
div data-line-number="240" class="react-line-number react-code-text" style="padding-right:16px">240241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
/**
* Take a database backup, verify it is readable, and record that it happened.
*
* DATABASE_URL="<url>" pnpm tsx scripts/backup.ts --out backups/
*
* Designed to run unattended from CI. Three properties matter more than speed:
*
* 1. It VERIFIES what it wrote. A backup nobody has read back is a file, not a
* backup. This one re-reads the dump, checks the objects it must contain,
* and refuses to report success on a truncated or empty file.
* 2. It RECORDS the run in `job_runs`, the same table the retention sweep uses.
* That is what makes a missing backup visible: the admin overview reads the
* newest row's age, so a job that silently stops running turns the console
* amber without anyone having to remember to check.
* 3. It FAILS LOUDLY. A non-zero exit is what makes CI send the email. A backup
* script that swallows its own errors is worse than none, because it
* produces the paperwork of safety without the safety.
*
* The dump is plain SQL, gzipped: portable, greppable, and restorable with
* nothing but `psql`. Custom format would be smaller and would need `pg_restore`
* at exactly the right version during an incident, which is not a trade worth
* making at 3am.
*/
import { execFileSync, spawnSync } from "node:child_process";
import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { sql } from "drizzle-orm";
import { createDb } from "@inkloom/db/client";
import { BACKUP_JOB, recordJobRun } from "@inkloom/core/retention";
import { describeTarget, insideContainer, required } from "./_env";
import { verifyDumpFile } from "./_dump";
import { encryptBackup } from "./_crypto";
function arg(name: string): string | undefined {
const i = process.argv.indexOf(`--${name}`);
return i === -1 ? undefined : process.argv[i + 1];
}
/**
* `pg_dump`, wherever it lives.
*
* On CI it is on PATH. Locally it usually is not, but the Postgres container
* has it — and its version matches the server, which matters because pg_dump
* refuses to dump a server newer than itself.
*/
function pgDump(args: string[]): { stdout: Buffer; usedDocker: boolean } {
const direct = spawnSync("pg_dump", ["--version"], { encoding: "utf8" });
if (direct.status === 0) {
return {
stdout: execFileSync("pg_dump", args, { maxBuffer: 1024 * 1024 * 1024 }),
usedDocker: false,
};
}
const container = process.env.PG_CONTAINER ?? "inkloom-postgres";
// The URL is always the first argument; translate it for the container.
const inside = args.map((a, i) => (i === 0 ? insideContainer(a) : a));
return {
stdout: execFileSync("docker", ["exec", "-i", container, "pg_dump", ...inside], {
maxBuffer: 1024 * 1024 * 1024,
}),
usedDocker: true,
};
}
async function main() {
// Verify-only: check an existing archive without touching a database.
const verifyOnly = arg("verify");
if (verifyOnly) {
const { bytes, tables } = verifyDumpFile(verifyOnly);
console.log(
`\n ${verifyOnly}\n OK — ${(bytes / 1024 / 1024).toFixed(2)} MB, ${tables} tables\n`,
);
process.exit(0);
}
const url = required("DATABASE_URL");
const outDir = arg("out") ?? "backups";
const label = arg("label") ?? "manual";
const startedAt = new Date();
const started = Date.now();
if (!existsSync(outDir)) mkdirSync(outDir, { recursive: true });
const stamp = startedAt.toISOString().replace(/[:.]/g, "-");
const file = join(outDir, `inkloom-${label}-${stamp}.sql.gz`);
console.log(`\n source: ${describeTarget(url)}`);
console.log(` target: ${file}`);
/*
* A preflight that names the environment, without naming any secret.
*
* Written after a failed CI run that could not be diagnosed from here at all:
* the summary said "exit code 1", and every candidate cause — a missing key,
* a key too short, the wrong pg_dump, an unreachable host — produces the same
* line. None of the values below are sensitive: a length is not a key, and a
* hostname is not a connection string. Together they identify which of those
* it was on the first read, instead of after a round of guessing.
*/
const keyLength = (process.env.BACKUP_ENCRYPTION_KEY ?? "").length;
let pgDumpVersion = "NOT FOUND";
try {
const direct = spawnSync("pg_dump", ["--version"], { encoding: "utf8" });
pgDumpVersion =
direct.status === 0
? `${direct.stdout.trim()} (PATH)`
: `${execFileSync("docker", ["exec", "-i", process.env.PG_CONTAINER ?? "inkloom-postgres", "pg_dump", "--version"], { encoding: "utf8" }).trim()} (container)`;
} catch {
/* reported as NOT FOUND below */
}
console.log(` preflight:`);
console.log(` node ${process.version}`);
console.log(` pg_dump ${pgDumpVersion}`);
console.log(` database host ${new URL(url).hostname}`);
console.log(
` encryption key ${keyLength === 0 ? "NOT SET" : `set, ${keyLength} characters`}` +
`${keyLength > 0 && keyLength < 16 ? " <-- TOO SHORT, minimum is 16" : ""}`,
);
console.log(` CI ${process.env.CI ? "yes" : "no"}\n`);
const { db, pool } = createDb({ connectionString: url, max: 1 });
let status: "ok" | "failed" = "ok";
let finalFile = file;
let error: string | undefined;
let rowsCovered = 0;
try {
/*
* A database that was never provisioned is not a backup failure.
*
* Production has a Neon project and a Hyperdrive but has never had
* migrations run, so every nightly run failed on `SELECT COUNT(*) FROM
* users` and reported a backup outage for an environment that does not
* exist yet. That is noise, and noise is how a real outage gets ignored.
*
* The distinction is the MIGRATIONS TABLE, not the absence of tables. A
* database that has never been provisioned has no `drizzle.__drizzle_
* migrations`; one that HAS been provisioned and then lost its tables still
* has it, and that is a catastrophe which must fail loudly. Skipping on
* "no tables" alone would silently wave through a wiped production.
*/
const provisioned = (
await db.execute<{ exists: boolean }>(sql`
SELECT EXISTS (
SELECT 1 FROM information_schema.tables
WHERE table_schema = 'drizzle' AND table_name = '__drizzle_migrations'
) AS exists
`)
).rows[0]?.exists;
if (!provisioned) {
console.log(
"\n This database has never been migrated — there is nothing to back up yet.\n" +
" Skipping, and NOT recording a failure. Once migrations run here, backups\n" +
" start on their own.\n",
);
if (process.env.CI) {
console.log(
`::notice title=Nothing to back up::${label} has no schema yet ` +
"(migrations have never run). Skipped, not failed.",
);
}
await pool.end();
process.exit(0);
}
// What the live database holds, so the dump can be checked against it.
const live = (
await db.execute<Record<string, string>>(sql`
SELECT (SELECT COUNT(*)::text FROM users) AS users,
(SELECT COUNT(*)::text FROM credit_ledger) AS ledger,
(SELECT COUNT(*)::text FROM audit_events) AS audit,
(SELECT COUNT(*)::text FROM information_schema.tables
WHERE table_schema = 'public' AND table_type = 'BASE TABLE') AS tables
`)
).rows[0]!;
rowsCovered = Number(live.users) + Number(live.ledger) + Number(live.audit);
console.log(
` live: ${live.tables} tables, ${live.users} users, ${live.ledger} ledger, ${live.audit} audit`,
);
// --- take it --------------------------------------------------------
const { stdout, usedDocker } = pgDump([
url,
"--no-owner",
"--no-privileges",
"--format=plain",
"--compress=6",
]);
writeFileSync(file, stdout);
console.log(` pg_dump via ${usedDocker ? "container" : "PATH"}`);
// --- verify it ------------------------------------------------------
const { bytes, userRows } = verifyDumpFile(file);
// Row counts, read back out of the dump's own COPY block.
if (userRows !== null && userRows !== Number(live.users)) {
console.log(
` WARN users in dump (${userRows}) differs from live (${live.users}) —` +
" expected if writes landed during the dump",
);
}
console.log(
` verified: ${(bytes / 1024 / 1024).toFixed(2)} MB, all required objects present,` +
` not truncated`,
);
/*
* Encrypt LAST, after verification has read the real dump.
*
* Order matters: verifying the plaintext proves the backup is good, and
* encrypting afterwards means what leaves this machine is ciphertext. Doing
* it the other way round would mean either verifying nothing, or holding
* the key in order to check — and the whole point is that the storage
* provider never sees plaintext.
*
* Unencrypted is allowed for a local drill, and says so loudly. It is not
* allowed to be the silent default in CI, where the file travels.
*/
const passphrase = process.env.BACKUP_ENCRYPTION_KEY;
if (passphrase) {
const sealed = encryptBackup(readFileSync(file), passphrase);
writeFileSync(`${file}.enc`, sealed);
unlinkSync(file);
finalFile = `${file}.enc`;
console.log(
` encrypted: AES-256-GCM, ${(sealed.length / 1024 / 1024).toFixed(2)} MB ->` +
` ${finalFile}`,
);
} else if (process.env.CI) {
throw new Error(
"BACKUP_ENCRYPTION_KEY is not set. A dump holds every address, profile and " +
"ledger entry; it must not leave this runner in plaintext.",
);
} else {
console.log(" NOT ENCRYPTED — set BACKUP_ENCRYPTION_KEY before storing this anywhere");
}
} catch (e) {
status = "failed";
error = e instanceof Error ? e.message : String(e);
console.error(`\n BACKUP FAILED: ${error}`);
/*
* Put the reason on the run summary, not only in the step log.
*
* Without this a failed run shows "Process completed with exit code 1" on
* the summary page and the actual cause is buried several clicks deep in a
* collapsed step. The first real failure of this workflow cost exactly that
* detour, which is the wrong thing to be doing during a backup outage.
*/
if (process.env.CI) {
console.log(`::error title=Backup failed::${error.replace(/\n/g, " ")}`);
}
} finally {
/*
* Record the run either way, and never let recording fail the backup.
*
* A `failed` row is what turns a broken backup into something the admin
* overview shows, rather than a CI email nobody opened. If the database is
* what broke, this insert cannot work either — staleness is the backstop
* that depends on nothing working.
*/
await recordJobRun(db, {
job: BACKUP_JOB,
startedAt,
finishedAt: new Date(),
status,
durationMs: Date.now() - started,
removed: rowsCovered,
steps: [{ name: "pg_dump", file: finalFile, label, ok: status === "ok" }],
error,
}).catch((e: unknown) => console.error(` WARN could not record the run: ${String(e)}`));
await pool.end();
}
console.log(
`\n ${status === "ok" ? "BACKUP OK" : "BACKUP FAILED"} in ${((Date.now() - started) / 1000).toFixed(1)}s\n`,
);
// Non-zero is what makes CI shout.
process.exit(status === "ok" ? 0 : 1);
}
void main();