Repository navigation
Expand file tree
/
Copy pathget.js
More file actions
285 lines (266 loc) · 13 KB
/
Copy pathget.js
File metadata and controls
285 lines (266 loc) · 13 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
// /get and /get.ps1 — serve the installer script and count the request.
//
// ─────────────────────────── PRIVACY POSTURE ────────────────────────────
// This counts REQUESTS TO OUR WEB SERVER. It is not telemetry: nothing is
// compiled into the xerj binary, nothing runs on the user's machine, and
// nothing is sent back after the install. `curl` executes no JavaScript, so
// there is no beacon, no cookie and no storage on the client side either.
//
// What is recorded, per request:
// ts ISO-8601 timestamp
// country request.cf.country — Cloudflare's 2-letter country code
// ua User-Agent header, truncated to 200 chars
// kind installer | browser | bot | unknown (derived from ua)
// os/arch coarse guess derived from ua only (may be "unknown")
// script "get" or "get.ps1"
// ver the XERJ_VERSION-ish ?v= query param, if the caller sent one
//
// What is deliberately NOT recorded:
// - NO IP address. Not cf-connecting-ip, not x-forwarded-for, not a hash
// of either. functions/lead.js DOES store `ip`; that was a deliberate
// divergence, not an oversight — a lead is a person who typed their
// email in, an installer request is not.
// - NO cookie, NO Set-Cookie, NO localStorage, NO unique or stable
// identifier of any kind. Two installs by the same person are
// indistinguishable from two installs by two people, by design.
// - NO Referer, NO Accept-Language, NO TLS/JA3 fingerprint, no
// request.cf fields beyond `country` (city, ASN, postal code and
// latitude/longitude are all available here and all discarded).
// - NO Ray ID / request id.
// The result is a count with a coarse platform and country breakdown, and
// nothing that can follow a person between two requests. That is the entire
// intended capability. If you extend this file, keep it that way.
//
// ──────────────────────────── FAIL-OPEN RULE ────────────────────────────
// The install must never depend on the counter. The script body is resolved
// and the Response is built FIRST; the R2 write is handed to waitUntil() so
// it runs after the response is already on its way, and every failure path
// (no binding, throw, timeout) is swallowed. A broken counter degrades to
// "we learn nothing", never to "the install breaks".
//
// ───────────────────────────── DEPLOYMENT ───────────────────────────────
// Bucket bound as INSTALLS (see wrangler.toml). Until that bucket exists the
// function serves the script and logs one line to the Pages console —
// working install, no counting.
//
// landing/_routes.json is what guarantees this function actually owns /get
// rather than the static asset shadowing it. If you delete that file, the
// counter may silently stop being invoked while /get keeps working.
const CT = 'text/plain; charset=utf-8';
// Cache-Control on the installer used to be `public, max-age=300` (set in
// landing/_headers). A cached response never reaches this function, so a
// counter behind a 5-minute cache would undercount by exactly the cache hit
// rate — an unknown number. no-store is the price of the count; the body is
// ~8 KB and `curl | sh` fetches it once per install.
const HEADERS = {
'content-type': CT,
'cache-control': 'no-store',
'x-content-type-options': 'nosniff',
// The installer is a script, not a page: keep it out of search results
// (GSC listed /get and /get.ps1 as crawled, not indexed). Fetching is unaffected.
'x-robots-tag': 'noindex',
};
// Bot/crawler UA substrings (lowercase). Kept deliberately short: this
// classifies, it does not block. Anything unmatched stays "unknown" rather
// than being quietly counted as a human.
const BOT_HINTS = [
'bot', 'crawler', 'spider', 'slurp', 'facebookexternalhit', 'preview',
'monitor', 'uptime', 'scanner', 'python-requests', 'go-http-client',
'okhttp', 'java/', 'libwww-perl', 'headlesschrome', 'axios', 'node-fetch',
];
/** Coarse request class from the UA alone. No other input. */
function classify(ua) {
const s = ua.toLowerCase();
if (!s) return 'unknown';
// Order matters: curl/wget/powershell are checked first because some
// scanners advertise "curl" *and* "bot"; an installer-shaped fetch is the
// thing we are counting, so it wins ties only when no bot hint is present.
const looksBot = BOT_HINTS.some((h) => s.includes(h));
if (!looksBot && /^(curl|wget|fetch)\//.test(s)) return 'installer';
if (!looksBot && (s.includes('powershell') || s.includes('windowspowershell'))) return 'installer';
if (looksBot) return 'bot';
if (s.startsWith('mozilla/')) return 'browser';
return 'unknown';
}
/**
* Coarse OS/arch guess from the UA. Returns "unknown" freely — a wrong guess
* is worse than no guess, because the platform mix is used to decide which
* targets get built.
*
* Note the honest limit: `curl/8.5.0` (the overwhelmingly common installer
* UA) carries NO platform at all, so most installer rows are unknown/unknown.
* The platform breakdown that matters still comes from the per-asset release
* download counts (metrics/release-downloads.jsonl), not from here.
*/
function platform(ua) {
const s = ua.toLowerCase();
let os = 'unknown';
if (s.includes('windows') || s.includes('win32') || s.includes('win64'
8000
) || s.includes('mingw')) os = 'windows';
else if (s.includes('mac os') || s.includes('macos') || s.includes('darwin')) os = 'macos';
else if (s.includes('android')) os = 'android';
else if (s.includes('linux')) os = 'linux';
else if (s.includes('freebsd')) os = 'freebsd';
let arch = 'unknown';
if (s.includes('aarch64') || s.includes('arm64')) arch = 'aarch64';
else if (s.includes('x86_64') || s.includes('x64') || s.includes('amd64') || s.includes('win64')) arch = 'x86_64';
return { os, arch };
}
/** Which script this request is for. Anything else is not ours to serve. */
function scriptFor(pathname) {
if (pathname === '/get') return 'get';
if (pathname === '/get.sh') return 'get'; // alias: same shell installer
if (pathname === '/get.ps1') return 'get.ps1';
return null;
}
/**
* Write one immutable record. One object per request: no read-modify-write,
* so concurrent installs cannot lose each other's counts. Aggregation happens
* at read time (scripts/adoption-snapshot.sh, or the ?token= export below).
* Date-prefixed so a range read never has to list the whole bucket.
*/
async function record(env, req, script) {
if (!env || !env.INSTALLS) {
console.log('[get] no INSTALLS binding; not counting', script);
return;
}
const ua = (req.headers.get('user-agent') || '').slice(0, 200);
const { os, arch } = platform(ua);
const now = new Date();
const day = now.toISOString().slice(0, 10);
let ver = '';
try { ver = (new URL(req.url).searchParams.get('v') || '').slice(0, 40); } catch (_) {}
const row = {
ts: now.toISOString(),
script,
kind: classify(ua),
country: (req.cf && req.cf.country) || '',
os,
arch,
ua,
...(ver ? { ver } : {}),
};
// crypto.randomUUID() here is a KEY, not an identity: it is generated
// server-side per request and never returned to the client, so it cannot
// correlate two requests. It exists only so two simultaneous writes do not
// overwrite one another.
await env.INSTALLS.put(
`installs/${day}/${now.toISOString()}_${crypto.randomUUID()}.json`,
JSON.stringify(row),
{ httpMetadata: { contentType: 'application/json' } },
);
}
export async function onRequestGet(context) {
const { request, env } = context;
const url = new URL(request.url);
const script = scriptFor(url.pathname);
if (!script) return new Response('not found\n', { status: 404, headers: HEADERS });
// Export path: ?token=… returns the aggregate instead of the script. Guarded
// exactly like lead.js — needs BOTH a caller token and a configured secret,
// so an unset secret can never open the export.
const token = url.searchParams.get('token');
if (token && env && env.INSTALLS_TOKEN && token === env.INSTALLS_TOKEN) {
return exportStats(env, url);
}
// 1. Resolve the script body. env.ASSETS bypasses Functions and reads the
// static file straight out of the deployment, so landing/get stays the
// single source of truth — this function never carries a copy that can
// drift from the file people audit at https://xerj.org/get.
let body;
try {
const asset = await env.ASSETS.fetch(new URL(`/${script}`, url.origin).toString());
if (!asset.ok) throw new Error(`asset ${script} -> ${asset.status}`);
body = await asset.text();
} catch (e) {
// Nothing sane to fall back to: serving a stale inlined copy of an
// installer would be worse than failing loudly, and redirecting to a
// third-party raw-file host would change what `curl | sh` trusts.
// The body below is valid in BOTH target shells — `#` starts a comment and
// `exit 1` is a statement in sh and in PowerShell alike — so whichever of
// the two pipelines receives it stops with a non-zero status and a message,
// instead of executing a fragment of an error page.
console.log('[get] asset fetch failed:', String(e));
return new Response(
`# xerj: the installer could not be served right now.\n` +
`# Install directly from the release page instead:\n` +
`# https://github.com/xerj-org/xerj/releases/latest\n` +
`exit 1\n`,
{ status: 503, headers: HEADERS },
);
}
// 2. Count AFTER the body is in hand and OFF the response path. waitUntil
// keeps the worker alive for the write without delaying a single byte to
// the installer. Errors are swallowed here, on purpose.
// (context.waitUntil, not a destructured copy — never rely on the method
// staying bound after destructuring.)
try {
const p = record(env, request, script).catch((e) => console.log('[get] count failed:', String(e)));
if (typeof context.waitUntil === 'function') context.waitUntil(p);
} catch (e) {
console.log('[get] count dispatch failed:', String(e));
}
return new Response(body, { headers: HEADERS });
}
// HEAD is what link checkers and some proxies send. Serve the right headers
// without a body, and do not count it — it is not an install attempt.
export async function onRequestHead(context) {
const url = new URL(context.request.url);
if (!scriptFor(url.pathname)) return new Response(null, { status: 404, headers: HEADERS });
return new Response(null, { headers: HEADERS });
}
/**
* Token-guarded read-out. Returns aggregates by default (that is all anyone
* should need); ?format=raw dumps the rows so they can be re-aggregated.
* Bounded by ?days= (default 30, max 365) so this can never walk an unbounded
* bucket.
*/
async function exportStats(env, url) {
const jsonHeaders = { 'content-type': 'application/json', 'cache-control': 'no-store' };
if (!env.INSTALLS) {
return new Response(JSON.stringify({ ok: false, error: 'no INSTALLS binding' }), { status: 503, headers: jsonHeaders });
}
const days = Math.min(Math.max(parseInt(url.searchParams.get('days') || '30', 10) || 30, 1), 365);
const today = new Date();
const wanted = [];
for (let i = 0; i < days; i++) {
const d = new Date(today.getTime() - i * 86400000);
wanted.push(d.toISOString().slice(0, 10));
}
const rows = [];
for (const day of wanted) {
let cursor;
do {
const page = await env.INSTALLS.list({ prefix: `installs/${day}/`, limit: 1000, cursor });
for (const obj of page.objects) {
const o = await env.INSTALLS.get(obj.key);
if (!o) continue;
try { rows.push(JSON.parse(await o.text())); } catch (_) {}
}
cursor = page.truncated ? page.cursor : undefined;
} while (cursor);
}
if (url.searchParams.get('format') === 'raw') {
return new Response(JSON.stringify({ ok: true, days, count: rows.length, rows }, null, 2), { headers: jsonHeaders });
}
const bump = (m, k) => { m[k || 'unknown'] = (m[k || 'unknown'] || 0) + 1; };
const by_day = {}, by_kind = {}, by_country = {}, by_os = {}, by_script = {};
for (const r of rows) {
bump(by_day, String(r.ts || '').slice(0, 10));
bump(by_kind, r.kind);
bump(by_country, r.country);
bump(by_os, r.os);
bump(by_script, r.script);
}
const installerish = rows.filter((r) => r.kind === 'installer').length;
return new Response(JSON.stringify({
ok: true,
window_days: days,
total_requests: rows.length,
installer_shaped_requests: installerish,
caveat:
'A request to /get is a request, not an install. It does not prove the ' +
'download succeeded, the checksum matched, or the binary was ever run. ' +
'kind=installer is a User-Agent heuristic and can be spoofed. Compare ' +
'against metrics/release-downloads.jsonl before quoting either number.',
by_day, by_script, by_kind, by_os, by_country,
}, null, 2), { headers: jsonHeaders });
}