8000 Device discovery: per-tenant time budget, log slow and skipped tenants by klehmann · Pull Request #122 · klehmann/MindooDB · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 55 additions & 1 deletion src/__tests__/DeviceDiscovery.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/**
* Device discovery: challenge / discover without a tenant id in the path.
*/
import { describe, expect, test } from "@jest/globals";
import { describe, expect, jest, test } from "@jest/globals";
import { DeviceDiscoveryAuthError, DeviceDiscoveryService } from "../server/DeviceDiscoveryService";
import { NodeCryptoAdapter } from "../node/crypto/NodeCryptoAdapter";
import { BaseMindooTenantFactory } from "../core/BaseMindooTenantFactory";
Expand Down Expand Up @@ -128,6 +128,60 @@ describe("DeviceDiscoveryService", () => {
expect(result.tenants[0]?.wrappedPublicInfosKey).toBeTruthy();
}, 60000);

test("a tenant whose lookup hangs is skipped instead of holding the answer", async () => {
const cryptoAdapter = new NodeCryptoAdapter();
const factory = new BaseMindooTenantFactory(
new InMemoryContentAddressedStoreFactory(),
cryptoAdapter,
);
const user = await factory.createUserId("cn=bob/o=discover", "bob-pass-123");
const subtle = cryptoAdapter.getSubtle();
const signingKey = await subtle.importKey(
"pkcs8",
await decryptPrivateKey(cryptoAdapter, user.userSigningKeyPair.privateKey, "bob-pass-123", "signing"),
{ name: "Ed25519" },
false,
["sign"],
);
const aesKey = cryptoAdapter.getRandomValues(new Uint8Array(32));
const granted = {
directory: {
getUserBySigningPublicKey: async () => ({
username: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
signingPublicKey: user.userSigningKeyPair.publicKey,
encryptionPublicKey: user.userEncryptionKeyPair.publicKey,
details: null,
}),
},
context: { config: { adminSigningPublicKey: "admin-sign", adminEncryptionPublicKey: "admin-enc" } },
};
const hanging = {
directory: { getUserBySigningPublicKey: () => new Promise(() => {}) },
context: granted.context,
};
const warn = jest.spyOn(console, "warn").mockImplementation(() => {});
try {
const service = new DeviceDiscoveryService(
{
listTenants: () => ["tenant-hangs", "tenant-a"],
getTenant: async (id: string) => (id === "tenant-hangs" ? hanging : granted),
getPublicInfosKeysForTenant: async () => [aesKey],
} as never,
cryptoAdapter,
{ tenantLookupTimeoutMs: 200 },
);
const challenge = service.createChallenge(user.userSigningKeyPair.publicKey);
const sigBuf = await subtle.sign("Ed25519", signingKey, new TextEncoder().encode(challenge));
const started = Date.now();
const result = await service.discover(challenge, new Uint8Array(sigBuf));
expect(Date.now() - started).toBeLessThan(5_000);
expect(result.tenants.map((t) => t.tenantId)).toEqual(["tenant-a"]);
expect(warn.mock.calls.some(([line]) => String(line).includes("tenant-hangs: skipped"))).toBe(true);
} finally {
warn.mockRestore();
}
}, 60000);

test("rejects invalid signatures", async () => {
const cryptoAdapter = new NodeCryptoAdapter();
const subtle = cryptoAdapter.getSubtle();
Expand Down
112 changes: 87 additions & 25 deletions src/server/DeviceDiscoveryService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,31 @@ import type { TenantManager } from "./TenantManager";

const CHALLENGE_TTL_MS = 5 * 60 * 1000;
const MAX_CHALLENGES = 10_000;
/**
* Budget per tenant for "is this signing key granted here?". One tenant whose
* directory hangs or rebuilds slowly must not hold the whole request: behind
* a proxy (Cloudflare gives up after 100–120 s) the device would only see a
* 5xx page and never learn about the tenants that did answer.
*/
const DEFAULT_TENANT_LOOKUP_TIMEOUT_MS = 10_000;
const DEFAULT_TENANT_LOOKUP_CONCURRENCY = 4;
/** Lookups slower than this are logged with the tenant id, to find the culprit. */
const SLOW_TENANT_LOOKUP_MS = 2_000;

export interface DeviceDiscoveryOptions {
tenantLookupTimeoutMs?: number;
tenantLookupConcurrency?: number;
}

class TenantLookupTimeout extends Error {}

function withTimeout<T>(promise: Promise<T>, ms: number): Promise<T> {
let timer: ReturnType<typeof setTimeout> | undefined;
const timeout = new Promise<never>((_, reject) => {
timer = setTimeout(() => reject(new TenantLookupTimeout(`no answer within ${ms} ms`)), ms);
});
return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
}

interface DeviceChallenge {
challenge: string;
Expand All @@ -38,10 +63,20 @@ export class DeviceDiscoveryAuthError extends Error {
export class DeviceDiscoveryService {
private challenges = new Map<string, DeviceChallenge>();

private readonly tenantLookupTimeoutMs: number;
private readonly tenantLookupConcurrency: number;

constructor(
private readonly tenantManager: TenantManager,
private readonly cryptoAdapter: CryptoAdapter,
) {}
options: DeviceDiscoveryOptions = {},
) {
this.tenantLookupTimeoutMs = options.tenantLookupTimeoutMs ?? DEFAULT_TENANT_LOOKUP_TIMEOUT_MS;
this.tenantLookupConcurrency = Math.max(
1,
options.tenantLookupConcurrency ?? DEFAULT_TENANT_LOOKUP_CONCURRENCY,
);
}

/**
* Issue a short-lived, single-use challenge bound to `signingPublicKey`.
Expand Down Expand Up @@ -107,31 +142,58 @@ export class DeviceDiscoveryService {
};
const matches: Match[] = [];

for (const tenantId of tenantIds) {
try {
const loaded = await this.tenantManager.getTenant(tenantId);
const lookup = await loaded.directory.getUserBySigningPublicKey(signingPublicKey);
if (!lookup?.encryptionPublicKey) continue;

matches.push({
tenantId,
adminSigningPublicKey: loaded.context.config.adminSigningPublicKey,
adminEncryptionPublicKey: loaded.context.config.adminEncryptionPublicKey,
encryptionPublicKey: lookup.encryptionPublicKey,
username_hash:
typeof lookup.username === "string" && /^[a-f0-9]{64}$/i.test(lookup.username)
? lookup.username
: undefined,
identity_hashes: lookup.identityHashes,
joinResponseWrapped:
typeof lookup.joinResponseWrapped === "string" && lookup.joinResponseWrapped.trim()
? lookup.joinResponseWrapped.trim()
: undefined,
});
} catch {
// Tenant missing directory / not loadable — skip.
const lookupTenant = async (tenantId: string): Promise<Match | null> => {
const loaded = await this.tenantManager.getTenant(tenantId);
const lookup = await loaded.directory.getUserBySigningPublicKey(signingPublicKey);
if (!lookup?.encryptionPublicKey) return null;
return {
tenantId,
adminSigningPublicKey: loaded.context.config.adminSigningPublicKey,
adminEncryptionPublicKey: loaded.context.config.adminEncryptionPublicKey,
encryptionPublicKey: lookup.encryptionPublicKey,
username_hash:
typeof lookup.username === "string" && /^[a-f0-9]{64}$/i.test(lookup.username)
? lookup.username
: undefined,
identity_hashes: lookup.identityHashes,
joinResponseWrapped:
typeof lookup.joinResponseWrapped === "string" && lookup.joinResponseWrapped.trim()
? lookup.joinResponseWrapped.trim()
: undefined,
};
};

// Tenants are checked a few at a time, each within its own time budget.
// A lookup that times out keeps running in the background (it cannot be
// cancelled) but no longer delays the answer.
let next = 0;
const worker = async (): Promise<void> => {
while (next < tenantIds.length) {
const tenantId = tenantIds[next++]!;
const started = Date.now();
try {
const match = await withTimeout(lookupTenant(tenantId), this.tenantLookupTimeoutMs);
if (match) matches.push(match);
const elapsed = Date.now() - started;
if (elapsed >= SLOW_TENANT_LOOKUP_MS) {
console.warn(`[DeviceDiscovery] tenant ${tenantId}: lookup took ${elapsed} ms`);
}
} catch (error) {
if (error instanceof TenantLookupTimeout) {
console.warn(
`[DeviceDiscovery] tenant ${tenantId}: skipped, ${error.message} (directory lookup still running)`,
);
}
// Otherwise: tenant missing directory / not loadable — skip.
}
}
}
};
await Promise.all(
Array.from({ length: Math.min(this.tenantLookupConcurrency, tenantIds.length) }, worker),
);

// Workers finish in any order; answer in the server's tenant order.
matches.sort((a, b) => tenantIds.indexOf(a.tenantId) - tenantIds.indexOf(b.tenantId));

if (matches.length === 0) {
return { tenants: [] };
Expand Down
0