Repository navigation
Expand file tree
/
Copy pathmemdir.ts
More file actions
507 lines (471 loc) · 20.7 KB
/
Copy pathmemdir.ts
File metadata and controls
507 lines (471 loc) · 20.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
import { feature } from 'bun:bundle'
import { join } from 'path'
import { getFsImplementation } from '../utils/fsOperations.js'
import { getAutoMemPath, isAutoMemoryEnabled } from './paths.js'
/* eslint-disable @typescript-eslint/no-require-imports */
const teamMemPaths = feature('TEAMMEM')
? (require('./teamMemPaths.js') as typeof import('./teamMemPaths.js'))
: null
import { getKairosActive, getOriginalCwd } from '../bootstrap/state.js'
import { getFeatureValue_CACHED_MAY_BE_STALE } from '../services/analytics/growthbook.js'
/* eslint-enable @typescript-eslint/no-require-imports */
import {
type AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
logEvent,
} from '../services/analytics/index.js'
import { GREP_TOOL_NAME } from '../tools/GrepTool/prompt.js'
import { isReplModeEnabled } from '../tools/REPLTool/constants.js'
import { logForDebugging } from '../utils/debug.js'
import { hasEmbeddedSearchTools } from '../utils/embeddedTools.js'
import { isEnvTruthy } from '../utils/envUtils.js'
import { formatFileSize } from '../utils/format.js'
import { getProjectDir } from '../utils/sessionStorage.js'
import { getInitialSettings } from '../utils/settings/settings.js'
import {
MEMORY_FRONTMATTER_EXAMPLE,
TRUSTING_RECALL_SECTION,
TYPES_SECTION_INDIVIDUAL,
WHAT_NOT_TO_SAVE_SECTION,
WHEN_TO_ACCESS_SECTION,
} from './memoryTypes.js'
export const ENTRYPOINT_NAME = 'MEMORY.md'
export const MAX_ENTRYPOINT_LINES = 200
// ~125 chars/line at 200 lines. At p97 today; catches long-line indexes that
// slip past the line cap (p100 observed: 197KB under 200 lines).
export const MAX_ENTRYPOINT_BYTES = 25_000
const AUTO_MEM_DISPLAY_NAME = 'auto memory'
export type EntrypointTruncation = {
content: string
lineCount: number
byteCount: number
wasLineTruncated: boolean
wasByteTruncated: boolean
}
/**
* Truncate MEMORY.md content to the line AND byte caps, appending a warning
* that names which cap fired. Line-truncates first (natural boundary), then
* byte-truncates at the last newline before the cap so we don't cut mid-line.
*
* Shared by buildMemoryPrompt and claudemd getMemoryFiles (previously
* duplicated the line-only logic).
*/
export function truncateEntrypointContent(raw: string): EntrypointTruncation {
const trimmed = raw.trim()
const contentLines = trimmed.split('\n')
const lineCount = contentLines.length
const byteCount = trimmed.length
const wasLineTruncated = lineCount > MAX_ENTRYPOINT_LINES
// Check original byte count — long lines are the failure mode the byte cap
// targets, so post-line-truncation size would understate the warning.
const wasByteTruncated = byteCount > MAX_ENTRYPOINT_BYTES
if (!wasLineTruncated && !wasByteTruncated) {
return {
content: trimmed,
lineCount,
byteCount,
wasLineTruncated,
wasByteTruncated,
}
}
let truncated = wasLineTruncated
? contentLines.slice(0, MAX_ENTRYPOINT_LINES).join('\n')
: trimmed
if (truncated.length > MAX_ENTRYPOINT_BYTES) {
const cutAt = truncated.lastIndexOf('\n', MAX_ENTRYPOINT_BYTES)
truncated = truncated.slice(0, cutAt > 0 ? cutAt : MAX_ENTRYPOINT_BYTES)
}
const reason =
wasByteTruncated && !wasLineTruncated
? `${formatFileSize(byteCount)} (limit: ${formatFileSize(MAX_ENTRYPOINT_BYTES)}) — index entries are too long`
: wasLineTruncated && !wasByteTruncated
? `${lineCount} lines (limit: ${MAX_ENTRYPOINT_LINES})`
: `${lineCount} lines and ${formatFileSize(byteCount)}`
return {
content:
truncated +
`\n\n> WARNING: ${ENTRYPOINT_NAME} is ${reason}. Only part of it was loaded. Keep index entries to one line under ~200 chars; move detail into topic files.`,
lineCount,
byteCount,
wasLineTruncated,
wasByteTruncated,
}
}
/* eslint-disable @typescript-eslint/no-require-imports */
const teamMemPrompts = feature('TEAMMEM')
? (require('./teamMemPrompts.js') as typeof import('./teamMemPrompts.js'))
: null
/* eslint-enable @typescript-eslint/no-require-imports */
/**
* Shared guidance text appended to each memory directory prompt line.
* Shipped because Claude was burning turns on `ls`/`mkdir -p` before writing.
* Harness guarantees the directory exists via ensureMemoryDirExists().
*/
export const DIR_EXISTS_GUIDANCE =
'This directory already exists — write to it directly with the Write tool (do not run mkdir or check for its existence).'
export const DIRS_EXIST_GUIDANCE =
'Both directories already exist — write to them directly with the Write tool (do not run mkdir or check for their existence).'
/**
* Ensure a memory directory exists. Idempotent — called from loadMemoryPrompt
* (once per session via systemPromptSection cache) so the model can always
* write without checking existence first. FsOperations.mkdir is recursive
* by default and already swallows EEXIST, so the full parent chain
* (~/.claude/projects/<slug>/memory/) is created in one call with no
* try/catch needed for the happy path.
*/
export async function ensureMemoryDirExists(memoryDir: string): Promise<void> {
const fs = getFsImplementation()
try {
await fs.mkdir(memoryDir)
} catch (e) {
// fs.mkdir already handles EEXIST internally. Anything reaching here is
// a real problem (EACCES/EPERM/EROFS) — log so --debug shows why. Prompt
// building continues either way; the model's Write will surface the
// real perm error (and FileWriteTool does its own mkdir of the parent).
const code =
e instanceof Error && 'code' in e && typeof e.code === 'string'
? e.code
: undefined
logForDebugging(
`ensureMemoryDirExists failed for ${memoryDir}: ${code ?? String(e)}`,
{ level: 'debug' },
)
}
}
/**
* Log memory directory file/subdir counts asynchronously.
* Fire-and-forget — doesn't block prompt building.
*/
function logMemoryDirCounts(
memoryDir: string,
baseMetadata: Record<
string,
| number
| boolean
| AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS
>,
): void {
const fs = getFsImplementation()
void fs.readdir(memoryDir).then(
dirents => {
let fileCount = 0
let subdirCount = 0
for (const d of dirents) {
if (d.isFile()) {
fileCount++
} else if (d.isDirectory()) {
subdirCount++
}
}
logEvent('tengu_memdir_loaded', {
...baseMetadata,
total_file_count: fileCount,
total_subdir_count: subdirCount,
})
},
() => {
// Directory unreadable — log without counts
logEvent('tengu_memdir_loaded', baseMetadata)
},
)
}
/**
* Build the typed-memory behavioral instructions (without MEMORY.md content).
* Constrains memories to a closed four-type taxonomy (user / feedback / project /
* reference) — content that is derivable from the current project state (code
* patterns, architecture, git history) is explicitly excluded.
*
* Individual-only variant: no `## Memory scope` section, no <scope> tags
* in type blocks, and team/private qualifiers stripped from examples.
*
* Used by both buildMemoryPrompt (agent memory, includes content) and
* loadMemoryPrompt (system prompt, content injected via user context instead).
*/
export function buildMemoryLines(
displayName: string,
memoryDir: string,
extraGuidelines?: string[],
skipIndex = false,
): string[] {
const howToSave = skipIndex
? [
'## How to save memories',
'',
'Write each memory to its own file (e.g., `user_role.md`, `feedback_testing.md`) using this frontmatter format:',
'',
...MEMORY_FRONTMATTER_EXAMPLE,
'',
'- Keep the name, description, and type fields in memory files up-to-date with the content',
'- Organize memory semantically by topic, not chronologically',
'- Update or remove memories that turn out to be wrong or outdated',
'- Do not write duplicate memories. First check if there is an existing memory you can update before writing a new one.',
]
: [
'## How to save memories',
'',
'Saving a memory is a two-step process:',
'',
'**Step 1** — write the memory to its own file (e.g., `user_role.md`, `feedback_testing.md`) using this frontmatter format:',
'',
...MEMORY_FRONTMATTER_EXAMPLE,
'',
`**Step 2** — add a pointer to that file in \`${ENTRYPOINT_NAME}\`. \`${ENTRYPOINT_NAME}\` is an index, not a memory — each entry should be one line, under ~150 characters: \`- [Title](file.md) — one-line hook\`. It has no frontmatter. Never write memory content directly into \`${ENTRYPOINT_NAME}\`.`,
'',
`- \`${ENTRYPOINT_NAME}\` is always loaded into your conversation context — lines after ${MAX_ENTRYPOINT_LINES} will be truncated, so keep the index concise`,
'- Keep the name, description, and type fields in memory files up-to-date with the content',
'- Organize memory semantically by topic, not chronologically',
'- Update or remove memories that turn out to be wrong or outdated',
'- Do not write duplicate memories. First check if there is an existing memory you can update before writing a new one.',
]
const lines: string[] = [
`# ${displayName}`,
'',
`You have a persistent, file-based memory system at \`${memoryDir}\`. ${DIR_EXISTS_GUIDANCE}`,
'',
"You should build up this memory system over time so that future conversations can have a complete picture of who the user is, how they'd like to collaborate with you, what behaviors to avoid or repeat, and the context behind the work the user gives you.",
'',
'If the user explicitly asks you to remember something, save it immediately as whichever type fits best. If they ask you to forget something, find and remove the relevant entry.',
'',
...TYPES_SECTION_INDIVIDUAL,
...WHAT_NOT_TO_SAVE_SECTION,
'',
...howToSave,
'',
...WHEN_TO_ACCESS_SECTION,
'',
...TRUSTING_RECALL_SECTION,
'',
'## Memory and other forms of persistence',
'Memory is one of several persistence mechanisms available to you as you assist the user in a given conversation. The distinction is often that memory can be recalled in future conversations and should not be used for persisting information that is only useful within the scope of the current conversation.',
'- When to use or update a plan instead of memory: If you are about to start a non-trivial implementation task and would like to reach alignment with the user on your approach you should use a Plan rather than saving this information to memory. Similarly, if you already have a plan within the conversation and you have changed your approach persist that change by updating the plan rather than saving a memory.',
'- When to use or update tasks instead of memory: When you need to break your work in current conversation into discrete steps or keep track of your progress use tasks instead of saving to memory. Tasks are great for persisting information about the work that needs to be done in the current conversation, but memory should be reserved for information that will be useful in future conversations.',
'',
...(extraGuidelines ?? []),
'',
]
lines.push(...buildSearchingPastContextSection(memoryDir))
return lines
}
/**
* Build the typed-memory prompt with MEMORY.md content included.
* Used by agent memory (which has no getClaudeMds() equivalent).
*/
export function buildMemoryPrompt(params: {
displayName: string
memoryDir: string
extraGuidelines?: string[]
}): string {
const { displayName, memoryDir, extraGuidelines } = params
const fs = getFsImplementation()
const entrypoint = memoryDir + ENTRYPOINT_NAME
// Directory creation is the caller's responsibility (loadMemoryPrompt /
// loadAgentMemoryPrompt). Builders only read, they don't mkdir.
// Read existing memory entrypoint (sync: prompt building is synchronous)
let entrypointContent = ''
try {
// eslint-disable-next-line custom-rules/no-sync-fs
entrypointContent = fs.readFileSync(entrypoint, { encoding: 'utf-8' })
} catch {
// No memory file yet
}
const lines = buildMemoryLines(displayName, memoryDir, extraGuidelines)
if (entrypointContent.trim()) {
const t = truncateEntrypointContent(entrypointContent)
const memoryType = displayName === AUTO_MEM_DISPLAY_NAME ? 'auto' : 'agent'
logMemoryDirCounts(memoryDir, {
content_length: t.byteCount,
line_count: t.lineCount,
was_truncated: t.wasLineTruncated,
was_byte_truncated: t.wasByteTruncated,
memory_type:
memoryType as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
})
lines.push(`## ${ENTRYPOINT_NAME}`, '', t.content)
} else {
lines.push(
`## ${ENTRYPOINT_NAME}`,
'',
`Your ${ENTRYPOINT_NAME} is currently empty. When you save new memories, they will appear here.`,
)
}
return lines.join('\n')
}
/**
* Assistant-mode daily-log prompt. Gated behind feature('KAIROS').
*
* Assistant sessions are effectively perpetual, so the agent writes memories
* append-only to a date-named log file rather than maintaining MEMORY.md as
* a live index. A separate nightly /dream skill distills logs into topic
* files + MEMORY.md. MEMORY.md is still loaded into context (via claudemd.ts)
* as the distilled index — this prompt only changes where NEW memories go.
*/
function buildAssistantDailyLogPrompt(skipIndex = false): string {
const memoryDir = getAutoMemPath()
// Describe the path as a pattern rather than inlining today's literal path:
// this prompt is cached by systemPromptSection('memory', ...) and NOT
// invalidated on date change. The model derives the current date from the
// date_change attachment (appended at the tail on midnight rollover) rather
// than the user-context message — the latter is intentionally left stale to
// preserve the prompt cache prefix across midnight.
const logPathPattern = join(memoryDir, 'logs', 'YYYY', 'MM', 'YYYY-MM-DD.md')
const lines: string[] = [
'# auto memory',
'',
`You have a persistent, file-based memory system found at: \`${memoryDir}\``,
'',
"This session is long-lived. As you work, record anything worth remembering by **appending** to today's daily log file:",
'',
`\`${logPathPattern}\``,
'',
"Substitute today's date (from `currentDate` in your context) for `YYYY-MM-DD`. When the date rolls over mid-session, start appending to the new day's file.",
'',
'Write each entry as a short timestamped bullet. Create the file (and parent directories) on first write if it does not exist. Do not rewrite or reorganize the log — it is append-only. A separate nightly process distills these logs into `MEMORY.md` and topic files.',
'',
'## What to log',
'- User corrections and preferences ("use bun, not npm"; "stop summarizing diffs")',
'- Facts about the user, their role, or their goals',
'- Project context that is not derivable from the code (deadlines, incidents, decisions and their rationale)',
'- Pointers to external systems (dashboards, Linear projects, Slack channels)',
'- Anything the user explicitly asks you to remember',
'',
...WHAT_NOT_TO_SAVE_SECTION,
'',
...(skipIndex
? []
: [
`## ${ENTRYPOINT_NAME}`,
`\`${ENTRYPOINT_NAME}\` is the distilled index (maintained nightly from your logs) and is loaded into your context automatically. Read it for orientation, but do not edit it directly — record new information in today's log instead.`,
'',
]),
...buildSearchingPastContextSection(memoryDir),
]
return lines.join('\n')
}
/**
* Build the "Searching past context" section if the feature gate is enabled.
*/
export function buildSearchingPastContextSection(autoMemDir: string): string[] {
if (!getFeatureValue_CACHED_MAY_BE_STALE('tengu_coral_fern', false)) {
return []
}
const projectDir = getProjectDir(getOriginalCwd())
// Ant-native builds alias grep to embedded ugrep and remove the dedicated
// Grep tool, so give the model a real shell invocation there.
// In REPL mode, both Grep and Bash are hidden from direct use — the model
// calls them from inside REPL scripts, so the grep shell form is what it
// will write in the script anyway.
const embedded = hasEmbeddedSearchTools() || isReplModeEnabled()
const memSearch = embedded
? `grep -rn "<search term>" ${autoMemDir} --include="*.md"`
: `${GREP_TOOL_NAME} with pattern="<search term>" path="${autoMemDir}" glob="*.md"`
const transcriptSearch = embedded
? `grep -rn "<search term>" ${projectDir}/ --include="*.jsonl"`
: `${GREP_TOOL_NAME} with pattern="<search term>" path="${projectDir}/" glob="*.jsonl"`
return [
'## Searching past context',
'',
'When looking for past context:',
'1. Search topic files in your memory directory:',
'```',
memSearch,
'```',
'2. Session transcript logs (last resort — large files, slow):',
'```',
transcriptSearch,
'```',
'Use narrow search terms (error messages, file paths, function names) rather than broad keywords.',
'',
]
}
/**
* Load the unified memory prompt for inclusion in the system prompt.
* Dispatches based on which memory systems are enabled:
* - auto + team: combined prompt (both directories)
* - auto only: memory lines (single directory)
* Team memory requires auto memory (enforced by isTeamMemoryEnabled), so
* there is no team-only branch.
*
* Returns null when auto memory is disabled.
*/
export async function loadMemoryPrompt(): Promise<string | null> {
const autoEnabled = isAutoMemoryEnabled()
const skipIndex = getFeatureValue_CACHED_MAY_BE_STALE(
'tengu_moth_copse',
false,
)
// KAIROS daily-log mode takes precedence over TEAMMEM: the append-only
// log paradigm does not compose with team sync (which expects a shared
// MEMORY.md that both sides read + write). Gating on `autoEnabled` here
// means the !autoEnabled case falls through to the tengu_memdir_disabled
// telemetry block below, matching the non-KAIROS path.
if (feature('KAIROS') && autoEnabled && getKairosActive()) {
logMemoryDirCounts(getAutoMemPath(), {
memory_type:
'auto' as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
})
return buildAssistantDailyLogPrompt(skipIndex)
}
// Cowork injects memory-policy text via env var; thread into all builders.
const coworkExtraGuidelines =
process.env.CLAUDE_COWORK_MEMORY_EXTRA_GUIDELINES
const extraGuidelines =
coworkExtraGuidelines && coworkExtraGuidelines.trim().length > 0
? [coworkExtraGuidelines]
: undefined
if (feature('TEAMMEM')) {
if (teamMemPaths!.isTeamMemoryEnabled()) {
const autoDir = getAutoMemPath()
const teamDir = teamMemPaths!.getTeamMemPath()
// Harness guarantees these directories exist so the model can write
// without checking. The prompt text reflects this ("already exists").
// Only creating teamDir is sufficient: getTeamMemPath() is defined as
// join(getAutoMemPath(), 'team'), so recursive mkdir of the team dir
// creates the auto dir as a side effect. If the team dir ever moves
// out from under the auto dir, add a second ensureMemoryDirExists call
// for autoDir here.
await ensureMemoryDirExists(teamDir)
logMemoryDirCounts(autoDir, {
memory_type:
'auto' as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
})
logMemoryDirCounts(teamDir, {
memory_type:
'team' as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
})
return teamMemPrompts!.buildCombinedMemoryPrompt(
extraGuidelines,
skipIndex,
)
}
}
if (autoEnabled) {
const autoDir = getAutoMemPath()
// Harness guarantees the directory exists so the model can write without
// checking. The prompt text reflects this ("already exists").
await ensureMemoryDirExists(autoDir)
logMemoryDirCounts(autoDir, {
memory_type:
'auto' as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
})
return buildMemoryLines(
'auto memory',
autoDir,
extraGuidelines,
skipIndex,
).join('\n')
}
logEvent('tengu_memdir_disabled', {
disabled_by_env_var: isEnvTruthy(
process.env.CLAUDE_CODE_DISABLE_AUTO_MEMORY,
),
disabled_by_setting:
!isEnvTruthy(process.env.CLAUDE_CODE_DISABLE_AUTO_MEMORY) &&
getInitialSettings().autoMemoryEnabled === false,
})
// Gate on the GB flag directly, not isTeamMemoryEnabled() — that function
// checks isAutoMemoryEnabled() first, which is definitionally false in this
// branch. We want "was this user in the team-memory cohort at all."
if (getFeatureValue_CACHED_MAY_BE_STALE('tengu_herring_clock', false)) {
logEvent('tengu_team_memdir_disabled', {})
}
return null
}