Repository navigation
Expand file tree
/
Copy pathcoli
More file actions
executable file
·3700 lines (3519 loc) · 193 KB
/
Copy pathcoli
File metadata and controls
executable file
·3700 lines (3519 loc) · 193 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env python3
"""
colibri — tiny engine, immense model.
Run GLM-5.2 (744B) locally on CPU with roughly 15-26 GB of RAM.
coli setup one step: find the hardware, pick a model, build or fetch
the engine, download, then start the dashboard
coli start / status / logs start, inspect, or read the log of what setup configured
coli chat interactive chat (loads the model once)
coli serve OpenAI-compatible HTTP API (persistent engine)
coli run "prompt" one-shot generation
coli info model, RAM, disk, and configuration status
coli plan Disk / RAM / VRAM resource plan
coli mirror Plan, stage, or verify a learned partial mirror
coli doctor installation and execution-plan diagnostics
coli bench [task...] quality benchmarks (MMLU/HellaSwag/...)
coli convert convert GLM-5.2-FP8 to int4, one shard at a time
coli build build the engine
Configuration through environment variables or flags (also valid after the subcommand):
COLI_MODEL=<dir> model directory (required; or pass --model <dir>)
COLI_MODEL_MIRROR=<dir> second copy of the model on another drive: expert reads
are split across both SSDs (COLI_DISK_WEIGHTS=9,3 sets the
primary,mirror bandwidth ratio; default: measured at startup)
--ram N RAM budget in GB (automatically sizes the expert cache)
--repin N adapt RAM/VRAM experts every N tokens
--topp P adaptive expert top-p --topk N fixed top-k
--ngen N maximum response tokens --cap N cache slots/layer
"""
import os, sys, subprocess, argparse, json, math, time, signal, shutil, threading, re, codecs, tempfile, textwrap, struct, shlex
# input() only gets line editing and UP/DOWN history once the readline module
# has been LOADED — without it the terminal's arrow-key escape sequences are
# echoed raw into the prompt (#922). Guarded: Windows has no readline module
# and the console provides its own editing; on FreeBSD Python links libedit,
# which this activates the same way.
try:
import readline
except ImportError:
readline = None
# The engine mmaps every shard (144+ files); macOS default RLIMIT_NOFILE is 256.
if sys.platform != "win32":
try:
import resource
_soft, _hard = resource.getrlimit(resource.RLIMIT_NOFILE)
_want = min(65536 if _hard == resource.RLIM_INFINITY else _hard, 65536)
if _soft < _want:
resource.setrlimit(resource.RLIMIT_NOFILE, (_want, _hard))
except (ImportError, ValueError, OSError):
pass
# Windows: forza output UTF-8 (console cp1252 tronca Unicode box-drawing/emoji)
if sys.platform == "win32":
for s in (sys.stdout, sys.stderr):
try: s.reconfigure(encoding="utf-8")
except (AttributeError, OSError): pass
# realpath, not abspath: on a merged-/usr system /bin and /sbin are symlinks
# to /usr/bin, and an installed launcher invoked as /bin/coli would otherwise
# derive /libexec/colibri instead of /usr/libexec/colibri (#1689).
HERE = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, HERE)
# version.py sits next to this script in a source checkout, but an installed
# layout puts the launcher in $(PREFIX)/bin while the support modules live in
# $(PREFIX)/libexec/colibri — and a packager may strip it entirely (#575,
# FreeBSD port). The launcher must never crash over a version string.
try:
from version import __version__ as _version
except ModuleNotFoundError:
sys.path.insert(0, os.path.join(os.path.dirname(HERE), "libexec", "colibri"))
try:
from version import __version__ as _version
except ModuleNotFoundError:
_version = "unknown"
from family_registry import (FamilyConfigError, UnknownFamilyError, all_families,
display_for, family_by_id, family_for_config, resolve_model,
tuning_replay_prompt)
_TUNE_ROTATION_PROMPT = (
"A service has high median throughput but poor tail latency. Explain how "
"you would distinguish a CPU, memory, and storage bottleneck."
)
# Run-in-place (source checkout, "cd c && ./coli ..."): the engine, the
# support modules (resource_plan.py, doctor.py, autotune.py, openai_server.py) and
# tools/ all live next to this script — unchanged from before.
#
# Installed layout ("make install"): this script is $(PREFIX)/bin/coli,
# while the engine binaries and support files live in
# $(PREFIX)/libexec/colibri, since they aren't meant to be run directly
# by users. COLI_ENGINE overrides the engine path explicitly if neither
# guess is right (e.g. a custom packaging layout).
_EXE = ".exe" if sys.platform == "win32" else ""
_LIBEXEC = os.path.join(os.path.dirname(HERE), "libexec", "colibri")
_here_colibri = os.path.join(HERE, "colibri" + _EXE)
_here_glm = os.path.join(HERE, "glm" + _EXE)
if os.environ.get("COLI_ENGINE"):
GLM = os.environ["COLI_ENGINE"]
TOOLS = os.path.join(os.path.dirname(GLM), "tools")
elif os.path.exists(_here_colibri):
GLM = _here_colibri
TOOLS = os.path.join(HERE, "tools")
elif os.path.exists(_here_glm):
GLM = _here_glm
TOOLS = os.path.join(HERE, "tools")
else:
GLM = os.path.join(_LIBEXEC, "colibri" + _EXE)
TOOLS = os.path.join(_LIBEXEC, "tools")
sys.path.insert(0, _LIBEXEC) # so `import resource_plan`, `doctor`, `openai_server` still resolve
# No invented default: a path from whoever's machine this was written on is worse than
# no path at all, because every downstream error then names a directory the user never
# chose and cannot act on (#724). None here, and each command says so in its own voice.
DEF_MODEL = os.environ.get("COLI_MODEL")
NO_MODEL_HINT = "pass --model <dir>, or set COLI_MODEL=<dir>"
END = b"\x01\x01END\x01\x01\n"
READY = b"\x01\x01READY\x01\x01\n"
# ---------- palette & stile ----------
def _c(n): return f"\033[38;5;{n}m"
class C:
teal=_c(37); cyan=_c(80); mag=_c(170); org=_c(208); grn=_c(78); yel=_c(179)
dim="\033[2m"; b="\033[1m"; r="\033[0m"; gray=_c(242); dgray=_c(238)
@staticmethod
def off():
for k,v in vars(C).items():
if isinstance(v,str) and v.startswith("\033"): setattr(C,k,"")
TTY = sys.stdout.isatty() or os.environ.get("COLI_COLOR")=="1"
if not TTY: C.off()
# ---------- colibri 8-bit (pixel art, 2 pixel verticali per carattere) ----------
SPRITE = [
"....MMM.........",
"...MMMMM..w.....",
"....MMMM.ww.....",
"OOOOTTeTCC......",
"....TTTTTCC.....",
".....TTTTCC.....",
"......TTCC......",
".......TC.......",
"........C.......",
"................",
]
PAL = {"M":170, "T":37, "C":80, "O":208, "e":231, "w":80, ".":None}
def sprite_lines():
if not TTY:
return [" (\\ ", " )·> ", " / \\ ", " ", " "]
out=[]
for y in range(0,len(SPRITE),2):
top, bot = SPRITE[y], SPRITE[y+1] if y+1<len(SPRITE) else "."*len(SPRITE[y])
row=""
for x in range(len(top)):
ct, cb = PAL.get(top[x]), PAL.get(bot[x])
if ct is None and cb is None: row+= "\033[0m "
elif ct is not None and cb is None: row+= f"\033[38;5;{ct}m\033[49m▀"
elif ct is None and cb is not None: row+= f"\033[38;5;{cb}m\033[49m▄"
else: row+= f"\033[38;5;{ct}m\033[48;5;{cb}m▀"
out.append(row+"\033[0m")
return out
# Display names for the banner, keyed on the RAW model_type -- deliberately not
# on model_arch(), which returns "glm" for everything it does not recognise.
# That fallback is right for picking an engine (colibri.c is the general path)
# and wrong for telling someone what they loaded: it is why the banner said
# GLM-5.2 over an Inkling, a Kimi K3 and a DeepSeek V4 alike.
#
# Parameter counts are the roster's, i.e. the README's, so the two cannot drift
# apart quietly. A model_type absent from this table is NOT forced into a name
# here -- it prints its own model_type and its measured geometry instead.
_BANNER_MODELS = tuple((family.model_types[0], family.display_name, family.display_scale)
for family in all_families())
def model_banner_line(model):
"""Third banner line: what is actually loaded, or the generic tagline.
Reads config.json and stats the shards. No safetensors headers are parsed:
this runs before every command, and a banner may not cost a scan of a
400 GB checkpoint.
"""
# Senza un modello non si nomina un modello. Diceva "GLM-5.2 · 744B", che
# era la famiglia di punta quando e' stata scritta e che dal #1367 non si
# chiama nemmeno piu' cosi'; una riga che invecchia perche' cita un caso
# invece di descrivere il programma. Il conto viene dal registry: aggiungere
# una famiglia lo aggiorna, dimenticarsene non lo sbaglia.
generic = (f"{len(_BANNER_MODELS)} model families · "
f"MoE experts streamed from disk · CPU")
if not model:
return generic
if is_image_model(model):
return image_banner_line(model) or generic
if is_decision_model(model):
try:
resolved = resolve_model(model)
name, scale = display_for(resolved)
size = os.path.getsize(os.path.join(model, "model.safetensors"))
except (FamilyConfigError, UnknownFamilyError, OSError):
return generic
return " · ".join(bit for bit in (name, scale, "decision model",
f"{size/1e6:.0f} MB on disk" if size else "") if bit)
try:
with open(os.path.join(model, "config.json"), encoding="utf-8") as f:
cfg = json.load(f)
except (OSError, ValueError, TypeError):
return generic
kind = (cfg.get("model_type") or "").lower()
geometry = cfg
try:
resolved = resolve_model(model)
family = resolved.descriptor
# Size-aware: a family's model_type can span checkpoints an order of
# magnitude apart (Qwen3.6-35B and Qwen3.8-2.4T both resolve to
# qwen36). display_for names the geometry actually on disk, and
# falls back to the model_type when it recognises none.
name, scale = display_for(resolved)
geometry = resolved.family_config
# display_scale e' la taglia del checkpoint di riferimento. Un REAP ha
# lo stesso model_type e meno esperti: "284B" su un 150B e' #1367
# di nuovo. Quando la geometria non e' quella del riferimento si
# lascia parlare la geometria, che e' misurata e non puo' mentire.
experts = cfg.get("n_routed_experts")
if family.reference_experts and experts and experts != family.reference_experts:
scale = ""
except (FamilyConfigError, UnknownFamilyError):
# Unknown checkpoint: say so with its own words rather than guess.
name = kind or "unknown model"
scale = ""
bits = [name]
experts = geometry.get("n_routed_experts") or geometry.get("num_experts")
if scale:
bits.append(f"{scale} MoE" if experts else scale)
else:
layers = geometry.get("num_hidden_layers")
if layers and experts:
bits.append(f"{layers}L x {experts}E MoE")
elif layers:
bits.append(f"{layers} layer")
try:
size = sum(os.path.getsize(os.path.join(model, x))
for x in os.listdir(model) if x.endswith(".safetensors"))
if size >= 1e9:
# One decimal below 10 GB so a small checkpoint does not read as
# "0 GB"; whole numbers above, where a decimal is noise.
bits.append(f"{size/1e9:.1f} GB on disk" if size < 1e10
else f"{size/1e9:.0f} GB on disk")
elif size:
bits.append(f"{size/1e6:.0f} MB on disk")
except OSError:
pass
return " · ".join(bits)
def is_image_model(model):
"""A diffusers pipeline (model_index.json, no root config.json): the image
REPL, the image planner and the images endpoint instead of chat."""
try:
from image_engine import is_image_model_dir
except ImportError:
return False
try:
return is_image_model_dir(model)
except (OSError, TypeError):
return False
def is_decision_model(model):
"""A decision checkpoint (Laya: rl_agent_config.json, no root config.json;
GLiNER2.5-Decide: a GLiNER2 extractor config.json): served by coli serve /
coli web on POST /v1/systemone, never chatted with."""
try:
return resolve_model(model).descriptor.modality == "decision"
except (FamilyConfigError, UnknownFamilyError, OSError, TypeError, ValueError):
return False
# What bounds the tokens a decision engine reads, and the variable that moves it.
DECISION_LENGTH = {
"laya": ("at most its checkpoint's max_len per question", "COLI_LAYA_MAX_LEN"),
"gliner_decide": ("every question and the state in one sequence of at most 4096 tokens",
"COLI_GLINER_MAX_LEN"),
}
def is_gliner_decide(model):
try:
return resolve_model(model).descriptor.id == "gliner_decide"
except (FamilyConfigError, UnknownFamilyError, OSError, TypeError, ValueError):
return False
def decision_head_line(model, resolved):
"""`coli info`'s line for a decision head over a chat model (Clef): what it
adds, and that the chat stays."""
from family_registry import decision_head_of
head = decision_head_of(resolved)
try:
with open(os.path.join(model, head.files[0]), encoding="utf-8") as f:
cfg = json.load(f)
shape = (f"{cfg.get('routing_layers')} routing + {cfg.get('layers')} decoder layers, "
f"width {cfg.get('width')}")
except (OSError, ValueError, AttributeError):
shape = "head config unreadable"
return (f"{head.display_name} joint schema head ({shape}): POST /v1/systemone answers "
f"natively; chat, completions and messages work as on any {resolved.descriptor.id} model")
def decision_refusal(family):
"""What every chatting command says about a decision model, and where to go.
Laya ships a calibration; GLiNER2.5-Decide gives its classifier's softmax."""
answers = "calibrated probabilities" if family.id == "laya" else "a probability per option"
return (f"{C.yel}{family.display_name} is a decision model:{C.r} it answers typed questions "
f"(choice, score, noul) with {answers} and does not chat.\n"
f" Start it with: coli serve --model <dir> (or coli web for the dashboard)\n"
f" then POST /v1/systemone (docs/systemone.md, \"Decision engines\")")
def image_banner_line(model):
"""Banner line of an image model: name, what it does, bytes on disk. Sizes
come from stat() alone; the banner must never parse a header."""
try:
name = resolve_model(model).descriptor.display_name
except (FamilyConfigError, UnknownFamilyError, OSError):
return None
size = 0
for part in ("text_encoder", "transformer", "vae"):
folder = os.path.join(model, part)
try:
size += sum(os.path.getsize(os.path.join(folder, x)) for x in os.listdir(folder)
if x.endswith(".safetensors"))
except OSError:
pass
bits = [name, "text-to-image"]
if size >= 1e9: bits.append(f"{size/1e9:.1f} GB on disk")
elif size: bits.append(f"{size/1e6:.0f} MB on disk")
return " · ".join(bits)
def banner(sub="", *, model=None):
# model= is keyword-only on purpose: banner(sub) is called from a dozen
# places and from open PRs, and a second positional argument here would
# silently be read as a path.
sp=sprite_lines()
txt=[
f"{C.teal}{C.b}colibri{C.r} {C.dim}v{_version}{C.r}",
f"{C.dim}tiny engine, immense model{C.r}",
f"{C.gray}{model_banner_line(model)}{C.r}",
f"{C.dgray}{sub}{C.r}" if sub else "",
"",
]
print()
for i,s in enumerate(sp):
t = txt[i] if i<len(txt) else ""
print(f" {s} {t}")
print(f" {C.dgray}{'─'*58}{C.r}")
def hline(w): return f"{C.dgray}{'─'*w}{C.r}"
# ---------- util ----------
def term_w(): return min(shutil.get_terminal_size((80,20)).columns, 100)
def _read_pasted_lines(stream):
"""Collect complete lines already queued after an interactive paste.
``input()`` is line-oriented on libedit (including the FreeBSD build), so a
pasted block can return its first line while the remaining lines stay in
the terminal input queue. Read only data that is immediately ready; a
normal Enter therefore remains a one-line prompt and typing for the next
turn is not blocked here.
"""
lines = []
try:
if sys.platform == "win32":
import msvcrt
pending = []
while msvcrt.kbhit():
ch = msvcrt.getwch()
if ch in ("\r", "\n"):
lines.append("".join(pending))
pending = []
elif ch not in ("\x00", "\xe0"):
pending.append(ch)
if pending:
lines.append("".join(pending))
return lines
import select
while select.select([stream], [], [], 0)[0]:
line = stream.readline()
if not line:
break
lines.append(line.rstrip("\r\n"))
except (ImportError, OSError, ValueError):
# Some stdin implementations (notably embedded consoles) cannot be
# polled. Keeping the first line is safer than blocking the chat loop.
pass
return lines
def read_prompt(prompt=""):
"""Read one prompt, retaining additional lines from a pasted block."""
first = input(prompt)
if not TTY:
return first
return "\n".join([first, *_read_pasted_lines(sys.stdin)])
def prompt_box_lines(message, width):
"""Wrap a prompt for the terminal box without losing explicit newlines."""
lines = []
for logical in message.splitlines() or [""]:
lines.extend(textwrap.wrap(logical, width=max(1, width),
replace_whitespace=False,
drop_whitespace=False) or [""])
return lines
def prompt_input_rows(message, columns):
"""Return the number of terminal rows occupied by the readline input."""
rows = 0
for index, logical in enumerate(message.splitlines() or [""]):
prefix = 6 if index == 0 else 0 # `` │ › `` before the first line
rows += max(1, (prefix + len(logical) + columns - 1) // columns)
return rows
def redraw_prompt_box(message, width):
"""Replace readline's raw echo with a wrapped, multiline prompt box."""
clean = message.strip("\r\n")
columns = shutil.get_terminal_size((80,20)).columns
used = prompt_input_rows(clean, columns)
sys.stdout.write(f"\x1b[{used}A\x1b[0J")
inner = width - 3
for index, line in enumerate(prompt_box_lines(clean, inner)):
prefix = f"{C.teal}{C.b}›{C.r}" if index == 0 else " "
print(f" {C.dgray}│{C.r} {prefix} {line}{' '*(inner-len(line))}{C.dgray}│{C.r}")
print(f" {C.dgray}╰{'─'*width}╯{C.r}")
return clean
def model_arch(model):
return resolve_model(model).descriptor.id
def engine_for(model):
family = resolve_model(model).descriptor
if os.environ.get("COLI_ENGINE"):
if (family.id != "glm" and
os.environ.get("COLI_DOCKER_GLM_ONLY") == "1"):
raise UnknownFamilyError(
f"this image contains only the GLM engine; {family.display_name} "
"needs a full release archive or source build")
return GLM
if family.id == "glm":
return GLM
name = family.engine_artifact
local = os.path.join(HERE, name + _EXE)
return local if os.path.exists(local) else os.path.join(_LIBEXEC, name + _EXE)
def require_model(model, file="tokenizer.json", engine=None, target="", build="coli build"):
"""Shared model-directory / required-file / engine presence check.
`need_model` and `need_worker_model` differ only in which file they require
(tokenizer.json vs config.json) and whether non-GLM families are allowed
(expert workers live in the GLM engine only). The exit wording and
the check order are shared so the two validators cannot drift apart.
"""
if not model:
sys.exit(f"{C.yel}no model directory given.{C.r}\n {NO_MODEL_HINT}")
if not os.path.isdir(model):
sys.exit(f"{C.yel}model not found:{C.r} {model}\n set COLI_MODEL or use --model")
if file and not os.path.exists(os.path.join(model, file)):
sys.exit(f"{C.yel}{file} is missing from {model}{C.r}")
if engine and not os.path.exists(engine):
label = f"{target} " if target else ""
sys.exit(f"{C.yel}{label}engine is not built.{C.r} Run: {build}")
def need_model(model, engine=None):
# Directory/file checks come FIRST: resolving the family reads config.json,
# and a missing model dir must say "model not found", not "unsupported model".
# An image pipeline keeps its tokenizer in processor/, not at the root.
require_model(model, file=None)
tokenizer = "tokenizer.json"
if is_image_model(model) or is_decision_model(model):
# a decision checkpoint keeps it in tokenizer/ (Laya)
try: tokenizer = resolve_model(model).descriptor.tokenizer_file
except (FamilyConfigError, UnknownFamilyError): pass # engine_for says why, below
require_model(model, file=tokenizer)
engine = engine or engine_for(model)
target = resolve_model(model).descriptor.build_target
require_model(model, file=tokenizer, engine=engine, target=target,
build=f"make -C c {target}")
# One-shot runs keep the historic 1024. Interactive sessions get 16384, because
# the browser and the TUI both offer a per-request limit and the server treats
# --ngen as a hard ceiling it clamps to (#260) -- so a low ceiling makes the
# user's own control inert instead of merely cautious. Still a ceiling, not a
# target: generation ends at EOS either way.
def ngen_for(a, interactive=False, family=None):
if getattr(a, "ngen", None): return a.ngen
if family is not None:
if isinstance(family, str): family = family_by_id(family)
limits = family.limits
return limits.interactive_max_output if interactive else limits.default_max_output
return 16384 if interactive else 1024
_PROFILE_CAP_ENV = "COLI_PROFILE_CAP"
_PLAN_CAP_ENV = "COLI_PLAN_CAP"
def apply_measured_profile(env, profile, explicit_env, explicit_cap):
"""Apply scheduling/resource results while preserving CLI precedence."""
from autotune import apply_profile
result=apply_profile(env,profile,explicit_env)
# This is a launcher-private bridge for engines whose cache capacity is an
# argv value rather than an environment variable. It must never leak from a
# parent shell or override an explicit --cap.
result.pop(_PROFILE_CAP_ENV,None)
measured=profile.get("winner",{}).get("cap")
if explicit_cap is None and isinstance(measured,int) and not isinstance(measured,bool):
result[_PROFILE_CAP_ENV]=str(measured)
return result
def cap_for_launch(explicit_cap, env, fallback):
"""Resolve a measured cap for direct-engine launchers; --cap always wins."""
if explicit_cap is not None:
return explicit_cap
try:
measured=int(env.pop(_PROFILE_CAP_ENV,""))
except (TypeError,ValueError):
measured=0
try:
planned=int(env.pop(_PLAN_CAP_ENV,""))
except (TypeError,ValueError):
planned=0
return measured if measured>=1 else planned if planned>=1 else fallback
def operator_cap(a, arch):
"""Return a real operator cap, including GLM's documented CAP channel."""
cap=getattr(a,"cap",None)
if cap is not None or arch!="glm":
return cap
try:
cap=int(os.environ.get("CAP","0"))
except (TypeError,ValueError):
cap=0
return cap if cap else None
def env_for_engine(a, arch, plan=None):
if arch == "glm":
return env_for(a)
explicit_env = set(os.environ)
env = os.environ.copy()
env.pop(_PROFILE_CAP_ENV,None)
env.pop(_PLAN_CAP_ENV,None)
family=family_by_id(arch)
if family.modality == "image" and getattr(a, "ctx", None):
sys.exit(f"{C.yel}--ctx does not apply to {family.display_name}:{C.r} an image model has "
f"no context window (the prompt goes through its text encoder once)")
if family.modality == "decision" and getattr(a, "ctx", None):
reads, knob = DECISION_LENGTH.get(family.id, ("what its engine allows", "its engine's own variable"))
sys.exit(f"{C.yel}--ctx does not apply to {family.display_name}:{C.r} a decision model reads "
f"{reads} ({knob} overrides it)")
if getattr(a,"ram",0): explicit_env.add("RAM_GB")
# OMP_NUM_THREADS for the sister engines that do not size their own team.
#
# #805 set this from physical cores "on every platform" -- but only inside
# env_for(), which this function calls only for glm. inkling, kimi_k3,
# olmoe and deepseek_v4 all took the branch below and got libgomp's
# default, nproc, i.e. LOGICAL cores. On any SMT host that is a 2x
# over-subscription of a memory-bound int4 GEMV, which is the collapse
# measured in #718 (2.3x on Zen3) and again here: an i7-1355U reports 6
# physical and 12 logical, and DeepSeek V4 was running the 12.
#
# Same defect shape as the one #805 fixed, one level out: a mechanism that
# reaches one engine and not its siblings.
#
# DeepSeek V4 is now the exception: its C runtime sizes the team from
# logical CPUs minus its expert-loader reservation. Setting the physical
# count here makes that runtime mistake a launcher default for a user
# override and bypass its loader-aware policy. The other engines still
# need the physical-core default below.
#
# setdefault, so an explicit OMP_NUM_THREADS still wins.
if not env.get("COLI_NO_OMP_TUNE"):
if arch != "deepseek_v4":
from resource_plan import physical_cpu_count
env.setdefault("OMP_NUM_THREADS", str(physical_cpu_count()))
else:
env.setdefault("OMP_WAIT_POLICY", "active")
env.setdefault("GOMP_SPINCOUNT", "200000")
env.setdefault("OMP_DYNAMIC", "FALSE")
if sys.platform != "win32":
env.setdefault("OMP_PROC_BIND", "close")
env.setdefault("OMP_PLACES", "cores")
env["NGEN"] = str(ngen_for(a, family=arch))
if a.temp is not None: env["COLI_TEMP"] = str(a.temp)
# SNAP is how every engine but GLM learns its model directory. env_for()
# sets it for GLM and openai_server.py sets it for the gateway, so `coli
# chat` and `coli serve` worked while `coli run` handed the sister engines
# an environment without it: OLMoE exited with "started without a model"
# (#1501, #1600). Set it here, once, for all of them. --model is the
# directory, same as env_for(): a leftover SNAP must not load another model.
if getattr(a, "model", None):
env["SNAP"] = os.path.abspath(a.model)
if arch == "olmoe":
env["CHAT"] = "1"
env["MAX_NEW"] = str(ngen_for(a, family=arch))
# olmoe reads COLI_TEMP like every other engine (line above). Setting the
# legacy TEMP alias here also clobbered %TEMP% with "0.7" for every child
# of the chat process on Windows — and olmoe never read COLI_TEMP, so the
# ONLY working channel was a poisoned system variable (#509's exact shape).
# --ram reaches the engines that read it. kimi_k3 gained a RAM budget in
# #855; before that `grep -c RAM_GB c/kimi_k3.c` returned 0, so `coli chat
# --ram 242` on Kimi K3 set an environment variable nobody looked at and the
# user's session ran itself out of memory with the flag apparently set.
if arch in ("deepseek_v4", "deepseek_v41", "kimi", "glm53", "olmoe", "mimo"):
if a.ram: env["RAM_GB"] = str(a.ram)
if arch == "glm53":
# I densi vanno a int4 di default: sono 9,7 B parametri su 321, e in
# BF16 sono 19 GB che nessuno recupera. Chi vuole la precisione piena
# mette GLM53_BITS=32, e questo setdefault gli lascia la scelta.
env.setdefault("GLM53_BITS", "4")
# Budget della cache degli esperti. Se l'utente ha dato --ram, si lascia
# spazio a densi, stato e sistema invece di prendersi tutto.
if a.ram:
try:
spare = max(2.0, float(a.ram) - 8.0)
env.setdefault("GLM53_EXPERT_GB", f"{spare:.1f}")
except (TypeError, ValueError):
pass
if a.ctx:
limits = family_by_id(arch).limits
# #1376: the engine clamps silently to its hard ceiling. A flag that is
# accepted and then ignored is worse than one refused with the number.
if limits.max_context and a.ctx > limits.max_context:
sys.exit(f"{C.yel}--ctx {a.ctx} exceeds what {family_by_id(arch).display_name} "
f"supports ({limits.max_context} tokens).{C.r}\n"
f" Pass --ctx {limits.max_context} or less.")
env[limits.context_env] = str(a.ctx)
if arch == "deepseek_v4":
# Speculation stays opt-in. Real multi-turn chat measured only 1/15
# accepted prompt-lookup candidates; recurrent-state replay dominated
# the visible decode. V4_DRAFT=5 remains available for repeated code.
env.setdefault("V4_DRAFT", "0")
# Full three-stage DSpark remains opt-in. In a real multi-turn chat it
# accepted only 10/24 candidates and rejected-suffix replay turned a
# 14-token answer into 495 seconds. V4_MTP=1 still enables A/B runs;
# exact zero-I/O prompt lookup above remains enabled by default.
env.setdefault("V4_MTP", "0")
env.setdefault("V4_MTP_DRAFT", "3")
env.setdefault("V4_MTP_GB", "0.45")
env.setdefault("V4_MTP_MISS", "96")
env.setdefault("V4_MTP_MIN", "3")
env.setdefault("V4_MTP_CONF", "0.55")
# GPU mirroring of the MTP draft experts stays opt-in: the CUDA kernels
# accumulate fp32 differently from the CPU refs and speculative
# acceptance needs a bit-exact draft. V4_MTP_GPU=1 mirrors GLM's
# COLI_CUDA_MTP switch for CUDA-driven speculative decoding.
env.setdefault("V4_MTP_GPU", "0")
# getattr: tests build partial Namespaces for env_for_engine.
gpu = getattr(a, "gpu", None)
vram = getattr(a, "vram", None)
if not family.supports_accelerator:
if gpu not in (None,"none"):
sys.exit(f"{C.yel}{family.display_name} currently supports CPU only; --gpu is unavailable{C.r}")
if vram and gpu!="none":
sys.exit(f"{C.yel}{family.display_name} currently supports CPU only; --vram is unavailable{C.r}")
if gpu is not None:
env.pop("COLI_GPU", None); env.pop("COLI_GPUS", None)
if gpu == "none":
env["COLI_CUDA"] = "0"
env.pop("CUDA_EXPERT_GB", None); env.pop("CUDA_DENSE", None)
else:
if arch == "deepseek_v4":
if not dsv4_cuda_available(a.model):
sys.exit(f"{C.yel}--gpu needs the CUDA build:{C.r} {dsv4_cuda_build_hint()}")
elif not cuda_binary(engine_for_gpu_check(a)):
sys.exit(f"{C.yel}--gpu needs the CUDA build:{C.r} the engine binary is CPU-only")
env["COLI_CUDA"] = "1"
if gpu != "auto":
env["COLI_GPUS"] = gpu
env.setdefault("CUDA_DENSE", "1")
if vram and gpu != "none":
if arch == "deepseek_v4":
if not dsv4_cuda_available(a.model):
sys.exit(f"{C.yel}--vram needs the CUDA build:{C.r} {dsv4_cuda_build_hint()}")
elif not cuda_binary(engine_for_gpu_check(a)):
sys.exit(f"{C.yel}--vram needs the CUDA build:{C.r} the engine binary is CPU-only")
env["COLI_CUDA"] = "1"
env["CUDA_EXPERT_GB"] = str(vram)
if getattr(a, "auto_tier", False) and family.modality in ("image", "decision"):
# The tier plan places experts and KV state; an image or decision model
# has neither, and build_plan would stop on "no safetensors shards".
print(f" {C.dim}[PLAN] --auto-tier has nothing to place for "
f"{family.display_name}; coli plan shows its RAM{C.r}", file=sys.stderr)
elif getattr(a, "auto_tier", False):
# GLM has historically applied both the resource plan and its measured
# profile inside env_for(). The sibling launcher returned above that
# code path, so #1196 could measure and save a perfectly valid Kimi,
# Inkling, OLMoE, Qwen or V4 profile that chat/serve never loaded.
from resource_plan import build_plan, environment_for_plan
try:
if plan is None:
ram,ctx,devices,vram=resource_request(a,env)
plan=build_plan(a.model,ram,ctx,devices,vram,policy=a.policy,
kv_slots=requested_kv_slots(a,env))
plan_must_not_choose_omp=("OMP_NUM_THREADS" not in env
and (arch == "deepseek_v4"
or env.get("COLI_NO_OMP_TUNE")))
gpu_off_by_request = env.get("COLI_CUDA") == "0"
cuda_on = plan_cuda_enabled(a, arch, env)
env=environment_for_plan(plan,env,cuda_enabled=cuda_on)
except (OSError,ValueError,json.JSONDecodeError) as error:
sys.exit(f"{C.yel}invalid resource plan:{C.r} {error}")
# Outside the try on purpose: this only prints, and a display helper
# that stumbled inside it would be reported as "invalid resource plan"
# and take the launch down with it.
if not cuda_on: report_unapplied_vram_tier(plan, gpu_off_by_request)
if plan_must_not_choose_omp:
# environment_for_plan supplies the physical-core default used by
# the other engines. V4 deliberately computes logical CPUs minus
# its loader reservation at startup; COLI_NO_OMP_TUNE likewise
# promises no automatic team. Keep those baselines unless a
# measured profile (V4 only, below) or the user explicitly chooses
# a team.
env.pop("OMP_NUM_THREADS",None)
if not getattr(a, "no_tune_profile", False):
from autotune import load_profile
profile=load_profile(plan,a.model,engine_for(a.model))
if profile:
env=apply_measured_profile(env,profile,explicit_env,
getattr(a,"cap",None))
gain=100.0*profile["gain"]
print(f" {C.dim}[TUNE] applied measured profile · +{gain:.1f}% "
f"calibration throughput{C.r}",file=sys.stderr)
# Kimi's CUDA path is gated on K3_CUDA, not COLI_CUDA. --gpu / --vram /
# auto-tier write COLI_CUDA=1 after proving the build; without this copy
# the flag was accepted and the experts still stayed on the CPU.
if arch == "kimi":
if env.get("COLI_CUDA") == "0":
env["K3_CUDA"] = "0"
elif env.get("COLI_CUDA") == "1" and "K3_CUDA" not in explicit_env:
env["K3_CUDA"] = "1"
return env
def dsv4_cuda_available(model=None):
# Windows loads the tier as a runtime DLL next to the engine; Linux links
# it straight into the binary (nvcc, -lcudart), so the DLL check there
# rejected every valid `make deepseek-v4 CUDA=1` build (#1219). Detect the
# Linux build through the dynamic libcudart entry added by the in-tree
# CUDA=1 target. DeepSeek V4 has no HIP backend, so libamdhip64 alone must
# not make a CPU engine look GPU-capable.
eng = engine_for(model) if model else GLM
if sys.platform == "win32":
# backend_loader_dsv4.c tries the DeepGEMM build first and the generic
# one second, so either next to the engine starts the tier (the same
# two names `coli doctor` accepts).
return any(os.path.exists(os.path.join(os.path.dirname(eng), name))
for name in ("coli_cuda_dsv4_dg.dll", "coli_cuda_dsv4.dll"))
if sys.platform == "linux" and os.path.exists(eng):
try:
linked = subprocess.run(["ldd", eng], capture_output=True, text=True, timeout=3)
return any("libcudart" in line and "not found" not in line
for line in linked.stdout.splitlines())
except (OSError, subprocess.SubprocessError):
return False
return False
def dsv4_cuda_build_hint():
if sys.platform == "win32":
return "build coli_cuda_dsv4.dll (make cuda-dsv4-dll)"
if sys.platform == "linux":
return "make deepseek-v4 CUDA=1 (this binary is CPU-only)"
return "the DeepSeek V4 CUDA tier is supported only on Linux and Windows"
def need_worker_model(model):
require_model(model, file="config.json")
family = resolve_model(model).descriptor
if family.id != "glm":
sys.exit(f"{C.yel}cluster expert workers currently support the GLM engine only;{C.r} "
f"{family.display_name} models cannot serve experts")
engine = engine_for(model)
require_model(model, file="config.json", engine=engine, target=family.build_target,
build=f"make -C c {family.build_target}")
return engine
def engine_for_gpu_check(a):
"""The family's engine path for cuda_binary(), or None (the GLM default)
when the model cannot be resolved here: the check must not turn a missing
config.json into a different error than the one need_model() gives."""
model = getattr(a, "model", None)
if not model: return None
try: return engine_for(model)
except (FamilyConfigError, UnknownFamilyError, OSError): return None
def cuda_binary(engine=None):
"""Was `engine` (default: the GLM binary) built with the GPU backend?
Callers that know the family pass its engine: checking colibri.exe for a
qwen36 run refused --gpu on a correct qwen36 CUDA build (#1533)."""
engine = engine or GLM
if not os.path.exists(engine): return False
if sys.platform == "linux":
try:
linked=subprocess.run(["ldd",engine],capture_output=True,text=True,timeout=3)
# Detect both CUDA (libcudart) and HIP (libamdhip64) builds.
return any(("libcudart" in line or "libamdhip64" in line) and "not found" not in line
for line in linked.stdout.splitlines())
except (OSError,subprocess.SubprocessError): return False
if sys.platform == "win32":
# Windows CUDA_DLL/HIP_DLL hosts never link the GPU runtime directly:
# the engine LoadLibrary's its backend (backend_loader.c). The host
# compiles exactly one basename -- coli_hip.dll or coli_cuda.dll --
# and that is the file that must sit next to it. The GLM/Qwen banner
# is not a GPU-build marker: Kimi K3 CUDA_DLL builds link the same
# loader without printing it, and a HIP host that only had coli_hip.dll
# used to be refused as CPU-only.
try:
with open(engine,"rb") as f: image=f.read()
except OSError: return False
from doctor import windows_backend_dll
expected=windows_backend_dll(image)
if not expected: return False
return os.path.exists(os.path.join(os.path.dirname(engine),expected))
return False
def resource_request(a, env):
from family_registry import default_context
resolved=resolve_model(a.model) if getattr(a,"model",None) else None
family=resolved.descriptor if resolved else family_by_id("glm")
# a Clef checkpoint plans for its own 16384 (family_registry.default_context)
fallback=default_context(resolved) if resolved else family.limits.default_context
ctx=a.ctx or int(env.get(family.limits.context_env,fallback))
if ctx<1 or ctx>family.limits.max_context:
raise ValueError(f"--ctx must be between 1 and {family.limits.max_context} "
f"for {family.display_name}")
def num(v):
try: return float(v)
except ValueError: return 0.0 # "auto": the planner sizes it; the engine reads the string itself
ram=a.ram or num(env.get("RAM_GB",0))
vram=a.vram or num(env.get("CUDA_EXPERT_GB",0))
gpu=a.gpu
if gpu is None:
gpu=env.get("COLI_GPUS",env.get("COLI_GPU","auto"))
devices=None if gpu=="auto" else ([] if gpu=="none" else
[int(value) for value in gpu.split(",")])
return ram,ctx,devices,vram
def requested_kv_slots(a, env):
value=getattr(a,"kv_slots",None)
if value is None: value=int(env.get("COLI_KV_SLOTS","1"))
return value
def plan_cuda_enabled(a, arch, env):
"""Should --auto-tier apply the VRAM tier that `coli plan` printed?
Yes exactly when the engine that is about to run was built with a GPU
backend, which is the question the GLM path has always asked
(`has_cuda=cuda_binary()`), and which is what --auto-tier documents: apply
the plan.
The sibling path used to answer a narrower question -- "did the user pass
--gpu or --vram" -- and the comment that guarded it was right at the time:
cuda_binary() could only inspect the GLM binary, so a perfectly good qwen36
CUDA build looked CPU-only to it and auto-tier had to assume the worst
rather than risk launching a CPU-only sibling as a CUDA one. Since #1533
the check takes the engine that will actually run, so the thing that
contract was protecting against can be TESTED instead of assumed, and
assuming it costs the user the tier they were just shown. On the box in
#1581 that was 11.8 tok/s against 21, with nothing on screen to explain it.
A CPU-only build still never gets a CUDA launch: that is what the check
below says. `--gpu none` (COLI_CUDA=0) is still the off switch, and an
explicit --gpu/--vram has already proved the build before we get here.
"""
value = env.get("COLI_CUDA")
if value == "1": return True # --gpu / --vram validated the build above
if value == "0": return False # --gpu none, or the user set it by hand
# DeepSeek V4 links its tier differently on each platform, so it has its own
# probe; using cuda_binary() here would reject valid V4 CUDA builds (#1219).
if arch == "deepseek_v4": return dsv4_cuda_available(getattr(a, "model", None))
return cuda_binary(engine_for_gpu_check(a))
def report_unapplied_vram_tier(plan, gpu_off_by_request=False):
"""Say so when a real VRAM tier from the plan is not going into the launch.
After plan_cuda_enabled() there are only two ways to get here, and the user
can act on exactly one of them, so the notice says which: the GPU was turned
off by request, or the engine binary has no GPU backend to turn on.
Being told matters because `coli plan` and `coli doctor` print the VRAM tier
as part of the plan and --auto-tier is documented as applying that plan. In
#1581 the tier vanished with nothing on screen connecting the two, and the
reporter only found it by watching nvidia-smi.
The test for "real" is the planner's own: a device qualified to drive
place
DF08
ment, and a budget above zero. Anything else was not going to be
applied with the GPU on either, and saying so would be noise.
"""
try:
from resource_plan import plans_placement, GB
vram = plan["tiers"]["vram"]
devices = [d for d in vram["devices"] if plans_placement(d)]
budget = vram["budget_bytes"]
except (ImportError, KeyError, TypeError, AttributeError):
return
if not devices or budget <= 0:
return
where = ", ".join(f"{d['index']}:{d.get('name') or 'GPU'}" for d in devices)
why = ("--gpu none turns the GPU off, so the tier is not applied. Drop it "
"to use the plan as printed."
if gpu_off_by_request else
"the engine binary has no GPU backend, so there is nothing to apply "
"it with. Rebuild the engine with CUDA=1 to use it.")
print(f" {C.yel}[PLAN] the plan's VRAM tier is not being applied{C.r}\n"
f" {C.dim}{budget / GB:.1f} GB on {where} stays unused: {why}{C.r}",
file=sys.stderr)
def env_for(a):
explicit_env = set(os.environ)
e = dict(os.environ, SNAP=a.model)
e.pop(_PROFILE_CAP_ENV,None)
e.pop(_PLAN_CAP_ENV,None)
if getattr(a,"ram",0): explicit_env.add("RAM_GB")
# A GLM resource winner is a measured pair (RAM ceiling + argv cap). An
# explicit cap asks the engine to size against the current RAM plan, not a
# smaller ceiling remembered from that old pair.
requested_cap=operator_cap(a,"glm")
if requested_cap is not None: explicit_env.add("RAM_GB")
# OMP_NUM_THREADS on EVERY platform, not only Windows.
#
# glm.c's self-exec tuning (colibri.c, the COLI_OMP_TUNED block) sets
# OMP_WAIT_POLICY and GOMP_SPINCOUNT but never the thread count -- and it
# skips itself entirely when COLI_CUDA or COLI_METAL is on. So on Linux
# nothing ever sets OMP_NUM_THREADS and libgomp falls back to nproc, i.e.
# LOGICAL cores. On any SMT host that over-subscribes by 2x, which is
# precisely what physical_cpu_count()'s own docstring warns about:
# "two SMT siblings share one AVX-512 unit and contend".
#
# Measured, GLM-5.2 744B on 4x RTX A6000 + EPYC 7402P (24 cores / 48
# threads), CTX=32768, .coli_usage restored between runs:
#
# OMP_NUM_THREADS routed CPU read decode per thread
# 4 6.48 GB/s 1.28 tok/s 1.62 GB/s
# 8 12.50 1.97 1.56
# 16 23.97 2.63 1.49
# 24 (physical) 30.10 2.90 1.25
# 48 (logical) 17.35 2.06 0.36 <- today's default
#
# setdefault, so an explicit OMP_NUM_THREADS still wins, and the whole
# thing stays behind COLI_NO_OMP_TUNE like the rest of the OMP tuning.
if not e.get("COLI_NO_OMP_TUNE"):
from resource_plan import physical_cpu_count
e.setdefault("OMP_NUM_THREADS", str(physical_cpu_count()))
if sys.platform == "win32":
# COLI_NO_OMP_TUNE spegne SOLO il blocco OMP (stesso perimetro del
# self-exec di glm.c; presence-based come nel motore: impostarla a
# qualsiasi valore, anche 0, disattiva). I default I/O piu' sotto
# restano attivi: kill-switch dedicati = le var stesse (DIRECT=0 ecc.)
if not e.get("COLI_NO_OMP_TUNE"):
# parita' col tuning OMP self-exec di glm.c (solo Linux/FreeBSD, e
# comunque saltato sotto CUDA/Metal): libgomp legge queste variabili
# prima di main, quindi su Windows vanno nell'ambiente del figlio.
# niente OMP_PROC_BIND/OMP_PLACES: la libgomp di MinGW non supporta
# l'affinity su Windows ("Affinity not supported on this configuration")
for k, v in (("OMP_WAIT_POLICY", "active"),
("GOMP_SPINCOUNT", "200000"),
("OMP_DYNAMIC", "FALSE")):
e.setdefault(k, v)
# Default Windows misurati sul box di riferimento (docs/tuning-9950x3d-5090.md),
# tutti lossless e tutti setdefault (un override esplicito vince sempre):
# - DIRECT=1: 10.7 GB/s O_DIRECT vs 9.0 buffered (iobench, anche a cache
# calda); nel motore 0.48 -> 1.02 tok/s. Upstream #162: 1.47x.
# - PIPE=1: overlap load/matmul, +8% sopra DIRECT (byte-identico, riordina
# solo l'I/O). PIPE_WORKERS resta al default 8 (sweep 4/8/16 piatto).
# - PILOT_REAL=1: prefetch cross-layer con load veri (unico prefetch
# funzionante su Windows: fadvise e' no-op), +11%, hit rate +19 punti.
e.setdefault("DIRECT", "1")
e.setdefault("PIPE", "1")
e.setdefault("PILOT_REAL", "1")
e["COLI_POLICY"]=a.policy
# Explicit request only. Absent flag => variable absent => engine default,
# which is the current path. Never set from discovery or from a plan.
if getattr(a, "xdna", False): e["COLI_XDNA"]="1"
if getattr(a, "cluster_workers", None): e["CLUSTER_WORKERS"] = a.cluster_workers
if getattr(a, "cluster_coordinator", None): e["CLUSTER_COORDINATOR"] = a.cluster_coordinator
if a.ram: e["RAM_GB"]=str(a.ram)
e["NGEN"]=str(ngen_for(a))
if a.topp: e["TOPP"]=str(a.topp)
if a.topk: e["TOPK"]=str(a.topk)
if a.temp is not None: e["COLI_TEMP"]=str(a.temp) # 0 = greedy; default motore: 1.0 + nucleus 0.95
# COLI_TEMP, non TEMP: ROCm e Windows leggono $TEMP come dir temporanea (#509)
if a.repin: e["REPIN"]=str(a.repin)
if a.ctx: e["CTX"]=str(a.ctx)
if a.auto_tier:
from resource_plan import build_plan, environment_for_plan, format_bytes
if a.gpu is not None:
e.pop("COLI_GPU",None); e.pop("COLI_GPUS",None)
if a.gpu=="none":
e["COLI_CUDA"]="0"; e.pop("CUDA_EXPERT_GB",None); e.pop("CUDA_DENSE",None)
else: e.pop("COLI_CUDA",None)
elif e.get("COLI_CUDA")=="0":
e.pop("COLI_GPU",None); e.pop("COLI_GPUS",None)
e.pop("CUDA_EXPERT_GB",None); e.pop("CUDA_DENSE",None)
if a.vram and a.gpu!="none": e["CUDA_EXPERT_GB"]=str(a.vram)
try:
ram,ctx,devices,vram=resource_request(a,e)
plan=build_plan(a.model,ram,ctx,devices,vram,policy=a.policy,
kv_slots=requested_kv_slots(a,e))
except (OSError,ValueError,json.JSONDecodeError) as error: