Repository navigation
Expand file tree
/
Copy pathods-preflight.sh
More file actions
executable file
·346 lines (306 loc) · 12 KB
/
Copy pathods-preflight.sh
File metadata and controls
executable file
·346 lines (306 loc) · 12 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
#!/bin/bash
# ODS Pre-flight Check
# Validates all services start correctly before user interaction
# Backend-aware: detects AMD vs NVIDIA (both use llama-server)
# Usage: ./ods-preflight.sh
# ./ods-preflight.sh --install-env # Linux install environment report (JSON: see scripts/linux-install-preflight.sh --help)
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ODS_DIR="$SCRIPT_DIR"
case "${1:-}" in
--install-env|--env-report)
shift
exec "$SCRIPT_DIR/scripts/linux-install-preflight.sh" "$@"
;;
esac
LOG_FILE="$ODS_DIR/preflight-$(date +%Y%m%d-%H%M%S).log"
# Safe .env loading (no eval; use lib/safe-env.sh)
[[ -f "$ODS_DIR/lib/safe-env.sh" ]] && . "$ODS_DIR/lib/safe-env.sh"
load_env_file "$ODS_DIR/.env"
# shellcheck source=lib/preflight-llm-route.sh
. "$ODS_DIR/lib/preflight-llm-route.sh"
SERVICE_HOST="${SERVICE_HOST:-localhost}"
# Bound every health probe so a listening-but-wedged service can't hang the
# preflight (mirrors scripts/ods-preflight.sh).
CURL_HEALTH_FLAGS=(--connect-timeout 3 --max-time 10)
# Auto-detect backend from .env or hardware probing.
# Priority: .env setting → nvidia-smi → AMD sysfs (any card).
# On dual-GPU systems (AMD iGPU + NVIDIA dGPU) we must prefer
# NVIDIA when present, since it is always the inference target.
# WSL2 keeps the Microsoft kernel release string; ODS_PROC_VERSION_FILE is a
# test hook, matching scripts/detect-hardware.sh.
is_wsl() {
local version_file="${ODS_PROC_VERSION_FILE:-/proc/version}"
[[ -f "$version_file" ]] && grep -qi microsoft "$version_file" 2>/dev/null
}
detect_backend() {
# 1. Trust .env if the installer already wrote it.
if [[ "${GPU_BACKEND:-}" == "amd" ]]; then
echo "amd"
return
fi
if [[ "${GPU_BACKEND:-}" == "nvidia" ]]; then
echo "nvidia"
return
fi
# 2. Probe NVIDIA first (matches installer's detect_gpu order).
# Validate hardware via sysfs vendor ID before trusting nvidia-smi,
# which may be installed without NVIDIA hardware.
local _nvidia_hw=false
for _v in /sys/class/drm/card*/device/vendor; do
[[ "$(cat "$_v" 2>/dev/null)" == "0x10de" ]] && _nvidia_hw=true && break
done
# WSL2 passes the host GPU through its paravirtualized nvidia-smi bridge and
# normally exposes no matching /sys/class/drm vendor node, so a successful,
# non-empty query is the hardware witness there. This is the same rule
# installers/lib/detection.sh, scripts/detect-hardware.sh and the bootstrap
# preliminary check already apply.
if ! $_nvidia_hw && is_wsl && command -v nvidia-smi &> /dev/null; then
local _wsl_gpu_name=""
_wsl_gpu_name=$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null) \
|| _wsl_gpu_name=""
[[ -n "$_wsl_gpu_name" ]] && _nvidia_hw=true
fi
if $_nvidia_hw && command -v nvidia-smi &> /dev/null; then
if nvidia-smi --query-gpu=name --format=csv,noheader &> /dev/null; then
echo "nvidia"
return
fi
fi
# 3. Probe AMD sysfs — scan all DRM cards, not just card1.
for card_dir in /sys/class/drm/card*/device; do
[[ -d "$card_dir" ]] || continue
if [[ "$(cat "$card_dir/vendor" 2>/dev/null)" == "0x1002" ]]; then
echo "amd"
return
fi
done
# 4. No GPU detected — default to cpu.
echo "cpu"
}
BACKEND=$(detect_backend)
# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color
PASS=0
FAIL=0
WARN=0
log() {
echo -e "$1"
echo -e "$1" | sed $'s/\033\\[[0-9;]*m//g' >> "$LOG_FILE"
}
pass() { log "${GREEN}✓${NC} $1"; PASS=$((PASS+1)); }
fail() { log "${RED}✗${NC} $1"; FAIL=$((FAIL+1)); }
warn() { log "${YELLOW}⚠${NC} $1"; WARN=$((WARN+1)); }
echo "" > "$LOG_FILE"
log "========================================"
log "ODS Pre-flight Check"
log "Started: $(date)"
log "Backend: $BACKEND"
log "========================================"
log ""
# 1. Docker check
log "[1/8] Checking Docker..."
if command -v docker &> /dev/null; then
DOCKER_VERSION=$(docker --version | awk '{print $3}' | tr -d ',')
pass "Docker installed: $DOCKER_VERSION"
if docker info &> /dev/null; then
pass "Docker daemon running"
else
fail "Docker daemon not running — start with: sudo systemctl start docker"
fi
else
fail "Docker not installed"
fi
log ""
# 2. Docker Compose check
log "[2/8] Checking Docker Compose..."
if docker compose version &> /dev/null 2>&1 || docker-compose version &> /dev/null 2>&1; then
COMPOSE_VERSION=$(docker compose version 2>/dev/null | awk '{print $4}' || docker-compose version 2>/dev/null | head -1 | awk '{print $3}')
pass "Docker Compose available: $COMPOSE_VERSION"
else
fail "Docker Compose not found"
fi
log ""
# 3. GPU check — backend-aware
log "[3/8] Checking GPU..."
if [[ "$BACKEND" == "amd" ]]; then
# AMD: check sysfs for GPU and driver
GPU_FOUND=false
for card_dir in /sys/class/drm/card*/device; do
[[ -d "$card_dir" ]] || continue
vendor=$(cat "$card_dir/vendor" 2>/dev/null) || continue
if [[ "$vendor" == "0x1002" ]]; then
device_id=$(cat "$card_dir/device" 2>/dev/null || echo "unknown")
gtt_bytes=$(cat "$card_dir/mem_info_gtt_total" 2>/dev/null || echo "0")
gtt_gb=$(( gtt_bytes / 1073741824 ))
if lsmod 2>/dev/null | grep -q amdgpu; then
pass "AMD GPU detected ($device_id) — ${gtt_gb}GB GTT, amdgpu driver loaded"
else
warn "AMD GPU detected ($device_id) but amdgpu driver not loaded"
fi
# Check ROCm device access
if [[ -c /dev/kfd ]]; then
pass "ROCm device /dev/kfd accessible"
else
warn "/dev/kfd not found — ROCm containers may fail"
fi
if [[ -d /dev/dri ]]; then
pass "AMD GPU device nodes available (/dev/dri)"
fi
GPU_FOUND=true
break
fi
done
if [[ "$GPU_FOUND" == "false" ]]; then
warn "No AMD GPU detected via sysfs"
fi
elif [[ "$BACKEND" == "nvidia" ]]; then
# NVIDIA: check nvidia-smi
if command -v nvidia-smi &> /dev/null; then
GPU_INFO=""
if raw_gpu=$(nvidia-smi --query-gpu=name,memory.total --format=csv,noheader 2>/dev/null); then
GPU_INFO=$(echo "$raw_gpu" | head -1)
fi
if [ -n "$GPU_INFO" ]; then
pass "NVIDIA GPU detected: $GPU_INFO"
if docker info 2>/dev/null | grep -q "nvidia"; then
pass "NVIDIA Docker runtime available"
else
warn "NVIDIA Docker runtime not configured — GPU containers may fail"
fi
else
warn "nvidia-smi found but no GPU detected"
fi
else
warn "nvidia-smi not found — NVIDIA GPU features unavailable"
fi
else
pass "CPU mode — no GPU runtime required"
fi
log ""
# 4. LLM Endpoint check
# OLLAMA_PORT controls the external port for llama-server.
# Canonical default is 8080 (config/ports.json, docker-compose.base.yml).
# 11434 is only used on Strix Halo AMD installs where phase 06 writes
# OLLAMA_PORT=11434 to .env automatically — it will be picked up via the
# ${OLLAMA_PORT:-...} expansion below, so the fallback should be 8080.
log "[4/8] Checking LLM endpoint..."
if ods_preflight_uses_litellm; then
LLM_PORT="${LITELLM_PORT:-4000}"
LLM_ENDPOINTS=("http://${SERVICE_HOST}:${LLM_PORT}/health/readiness" "http://127.0.0.1:${LLM_PORT}/health/readiness" "http://127.0.0.1:${LLM_PORT}/v1/models")
LLM_SERVICE_NAME="LiteLLM gateway"
LLM_CONTAINER_MATCH="ods-litellm"
LLM_START_CMD="cd \"$ODS_DIR\" && ./ods-cli start litellm"
else
LLM_PORT="${OLLAMA_PORT:-${LLAMA_SERVER_PORT:-8080}}"
# Also probe the actual mapped port in case docker remapped it
EXTERNAL_PORT="$(docker port ods-llama-server 8080/tcp 2>/dev/null | head -1 | cut -d: -f2 || true)"
[[ -n "$EXTERNAL_PORT" ]] || EXTERNAL_PORT="$LLM_PORT"
LLM_ENDPOINTS=("http://${SERVICE_HOST}:${EXTERNAL_PORT}/health" "http://${SERVICE_HOST}:${EXTERNAL_PORT}/v1/models" "http://127.0.0.1:${EXTERNAL_PORT}/health" "http://127.0.0.1:${EXTERNAL_PORT}/v1/models" "http://127.0.0.1:${LLM_PORT}/health" "http://127.0.0.1:${LLM_PORT}/v1/models")
LLM_SERVICE_NAME="llama-server"
LLM_CONTAINER_MATCH="ods-llama-server"
LLM_START_CMD="cd \"$ODS_DIR\" && ./ods-cli start llama-server"
fi
LLM_FOUND=false
for ENDPOINT in "${LLM_ENDPOINTS[@]}"; do
if curl -sf "${CURL_HEALTH_FLAGS[@]}" "$ENDPOINT" &> /dev/null; then
pass "LLM endpoint ($LLM_SERVICE_NAME) responding at $ENDPOINT"
LLM_FOUND=true
break
fi
done
if [ "$LLM_FOUND" = false ]; then
# Check if container is running but model still loading
if docker ps --format '{{.Names}}' 2>/dev/null | grep -qi "${LLM_CONTAINER_MATCH}"; then
warn "$LLM_SERVICE_NAME container running but not responding yet (model may still be loading)"
else
fail "No LLM endpoint found — checked: ${LLM_ENDPOINTS[*]}"
warn "Start $LLM_SERVICE_NAME with: $LLM_START_CMD"
fi
fi
log ""
# 5. Whisper STT check
log "[5/8] Checking Whisper STT..."
WHISPER_PORT_RESOLVED="${WHISPER_PORT:-9000}"
WHISPER_ENDPOINTS=("http://${SERVICE_HOST}:${WHISPER_PORT_RESOLVED}" "http://127.0.0.1:${WHISPER_PORT_RESOLVED}")
WHISPER_FOUND=false
for ENDPOINT in "${WHISPER_ENDPOINTS[@]}"; do
if curl -sf "${CURL_HEALTH_FLAGS[@]}" "$ENDPOINT/health" &> /dev/null; then
pass "Whisper STT responding at $ENDPOINT"
WHISPER_FOUND=true
break
fi
done
if [ "$WHISPER_FOUND" = false ]; then
warn "Whisper STT not found — voice input will be unavailable"
fi
log ""
# 6. TTS check
log "[6/8] Checking TTS (Kokoro)..."
TTS_PORT_RESOLVED="${TTS_PORT:-8880}"
TTS_ENDPOINTS=("http://${SERVICE_HOST}:${TTS_PORT_RESOLVED}" "http://127.0.0.1:${TTS_PORT_RESOLVED}")
TTS_FOUND=false
for ENDPOINT in "${TTS_ENDPOINTS[@]}"; do
if curl -sf "${CURL_HEALTH_FLAGS[@]}" "$ENDPOINT/health" &> /dev/null; then
pass "TTS endpoint responding at $ENDPOINT"
TTS_FOUND=true
break
fi
done
if [ "$TTS_FOUND" = false ]; then
warn "TTS not found — voice output will be unavailable"
fi
log ""
# 7. Embeddings check
log "[7/8] Checking Embeddings..."
EMBEDDINGS_PORT_RESOLVED="${EMBEDDINGS_PORT:-8090}"
EMBEDDING_ENDPOINTS=("http://${SERVICE_HOST}:${EMBEDDINGS_PORT_RESOLVED}" "http://127.0.0.1:${EMBEDDINGS_PORT_RESOLVED}")
EMBEDDING_FOUND=false
for ENDPOINT in "${EMBEDDING_ENDPOINTS[@]}"; do
if curl -sf "${CURL_HEALTH_FLAGS[@]}" "$ENDPOINT/health" &> /dev/null; then
pass "Embeddings endpoint responding at $ENDPOINT"
EMBEDDING_FOUND=true
break
fi
done
if [ "$EMBEDDING_FOUND" = false ]; then
warn "Embeddings not found — RAG features will be unavailable"
fi
log ""
# 8. Dashboard check (replaces LiveKit — more useful for all backends)
log "[8/8] Checking Dashboard..."
DASHBOARD_PORT_RESOLVED="${DASHBOARD_PORT:-3001}"
DASHBOARD_ENDPOINTS=("http://${SERVICE_HOST}:${DASHBOARD_PORT_RESOLVED}" "http://127.0.0.1:${DASHBOARD_PORT_RESOLVED}")
DASHBOARD_FOUND=false
for ENDPOINT in "${DASHBOARD_ENDPOINTS[@]}"; do
if curl -sf "${CURL_HEALTH_FLAGS[@]}" "$ENDPOINT" &> /dev/null; then
pass "Dashboard responding at $ENDPOINT"
DASHBOARD_FOUND=true
break
fi
done
if [ "$DASHBOARD_FOUND" = false ]; then
warn "Dashboard not found at port ${DASHBOARD_PORT_RESOLVED}"
fi
log ""
# Summary
log "========================================"
log "Pre-flight Summary"
log "========================================"
log "$(printf "${GREEN}✓${NC} Passed: %d" "$PASS")"
log "$(printf "${RED}✗${NC} Failed: %d" "$FAIL")"
log "$(printf "${YELLOW}⚠${NC} Warnings: %d" "$WARN")"
log ""
if [ $FAIL -eq 0 ]; then
pass "Pre-flight PASSED — ODS is ready!"
EXIT_CODE=0
else
fail "Pre-flight FAILED — fix issues above before proceeding"
EXIT_CODE=1
fi
log ""
log "Full log: $LOG_FILE"
exit $EXIT_CODE