Repository navigation
Expand file tree
/
Copy pathDockerfile
More file actions
435 lines (362 loc) · 14.5 KB
/
Copy pathDockerfile
File metadata and controls
435 lines (362 loc) · 14.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
# syntax=docker/dockerfile:1
# Slow layers (APT, Rust, Python deps, ML models, runtime stack) live in Dockerfile.base.
# Override with local tags if registry images are missing:
# docker build --build-arg PYTHON_DEPS_IMAGE=myreg/python-deps --build-arg RUNTIME_BASE_IMAGE=myreg/runtime .
# Slim app image (no pre-baked BGE; ~1.3 GB smaller — model downloads on first use):
# docker build --build-arg PYTHON_DEPS_IMAGE=pipeshubai/pipeshub-ai-base:python-deps-slim -t pipeshubai/pipeshub-ai:slim .
ARG PYTHON_DEPS_IMAGE=pipeshubai/pipeshub-ai-base:python-deps
ARG RUNTIME_BASE_IMAGE=pipeshubai/pipeshub-ai-base:runtime
FROM ${PYTHON_DEPS_IMAGE} AS python-deps
# The base image bakes in dependencies as of its publish time. Reconcile with the
# current pyproject.toml so packages added since the base was published (e.g. new
# connector SDKs like opensearch-py) end up in the app image. uv skips
# already-satisfied packages, so this is a fast no-op when the base is current and
# installs only the delta otherwise. The base carries uv + the build toolchain
# (it is built FROM build-base in Dockerfile.base), so native wheels can still
# compile here when a new dependency needs it.
WORKDIR /app/python
COPY backend/python/pyproject.toml ./
RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked \
uv pip install --system -e . && \
crawl4ai-setup && \
playwright install chromium
FROM ${RUNTIME_BASE_IMAGE} AS runtime-base
# Presentation previews are converted to PDF at request time. The published
# runtime base historically included Writer and Calc only, leaving the soffice
# wrapper present but unable to load PPT/PPTX files.
# Install CJK fallback fonts until they are available in the published runtime
# base image. LibreOffice uses these when documents reference unavailable fonts.
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
apt-get update && apt-get install -y --no-install-recommends \
libreoffice-impress-nogui fonts-noto-cjk \
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
# -----------------------------------------------------------------------------
# Stage 1: Node.js Backend Build
# -----------------------------------------------------------------------------
FROM node:20-slim AS nodejs-backend
WORKDIR /app/backend
COPY backend/nodejs/apps/package*.json ./
COPY backend/nodejs/apps/tsconfig.json ./
# Install dependencies with architecture handling (npm ci: lockfile-speed + reproducible)
RUN --mount=type=cache,target=/root/.npm,sharing=locked \
set -e; \
ARCH=$(uname -m); \
echo "Building for architecture: $ARCH"; \
if [ "$ARCH" = "arm64" ] || [ "$ARCH" = "aarch64" ]; then \
echo "Detected ARM architecture"; \
npm ci --ignore-scripts && \
npm uninstall jpeg-recompress-bin mozjpeg imagemin-mozjpeg 2>/dev/null || true && \
npm install sharp --save || true; \
else \
echo "Detected x86 architecture"; \
npm ci; \
fi
COPY backend/nodejs/apps/src ./src
RUN npm run build && \
# Prune dev dependencies after build
npm prune --production && \
# Clean npm cache
npm cache clean --force
# -----------------------------------------------------------------------------
# Stage 2: Frontend Build (Next.js in `frontend/`)
# -----------------------------------------------------------------------------
# Static export so the Node.js API can serve files from `backend/dist/public`.
FROM node:20-slim AS frontend-build
WORKDIR /app/frontend
COPY frontend/package*.json ./
RUN --mount=type=cache,target=/root/.npm,sharing=locked \
npm config set legacy-peer-deps true && \
npm ci
COPY frontend/ ./
# Force static export for container builds by setting the env flag that
# `frontend-new/next.config.mjs` already uses to enable `output: 'export'`.
RUN ELECTRON_STATIC=1 npm run build && \
mkdir -p /out && \
cp -a out/. /out/
# -----------------------------------------------------------------------------
# Stage 3: Final Runtime Image
# -----------------------------------------------------------------------------
FROM runtime-base AS runtime
WORKDIR /app
# Point fastembed at the pre-populated cache we copy in below, matching the
# FASTEMBED_CACHE_PATH used at build time in the python-deps stage.
ENV FASTEMBED_CACHE_PATH=/root/.cache/fastembed
# Cap ML thread fan-out (PyTorch/OpenBLAS/MKL). runtime_threads.py propagates
# this to all libraries. Override via OMP_NUM_THREADS in .env if needed.
ENV OMP_NUM_THREADS=2
# Several third-party deps (talon, pysbd, ...) ship regex string literals with
# unescaped backslashes and hit Python 3.12+'s SyntaxWarning at import time.
# We don't control their source, and new deps can hit this same issue, so
# suppress the class globally rather than patching each import site.
ENV PYTHONWARNINGS="ignore::SyntaxWarning"
# Copy Python site-packages from build stage
COPY --from=python-deps /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages
COPY --from=python-deps /usr/local/bin /usr/local/bin
# Copy ML model data (dense HF embeddings + reranker, sparse fastembed, NLTK)
COPY --from=python-deps /root/.cache/huggingface /root/.cache/huggingface
COPY --from=python-deps /root/.cache/fastembed /root/.cache/fastembed
COPY --from=python-deps /root/nltk_data /root/nltk_data
# Copy Playwright browser binaries
COPY --from=python-deps /root/.cache/ms-playwright /root/.cache/ms-playwright
# Copy Node.js backend (already pruned)
COPY --from=nodejs-backend /app/backend/dist ./backend/dist
COPY --from=nodejs-backend /app/backend/src/modules/mail ./backend/src/modules/mail
COPY --from=nodejs-backend /app/backend/src/modules/api-docs/pipeshub-openapi.yaml ./backend/src/modules/api-docs/pipeshub-openapi.yaml
COPY --from=nodejs-backend /app/backend/node_modules ./backend/dist/node_modules
# Copy frontend build (normalized to /out by the selected frontend stage)
COPY --from=frontend-build /out ./backend/dist/public
# Copy Python application code
COPY backend/python/app/ /app/python/app/
# Copy the process monitor script
COPY <<'EOF' /app/process_monitor.sh
#!/bin/bash
# Process monitor script with parent-child process management
set -e
LOG_FILE="/app/process_monitor.log"
CHECK_INTERVAL=${CHECK_INTERVAL:-20}
NODEJS_PORT=${NODEJS_PORT:-3000}
EMBEDDING_PORT=${EMBEDDING_SERVER_PORT:-8002}
# PIDs of child processes
NODEJS_PID=""
SLACKBOT_PID=""
EMBEDDING_PID=""
DOCLING_PID=""
INDEXING_PID=""
CONNECTOR_PID=""
QUERY_PID=""
PARSING_PID=""
EXTRACTION_PID=""
PARSING_PORT=${PARSING_SERVICE_PORT:-8092}
EXTRACTION_PORT=${EXTRACTION_SERVICE_PORT:-8093}
log() {
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
}
start_nodejs() {
log "Starting Node.js service..."
cd /app/backend
node dist/index.js &
NODEJS_PID=$!
log "Node.js started with PID: $NODEJS_PID"
log "Waiting for Node.js health check..."
local MAX_RETRIES=30
local RETRY_COUNT=0
local HEALTH_CHECK_URL="http://localhost:${NODEJS_PORT}/api/v1/health"
while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
if curl -s -f "$HEALTH_CHECK_URL" > /dev/null 2>&1; then
log "Node.js health check passed!"
break
fi
RETRY_COUNT=$((RETRY_COUNT + 1))
log "Health check attempt $RETRY_COUNT/$MAX_RETRIES failed, retrying in 2 seconds..."
sleep 2
done
if [ $RETRY_COUNT -eq $MAX_RETRIES ]; then
log "ERROR: Node.js health check failed after $MAX_RETRIES attempts"
return 1
fi
}
start_slackbot() {
log "Starting Slack Bot service..."
cd /app/backend
node dist/integrations/slack-bot/src/index.js &
SLACKBOT_PID=$!
log "Slack Bot started with PID: $SLACKBOT_PID"
}
start_embedding() {
log "Starting Embedding service..."
cd /app/python
python -m app.embedding_main &
EMBEDDING_PID=$!
log "Embedding service started with PID: $EMBEDDING_PID"
log "Waiting for Embedding service health check..."
local MAX_RETRIES=120
local RETRY_COUNT=0
local HEALTH_CHECK_URL="http://localhost:${EMBEDDING_PORT}/health"
while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
if curl -s -f "$HEALTH_CHECK_URL" > /dev/null 2>&1; then
log "Embedding service health check passed!"
break
fi
RETRY_COUNT=$((RETRY_COUNT + 1))
log "Health check attempt $RETRY_COUNT/$MAX_RETRIES failed, retrying in 2 seconds..."
sleep 2
done
if [ $RETRY_COUNT -eq $MAX_RETRIES ]; then
log "ERROR: Embedding service health check failed after $MAX_RETRIES attempts"
return 1
fi
}
start_docling() {
log "Starting Docling service..."
cd /app/python
python -m app.docling_main &
DOCLING_PID=$!
log "Docling started with PID: $DOCLING_PID"
}
start_indexing() {
log "Starting Indexing service..."
cd /app/python
python -m app.indexing_main &
INDEXING_PID=$!
log "Indexing started with PID: $INDEXING_PID"
}
start_connector() {
log "Starting Connector service..."
cd /app/python
python -m app.connectors_main &
CONNECTOR_PID=$!
log "Connector started with PID: $CONNECTOR_PID"
log "Waiting for Connector health check..."
local MAX_RETRIES=60
local RETRY_COUNT=0
local HEALTH_CHECK_URL="http://localhost:8088/health"
while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
if curl -s -f "$HEALTH_CHECK_URL" > /dev/null 2>&1; then
log "Connector health check passed!"
break
fi
RETRY_COUNT=$((RETRY_COUNT + 1))
log "Health check attempt $RETRY_COUNT/$MAX_RETRIES failed, retrying in 2 seconds..."
sleep 2
done
if [ $RETRY_COUNT -eq $MAX_RETRIES ]; then
log "ERROR: Connector health check failed after $MAX_RETRIES attempts"
return 1
fi
}
start_query() {
log "Starting Query service..."
cd /app/python
python -m app.query_main &
QUERY_PID=$!
log "Query started with PID: $QUERY_PID"
}
start_parsing() {
log "Starting Parsing service (port ${PARSING_PORT})..."
cd /app/python
PARSING_SERVICE_PORT=${PARSING_PORT} python -m app.parsing_main &
PARSING_PID=$!
log "Parsing service started with PID: $PARSING_PID"
log "Waiting for Parsing service health check..."
local MAX_RETRIES=60
local RETRY_COUNT=0
while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
if curl -s -f "http://localhost:${PARSING_PORT}/health" > /dev/null 2>&1; then
log "Parsing service health check passed!"
break
fi
RETRY_COUNT=$((RETRY_COUNT + 1))
log "Health check attempt $RETRY_COUNT/$MAX_RETRIES failed, retrying in 2 seconds..."
sleep 2
done
if [ $RETRY_COUNT -eq $MAX_RETRIES ]; then
log "ERROR: Parsing service health check failed after $MAX_RETRIES attempts"
return 1
fi
}
start_extraction() {
log "Starting Extraction service (port ${EXTRACTION_PORT})..."
cd /app/python
EXTRACTION_SERVICE_PORT=${EXTRACTION_PORT} python -m app.extraction_main &
EXTRACTION_PID=$!
log "Extraction service started with PID: $EXTRACTION_PID"
log "Waiting for Extraction service health check..."
local MAX_RETRIES=60
local RETRY_COUNT=0
while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
if curl -s -f "http://localhost:${EXTRACTION_PORT}/health" > /dev/null 2>&1; then
log "Extraction service health check passed!"
break
fi
RETRY_COUNT=$((RETRY_COUNT + 1))
log "Health check attempt $RETRY_COUNT/$MAX_RETRIES failed, retrying in 2 seconds..."
sleep 2
done
if [ $RETRY_COUNT -eq $MAX_RETRIES ]; then
log "ERROR: Extraction service health check failed after $MAX_RETRIES attempts"
return 1
fi
}
check_process() {
local pid=$1
local name=$2
if [ -z "$pid" ] || ! kill -0 "$pid" 2>/dev/null; then
log "WARNING: $name (PID: $pid) is not running!"
return 1
fi
return 0
}
cleanup() {
log "Shutting down all services..."
[ -n "$NODEJS_PID" ] && kill "$NODEJS_PID" 2>/dev/null || true
[ -n "$SLACKBOT_PID" ] && kill "$SLACKBOT_PID" 2>/dev/null || true
[ -n "$EMBEDDING_PID" ] && kill "$EMBEDDING_PID" 2>/dev/null || true
[ -n "$DOCLING_PID" ] && kill "$DOCLING_PID" 2>/dev/null || true
[ -n "$INDEXING_PID" ] && kill "$INDEXING_PID" 2>/dev/null || true
[ -n "$CONNECTOR_PID" ] && kill "$CONNECTOR_PID" 2>/dev/null || true
[ -n "$QUERY_PID" ] && kill "$QUERY_PID" 2>/dev/null || true
[ -n "$PARSING_PID" ] && kill "$PARSING_PID" 2>/dev/null || true
[ -n "$EXTRACTION_PID" ] && kill "$EXTRACTION_PID" 2>/dev/null || true
wait
log "All services stopped."
exit 0
}
trap cleanup SIGTERM SIGINT SIGQUIT
log "=== Process Monitor Starting ==="
start_nodejs
start_slackbot
start_embedding
start_connector
start_query
start_docling
# Conditionally start the standalone Parsing and Extraction services.
# Set USE_PARSING_SERVICE=true in the environment to enable them.
# Must start (and pass health checks) *before* Indexing: on startup Indexing
# recovers in-progress records immediately and, when USE_PARSING_SERVICE=true,
# routes them through the Parsing service — if that service isn't up yet those
# recovery calls fail with connection errors / 503s.
if [ "${USE_PARSING_SERVICE:-false}" = "true" ]; then
log "USE_PARSING_SERVICE=true — starting Parsing and Extraction services"
start_parsing
start_extraction
else
log "USE_PARSING_SERVICE not set — skipping Parsing and Extraction services"
fi
start_indexing
log "All services started. Beginning monitoring cycle (checking every ${CHECK_INTERVAL}s)..."
while true; do
sleep "$CHECK_INTERVAL"
if ! check_process "$NODEJS_PID" "Node.js"; then
start_nodejs
fi
if [ -n "$SLACKBOT_PID" ] && ! check_process "$SLACKBOT_PID" "Slack Bot"; then
start_slackbot
fi
if ! check_process "$EMBEDDING_PID" "Embedding"; then
start_embedding
fi
if ! check_process "$DOCLING_PID" "Docling"; then
start_docling
fi
if ! check_process "$INDEXING_PID" "Indexing"; then
start_indexing
fi
if ! check_process "$CONNECTOR_PID" "Connector"; then
start_connector
fi
if ! check_process "$QUERY_PID" "Query"; then
start_query
fi
if [ "${USE_PARSING_SERVICE:-false}" = "true" ]; then
if ! check_process "$PARSING_PID" "Parsing"; then
start_parsing
fi
if ! check_process "$EXTRACTION_PID" "Extraction"; then
start_extraction
fi
3545
fi
done
EOF
RUN chmod +x /app/process_monitor.sh
EXPOSE 3000 8002 8092 8093
CMD ["/app/process_monitor.sh"]