8000 Purge generated exports on a configurable retention by jasonmadigan · Pull Request #203 · tracefinity/tracefinity · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ By default, Tracefinity uses [IS-Net](https://github.com/xuebinqin/DIS) for loca
| `MAX_UPLOAD_MB` | `20` | Maximum compressed upload size in megabytes |
| `MAX_IMAGE_PIXELS` | `64000000` | Maximum decoded pixels accepted before downscaling |
| `STL_GENERATION_CONCURRENCY` | unlimited | Process-wide maximum STL generation jobs; excess jobs wait up to 5 seconds, then receive 503 |
| `STL_RETENTION_HOURS` | `24` | Hours generated STL/3MF/zip exports are kept before a background sweep deletes them; a bin page regenerates them on the next visit or export download. `0` keeps exports forever |
| `TRACEFINITY_ONNX_PROVIDER` | `auto` | Local ONNX provider: `auto`, `cuda`, or `cpu` |
| `GEMINI_IMAGE_MODEL` | `gemini-3.1-flash-image-preview` | Gemini model for mask generation (see below) |
| `TOOL_LABEL_PROVIDER` | `none` | Optional automatic tool naming. Set to `ollama` for local vision naming |
Expand Down
34 changes: 28 additions & 6 deletions backend/app/api/routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -644,6 +644,13 @@ def cached_response() -> GenerateResponse | None:
if not (output_path.exists() and hash_path.exists() and hash_path.read_text() == input_hash):
return None
part_paths = sorted(user_path.glob(f"outputs/{entity_id}_part*.stl"))
# cache hits rewrite nothing, so refresh mtimes or the retention
# sweep could purge artefacts a live page was just handed urls for
for artefact in (output_path, hash_path, threemf_path, zip_path, insert_path, *part_paths):
try:
os.utime(artefact)
except FileNotFoundError:
pass
stl_urls = [f"/storage/{user_id}/outputs/{p.name}" for p in part_paths]
insert_stl_url = (
f"/storage/{user_id}/outputs/{entity_id}_insert.stl"
Expand Down Expand Up @@ -1425,8 +1432,12 @@ async def download_stl(request: Request, session_id: str, user_id: str = Depends
if not session or not session.stl_path:
raise HTTPException(status_code=404, detail="stl not found")

stl_abs = _abs(session.stl_path)
if not Path(stl_abs).exists():
raise HTTPException(status_code=404, detail="stl expired; regenerate the bin")

return FileResponse(
_abs(session.stl_path),
stl_abs,
media_type="application/sla",
filename=f"tracefinity-{session_id[:8]}.stl",
)
Expand All @@ -1437,6 +1448,10 @@ async def download_zip(request: Request, session_id: str, user_id: str = Depends
up = _user_path(user_id)
zip_path = up / "outputs" / f"{session_id}_parts.zip"
if not zip_path.exists():
user_sessions, _, _ = get_stores(user_id)
session = user_sessions.get(session_id)
if session and session.stl_path:
raise HTTPException(status_code=404, detail="zip expired; regenerate the bin")
raise HTTPException(status_code=404, detail="zip not found")

return FileResponse(
Expand All @@ -1455,7 +1470,7 @@ async def download_threemf(request: Request, session_id: str, user_id: str = Dep

threemf_path = Path(_abs(session.stl_path)).with_suffix(".3mf")
if not threemf_path.exists():
raise HTTPException(status_code=404, detail="3mf not found")
raise HTTPException(status_code=404, detail="3mf expired; regenerate the bin")

return FileResponse(
str(threemf_path),
Expand Down Expand Up @@ -2190,8 +2205,11 @@ async def download_bin_stl(request: Request, bin_id: str, user_id: str = Depends
bin_data = user_bins.get(bin_id)
if not bin_data or not bin_data.stl_path:
raise HTTPException(status_code=404, detail="stl not found")
stl_abs = _abs(bin_data.stl_path)
if not Path(stl_abs).exists():
raise HTTPException(status_code=404, detail="stl expired; regenerate the bin")
return FileResponse(
_abs(bin_data.stl_path),
stl_abs,
media_type="application/sla",
filename=f"{_bin_stem(bin_data)}.stl",
)
Expand All @@ -2201,10 +2219,12 @@ async def download_bin_stl(request: Request, bin_id: str, user_id: str = Depends
async def download_bin_zip(request: Request, bin_id: str, user_id: str = Depends(get_user_id)):
_, _, user_bins = get_stores(user_id)
up = _user_path(user_id)
bin_data = user_bins.get(bin_id)
zip_path = up / "outputs" / f"{bin_id}_parts.zip"
if not zip_path.exists():
if bin_data and bin_data.stl_path:
raise HTTPException(status_code=404, detail="zip expired; regenerate the bin")
raise HTTPException(status_code=404, detail="zip not found")
bin_data = user_bins.get(bin_id)
fname = f"{_bin_stem(bin_data)}-parts.zip" if bin_data else f"{bin_id[:8]}-parts.zip"
return FileResponse(
str(zip_path),
Expand All @@ -2221,7 +2241,7 @@ async def download_bin_threemf(request: Request, bin_id: str, user_id: str = Dep
raise HTTPException(status_code=404, detail="3mf not found")
threemf_path = Path(_abs(bin_data.stl_path)).with_suffix(".3mf")
if not threemf_path.exists():
raise HTTPException(status_code=404, detail="3mf not found")
raise HTTPException(status_code=404, detail="3mf expired; regenerate the bin")
return FileResponse(
str(threemf_path),
media_type="application/vnd.ms-package.3dmanufacturing-3dmodel+xml",
Expand All @@ -2233,10 +2253,12 @@ async def download_bin_threemf(request: Request, bin_id: str, user_id: str = Dep
async def download_bin_insert(request: Request, bin_id: str, user_id: str = Depends(get_user_id)):
_, _, user_bins = get_stores(user_id)
up = _user_path(user_id)
bin_data = user_bins.get(bin_id)
insert_path = up / "outputs" / f"{bin_id}_insert.stl"
if not insert_path.exists():
if bin_data and bin_data.stl_path:
raise HTTPException(status_code=404, detail="insert stl expired; regenerate the bin")
raise HTTPException(status_code=404, detail="insert stl not found")
bin_data = user_bins.get(bin_id)
fname = f"{_bin_stem(bin_data)}-insert.stl" if bin_data else f"{bin_id[:8]}-insert.stl"
return FileResponse(
str(insert_path),
Expand Down
2 changes: 2 additions & 0 deletions backend/app/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ class Settings(BaseSettings):
max_upload_mb: int = 20
max_image_pixels: int = Field(default=64_000_000, gt=0)
stl_generation_concurrency: Optional[int] = Field(default=None, gt=0)
# hours generated export artefacts are kept; 0 keeps them forever
stl_retention_hours: float = Field(default=24, ge=0)
log_level: str = "INFO"
proxy_secret: Optional[str] = None
cors_origins: list[str] = ["http://localhost:3000", "http://localhost:4001"]
Expand Down
13 changes: 13 additions & 0 deletions backend/app/main.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import asyncio
import logging
from posixpath import normpath

Expand All @@ -11,6 +12,7 @@
from starlette.responses import JSONResponse, Response

from app.config import settings
from app.services.output_retention import retention_loop
from app.services.store_errors import StoreClosedError

LOG_FORMAT = "%(asctime)s %(levelname)s %(name)s: %(message)s"
Expand Down Expand Up @@ -64,6 +66,15 @@ def _configure_uvicorn_logging():
h.setFormatter(fmt)


@app.on_event("startup")
async def _start_output_retention():
if settings.stl_retention_hours > 0:
# reference kept on app.state so the task is not garbage collected
app.state.output_retention_task = asyncio.create_task(
retention_loop(settings.storage_path, settings.stl_retention_hours)
)


class ProxySecretMiddleware(BaseHTTPMiddleware):
"""only trust user-scoped headers from an authenticated proxy."""

Expand Down Expand Up @@ -105,6 +116,8 @@ async def dispatch(self, request: Request, call_next):
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
# frontend blob downloads read the server-set filename cross-origin in dev
expose_headers=["Content-Disposition"],
)

app.mount("/storage", StaticFiles(directory=str(settings.storage_path)), name="storage")
Expand Down
60 changes: 60 additions & 0 deletions backend/app/services/output_retention.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
"""retention sweep for generated export artefacts.

STLs, 3MFs, part zips, and generation hash markers are regenerable from
stored polygons and bin config, so they are purged once older than the
configured retention. scope is deliberately tight: only files directly
inside each user's outputs/ dir with an export suffix. photos, traces,
tool/bin/session stores, and anything nested are never touched.
"""

from __future__ import annotations

import asyncio
import logging
import time
from pathlib import Path

logger = logging.getLogger(__name__)

SWEEP_INTERVAL_SECONDS = 15 * 60

# every file _run_generate writes ends in one of these
_ARTEFACT_SUFFIXES = {".stl", ".3mf", ".zip", ".hash"}


def sweep_expired_outputs(storage_path: Path, retention_hours: float, now: float | None = None) -> int:
"""delete export artefacts older than retention_hours; returns count removed.

retention_hours <= 0 disables the sweep entirely (keep forever).
"""
if retention_hours <= 0:
return 0
cutoff = (now if now is not None else time.time()) - retention_hours * 3600
removed = 0
for outputs_dir in sorted(storage_path.glob("*/outputs")):
if not outputs_dir.is_dir():
continue
for entry in outputs_dir.iterdir():
if entry.suffix not in _ARTEFACT_SUFFIXES:
continue
try:
if not entry.is_file() or entry.stat().st_mtime >= cutoff:
continue
entry.unlink()
except FileNotFoundError:
# generation or deletion raced the sweep; nothing left to do
continue
removed += 1
return removed


async def retention_loop(storage_path: Path, retention_hours: float) -> None:
"""periodic sweep; first pass runs immediately to catch crash leftovers."""
while True:
try:
removed = await asyncio.to_thread(sweep_expired_outputs, storage_path, retention_hours)
if removed:
logger.info("retention sweep removed %d expired export file(s)", removed)
except Exception:
logger.exception("retention sweep failed")
await asyncio.sleep(SWEEP_INTERVAL_SECONDS)
Loading
Loading
0