Repository navigation
Expand file tree
/
Copy pathurl_ingest.py
More file actions
282 lines (237 loc) · 10.2 KB
/
Copy pathurl_ingest.py
File metadata and controls
282 lines (237 loc) · 10.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
"""Fetch remote URLs into the knowledge base's ``raw/`` directory.
This module is the input-acquisition layer for ``openkb add <URL>``: it
takes an http(s) URL, decides whether it points at a PDF or an HTML
document (using both the HTTP ``Content-Type`` header and a magic-byte
sniff so a mistyped header doesn't fool us), saves a local file under
``raw/``, and hands the path back to the caller.
The caller (``openkb.cli.add``) then dispatches the saved path to the
normal local-file ingest pipeline (``add_single_file`` →
``convert_document`` → markitdown / PageIndex), so all the existing
short-vs-long-doc routing applies automatically based on the file
extension and page count.
PDF responses are streamed to disk in chunks (large papers can be tens
of MB). HTML responses are passed through trafilatura's main-content
extractor — saving the raw HTML directly would feed nav/footer/cookie
chrome into the LLM and produce noisy summaries.
"""
from __future__ import annotations
import re
import urllib.error
import urllib.request
from pathlib import Path
from urllib.parse import unquote, urlparse
import click
_USER_AGENT = "openkb/url-fetcher (+https://github.com/VectifyAI/OpenKB)"
_TIMEOUT_SECONDS = 30
_CHUNK_BYTES = 64 * 1024
_SNIFF_BYTES = 512
_HTML_MIN_EXTRACT_CHARS = 300
_MAX_FILENAME_STEM = 80
def looks_like_url(s: str) -> bool:
"""Cheap check used by ``openkb add`` to branch into URL ingest."""
return s.startswith(("http://", "https://"))
def _sniff_content_type(head: bytes, declared: str) -> str:
"""Return ``'pdf'``, ``'html'``, or ``'unknown'`` based on magic bytes,
falling back to the server's declared Content-Type.
Magic bytes win when they disagree with the header (some servers
mislabel PDFs as ``application/octet-stream`` or send HTML
interstitial pages with ``application/pdf``).
"""
if head.startswith(b"%PDF-"):
return "pdf"
stripped = head.lstrip(b" \t\r\n\xef\xbb\xbf") # strip BOM + leading whitespace
if stripped[:1] == b"<":
return "html"
declared_main = declared.split(";")[0].strip().lower()
if declared_main == "application/pdf":
return "pdf"
if declared_main.startswith("text/html") or declared_main == "application/xhtml+xml":
return "html"
return "unknown"
def _sanitize_filename(name: str, ext: str) -> str:
"""Make a filename safe for shell + filesystem use.
- URL-decodes percent escapes.
- Strips the existing extension **only when it matches the target
``ext``** — so ``"2509.11420"`` keeps its dot when we're saving a
``.pdf`` (the dot is part of the arxiv identifier, not an
extension).
- Replaces whitespace / parentheses / other non-``[a-zA-Z0-9._-]``
chars with ``-``, collapses repeated ``-``, and trims.
- Caps the stem at 80 chars to avoid filesystem limits.
- Returns ``<stem><ext>``, falling back to ``document<ext>`` if the
sanitized stem is empty.
"""
decoded = unquote(name)
if ext and decoded.lower().endswith(ext.lower()):
stem = decoded[: -len(ext)]
else:
stem = decoded
stem = re.sub(r"[^a-zA-Z0-9._-]+", "-", stem)
stem = re.sub(r"-+", "-", stem).strip("-._")
stem = stem[:_MAX_FILENAME_STEM].rstrip("-._")
return f"{stem}{ext}" if stem else f"document{ext}"
def _parse_content_disposition_filename(header: str | None) -> str | None:
"""Extract a filename hint from a ``Content-Disposition`` header.
Handles three forms (in priority order):
1. ``filename*=UTF-8''percent-encoded`` (RFC 5987)
2. ``filename="quoted with spaces.pdf"``
3. ``filename=unquoted-no-spaces.pdf``
"""
if not header:
return None
# RFC 5987 extended form first
m = re.search(r"filename\*=(?:[\w-]+'[\w-]*')?([^;]+)", header)
if m:
return unquote(m.group(1).strip())
# Quoted form (may contain spaces / parens / commas)
m = re.search(r'filename="([^"]+)"', header)
if m:
return m.group(1)
# Unquoted form (stops at whitespace or semicolon)
m = re.search(r"filename=([^\s;]+)", header)
if m:
return m.group(1)
return None
def _pdf_filename(url: str, content_disposition: str | None) -> str:
"""Derive a ``.pdf`` filename for a downloaded PDF.
Priority: ``Content-Disposition: filename=`` header → URL
8000
basename →
URL host. The result is run through :func:`_sanitize_filename`.
"""
cd_name = _parse_content_disposition_filename(content_disposition)
if cd_name:
return _sanitize_filename(cd_name, ".pdf")
parsed = urlparse(url)
last = (parsed.path.rsplit("/", 1)[-1] or parsed.netloc).strip()
return _sanitize_filename(last or "document", ".pdf")
def _unique_path(target: Path) -> Path:
"""Return ``target`` if it doesn't exist yet, otherwise append ``_2``,
``_3``, … to the stem until an unused name is found.
Prevents silent overwrites in ``raw/`` when two different URLs
sanitize to the same filename (e.g. two blog posts both titled
"Introduction" → both ``Introduction.md``).
"""
if not target.exists():
return target
stem = target.stem
suffix = target.suffix
parent = target.parent
for i in range(2, 10_000):
candidate = parent / f"{stem}_{i}{suffix}"
if not candidate.exists():
return candidate
raise RuntimeError(f"Could not find a free filename for {target} after 10k attempts")
def _download_pdf_chunked(response, head_bytes: bytes, target: Path) -> None:
"""Write the already-read ``head_bytes`` plus the remaining streamed
body to ``target``. Chunked so very large PDFs (50+ MB) don't sit in
RAM.
"""
with open(target, "wb") as fh:
if head_bytes:
fh.write(head_bytes)
while True:
chunk = response.read(_CHUNK_BYTES)
if not chunk:
break
fh.write(chunk)
def _extract_html(url: str, raw_dir: Path) -> Path | None:
"""Fetch the URL through trafilatura, extract the main content as
Markdown, and write it to ``raw/<title-slug>.md``.
Returns the saved path, or None if extraction failed entirely. A
short extraction (< 300 chars) is saved anyway but flagged on
stderr — pages that are JS-rendered or paywalled often produce
near-empty extractions.
"""
import trafilatura
raw_html = trafilatura.fetch_url(url)
if not raw_html:
click.echo(f" [ERROR] Could not fetch URL: {url}", err=True)
return None
markdown = trafilatura.extract(
raw_html,
output_format="markdown",
include_links=True,
)
if not markdown:
click.echo(
" [ERROR] No main content extracted — page may be empty, JS-rendered, or paywalled.",
err=True,
)
return None
if len(markdown) < _HTML_MIN_EXTRACT_CHARS:
click.echo(
f" [WARN] Only {len(markdown)} chars extracted — page may be "
f"JS-rendered or behind a login. Saving anyway; inspect the "
f"resulting wiki entry and use `openkb remove` if it's empty.",
err=True,
)
metadata = trafilatura.extract_metadata(raw_html)
title = (metadata.title if metadata else None) or url
filename = _sanitize_filename(title, ".md")
# Pick a non-colliding name — two blog posts titled "Introduction"
# would otherwise overwrite each other in raw/ and leave the hash
# registry pointing at stale bytes.
target = _unique_path(raw_dir / filename)
target.write_text(markdown, encoding="utf-8")
click.echo(
f" Extracted: {title!r}\n"
f" Saved: raw/{target.name} ({len(markdown) // 1024 or 1} KB clean markdown)"
)
return target
def fetch_url_to_raw(url: str, kb_dir: Path) -> Path | None:
"""Fetch ``url`` into ``<kb>/raw/`` and return the local path.
Routing is decided by HTTP ``Content-Type`` validated against magic
bytes (in case the server lies):
- PDF → urllib chunked download → ``raw/<sanitized>.pdf``
- HTML → trafilatura main-content extract → ``raw/<title-slug>.md``
- anything else → error, returns None
The caller then hands the saved path to ``add_single_file``, so the
existing PageIndex / markitdown routing by file extension and page
count takes over from there.
"""
raw_dir = kb_dir / "raw"
raw_dir.mkdir(parents=True, exist_ok=True)
click.echo(f"Downloading: {url}")
request = urllib.request.Request(
url,
headers={"User-Agent": _USER_AGENT, "Accept": "*/*"},
)
try:
response = urllib.request.urlopen(request, timeout=_TIMEOUT_SECONDS)
except urllib.error.HTTPError as exc:
click.echo(f" [ERROR] HTTP {exc.code} {exc.reason}", err=True)
return None
except urllib.error.URLError as exc:
click.echo(f" [ERROR] Network error: {exc.reason}", err=True)
return None
except Exception as exc:
click.echo(f" [ERROR] Fetch failed: {exc}", err=True)
return None
with response:
declared = response.headers.get("Content-Type", "") or ""
head_bytes = response.read(_SNIFF_BYTES)
actual = _sniff_content_type(head_bytes, declared)
if actual == "pdf":
# Derive the filename from the *post-redirect* URL — urllib
# follows redirects by default, so the user-typed URL may
# not be the URL that actually served the bytes (DOI / short
# link resolvers, mirror redirects, etc.). Falls back to the
# original input when the response doesn't expose a final
# URL.
final_url = response.geturl() or url
filename = _pdf_filename(
final_url,
response.headers.get("Content-Disposition"),
)
target = _unique_path(raw_dir / filename)
_download_pdf_chunked(response, head_bytes, target)
size_mb = target.stat().st_size / (1024 * 1024)
click.echo(f" Saved: raw/{target.name} ({size_mb:.1f} MB PDF)")
return target
if actual == "html":
return _extract_html(url, raw_dir)
click.echo(
f" [ERROR] Unsupported content type {declared!r} for URL ingest. "
"Download the file manually and pass its path to `openkb add` instead.",
err=True,
)
return None