8000 markscrub/src/files.ts at main · alahmadics/markscrub · GitHub
[go: up one dir, main page]

Skip to content

Latest commit

 

History

History
559 lines (511 loc) · 15.3 KB

files.ts

File metadata and controls

559 lines (511 loc) · 15.3 KB
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import JSZip from "jszip";
const execFileAsync = promisify(execFile);
export interface FileCleanResult {
buffer: Buffer;
actions: string[];
findings: Array<{
kind: "metadata" | "c2pa" | "frontmatter";
code: string;
message: string;
count?: number;
verifiable: boolean;
}>;
}
function extOf(path: string): string {
const i = path.lastIndexOf(".");
return i >= 0 ? path.slice(i + 1).toLowerCase() : "";
}
/** PNG: drop tEXt/zTXt/iTXt/eXIf and any chunk whose data mentions c2pa. */
export function cleanPng(buf: Buffer): FileCleanResult {
const actions: string[] = [];
const findings: FileCleanResult["findings"] = [];
if (buf.length < 8 || buf.subarray(0, 8).toString("hex") !== "89504e470d0a1a0a") {
return { buffer: buf, actions, findings };
}
const out: Buffer[] = [buf.subarray(0, 8)];
let offset = 8;
let dropped = 0;
let c2pa = 0;
while (offset + 8 <= buf.length) {
const length = buf.readUInt32BE(offset);
const type = buf.subarray(offset + 4, offset + 8).toString("ascii");
const total = 12 + length;
if (offset + total > buf.length) break;
const chunk = buf.subarray(offset, offset + total);
const data = buf.subarray(offset + 8, offset + 8 + length);
const dropMeta = ["tEXt", "zTXt", "iTXt", "eXIf"].includes(type);
const looksC2pa =
type === "caBX" ||
data.includes(Buffer.from("c2pa")) ||
data.includes(Buffer.from("C2PA")) ||
data.includes(Buffer.from("contentcredentials"));
if (dropMeta || looksC2pa) {
dropped += 1;
if (looksC2pa) c2pa += 1;
} else {
out.push(chunk);
}
offset += total;
}
if (dropped) {
actions.push(`png: removed ${dropped} metadata/ancillary chunk(s)`);
findings.push({
kind: "metadata",
code: "png_metadata",
message: `Removed ${dropped} PNG metadata chunk(s)`,
count: dropped,
verifiable: true,
});
}
if (c2pa) {
findings.push({
kind: "c2pa",
code: "png_c2pa",
message: `Removed ${c2pa} PNG chunk(s) that looked like C2PA/Content Credentials`,
count: c2pa,
verifiable: true,
});
}
return { buffer: Buffer.concat(out), actions, findings };
}
/** JPEG: drop APP1 (EXIF/XMP) and APP11 (often C2PA) segments. */
export function cleanJpeg(buf: Buffer): FileCleanResult {
const actions: string[] = [];
const findings: FileCleanResult["findings"] = [];
if (buf.length < 4 || buf[0] !== 0xff || buf[1] !== 0xd8) {
return { buffer: buf, actions, findings };
}
const out: number[] = [0xff, 0xd8];
let i = 2;
let dropped = 0;
let c2pa = 0;
while (i < buf.length) {
if (buf[i] !== 0xff) {
// Copy through SOS entropy-coded data as-is.
out.push(...buf.subarray(i));
break;
}
const marker = buf[i + 1];
if (marker === 0xd9) {
out.push(0xff, 0xd9);
break;
}
if (marker === 0xda) {
out.push(...buf.subarray(i));
break;
}
// Standalone markers
if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) {
out.push(0xff, marker);
i += 2;
continue;
}
if (i + 4 > buf.length) break;
const len = buf.readUInt16BE(i + 2);
const segEnd = i + 2 + len;
const seg = buf.subarray(i, segEnd);
const drop = marker === 0xe1 || marker === 0xeb; // APP1, APP11
const looksC2pa =
seg.includes(Buffer.from("c2pa")) ||
seg.includes(Buffer.from("C2PA")) ||
marker === 0xeb;
if (drop) {
dropped += 1;
if (looksC2pa) c2pa += 1;
} else {
out.push(...seg);
}
i = segEnd;
}
if (dropped) {
actions.push(`jpeg: removed ${dropped} APP segment(s)`);
findings.push({
kind: "metadata",
code: "jpeg_metadata",
message: `Removed ${dropped} JPEG APP segment(s)`,
count: dropped,
verifiable: true,
});
}
if (c2pa) {
findings.push({
kind: "c2pa",
code: "jpeg_c2pa",
message: `Removed ${c2pa} JPEG segment(s) associated with C2PA/XMP`,
count: c2pa,
verifiable: true,
});
}
return { buffer: Buffer.from(out), actions, findings };
}
const SVG_META_RE =
/<metadata[\s\S]*?<\/metadata>/gi;
const SVG_RDF_RE = /<rdf:RDF[\s\S]*?<\/rdf:RDF>/gi;
export function cleanSvg(text: string): {
text: string;
actions: string[];
findings: FileCleanResult["findings"];
} {
const actions: string[] = [];
const findings: FileCleanResult["findings"] = [];
let out = text;
const meta = out.match(SVG_META_RE)?.length ?? 0;
const rdf = out.match(SVG_RDF_RE)?.length ?? 0;
if (meta) {
out = out.replace(SVG_META_RE, "");
actions.push(`svg: removed ${meta} <metadata> block(s)`);
findings.push({
kind: "metadata",
code: "svg_metadata",
message: `Removed ${meta} SVG metadata block(s)`,
count: meta,
verifiable: true,
});
}
if (rdf) {
out = out.replace(SVG_RDF_RE, "");
actions.push(`svg: removed ${rdf} rdf:RDF block(s)`);
findings.push({
kind: "c2pa",
code: "svg_xmp",
message: `Removed ${rdf} XMP/RDF block(s)`,
count: rdf,
verifiable: true,
});
}
return { text: out, actions, findings };
}
const AI_FRONTMATTER_KEYS = [
"generator",
"ai",
"model",
"created_with",
"provenance",
"c2pa",
];
export function cleanMarkdown(text: string): {
text: string;
actions: string[];
findings: FileCleanResult["findings"];
} {
const actions: string[] = [];
const findings: FileCleanResult["findings"] = [];
if (!text.startsWith("---\n") && !text.startsWith("---\r\n")) {
return { text, actions, findings };
}
const end = text.indexOf("\n---", 4);
if (end < 0) return { text, actions, findings };
const fm = text.slice(4, end);
const body = text.slice(end + 4).replace(/^\r?\n/, "");
const lines = fm.split(/\r?\n/);
const kept: string[] = [];
let dropped = 0;
for (const line of lines) {
const key = line.split(":")[0]?.trim().toLowerCase() ?? "";
if (AI_FRONTMATTER_KEYS.some((k) => key === k || key.startsWith(`${k}_`))) {
dropped += 1;
continue;
}
kept.push(line);
}
if (!dropped) return { text, actions, findings };
const next =
kept.filter((l) => l.trim()).length > 0
? `---\n${kept.join("\n")}\n---\n${body}`
: body;
actions.push(`markdown: dropped ${dropped} AI-ish frontmatter key(s)`);
findings.push({
kind: "frontmatter",
code: "md_frontmatter",
message: `Dropped ${dropped} frontmatter key(s)`,
count: dropped,
verifiable: true,
});
return { text: next, actions, findings };
}
const HTML_META_RE =
/<meta\b[^>]*(name|property)\s*=\s*["'][^"']*(generator|ai|provenance|c2pa)[^"']*["'][^>]*>/gi;
const HTML_JSONLD_AI_RE =
/<script\b[^>]*type=["']application\/ld\+json["'][^>]*>[\s\S]*?(AI|Generative|c2pa|Provable)[\s\S]*?<\/script>/gi;
export function cleanHtml(text: string): {
text: string;
actions: string[];
findings: FileCleanResult["findings"];
} {
const actions: string[] = [];
const findings: FileCleanResult["findings"] = [];
let out = text;
const meta = out.match(HTML_META_RE)?.length ?? 0;
if (meta) {
out = out.replace(HTML_META_RE, "");
actions.push(`html: removed ${meta} AI-ish <meta> tag(s)`);
findings.push({
kind: "metadata",
code: "html_meta",
message: `Removed ${meta} HTML meta tag(s)`,
count: meta,
verifiable: true,
});
}
const jsonld = out.match(HTML_JSONLD_AI_RE)?.length ?? 0;
if (jsonld) {
out = out.replace(HTML_JSONLD_AI_RE, "");
actions.push(`html: removed ${jsonld} JSON-LD block(s) with AI hints`);
findings.push({
kind: "metadata",
code: "html_jsonld",
message: `Removed ${jsonld} JSON-LD block(s)`,
count: jsonld,
verifiable: true,
});
}
return { text: out, actions, findings };
}
async function cleanDocx(buf: Buffer): Promise<FileCleanResult> {
const actions: string[] = [];
const findings: FileCleanResult["findings"] = [];
const zip = await JSZip.loadAsync(buf);
let touched = 0;
for (const name of Object.keys(zip.files)) {
if (
name.startsWith("docProps/") ||
name.startsWith("customXml/") ||
name.includes("provenance")
) {
zip.remove(name);
touched += 1;
}
}
if (touched) {
actions.push(`docx: removed ${touched} docProps/customXml entr(y/ies)`);
findings.push({
kind: "metadata",
code: "docx_props",
message: `Removed ${touched} DOCX property/customXml entr(y/ies)`,
count: touched,
verifiable: true,
});
}
const out = await zip.generateAsync({ type: "nodebuffer" });
return { buffer: out, actions, findings };
}
async function cleanPdfWithExiftool(
inputPath: string,
outputPath: string,
): Promise<FileCleanResult | null> {
try {
await execFileAsync("exiftool", [
"-all=",
"-overwrite_original",
"-o",
outputPath,
inputPath,
]);
const { readFile } = await import("node:fs/promises");
const buffer = await readFile(outputPath);
return {
buffer,
actions: ["pdf: stripped metadata via exiftool"],
findings: [
{
kind: "metadata",
code: "pdf_exiftool",
message: "Stripped PDF metadata with exiftool",
verifiable: true,
},
],
};
} catch {
return null;
}
}
/**
* Best-effort PDF XMP / metadata scrub without exiftool.
* Removes XML packets and clears common Info dictionary string values.
* Does not rebuild xref tables — safe for many PDFs; prefer exiftool when available.
*/
export function cleanPdfBestEffort(buf: Buffer): FileCleanResult {
const actions: string[] = [];
const findings: FileCleanResult["findings"] = [];
let text = buf.toString("latin1");
let changed = 0;
const xmpRe =
/<\?xpacket[\s\S]*?<\?xpacket end=["'][ew]["']\?>/g;
const xmpMatches = text.match(xmpRe)?.length ?? 0;
if (xmpMatches) {
text = text.replace(xmpRe, "");
changed += xmpMatches;
actions.push(`pdf: removed ${xmpMatches} XMP packet(s) (best-effort)`);
findings.push({
kind: "metadata",
code: "pdf_xmp",
message: `Removed ${xmpMatches} embedded XMP packet(s)`,
count: xmpMatches,
verifiable: true,
});
}
8000
const metaStreamRe =
/\/Metadata\s+\d+\s+\d+\s+R/g;
const metaRefs = text.match(metaStreamRe)?.length ?? 0;
if (metaRefs) {
text = text.replace(metaStreamRe, "/Metadata null");
changed += metaRefs;
actions.push(`pdf: nulled ${metaRefs} /Metadata reference(s) (best-effort)`);
findings.push({
kind: "metadata",
code: "pdf_metadata_ref",
message: `Nulled ${metaRefs} /Metadata reference(s)`,
count: metaRefs,
verifiable: true,
});
}
// Clear common document info strings in-place (same length via spaces).
for (const key of ["Title", "Author", "Subject", "Creator", "Producer", "Keywords"]) {
const re = new RegExp(`/${key}\\s*\\((?:\\\\.|[^\\\\)])*\\)`, "g");
text = text.replace(re, (m) => {
changed += 1;
return "/" + key + " (" + " ".repeat(Math.max(0, m.length - key.length - 4)) + ")";
});
}
if (!changed) {
findings.push({
kind: "metadata",
code: "pdf_best_effort_noop",
message:
"No XMP/Info patterns found for best-effort PDF scrub. Install exiftool for deeper cleaning.",
verifiable: false,
});
return { buffer: buf, actions, findings };
}
actions.push(
"pdf: best-effort scrub applied (install exiftool for fuller metadata removal)",
);
return { buffer: Buffer.from(text, "latin1"), actions, findings };
}
export async function cleanFileBuffer(
inputPath: string,
buf: Buffer,
textHint?: string,
): Promise<FileCleanResult> {
const ext = extOf(inputPath);
if (ext === "png") return cleanPng(buf);
if (ext === "jpg" || ext === "jpeg") return cleanJpeg(buf);
if (ext === "svg") {
const src = textHint ?? buf.toString("utf8");
const cleaned = cleanSvg(src);
return {
buffer: Buffer.from(cleaned.text, "utf8"),
actions: cleaned.actions,
findings: cleaned.findings,
};
}
if (ext === "md" || ext === "markdown") {
const src = textHint ?? buf.toString("utf8");
const cleaned = cleanMarkdown(src);
return {
buffer: Buffer.from(cleaned.text, "utf8"),
actions: cleaned.actions,
findings: cleaned.findings,
};
}
if (ext === "html" || ext === "htm") {
const src = textHint ?? buf.toString("utf8");
const cleaned = cleanHtml(src);
return {
buffer: Buffer.from(cleaned.text, "utf8"),
actions: cleaned.actions,
findings: cleaned.findings,
};
}
if (ext === "docx") return cleanDocx(buf);
if (ext === "pdf") {
const { mkdtemp, writeFile, rm } = await import("node:fs/promises");
const { tmpdir } = await import("node:os");
const { join } = await import("node:path");
const dir = await mkdtemp(join(tmpdir(), "markscrub-"));
const inPath = join(dir, "in.pdf");
const outPath = join(dir, "out.pdf");
try {
await writeFile(inPath, buf);
const viaTool = await cleanPdfWithExiftool(inPath, outPath);
if (viaTool) return viaTool;
return cleanPdfBestEffort(buf);
} finally {
await rm(dir, { recursive: true, force: true });
}
}
return { buffer: buf, actions: [], findings: [] };
}
export function inspectFileHints(
inputPath: string,
buf: Buffer,
): FileCleanResult["findings"] {
const findings: FileCleanResult["findings"] = [];
const ext = extOf(inputPath);
const asText =
["md", "markdown", "html", "htm", "svg", "txt"].includes(ext)
? buf.toString("utf8")
: "";
if (buf.includes(Buffer.from("c2pa")) || buf.includes(Buffer.from("C2PA"))) {
findings.push({
kind: "c2pa",
code: "c2pa_bytes",
message: "Byte pattern suggests C2PA / Content Credentials metadata",
verifiable: true,
});
}
if (ext === "png" || ext === "jpg" || ext === "jpeg") {
if (ext === "png" && (buf.includes(Buffer.from("tEXt")) || buf.includes(Buffer.from("iTXt")))) {
findings.push({
kind: "metadata",
code: "png_text_chunks",
message: "PNG contains textual metadata chunks",
verifiable: true,
});
}
}
if (ext === "pdf") {
const latin = buf.toString("latin1");
if (latin.includes("<?xpacket") || latin.includes("/Metadata")) {
findings.push({
kind: "metadata",
code: "pdf_meta_present",
message: "PDF appears to contain XMP or /Metadata references",
verifiable: true,
});
}
}
if (asText) {
if (/<metadata[\s\S]*?<\/metadata>/i.test(asText) || /<rdf:RDF[\s\S]*?<\/rdf:RDF>/i.test(asText)) {
findings.push({
kind: "metadata",
code: "svg_meta_present",
message: "SVG metadata / XMP present",
verifiable: true,
});
}
if (asText.startsWith("---")) {
findings.push({
kind: "frontmatter",
code: "md_frontmatter_present",
message: "Markdown frontmatter present (may include AI keys)",
verifiable: true,
});
}
if (
/<meta\b[^>]*(name|property)\s*=\s*["'][^"']*(generator|ai|provenance|c2pa)[^"']*["'][^>]*>/i.test(
asText,
)
) {
findings.push({
kind: "metadata",
code: "html_meta_present",
message: "HTML contains AI-ish meta tags",
verifiable: true,
});
}
}
return findings;
}
0