Repository navigation
Expand file tree
/
Copy patharxiv.ts
More file actions
113 lines (93 loc) · 3.21 KB
/
Copy patharxiv.ts
File metadata and controls
113 lines (93 loc) · 3.21 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
import { API_URLS } from '@/config';
import { createCircuitBreaker } from '@/utils';
export interface ArxivPaper {
id: string;
title: string;
summary: string;
authors: string[];
published: Date;
updated: Date;
categories: string[];
link: string;
pdfLink: string;
}
const breaker = createCircuitBreaker<ArxivPaper[]>({ name: 'ArXiv Papers' });
// Parse ArXiv Atom XML response
function parseArxivXML(xmlText: string): ArxivPaper[] {
const parser = new DOMParser();
const xmlDoc = parser.parseFromString(xmlText, 'text/xml');
const entries = xmlDoc.querySelectorAll('entry');
const papers: ArxivPaper[] = [];
entries.forEach((entry) => {
const id = entry.querySelector('id')?.textContent || '';
const title = entry.querySelector('title')?.textContent?.trim() || '';
const summary = entry.querySelector('summary')?.textContent?.trim() || '';
const authors: string[] = [];
entry.querySelectorAll('author name').forEach((authorEl) => {
const name = authorEl.textContent?.trim();
if (name) authors.push(name);
});
const published = entry.querySelector('published')?.textContent || '';
const updated = entry.querySelector('updated')?.textContent || '';
const categories: string[] = [];
entry.querySelectorAll('category').forEach((catEl) => {
const term = catEl.getAttribute('term');
if (term) categories.push(term);
});
// Find links
let link = id;
let pdfLink = id.replace('/abs/', '/pdf/') + '.pdf';
entry.querySelectorAll('link').forEach((linkEl) => {
const rel = linkEl.getAttribute('rel');
const href = linkEl.getAttribute('href');
if (href) {
if (rel === 'alternate') link = href;
if (linkEl.getAttribute('title') === 'pdf') pdfLink = href;
}
});
papers.push({
id: id.split('/').pop() || id,
title,
summary,
authors,
published: new Date(published),
updated: new Date(updated),
categories,
link,
pdfLink,
});
});
return papers;
}
export async function fetchArxivPapers(
category: string = 'cs.AI',
maxResults: number = 50
): Promise<ArxivPaper[]> {
return breaker.execute(async () => {
const response = await fetch(API_URLS.arxiv(category, maxResults));
if (!response.ok) throw new Error(`HTTP ${response.status}`);
const xmlText = await response.text();
return parseArxivXML(xmlText);
}, []);
}
// Fetch papers from multiple AI/ML categories
export async function fetchAllAIPapers(): Promise<ArxivPaper[]> {
const categories = ['cs.AI', 'cs.LG', 'cs.CL']; // AI, Machine Learning, Computation & Language
const results = await Promise.allSettled(
categories.map(cat => fetchArxivPapers(cat, 20))
);
const allPapers: ArxivPaper[] = [];
results.forEach((result) => {
if (result.status === 'fulfilled') {
allPapers.push(...result.value);
}
});
// Remove duplicates by ID and sort by published date
const uniquePapers = Array.from(
new Map(allPapers.map(p => [p.id, p])).values()
).sort((a, b) => b.published.getTime() - a.published.getTime());
return uniquePapers.slice(0, 50); // Return top 50 most recent
}
export function getArxivStatus(): string {
return breaker.getStatus();
}