Repository navigation
Expand file tree
/
Copy pathfetch.js
More file actions
71 lines (67 loc) · 2.37 KB
/
Copy pathfetch.js
File metadata and controls
71 lines (67 loc) · 2.37 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
const { promises: fs } = require("fs");
const childProcess = require("child_process");
const path = require("path");
const tests = {
Amazon: "https://www.amazon.com/",
BBC: "https://www.bbc.co.uk/",
Bootstrap: "https://getbootstrap.com/docs/3.4/css/",
Bing: "https://www.bing.com/",
"Coding Horror": "https://blog.codinghorror.com/",
"ECMA-262": "https://www.ecma-international.org/ecma-262/10.0/index.html",
Google: "https://www.google.com/",
"Hacker News": "https://news.ycombinator.com/",
"NY Times": "https://www.nytimes.com/",
Reddit: "https://www.reddit.com/",
"Stack Overflow": "https://www.stackoverflow.com/",
Twitter: "https://twitter.com/",
Wikipedia: "https://en.wikipedia.org/wiki/Soil",
};
const fetchTest = (name, url) =>
new Promise((resolve, reject) => {
// Use curl to follow redirects without needing a Node.js library.
childProcess.execFile(
"curl",
[
"-H",
`User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; rv:71.0) Gecko/20100101 Firefox/71.0`,
"-H",
"Accept: */*",
"-fLSs",
url,
],
(error, stdout, stderr) => {
if (error) {
return reject(error);
}
if (stderr) {
return reject(new Error(`stderr: ${stderr}`));
}
resolve([name, stdout]);
}
);
});
(async () => {
const existing = await fs.readdir(path.join(__dirname, "tests"));
await Promise.all(
existing.map((e) => fs.unlink(path.join(__dirname, "tests", e)))
);
// Format after fetching as formatting is synchronous and can take so long that connections get dropped by server due to inactivity.
for (const [name, html] of await Promise.all(
Object.entries(tests).map(([name, url]) => fetchTest(name, url))
)) {
// Apply some fixes to HTML.
const fixed = html
// Fix early termination of conditional comment in Amazon.
.replace("--></style>\n<![endif]-->", "</style>\n<![endif]-->")
// Fix closing of void tag in Amazon.
.replace(/><\/hr>/g, "/>")
// Fix extra '</div>' in BBC.
.replace(
"</a></span></small></div></div></div></footer>",
"</a></span></small></div></div></footer>"
)
// Fix broken attribute value in Stack Overflow.
.replace('height=151"', 'height="151"');
await fs.writeFile(path.join(__dirname, "tests", name), fixed);
}
})().catch(console.error);