Repository navigation
Expand file tree
/
Copy pathscrape.py
More file actions
executable file
·187 lines (136 loc) · 4.96 KB
/
Copy pathscrape.py
File metadata and controls
executable file
·187 lines (136 loc) · 4.96 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
#!/usr/bin/env python
import re
import sys
import httpx
import pypandoc
from bs4 import BeautifulSoup
from playwright.sync_api import sync_playwright
from aider import __version__
aider_user_agent = f"Aider/{__version__} +https://aider.chat"
PLAYWRIGHT_INFO = """
For better web scraping, install Playwright chromium with this command in your terminal:
playwright install --with-deps chromium
See https://aider.chat/docs/install.html#enable-playwright-optional for more info.
"""
class Scraper:
pandoc_available = None
playwright_available = None
playwright_instructions_shown = False
def __init__(self, print_error=None):
if print_error:
self.print_error = print_error
else:
self.print_error = print
def scrape_with_playwright(self, url):
with sync_playwright() as p:
try:
browser = p.chromium.launch()
except Exception as e:
self.playwright_available = False
self.print_error(e)
return
page = browser.new_page()
user_agent = page.evaluate("navigator.userAgent")
user_agent = user_agent.replace("Headless", "")
user_agent = user_agent.replace("headless", "")
user_agent += " " + aider_user_agent
page = browser.new_page(user_agent=user_agent)
page.goto(url)
content = page.content()
browser.close()
return content
def try_playwright(self):
if self.playwright_available is not None:
return
with sync_playwright() as p:
try:
p.chromium.launch()
self.playwright_available = True
except Exception:
self.playwright_available = False
def get_playwright_instructions(self):
if self.playwright_available in (True, None):
return
if self.playwright_instructions_shown:
return
self.playwright_instructions_shown = True
return PLAYWRIGHT_INFO
def scrape_with_httpx(self, url):
headers = {"User-Agent": f"Mozilla./5.0 ({aider_user_agent})"}
try:
with httpx.Client(headers=headers) as client:
response = client.get(url)
response.raise_for_status()
return response.text
except httpx.HTTPError as http_err:
self.print_error(f"HTTP error occurred: {http_err}")
except Exception as err:
self.print_error(f"An error occurred: {err}")
return None
def scrape(self, url):
self.try_playwright()
if self.playwright_available:
content = self.scrape_with_playwright(url)
else:
content = self.scrape_with_httpx(url)
if not content:
return
self.try_pandoc()
content = self.html_to_markdown(content)
# content = html_to_text(content)
return content
def try_pandoc(self):
if self.pandoc_available:
return
try:
pypandoc.get_pandoc_version()
self.pandoc_available = True
return
except OSError:
pass
pypandoc.download_pandoc(delete_installer=True)
self.pandoc_available = True
def html_to_markdown(self, page_source):
soup = BeautifulSoup(page_source, "html.parser")
soup = slimdown_html(soup)
page_source = str(soup)
md = pypandoc.convert_text(page_source, "markdown", format="html")
md = re.sub(r"</div>", " ", md)
md = re.sub(r"<div>", " ", md)
md = re.sub(r"\n\s*\n", "\n\n", md)
return md
def slimdown_html(soup):
for svg in soup.find_all("svg"):
svg.decompose()
if soup.img:
soup.img.decompose()
for tag in soup.find_all(href=lambda x: x and x.startswith("data:")):
tag.decompose()
for tag in soup.find_all(src=lambda x: x and x.startswith("data:")):
tag.decompose()
for tag in soup.find_all(True):
for attr in list(tag.attrs):
if attr != "href":
tag.attrs.pop(attr, None)
return soup
# Adapted from AutoGPT, MIT License
#
# https://github.com/Significant-Gravitas/AutoGPT/blob/fe0923ba6c9abb42ac4df79da580e8a4391e0418/autogpts/autogpt/autogpt/commands/web_selenium.py#L173
def html_to_text(page_source: str) -> str:
soup = BeautifulSoup(page_source, "html.parser")
for script in soup(["script", "style"]):
script.extract()
text = soup.get_text()
lines = (line.strip() for line in text.splitlines())
chunks = (phrase.strip() for line in lines for phrase in line.split(" "))
text = "\n".join(chunk for chunk in chunks if chunk)
return text
def main(url):
scraper = Scraper()
content = scraper.scrape(url)
print(content)
if __name__ == "__main__":
if len(sys.argv) < 2:
print("Usage: python playw.py <URL>")
sys.exit(1)
main(sys.argv[1])