Repository navigation
Expand file tree
/
Copy pathhelp.py
More file actions
executable file
·120 lines (88 loc) · 3.11 KB
/
Copy pathhelp.py
File metadata and controls
executable file
·120 lines (88 loc) · 3.11 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
#!/usr/bin/env python
import os
import warnings
from pathlib import Path
import importlib_resources
from aider import __version__
from aider.dump import dump # noqa: F401
from aider.help_pats import exclude_website_pats
warnings.simplefilter("ignore", category=FutureWarning)
def get_package_files():
for path in importlib_resources.files("aider.website").iterdir():
if path.is_file():
yield path
elif path.is_dir():
for subpath in path.rglob("*.md"):
yield subpath
def fname_to_url(filepath):
website = "website/"
index = "/index.md"
md = ".md"
docid = ""
if filepath.startswith("website/_includes/"):
pass
elif filepath.startswith(website):
docid = filepath[len(website) :]
if filepath.endswith(index):
filepath = filepath[: -len(index)] + "/"
elif filepath.endswith(md):
filepath = filepath[: -len(md)] + ".html"
docid = "https://aider.chat/" + filepath
return docid
def get_index():
from llama_index.core import (
Document,
StorageContext,
VectorStoreIndex,
load_index_from_storage,
)
from llama_index.core.node_parser import MarkdownNodeParser
dname = Path.home() / ".aider" / "caches" / ("help." + __version__)
if dname.exists():
storage_context = StorageContext.from_defaults(
persist_dir=dname,
)
index = load_index_from_storage(storage_context)
else:
parser = MarkdownNodeParser()
nodes = []
for fname in get_package_files():
fname = Path(fname)
if any(fname.match(pat) for pat in exclude_website_pats):
continue
doc = Document(
text=importlib_resources.files("aider.website")
.joinpath(fname)
.read_text(encoding="utf-8"),
metadata=dict(
filename=fname.name,
extension=fname.suffix,
url=fname_to_url(str(fname)),
),
)
nodes += parser.get_nodes_from_documents([doc])
index = VectorStoreIndex(nodes, show_progress=True)
dname.parent.mkdir(parents=True, exist_ok=True)
index.storage_context.persist(dname)
return index
class Help:
def __init__(self):
from llama_index.core import Settings
from llama_index.embeddings.huggingface import HuggingFaceEmbedding
os.environ["TOKENIZERS_PARALLELISM"] = "true"
Settings.embed_model = HuggingFaceEmbedding(model_name="BAAI/bge-small-en-v1.5")
index = get_index()
self.retriever = index.as_retriever(similarity_top_k=20)
def ask(self, question):
nodes = self.retriever.retrieve(question)
context = f"""# Question: {question}
# Relevant docs:
"""
for node in nodes:
url = node.metadata.get("url", "")
if url:
url = f' from_url="{url}"'
context += f"<doc{url}>\n"
context += node.text
context += "\n</doc>\n\n"
return context