Repository navigation
Expand file tree
/
Copy pathgemini.py
More file actions
91 lines (73 loc) · 2.99 KB
/
Copy pathgemini.py
File metadata and controls
91 lines (73 loc) · 2.99 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
import subprocess
import json
import http.client
import mimetypes
import os
def get_text_files():
text_files = []
# List all files tracked by Git
git_files_proc = subprocess.run(['git', 'ls-files'], stdout=subprocess.PIPE, text=True)
for file in git_files_proc.stdout.strip().split('\n'):
# Check MIME type for each file
mime_check_proc = subprocess.run(['file', '--mime', file], stdout=subprocess.PIPE, text=True)
if 'text' in mime_check_proc.stdout:
text_files.append(file)
print(f"File count: {len(text_files)}")
return text_files
def get_file_contents(file):
# Read file content
with open(file, 'r') as f:
return f.read()
def main():
GEMINI_API_KEY = os.environ.get('GEMINI_API_KEY')
# Your prompt
prompt = "Document the data types and dataflow in this codebase in preparation to port a streaming implementation to rust:\n\n"
# Fetch all text files
text_files = get_text_files()
code_blocks = []
for file in text_files:
file_contents = get_file_contents(file)
# Create a code block for each text file
code_blocks.append(f"\n`{file}`\n\n```{file_contents}```\n")
# Construct the JSON payload
payload = json.dumps({
"contents": [{
"parts": [{
"text": prompt + "".join(code_blocks)
}]
}]
})
# Prepare the HTTP connection
conn = http.client.HTTPSConnection("generativelanguage.googleapis.com")
# Define headers
headers = {
'Content-Type': 'application/json',
'Content-Length': str(len(payload))
}
# Output the content length in bytes
print(f"Content Length in kilobytes: {len(payload.encode('utf-8')) / 1024:.2f} KB")
# Send a request to count the tokens
conn.request("POST", f"/v1beta/models/gemini-1.5-pro-latest:countTokens?key={GEMINI_API_KEY}", body=payload, headers=headers)
# Get the response
response = conn.getresponse()
if response.status == 200:
token_count = json.loads(response.read().decode('utf-8')).get('totalTokens')
print(f"Token count: {token_count}")
else:
print(f"Failed to get token count. Status code: {response.status}, Response body: {response.read().decode('utf-8')}")
# Prepare the HTTP connection
conn = http.client.HTTPSConnection("generativelanguage.googleapis.com")
conn.request("GET", f"/v1beta/models/gemini-1.5-pro-latest:streamGenerateContent?key={GEMINI_API_KEY}", body=payload, headers=headers)
# Get the response in a streaming manner
response = conn.getresponse()
if response.status == 200:
print("Successfully sent the data to the API.")
# Read the response in chunks
while chunk := response.read(4096):
print(chunk.decode('utf-8'))
else:
print(f"Failed to send the data to the API. Status code: {response.status}, Response body: {response.read().decode('utf-8')}")
# Close the connection
conn.close()
if __name__ == "__main__":
main()