Repository navigation
Expand file tree
/
Copy pathutils.py
More file actions
110 lines (78 loc) · 3.11 KB
/
Copy pathutils.py
File metadata and controls
110 lines (78 loc) · 3.11 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
import os
import sys
import time
import logging.config
import json
class Utils:
@staticmethod
def insert_record(client, dataset_id, table_id, record):
result = client.push_rows(dataset_id, table_id, [record], None)
if result.get('insertErrors', None):
print "Record: %s" % (json.dumps(record))
print "Error result: %s" % result
return False
return True
@staticmethod
def import_from_file(client, dataset_id, table_id, filename, single_tweet=False):
if single_tweet:
record = json.loads(Utils.read_file(SAMPLE_TWEET_FILE))
success = Utils.insert_record(client, dataset_id, table_id, record)
return success
row = 0
with open(filename, "r") as f:
for tweet in f:
record = json.loads(tweet)
# ignore delete records for now
if record.get("delete", None):
continue
record_scrubbed = Utils.scrub(record)
success = Utils.insert_record(client, dataset_id, table_id, record_scrubbed)
if not success:
print "Failed row: %s %s" % (row, json.dumps(record))
return
else:
print "Processed row: %s" % row
row = row + 1
@staticmethod
def scrub(d):
# d.iteritems isn't used as you can't del or the iterator breaks.
for key, value in d.items():
if value is None:
del d[key]
elif key == 'coordinates':
del d[key]
elif key == 'attributes': # in 'place' object
del d[key]
elif key == 'bounding_box': # in 'place' object
del d[key]
elif key == 'retweeted_status':
del d[key]
elif key == 'created_at':
d[key] = Utils.convert_timestamp(value)
elif isinstance(value, dict):
Utils.scrub(value)
return d # For convenience
@staticmethod
def convert_timestamp(str):
ts = time.strptime(str,'%a %b %d %H:%M:%S +0000 %Y')
ts = time.strftime('%Y-%m-%d %H:%M:%S', ts)
return ts
@staticmethod
def read_file(fn):
data = ""
with open(fn, "r") as f:
for line in f:
data = data + line
return data
@staticmethod
def generate_schema_from_tweet():
record_str = read_file(SAMPLE_TWEET_FILE)
39CF
record = json.loads(record_str)
schema_str = schema_from_record(record)
return schema_str
@staticmethod
def enable_logging():
LOGGING_CONFIG = os.path.join(os.path.dirname(__file__), "logs", "logging.conf")
logging.config.fileConfig(LOGGING_CONFIG)
root = logging.getLogger("root")
return root