Repository navigation
Expand file tree
/
Copy pathstream.py
More file actions
133 lines (105 loc) · 4.07 KB
/
Copy pathstream.py
File metadata and controls
133 lines (105 loc) · 4.07 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from gevent import monkey
monkey.patch_all()
import json
import gc
import riak
from bottle import Bottle, abort, request
from bottle.ext.websocket import websocket
from bottle.ext.mongo import MongoPlugin
from pandas import DataFrame
from mining.utils import conf
from mining.utils._pandas import df_generate, DataFrameSearchColumn
stream_app = Bottle()
mongo = MongoPlugin(
uri=conf("mongodb")["uri"],
db=conf("mongodb")["db"],
json_mongo=True)
stream_app.install(mongo)
@stream_app.route('/data/<slug>', apply=[websocket])
def data(ws, mongodb, slug):
if not ws:
abort(400, 'Expected WebSocket request.')
MyClient = riak.RiakClient(protocol=conf("riak")["protocol"],
http_port=conf("riak")["http_port"],
host=conf("riak")["host"])
MyBucket = MyClient.bucket(conf("riak")["bucket"])
element = mongodb['element'].find_one({'slug': slug})
element['page_limit'] = 50
if request.GET.get('limit', True) is False:
element['page_limit'] = 9999999999
data = MyBucket.get(element.get('cube')).data or {}
columns = data.get('columns') or []
fields = columns
if request.GET.get('fields', None):
fields = request.GET.get('fields').split(',')
cube_last_update = mongodb['cube'].find_one({'slug': element.get('cube')})
ws.send(json.dumps({'type': 'last_update',
'data': str(cube_last_update.get('lastupdate', ''))}))
ws.send(json.dumps({'type': 'columns', 'data': fields}))
filters = [i[0] for i in request.GET.iteritems()
if len(i[0].split('filter__')) > 1]
if element['type'] == 'grid':
page = int(request.GET.get('page', 1))
page_start = 0
page_end = element['page_limit']
if page >= 2:
page_end = element['page_limit'] * page
page_start = page_end - element['page_limit']
else:
page_start = None
page_end = None
df = DataFrame(data.get('data') or {}, columns=fields)
if len(filters) >= 1:
for f in filters:
s = f.split('__')
field = s[1]
operator = s[2]
value = request.GET.get(f)
if operator == 'like':
df = df[df[field].str.contains(value)]
elif operator == 'regex':
df = DataFrameSearchColumn(df, field, value, operator)
else:
df = df.query(df_generate(df, value, f))
groupby = []
if request.GET.get('groupby', None):
groupby = request.GET.get('groupby', ).split(',')
if len(groupby) >= 1:
df = DataFrame(df.groupby(groupby).grouper.get_group_levels())
if request.GET.get('orderby',
element.get('orderby', None)) and request.GET.get(
'orderby', element.get('orderby', None)) in fields:
orderby = request.GET.get('orderby', element.get('orderby', ''))
if type(orderby) == str:
orderby = orderby.split(',')
orderby__order = request.GET.get('orderby__order',
element.get('orderby__order', ''))
if type(orderby__order) == str:
orderby__order = orderby__order.split(',')
ind = 0
for orde in orderby__order:
if orde == '0':
orderby__order[ind] = False
else:
orderby__order[ind] = True
ind += 1
df = df.sort(orderby, ascending=orderby__order)
ws.send(json.dumps({'type': 'max_page', 'data': len(df)}))
# CLEAN MEMORY
del filters, fields, columns
gc.collect()
categories = []
for i in df.to_dict(outtype='records')[page_start:page_end]:
if element.get('categories', None):
categories.append(i[element.get('categories')])
ws.send(json.dumps({'type': 'data', 'data': i}))
# CLEAN MEMORY
del df
gc.collect()
ws.send(json.dumps({'type': 'categories', 'data': categories}))
ws.send(json.dumps({'type': 'close'}))
# CLEAN MEMORY
del categories
gc.collect()