Repository navigation
Expand file tree
/
Copy pathexport.py
More file actions
125 lines (101 loc) · 3.82 KB
/
Copy pathexport.py
File metadata and controls
125 lines (101 loc) · 3.82 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from gevent import monkey
monkey.patch_all()
import json
import gc
import riak
from bottle import Bottle, request, response
from bottle.ext.mongo import MongoPlugin
from pandas import DataFrame
from mining.settings import PROJECT_PATH
from mining.utils import conf
from mining.utils._pandas import df_generate, DataFrameSearchColumn
export_app = Bottle()
mongo = MongoPlugin(
uri=conf("mongodb")["uri"],
db=conf("mongodb")["db"],
json_mongo=True)
export_app.install(mongo)
@export_app.route('/data/<slug>.<ext>')
def data(mongodb, slug, ext='xls'):
MyClient = riak.RiakClient(protocol=conf("riak")["protocol"],
http_port=conf("riak")["http_port"],
host=conf("riak")["host"])
MyBucket = MyClient.bucket(conf("riak")["bucket"])
element = mongodb['element'].find_one({'slug': slug})
element['page_limit'] = 50
if request.GET.get('limit', True) is False:
element['page_limit'] = 9999999999
data = MyBucket.get(element.get('cube')).data or {}
columns = data.get('columns') or []
fields = columns
if request.GET.get('fields', None):
fields = request.GET.get('fields').split(',')
filters = [i[0] for i in request.GET.iteritems()
if len(i[0].split('filter__')) > 1]
if element['type'] == 'grid':
page = int(request.GET.get('page', 1))
page_start = 0
page_end = element['page_limit']
if page >= 2:
page_end = element['page_limit'] * page
page_start = page_end - element['page_limit']
else:
page_start = None
page_end = None
df = DataFrame(data.get('data') or {}, columns=fields)
if len(filters) >= 1:
for f in filters:
s = f.split('__')
field = s[1]
operator = s[2]
value = request.GET.get(f)
if operator == 'like':
df = df[df[field].str.contains(value)]
elif operator == 'regex':
df = DataFrameSearchColumn(df, field, value, operator)
else:
df = df.query(df_generate(df, value, f))
groupby = []
if request.GET.get('groupby', None):
groupby = request.GET.get('groupby', ).split(',')
if len(groupby) >= 1:
df = DataFrame(df.groupby(groupby).grouper.get_group_levels())
if request.GET.get('orderby',
element.get('orderby', None)) and request.GET.get(
'orderby', element.get('orderby', None)) in fields:
orderby = request.GET.get('orderby', element.get('orderby', ''))
if type(orderby) == str:
orderby = orderby.split(',')
orderby__order = request.GET.get('orderby__order',
element.get('orderby__order', ''))
if type(orderby__order) == str:
orderby__order = orderby__order.split(',')
ind = 0
for orde in orderby__order:
if orde == '0':
orderby__order[ind] = False
else:
orderby__order[ind] = True
ind += 1
df = df.sort(orderby, ascending=orderby__order)
# CLEAN MEMORY
del filters, fields, columns
gc.collect()
file_name = '{}/assets/exports/openmining-{}.{}'.format(
PROJECT_PATH, element.get('cube'), ext)
if ext == 'csv':
df.to_csv(file_name, sep=";")
contenttype = 'text/csv'
else:
df.to_excel(file_name)
contenttype = 'application/vnd.ms-excel'
response.set_header('charset', 'utf-8')
response.set_header('Content-disposition', 'attachment; '
'filename={}.{}'.format(element.get('cube'), ext))
response.content_type = contenttype
ifile = open(file_name, "r")
o = ifile.read()
ifile.close()
return o