Repository navigation
Expand file tree
/
Copy pathio.py
More file actions
145 lines (116 loc) · 4.2 KB
/
Copy pathio.py
File metadata and controls
145 lines (116 loc) · 4.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
# Copyright (c) 2018, NVIDIA CORPORATION.
from libgdf_cffi import libgdf, ffi
from .column import Column
from .numerical import NumericalColumn
from .dataframe import DataFrame
from .datetime import DatetimeColumn
import numpy as np
def _wrap_string(text):
if(text is None):
return ffi.NULL
else:
return ffi.new("char[]", text.encode())
def read_csv(filepath, lineterminator='\n',
delimiter=',', sep=None, delim_whitespace=False,
skipinitialspace=False, names=None, dtype=None,
skipfooter=0, skiprows=0, dayfirst=False):
"""
Read CSV data into dataframe.
Parameters
----------
filepath : str
Path of file to be read.
delimiter : char, default ','
Delimiter to be used.
delim_whitespace : bool, default False
Determines whether to use whitespace as delimiter.
lineterminator : char, default '\n'
Character to indicate end of line.
skipinitialspace : bool, default False
Skip spaces after delimiter.
names : list of str, default None
List of column names to be used.
dtype : list of str or dict of {col: dtype}, default None
List of data types in the same order of the column names
or a dictionary with column_name:dtype (pandas style).
skiprows : int, default 0
Number of rows to be skipped from the start of file.
skipfooter : int, default 0
Number of rows to be skipped at the bottom of file.
Returns
-------
GPU ``DataFrame`` object.
Example
-------
foo.txt : ::
50,50|40,60|30,70|20,80|
>>> import pygdf
>>> df = pygdf.read_csv('foo.txt', delimiter=',', lineterminator='|',
... names=['col1', 'col2'], dtype=['int64', 'int64'],
... skiprows=1, skipfooter=1)
>>> df
col1 col2
0 40 60
1 30 70
"""
# def read_csv(filepath, lineterminator='\n',
# delimiter=',', sep=None, delim_whitespace=False,
# skipinitialspace=False, names=None, dtype=None,
# skipfooter=0, skiprows=0, dayfirst=False):
if names is None or dtype is None:
msg = '''Automatic dtype detection not implemented:
Column names and dtypes must be specified.'''
raise TypeError(msg)
if isinstance(dtype, dict):
dtype_dict = True
elif isinstance(dtype, list):
dtype_dict = False
if len(dtype) != len(names):
msg = '''All column dtypes must be specified.'''
raise TypeError(msg)
else:
msg = '''dtype must be 'list' or 'dict' '''
raise TypeError(msg)
csv_reader = ffi.new('csv_read_arg*')
# Populate csv_reader struct
file_path = _wrap_string(filepath)
csv_reader.file_path = file_path
arr_names = []
arr_dtypes = []
for col_name in names:
arr_names.append(_wrap_string(col_name))
if dtype_dict:
arr_dtypes.append(_wrap_string(str(dtype[col_name])))
names_ptr = ffi.new('char*[]', arr_names)
csv_reader.names = names_ptr
if not dtype_dict:
for col_dtype in dtype:
arr_dtypes.append(_wrap_string(str(col_dtype)))
dtype_ptr = ffi.new('char*[]', arr_dtypes)
csv_reader.dtype = dtype_ptr
csv_reader.delimiter = delimiter.encode()
csv_reader.lineterminator = lineterminator.encode()
csv_reader.delim_whitespace = delim_whitespace
csv_reader.skipinitialspace = skipinitialspace
csv_reader.dayfirst = dayfirst
csv_reader.num_cols = len(names)
csv_reader.skiprows = skiprows
csv_reader.skipfooter = skipfooter
# Call read_csv
libgdf.read_csv(csv_reader)
out = csv_reader.data
if out == ffi.NULL:
raise ValueError("Failed to parse CSV")
# Extract parsed columns
outcols = []
for i in range(csv_reader.num_cols_out):
newcol = Column.from_cffi_view(out[i])
if(newcol.dtype == np.dtype('datetime64[ms]')):
outcols.append(newcol.view(DatetimeColumn, dtype='datetime64[ms]'))
else:
outcols.append(newcol.view(NumericalColumn, dtype=newcol.dtype))
# Build dataframe
df = DataFrame()
for k, v in zip(names, outcols):
df[k] = v
return df