Repository navigation
Expand file tree
/
Copy pathDataSet.lua
More file actions
316 lines (273 loc) · 9.78 KB
/
Copy pathDataSet.lua
File metadata and controls
316 lines (273 loc) · 9.78 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
--------------------------------------------------------------------------------
-- DataSet: a class to handle standard datasets.
--
-- Authors: Clement Farabet, Benoit Corda
--------------------------------------------------------------------------------
local lDataSet = torch.class('nn.DataSet')
function lDataSet:__init(...)
xlua.require('image',true)
self.nbSamples = 0
if select('#',...) > 0 then
self:load(...)
end
end
function lDataSet:size()
return self.nbSamples
end
function lDataSet:__tostring__()
str = 'DataSet:\n'
if self.nbSamples then
str = str .. ' + nb samples : '..self.nbSamples
else
str = str .. ' + empty set...'
end
return str
end
function lDataSet:load(...)
-- parse args
local args, dataSetFolder, nbSamplesRequired, cacheFile, channels,
sampleSize,padding
= xlua.unpack(
{...},
'DataSet.load', nil,
{arg='dataSetFolder', type='string', help='path to dataset', req=true},
{arg='nbSamplesRequired', type='number', help='number of patches to load', default='all'},
{arg='cacheFile', type='string', help='path to file to cache files'},
{arg='channels', type='number', help='nb of channels', default=1},
{arg='sampleSize', type='table', help='resize all sample: {c,w,h}'},
{arg='padding', type='boolean', help='center sample in w,h dont rescale'}
)
self.cacheFileName = cacheFile or self.cacheFileName
-- Clear current dataset
self:emptySet()
-- Then try to find if cache file exists
-- the base name of this file can be provided by useCacheFile()
-- and the suffixe is the nb of samples needed, 'all' if not specified
local fileName
local datasetLoadedFromFile = false
if (self.cacheFileName ~= nil) then
fileName = self.cacheFileName .. '-' .. nbSamplesRequired
if sys.filep(fileName) then
-- File found
print('<DataSet> Loading samples from cached file ' .. fileName)
f = torch.DiskFile(fileName, 'rw')
f:binary()
self:read(f)
f.close(f)
datasetLoadedFromFile = true
end
end
-- If dataset couldn't be loaded from cache, load it
if (datasetLoadedFromFile == false) then
self:append{dataSetFolder=dataSetFolder, channels=channels,
nbSamplesRequired=nbSamplesRequired,
sampleSize=sampleSize}
-- if cache name given, create it now
if (fileName ~= nil) then
print('<DataSet> Dumping dataset to cache file ' .. fileName .. ' for fast retrieval')
f = torch.DiskFile(fileName, 'rw')
f:binary()
self:write(f)
f.close(f)
end
end
end
function lDataSet:emptySet(dataSetFolder)
for i = 1,table.getn(self) do
self[i] = nil
end
self.nbSamples = 0
end
function lDataSet:apply(toapply)
print('<DataSet> Applying function to dataset')
for i=1,self.nbSamples do
xlua.progress(i, self.nbSamples)
self[i][1] = toapply(self[i][1])
end
end
function lDataSet:cropAndResize(side)
for i=1,self.nbSamples do
local newSample = torch.Tensor(1, side, side)
local initSide = math.min(self[i][1]:size()[1], self[i][1]:size()[2])
local x1 = math.floor((self[i][1]:size(3) - initSide) / 2)
local y1 = math.floor((self[i][1]:size(2) - initSide) / 2)
local x2 = x1 + initSide
local y2 = y1 + initSide
image.crop(newSample,self[i][1],x1,y1,x2,y2)
self[i][1] = newSample
end
end
function lDataSet:add(args)
local input = args.input
local output = args.output
self.nbSamples = self.nbSamples + 1
self[self.nbSamples] = {input, output}
end
function lDataSet:append(...)
-- parse args
local args, dataSetFolder, channels, nbSamplesRequired, useLabelPiped,
useDirAsLabel, nbLabels, sampleSize, padding
= xlua.unpack(
{...},
'DataSet:append', 'append a folder to the dataset object',
{arg='dataSetFolder', type='string', help='path to dataset', req=true},
{arg='channels', type='number', help='number of channels for the image to load', default=3},
{arg='nbSamplesRequired', type='number', help='max number of samples to load'},
{arg='useLabelPiped', type='boolean', help='flag to use the filename as output value',default=false},
{arg='useDirAsLabel', type='boolean', help='flag to use the directory as label',default=false},
{arg='nbLabels', type='number', help='how many classes (goes with useDirAsLabel)', default=1},
{arg='sampleSize', type='table', help='resize all sample: {c,w,h}'},
{arg='padding',type='boolean',help='do we padd all the inputs in w,h'}
)
-- parse args
local files = sys.dir(dataSetFolder)
print('<DataSet> Loading samples from ' .. args.dataSetFolder .. '/')
-- nb of samples to load:
local toLoad = table.getn(files)
if (nbSamplesRequired ~= nil and nbSamplesRequired ~= 'all') then
toLoad = math.min(toLoad, nbSamplesRequired)
end
local loaded = 0
for k,file in pairs(files) do
local input, inputs, rawOutput
-- disp progress
xlua.progress(k, toLoad)
if (string.find(file,'.png')) then
-- load the PNG into a new Tensor
pathToPng = sys.concat(dataSetFolder, file)
input = image.loadPNG(pathToPng,channels)
-- parse the file name and set the ouput from it
rawOutput = sys.split(string.gsub(file, ".png", ""),'|')
elseif (string.find(file,'.p[pgn]m')) then
-- load the PPM into a new Tensor
pathToPpm = sys.concat(dataSetFolder, file)
input = image.loadPPM(pathToPpm,channels)
-- parse the file name and set the ouput from it
rawOutput = sys.split(string.gsub(file, ".p[pgn]m", ""),'|')
elseif (string.find(file,'.jpg')) then
-- load the JPG into a new Tensor
pathToPpm = sys.concat(dataSetFolder, file)
input = image.load(pathToPpm,channels)
-- parse the file name and set the ouput from it
rawOutput = sys.split(string.gsub(file, ".jpg", ""),'|')
end
-- if image loaded then add into the set
if (input and rawOutput) then
table.remove(rawOutput,1) --remove file ID
-- put input in 3D tensor
input:resize(channels, input:size(2), input:size(3))
-- rescale ?
if sampleSize then
inputs = torch.Tensor(channels, sampleSize[2], sampleSize[3])
if padding then
offw = math.floor((sampleSize[2] - input[2])*0.5)
offh = math.floor((sampleSize[3] - input[3])*0.5)
if offw >= 0 and offh >= 0 then
inputs:narrow(2,offw,input[2]):narrow(3,offh,input[3]):copy(input)
else
print('reverse crop not implemented w,h must be larger than all data points')
end
else
image.scale(input, inputs, 'bilinear')
end
else
inputs = input
end
-- and generate output
local output = torch.Tensor(table.getn(rawOutput), 1)
for i,v in ipairs(rawOutput) do
output[i][1]=v
end
-- add input/output in the set
self.nbSamples = self.nbSamples + 1
self[self.nbSamples] = {inputs, output}
loaded = loaded + 1
if (loaded == toLoad) then
break
end
end
-- some cleanup, for memory
collectgarbage()
end
end
function lDataSet:appendDataSet(dataset)
print("<DataSet> Merging dataset of size = "..dataset:size()..
" into dataset of size = "..self:size())
for i = 1,dataset:size() do
self.nbSamples = self.nbSamples + 1
self[self.nbSamples] = {}
self[self.nbSamples][1] = torch.Tensor(dataset[i][1]):copy(dataset[i][1])
if (dataset[i][2] ~= nil) then
self[self.nbSamples][2] = torch.Tensor(dataset[i][2]):copy(dataset[i][2])
end
end
end
function lDataSet:popSubset(args)
-- parse args
local nElement = args.nElement
local ratio = args.ratio or 0.1
local subset = args.outputSet or nn.DataSet()
-- get nb of samples to pop
local start_index
if (nElement ~= nil) then
start_index = self:size() - nElement + 1
else
start_index = math.floor((1-ratio)*self:size()) + 1
end
-- info
print('<DataSet> Popping ' .. self:size() - start_index + 1 .. ' samples dataset')
-- extract samples
for i = self:size(), start_index, -1 do
subset.nbSamples = subset.nbSamples + 1
subset[subset.nbSamples] = {}
subset[subset.nbSamples][1] = torch.Tensor(self[i][1]):copy(self[i][1])
subset[subset.nbSamples][2] = torch.Tensor(self[i][2]):copy(self[i][2])
self[i] = nil
self.nbSamples = self.nbSamples - 1
end
-- return network
return subset
end
function lDataSet:resize(w,h)
self.resized = true
xlua.error('not implemented yet', 'DataSet')
end
function lDataSet:shuffle()
if (self.nbSamples == 0) then
print('Warning, trying to shuffle empty Dataset, no effect...')
return
end
local n = self.nbSamples
while n > 2 do
local k = math.random(n)
-- swap elements
self[n], self[k] = self[k], self[n]
n = n - 1
end
end
function lDataSet:display(nSamples,legend)
local samplesToShow = {}
for i = 1,nSamples do
table.insert(samplesToShow, self[i][1])
end
image.display{image=samplesToShow,gui=false,legend=legend}
end
function lDataSet:useCacheFile(fileName)
self.cacheFileName = fileName
end
function lDataSet:write(file)
file:writeBool(self.resized)
file:writeInt(self.nbSamples)
-- write all the samples
for i = 1,self.nbSamples do
file:writeObject(self[i])
end
end
function lDataSet:read(file)
self.resized = file:readBool()
self.nbSamples = file:readInt()
-- read all the samples
for i = 1,self.nbSamples do
self[i] = file:readObject()
end
end