1f2ca8ec
David Mayerich
added files for b...
|
1
2
|
//make sure that this header file is only loaded once
|
180d7f3a
David Mayerich
added binary file...
|
3
4
5
|
#ifndef RTS_BINARY_H
#define RTS_BINARY_H
|
e8eb202f
David Mayerich
added a new ENVI ...
|
6
|
#include "../envi/envi_header.h"
|
4dcc9392
heziqi
Ziqi's first chan...
|
7
|
#include "../math/vector.h"
|
180d7f3a
David Mayerich
added binary file...
|
8
9
|
#include <fstream>
#include <sys/stat.h>
|
15a31763
David Mayerich
set up buffer est...
|
10
|
#include <cstring>
|
5435830b
David Mayerich
linux chrono fixes
|
11
|
#include <chrono>
|
008ae990
David Mayerich
implemented Windo...
|
12
13
|
#ifdef _WIN32
#include <Windows.h>
|
15a31763
David Mayerich
set up buffer est...
|
14
15
|
#else
#include <unistd.h>
|
008ae990
David Mayerich
implemented Windo...
|
16
17
|
#endif
|
683f216a
David Mayerich
fixed? Linux comp...
|
18
|
#ifdef USE_CUDA
|
eae75592
David Mayerich
added GPU support...
|
19
20
21
22
23
|
//CUDA externs
void gpu_permute(char* dest, char* src, size_t sx, size_t sy, size_t sz, size_t d0, size_t d1, size_t d2, size_t typesize);
#include <stim/cuda/cudatools/error.h>
#endif
|
8a86bd56
David Mayerich
changed rts names...
|
24
|
namespace stim{
|
180d7f3a
David Mayerich
added binary file...
|
25
|
|
6db250f5
David Mayerich
adaptive streamin...
|
26
27
28
29
|
/// This class calculates the optimal setting for independent parameter b (batch size) for
/// minimizing the dependent parameter bps (bytes per second)
class stream_optimizer{
protected:
|
0a575bb1
David Mayerich
updated optimizat...
|
30
|
size_t Bps[2]; //bytes per second for the previous batch
|
6db250f5
David Mayerich
adaptive streamin...
|
31
32
33
|
size_t interval_B; //number of bytes processed this interval
size_t interval_ms; //number of milliseconds spent in the current interval
size_t n[2]; //current batch size (in bytes)
|
0a575bb1
David Mayerich
updated optimizat...
|
34
|
size_t h; //spacing used for finite difference calculations
|
6db250f5
David Mayerich
adaptive streamin...
|
35
36
37
|
size_t dn; //delta value (in bytes) for setting the batch size (minimum change in batch parameter)
size_t maxn; //maximum value for the batch size
|
0a575bb1
David Mayerich
updated optimizat...
|
38
39
|
double alpha; //alpha value controls the factor of the gradient that is used to calculate the next point (speed of convergence)
|
6db250f5
David Mayerich
adaptive streamin...
|
40
41
42
43
44
45
|
bool sample_step; //calculating the derivative (this class alternates between calculating dBps and B)
bool forward_diff; //evaluate the derivative using forward differences
size_t window_ms; //size of the interval (in milliseconds) integrated to get a reliable bps value
// This function rounds x to the nearest value within dB
|
0a575bb1
David Mayerich
updated optimizat...
|
46
47
48
49
50
|
size_t round_limit(double n0){
if(n0 < 0) return dn; //if n0 is less than zero, return the lowest possible n
size_t new_n = (size_t)(n0 + 0.5); //now n0 must be positive, so round it to the nearest integer
if(new_n > maxn) new_n = maxn; //limit the returned size of x to within the specified bounds
|
6db250f5
David Mayerich
adaptive streamin...
|
51
|
|
0a575bb1
David Mayerich
updated optimizat...
|
52
|
size_t lowest = new_n / dn;
|
6db250f5
David Mayerich
adaptive streamin...
|
53
|
size_t highest = lowest + dn;
|
0a575bb1
David Mayerich
updated optimizat...
|
54
|
size_t diff[2] = {new_n - lowest, highest - new_n}; //calculate the two differences
|
6db250f5
David Mayerich
adaptive streamin...
|
55
56
57
58
59
60
61
62
|
if(diff[0] < diff[1])
return lowest;
return highest;
}
public:
//constructor initializes a stream optimizer
|
c6251f8b
David Mayerich
allows user to de...
|
63
|
stream_optimizer(size_t min_batch_size, size_t max_batch_size, double a = 0.003, size_t probe_step = 5, size_t window = 2000){
|
0a575bb1
David Mayerich
updated optimizat...
|
64
65
|
//Bps = 0; //initialize to zero bytes per second processed
Bps[0] = Bps[1] = 0; //initialize the bits per second to 0
|
6db250f5
David Mayerich
adaptive streamin...
|
66
67
68
69
|
interval_B = 0; //zero bytes have been processed at initialization
interval_ms = 0; //no time has been spent on the batch so far
dn = min_batch_size; //set the minimum batch size as the minimum change in batch size
maxn = max_batch_size; //set the maximum batch size
|
0a575bb1
David Mayerich
updated optimizat...
|
70
|
n[0] = max_batch_size; //set B
|
8933935f
David Mayerich
updated VERBOSE i...
|
71
|
h = (max_batch_size / min_batch_size) / probe_step * dn;
|
0a575bb1
David Mayerich
updated optimizat...
|
72
73
74
75
|
std::cout<<"h = "<<h<<std::endl;
if(h < dn) h = dn;
alpha = a;
//n[0] = round_limit( (max_batch_size - min_batch_size)/2 );
|
6db250f5
David Mayerich
adaptive streamin...
|
76
77
78
79
80
|
window_ms = window; //minimum integration interval (for getting a reliable bps measure)
sample_step = true; //the first step is to calculate the derivative
forward_diff = true; //start with the forward difference (since we start at the maximum batch size)
}
|
c6251f8b
David Mayerich
allows user to de...
|
81
|
size_t update(size_t bytes_processed, size_t ms_spent, size_t& data_rate, bool VERBOSE = false){
|
0a575bb1
David Mayerich
updated optimizat...
|
82
83
|
interval_B += bytes_processed; //increment the number of bytes processed
interval_ms += ms_spent; //increment the number of milliseconds spent processing
|
c6251f8b
David Mayerich
allows user to de...
|
84
|
data_rate = interval_B / interval_ms;
|
0a575bb1
David Mayerich
updated optimizat...
|
85
86
87
88
89
90
|
//if we have sufficient information to evaluate the optimization function at this point
if(interval_ms < window_ms){ //if insufficient time has passed to get a reliable Bps measurement
return n[0];
}
else{ //if we have collected enough information for a reliable Bps estimate
|
c6251f8b
David Mayerich
allows user to de...
|
91
|
|
0a575bb1
David Mayerich
updated optimizat...
|
92
|
if(Bps[0] == 0){ //if n[0] hasn't been evaluated yet, this is the first step
|
c6251f8b
David Mayerich
allows user to de...
|
93
|
Bps[0] = data_rate; //set the initial Bps value
|
0a575bb1
David Mayerich
updated optimizat...
|
94
|
n[1] = n[0] - h; //set the position of the next sample point
|
c6251f8b
David Mayerich
allows user to de...
|
95
96
|
if(VERBOSE)
std::cout<<"Bps value at n = "<<n[0]<<" is "<<Bps[0]<<" Bps, probing n = "<<n[1]<<std::endl;
|
0a575bb1
David Mayerich
updated optimizat...
|
97
98
99
|
return n[1]; //return the probe point
}
else{
|
c6251f8b
David Mayerich
allows user to de...
|
100
|
Bps[1] = data_rate; //set the Bps for the current point (n[1])
|
0a575bb1
David Mayerich
updated optimizat...
|
101
102
103
104
105
106
107
108
109
|
double Bps_p; //allocate a variable for the derivative
//calculate the derivative
if(n[0] < n[1]){ //if the current point is less than the previous one (probably the most common)
Bps_p = ((double)Bps[1] - (double)Bps[0]) / (double)h; //calculate the derivative using the forward finite difference
}
else{
Bps_p = ((double)Bps[0] - (double)Bps[1]) / (double)h; //calculate the derivative using the backward finite difference
}
|
c6251f8b
David Mayerich
allows user to de...
|
110
111
|
if(VERBOSE)
std::cout<<" probed n = "<<n[1]<<" with "<<Bps[1]<<" Bps, gradient = "<<Bps_p<<" Bps"<<std::endl;
|
0a575bb1
David Mayerich
updated optimizat...
|
112
113
114
115
116
117
118
119
|
double new_n_precise = n[0] + alpha * Bps_p; //calculate the next point (snap to closest integer)
size_t new_n_nearest = round_limit(new_n_precise); //calculate the next point (given batch parameters)
if(new_n_nearest == n[0]){ //if the newest point is the same as the original point
Bps[0] = Bps[1]; //update the Bps
//if(n[0] == dn) n[1] = n[0] + h; //if we're on the left edge, probe forward
//else n[1] = n[0] - h; //otherwise probe backwards
|
c6251f8b
David Mayerich
allows user to de...
|
120
121
|
if(VERBOSE)
std::cout<<" staying at n = "<<n[0]<<" for now"<<std::endl;
|
0a575bb1
David Mayerich
updated optimizat...
|
122
123
124
125
126
127
128
129
|
//return n[1]; //return the probe point
Bps[0] = 0; //reset the Bps for the current point
return n[0]; //return the current point for a re-calculation
}
else{ //if the newest point is different from the original point
n[0] = new_n_nearest; //move to the new point
Bps[0] = 0; //set the Bps to zero (point hasn't been tested)
|
c6251f8b
David Mayerich
allows user to de...
|
130
131
|
if(VERBOSE)
std::cout<<" moving to n = "<<n[0]<<std::endl;
|
0a575bb1
David Mayerich
updated optimizat...
|
132
133
134
135
136
137
138
|
return n[0]; //return the new point
}
}
}
}
/*// this function updates the optimizer, given the number of bytes processed in an interval and time spent processing
|
6db250f5
David Mayerich
adaptive streamin...
|
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
|
size_t update(size_t bytes_processed, size_t ms_spent){
interval_B += bytes_processed; //increment the number of bytes processed
interval_ms += ms_spent; //increment the number of milliseconds spent processing
//if we have sufficient information to evaluate the optimization function at this point
if(interval_ms >= window_ms){ //if sufficient time has passed to get a reliable Bps measurement
size_t new_Bps = interval_B / interval_ms; //calculate the current Bps
if(sample_step){ //if this is a sample step, collect the information for Bps = f(n0)
Bps = new_Bps; //set the Bps to the evaluated value
n[1] = n[0] - dn; //reduce the batch size by one delta to take a second sample
if(n[1] == 0){ //if the resulting batch size is zero
n[1] = 2*dn; //we're at the left edge: set the new sample point to 2*dn
}
interval_B = interval_ms = 0; //start a new interval at the new sample point
sample_step = false; //next step will calculate the new batch size via optimization
return n[1]; //return the new batch size
}
else{ //if we have sufficient information to evaluate the derivative and optimize
double f = (double)new_Bps; //we have evaluated the function at this location
double fprime;
if(n[1] < n[0] ){ //if the new point is less than the previous point (usually the case)
fprime = (double)(Bps - new_Bps) / (double)dn; //calculate the forward difference
}
else{ //if the new point is larger (only happens at the minimum limit)
fprime = (double)(new_Bps - Bps) / (double)dn; //calculate the backward difference
}
size_t bestn = n[1] - (size_t)(f / fprime); //calculate the best value for B using Newton's method
n[0] = round_limit( (size_t)bestn ); //set the new dependent point
sample_step = true; //the next step will be a sample step
}
}
if(sample_step) return n[0];
return n[1]; //insufficient information, keep the same batch size
|
0a575bb1
David Mayerich
updated optimizat...
|
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
|
}*/
/*size_t update(size_t bytes_processed, size_t ms_spent){
interval_B += bytes_processed; //increment the number of bytes processed
interval_ms += ms_spent; //increment the number of milliseconds spent processing
//if( Bps[0] == 0 ){ //if the left boundary hasn't been processed
//if we have sufficient information to evaluate the optimization function at this point
if(interval_ms >= window_ms){
size_t new_Bps = interval_B / interval_ms; //calculate the current Bps
if(Bps[0] == 0) //if the left interval Bps hasn't been calculated
Bps[0] = interval_B / interval_ms; //that is the interval being processed
else
Bps[1] = interval_B / interval_ms; //otherwise the right interval is being processed
if(Bps[0] != 0 && Bps[1] != 0){ //if both intervals have been processed
}
}*/
|
6db250f5
David Mayerich
adaptive streamin...
|
198
|
|
c6251f8b
David Mayerich
allows user to de...
|
199
200
|
/*size_t update(size_t bytes_processed, size_t ms_spent, size_t& data_rate, bool VERBOSE){
size_t time = update(bytes_processed, ms_spent, VERBOSE);
|
0a575bb1
David Mayerich
updated optimizat...
|
201
|
data_rate = Bps[0];
|
6db250f5
David Mayerich
adaptive streamin...
|
202
|
return time;
|
c6251f8b
David Mayerich
allows user to de...
|
203
|
}*/
|
6db250f5
David Mayerich
adaptive streamin...
|
204
205
|
};
|
33d7d3cf
David Mayerich
documentation for...
|
206
207
208
209
210
211
212
|
/** This class manages the streaming of large multidimensional binary files.
* Generally these are hyperspectral files with 2 spatial and 1 spectral dimension. However, this class supports
* other dimensions via the template parameter D.
*
* @param T is the data type used to store data to disk (generally float or double)
* @param D is the dimension of the data (default 3)
*/
|
180d7f3a
David Mayerich
added binary file...
|
213
214
215
216
|
template< typename T, unsigned int D = 3 >
class binary{
protected:
|
180d7f3a
David Mayerich
added binary file...
|
217
218
219
|
std::fstream file; //file stream used for reading and writing
std::string name; //file name
|
63fc1df8
David Mayerich
added an inverse ...
|
220
|
unsigned long long R[D]; //resolution
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
221
|
unsigned long long header; //header size (in bytes)
|
f4c5d71b
David Mayerich
started working w...
|
222
|
unsigned char* mask; //pointer to a character array: 0 = background, 1 = foreground (or valid data)
|
180d7f3a
David Mayerich
added binary file...
|
223
|
|
63fc1df8
David Mayerich
added an inverse ...
|
224
|
double progress; //stores the progress on the current operation (accessible using a thread)
|
c6251f8b
David Mayerich
allows user to de...
|
225
|
size_t data_rate; //data rate (currently in Bps)
|
ee4dea28
David Mayerich
fixed errors in c...
|
226
|
|
008ae990
David Mayerich
implemented Windo...
|
227
|
size_t buffer_size; //available memory for processing large files
|
ee4dea28
David Mayerich
fixed errors in c...
|
228
|
|
33d7d3cf
David Mayerich
documentation for...
|
229
|
/// Private initialization function used to set default parameters in the data structure.
|
180d7f3a
David Mayerich
added binary file...
|
230
|
void init(){
|
15a31763
David Mayerich
set up buffer est...
|
231
|
std::memset(R, 0, sizeof(unsigned long long) * D); //initialize the resolution to zero
|
008ae990
David Mayerich
implemented Windo...
|
232
|
header = 0; //initialize the header size to zero
|
f4c5d71b
David Mayerich
started working w...
|
233
|
mask = NULL;
|
cbce396e
David Mayerich
added support for...
|
234
|
|
c6251f8b
David Mayerich
allows user to de...
|
235
236
237
|
progress = 0; //initialize the progress for any algorithm to zero
data_rate = 0; //initialize the data rate to zero
set_buffer_frac(); //set the maximum buffer size to the default
|
63fc1df8
David Mayerich
added an inverse ...
|
238
239
|
}
|
33d7d3cf
David Mayerich
documentation for...
|
240
|
/// Private helper function that returns the size of the file on disk using system functions.
|
b0fee590
heziqi
convert long int ...
|
241
|
long long int get_file_size(){
|
193ff00e
David Mayerich
dealth with preci...
|
242
|
#ifdef _WIN32
|
b0fee590
heziqi
convert long int ...
|
243
244
|
struct _stat64 results;
if(_stat64(name.c_str(), &results) == 0)
|
180d7f3a
David Mayerich
added binary file...
|
245
|
return results.st_size;
|
193ff00e
David Mayerich
dealth with preci...
|
246
247
248
249
250
|
#else
struct stat results;
if(stat(name.c_str(), &results) == 0)
return results.st_size;
#endif
|
180d7f3a
David Mayerich
added binary file...
|
251
252
253
|
else return 0;
}
|
33d7d3cf
David Mayerich
documentation for...
|
254
|
/// Private helper function that tests to make sure that the calculated data size specified by the structure is the same as the data size on disk.
|
180d7f3a
David Mayerich
added binary file...
|
255
|
bool test_file_size(){
|
b0fee590
heziqi
convert long int ...
|
256
|
long long int npts = 1; //initialize the number of data points to 1
|
87da9adc
David Mayerich
fixed issues with...
|
257
258
|
for(unsigned int i = 0; i<D; i++) //iterate over each dimension
npts *= R[i]; //compute the total number of data points in the file
|
b0fee590
heziqi
convert long int ...
|
259
|
long long int datasize = npts * sizeof(T);//multiply the sum by the size of the template parameter
|
87da9adc
David Mayerich
fixed issues with...
|
260
261
|
if(datasize + header == get_file_size()) return true; //if the byte size matches the file size, we're golden
|
180d7f3a
David Mayerich
added binary file...
|
262
263
264
265
|
else return false; //otherwise return an error
}
|
f92397d2
David Mayerich
fixed a Win32 bug...
|
266
267
268
269
270
271
|
/// Private helper function that resets the file pointer to the beginning of the data
void reset(){
file.seekg(header, std::ios_base::beg);
}
|
33d7d3cf
David Mayerich
documentation for...
|
272
273
274
|
/// Private helper file that opens a specified binary file.
/// @param filename is the name of the binary file to stream
|
180d7f3a
David Mayerich
added binary file...
|
275
276
277
278
|
bool open_file(std::string filename){
//open the file as binary for reading and writing
file.open(filename.c_str(), std::ios::in | std::ios::out | std::ios::binary);
|
ba72c5b0
David Mayerich
added OBJ class
|
279
280
|
//if the file isn't open, the user may only have read access
if(!file.is_open()){
|
3d0b6243
David Mayerich
fixed hsiproc bug...
|
281
|
std::cout<<"class STIM::BINARY - failed to open file, trying for read only"<<std::endl;
|
ba72c5b0
David Mayerich
added OBJ class
|
282
283
284
285
286
287
288
|
file.open(filename.c_str(), std::ios::in | std::ios::binary);
if(!file.is_open()){
std::cout<<" still unable to load the file"<<std::endl;
return false;
}
}
|
180d7f3a
David Mayerich
added binary file...
|
289
290
291
292
293
294
|
//if the file is successful
if(file){
name = filename; //set the name
if(test_file_size()) //test the file size
return true;
}
|
f92397d2
David Mayerich
fixed a Win32 bug...
|
295
|
|
180d7f3a
David Mayerich
added binary file...
|
296
297
298
|
return false;
}
|
724ec347
David Mayerich
simplified the EN...
|
299
300
|
|
f92397d2
David Mayerich
fixed a Win32 bug...
|
301
302
|
|
180d7f3a
David Mayerich
added binary file...
|
303
304
|
public:
|
008ae990
David Mayerich
implemented Windo...
|
305
306
307
308
309
|
//default constructor
binary(){
init();
}
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
310
|
double get_progress(){
|
63fc1df8
David Mayerich
added an inverse ...
|
311
|
return progress;
|
cbce396e
David Mayerich
added support for...
|
312
313
|
}
|
63fc1df8
David Mayerich
added an inverse ...
|
314
315
|
void reset_progress(){
progress = 0;
|
cbce396e
David Mayerich
added support for...
|
316
317
|
}
|
c6251f8b
David Mayerich
allows user to de...
|
318
319
320
321
|
size_t get_data_rate(){
return data_rate;
}
|
008ae990
David Mayerich
implemented Windo...
|
322
|
//specify the maximum fraction of available memory that this class will use for buffering
|
c6251f8b
David Mayerich
allows user to de...
|
323
|
void set_buffer_frac(double mem_frac = 0.5){ //default to 50%
|
008ae990
David Mayerich
implemented Windo...
|
324
325
326
327
328
|
#ifdef _WIN32
MEMORYSTATUSEX statex;
statex.dwLength = sizeof (statex);
GlobalMemoryStatusEx (&statex);
buffer_size = (size_t)(statex.ullAvailPhys * mem_frac);
|
15a31763
David Mayerich
set up buffer est...
|
329
330
331
332
|
#else
size_t pages = sysconf(_SC_PHYS_PAGES);
size_t page_size = sysconf(_SC_PAGE_SIZE);
buffer_size = (size_t)(pages * page_size * mem_frac);
|
008ae990
David Mayerich
implemented Windo...
|
333
334
335
|
#endif
}
|
c6251f8b
David Mayerich
allows user to de...
|
336
337
338
339
|
void set_buffer_raw(size_t bytes){
buffer_size = bytes;
}
|
33d7d3cf
David Mayerich
documentation for...
|
340
341
342
343
344
|
/// Open a binary file for streaming.
/// @param filename is the name of the binary file
/// @param r is a STIM vector specifying the size of the binary file along each dimension
/// @param h is the length (in bytes) of any header file (default zero)
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
345
|
bool open(std::string filename, vec<unsigned long long> r, unsigned long long h = 0){
|
180d7f3a
David Mayerich
added binary file...
|
346
|
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
347
|
for(unsigned long long i = 0; i < D; i++) //set the dimensions of the binary file object
|
180d7f3a
David Mayerich
added binary file...
|
348
349
|
R[i] = r[i];
|
87da9adc
David Mayerich
fixed issues with...
|
350
351
352
|
header = h; //save the header size
if(!open_file(filename)) return false; //open the binary file
|
180d7f3a
David Mayerich
added binary file...
|
353
|
|
f92397d2
David Mayerich
fixed a Win32 bug...
|
354
355
|
//reset();
|
180d7f3a
David Mayerich
added binary file...
|
356
357
358
|
return test_file_size();
}
|
33d7d3cf
David Mayerich
documentation for...
|
359
360
361
362
363
|
/// Creates a new binary file for streaming
/// @param filename is the name of the binary file to be created
/// @param r is a STIM vector specifying the size of the file along each dimension
/// @offset specifies how many bytes to offset the file (used to leave room for a header)
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
364
|
bool create(std::string filename, vec<unsigned long long> r, unsigned long long offset = 0){
|
f47168a2
heziqi
Ziqi added create...
|
365
366
367
368
369
|
std::ofstream target(filename.c_str(), std::ios::binary);
//initialize binary file
T p = 0;
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
370
|
for(unsigned long long i =0; i < r[0] * r[1] * r[2]; i++){
|
f47168a2
heziqi
Ziqi added create...
|
371
372
373
|
target.write((char*)(&p), sizeof(T));
}
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
374
|
for(unsigned long long i = 0; i < D; i++) //set the dimensions of the binary file object
|
f47168a2
heziqi
Ziqi added create...
|
375
376
377
378
379
380
381
382
383
|
R[i] = r[i];
header = offset; //save the header size
if(!open_file(filename)) return false; //open the binary file
return test_file_size();
}
|
33d7d3cf
David Mayerich
documentation for...
|
384
385
386
387
|
/// Writes a single page of data to disk. A page consists of a sequence of data of size R[0] * R[1] * ... * R[D-1].
/// @param p is a pointer to the data to be written
/// @param page is the page number (index of the highest-numbered dimension)
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
388
|
bool write_page( T * p, unsigned long long page){
|
f47168a2
heziqi
Ziqi added create...
|
389
390
391
392
393
394
|
if(p == NULL){
std::cout<<"ERROR: unable to write into file, empty pointer"<<std::endl;
exit(1);
}
|
f92397d2
David Mayerich
fixed a Win32 bug...
|
395
|
file.seekg(R[1] * R[0] * page * sizeof(T) + header, std::ios::beg); //seek to the desired location on disk
|
8157c392
David Mayerich
added parser and ...
|
396
|
file.write((char *)p, R[0] * R[1] * sizeof(T)); //write binary data
|
f47168a2
heziqi
Ziqi added create...
|
397
398
399
400
|
return true;
}
|
33d7d3cf
David Mayerich
documentation for...
|
401
402
403
404
|
/// Reads a page from disk. A page consists of a sequence of data of size R[0] * R[1] * ... * R[D-1].
/// @param p is a pointer to pre-allocated memory equal to the page size
/// @param page is the index of the page
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
405
|
bool read_page( T * p, unsigned long long page, bool PROGRESS = false){
|
63fc1df8
David Mayerich
added an inverse ...
|
406
407
|
if(PROGRESS) progress = 0;
|
c7da85e3
heziqi
he,ziqi add metho...
|
408
|
|
5d5ce39a
heziqi
Ziqi convert the ...
|
409
|
if (page >= R[2]){ //make sure the bank number is right
|
2ad0ce4f
David Mayerich
syntactic edits b...
|
410
|
std::cout<<"ERROR: page out of range"<<std::endl;
|
5d5ce39a
heziqi
Ziqi convert the ...
|
411
|
return false;
|
c7da85e3
heziqi
he,ziqi add metho...
|
412
413
|
}
|
f92397d2
David Mayerich
fixed a Win32 bug...
|
414
|
file.seekg(R[1] * R[0] * page * sizeof(T) + header, std::ios::beg); //write into memory from the binary file
|
2ad0ce4f
David Mayerich
syntactic edits b...
|
415
|
file.read((char *)p, R[0] * R[1] * sizeof(T));
|
63fc1df8
David Mayerich
added an inverse ...
|
416
|
if(PROGRESS) progress = 100;
|
5d5ce39a
heziqi
Ziqi convert the ...
|
417
|
return true;
|
c7da85e3
heziqi
he,ziqi add metho...
|
418
419
|
}
|
724ec347
David Mayerich
simplified the EN...
|
420
421
422
423
424
425
426
|
///Reads a line Z (slowest dimension) for a given XY value
/// @param p is a pointer to pre-allocated memory equal to the line size R[2]
/// @param x is the x coordinate
/// @param y is the y coordinate
|
735a2a24
David Mayerich
started testing o...
|
427
|
void read_line_2( T* p, size_t n, bool PROGRESS = false){
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
428
|
unsigned long long i;
|
724ec347
David Mayerich
simplified the EN...
|
429
|
|
63fc1df8
David Mayerich
added an inverse ...
|
430
431
|
if(PROGRESS) progress = 0;
|
735a2a24
David Mayerich
started testing o...
|
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
|
if ( n > R[0] * R[1]){ //make sure the sample and line number is right
std::cout<<"ERROR: sample or line out of range in "<<__FILE__<<" (line "<<__LINE__<<")"<<std::endl;
exit(1);
}
file.seekg(n * sizeof(T), std::ios::beg); //point to the certain sample and line
for (i = 0; i < R[2]; i++){ //for each band
file.read((char *)(p + i), sizeof(T));
file.seekg((R[1] * R[0] - 1) * sizeof(T), std::ios::cur); //go to the next band
if(PROGRESS) progress = (double)i / (double)R[2] * 100;
}
if(PROGRESS) progress = 100;
}
void read_line_2( T * p, unsigned long long x, unsigned long long y, bool PROGRESS = false){
read_line_2(p, y * R[0] + x, PROGRESS);
/*unsigned long long i;
if(PROGRESS) progress = 0;
|
724ec347
David Mayerich
simplified the EN...
|
451
452
453
454
455
456
457
458
459
460
|
if ( x >= R[0] || y >= R[1]){ //make sure the sample and line number is right
std::cout<<"ERROR: sample or line out of range"<<std::endl;
return false;
}
file.seekg((x + y * R[0]) * sizeof(T), std::ios::beg); //point to the certain sample and line
for (i = 0; i < R[2]; i++)
{
file.read((char *)(p + i), sizeof(T));
file.seekg((R[1] * R[0] - 1) * sizeof(T), std::ios::cur); //go to the next band
|
63fc1df8
David Mayerich
added an inverse ...
|
461
|
if(PROGRESS) progress = (double)i / (double)R[2] * 100;
|
724ec347
David Mayerich
simplified the EN...
|
462
|
}
|
63fc1df8
David Mayerich
added an inverse ...
|
463
|
if(PROGRESS) progress = 100;
|
724ec347
David Mayerich
simplified the EN...
|
464
|
|
735a2a24
David Mayerich
started testing o...
|
465
|
return true;*/
|
724ec347
David Mayerich
simplified the EN...
|
466
467
468
469
470
471
472
|
}
///Reads a line X (fastest dimension) for a given YZ value
/// @param p is a pointer to pre-allocated memory equal to the line size R[2]
/// @param x is the y coordinate
/// @param y is the z coordinate
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
473
|
bool read_line_0(T * p, unsigned long long y, unsigned long long z, bool PROGRESS = false){
|
724ec347
David Mayerich
simplified the EN...
|
474
475
|
//test to make sure the specified value is within range
if( y >= R[1] || z >= R[2] ){
|
735a2a24
David Mayerich
started testing o...
|
476
|
std::cout<<"ERROR: sample ("<<y<<", "<<z<<") out of range in "<<__FILE__<<" (line "<<__LINE__<<")"<<std::endl;
|
724ec347
David Mayerich
simplified the EN...
|
477
478
479
480
481
|
return false;
}
file.seekg((z * R[0] * R[1] + y * R[0]) * sizeof(T), std::ios::beg); //seek to the start of the line
file.read((char *)p, sizeof(T) * R[0]); //read the line
|
63fc1df8
David Mayerich
added an inverse ...
|
482
|
if(PROGRESS) progress = 100;
|
724ec347
David Mayerich
simplified the EN...
|
483
484
485
486
487
488
489
490
|
return true;
}
///Reads a line Y (second fastest dimension) for a given XZ value
/// @param p is a pointer to pre-allocated memory equal to the line size R[2]
/// @param x is the y coordinate
/// @param z is the z coordinate
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
491
|
bool read_line_1(T * p, unsigned long long x, unsigned long long z, bool PROGRESS = false){
|
63fc1df8
David Mayerich
added an inverse ...
|
492
|
if(PROGRESS) progress = 0;
|
724ec347
David Mayerich
simplified the EN...
|
493
494
|
//test to make sure the specified value is within range
if( x >= R[0] || z >= R[2] ){
|
735a2a24
David Mayerich
started testing o...
|
495
|
std::cout<<"ERROR: sample or line out of range in "<<__FILE__<<" (line "<<__LINE__<<")"<<std::endl;
|
724ec347
David Mayerich
simplified the EN...
|
496
497
498
499
|
return false;
}
file.seekg((z * R[0] * R[1] + x) * sizeof(T), std::ios::beg); //seek to the start of the line
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
500
|
for (unsigned long long i = 0; i < R[1]; i++){ //for each pixel in the line
|
724ec347
David Mayerich
simplified the EN...
|
501
502
|
file.read((char *)(p + i), sizeof(T)); //read the pixel
file.seekg((R[0] - 1) * sizeof(T), std::ios::cur); //seek to the next pixel in the line
|
63fc1df8
David Mayerich
added an inverse ...
|
503
|
if(PROGRESS) progress = (double)i / (double)R[1] * 100;
|
724ec347
David Mayerich
simplified the EN...
|
504
|
}
|
63fc1df8
David Mayerich
added an inverse ...
|
505
|
if(PROGRESS) progress = 100;
|
724ec347
David Mayerich
simplified the EN...
|
506
507
508
|
return true;
}
|
27b826a8
David Mayerich
added a spherical...
|
509
510
511
512
|
/// Reads a plane given a coordinate along the 0-axis (YZ plane)
/// @param p is a pointer to pre-allocated memory of size R[1] * R[2] * sizeof(T)
/// @param n is the 0-axis coordinate used to retrieve the plane
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
513
|
bool read_plane_0(T* p, unsigned long long n, bool PROGRESS = false){
|
63fc1df8
David Mayerich
added an inverse ...
|
514
|
if(PROGRESS) progress = 0;
|
724ec347
David Mayerich
simplified the EN...
|
515
|
if (n >= R[0]){ //make sure the number is within the possible range
|
735a2a24
David Mayerich
started testing o...
|
516
|
std::cout<<"ERROR: sample or line out of range in "<<__FILE__<<" (line "<<__LINE__<<")"<<std::endl;
|
724ec347
David Mayerich
simplified the EN...
|
517
518
|
return false;
}
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
519
|
unsigned long long jump = (R[0] - 1) * sizeof(T); //number of bytes to skip between samples
|
724ec347
David Mayerich
simplified the EN...
|
520
521
522
523
|
//seek to the start of the plane
file.seekg(n * sizeof(T), std::ios::beg);
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
524
525
|
unsigned long long N = R[1] * R[2];
for(unsigned long long i = 0; i<N; i++){
|
724ec347
David Mayerich
simplified the EN...
|
526
527
|
file.read((char*)(p+i), sizeof(T));
file.seekg(jump, std::ios::cur);
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
528
|
if(PROGRESS) progress = (double)(i+1) / N * 100;
|
724ec347
David Mayerich
simplified the EN...
|
529
530
|
}
|
724ec347
David Mayerich
simplified the EN...
|
531
532
533
534
535
|
return true;
}
|
27b826a8
David Mayerich
added a spherical...
|
536
537
538
539
|
/// Reads a plane given a coordinate along the 1-axis (XZ plane)
/// @param p is a pointer to pre-allocated memory of size R[0] * R[2] * sizeof(T)
/// @param n is the 1-axis coordinate used to retrieve the plane
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
540
|
bool read_plane_1(T* p, unsigned long long n, bool PROGRESS = false){
|
63fc1df8
David Mayerich
added an inverse ...
|
541
|
if(PROGRESS) progress = 0;
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
542
543
|
unsigned long long L = R[0] * sizeof(T); //caculate the number of bytes in a sample line
unsigned long long jump = R[0] * (R[1] - 1) * sizeof(T);
|
724ec347
David Mayerich
simplified the EN...
|
544
545
546
547
548
549
550
|
if (n >= R[1]){ //make sure the bank number is right
std::cout<<"ERROR read_plane_1: page out of range"<<std::endl;
return false;
}
file.seekg(R[0] * n * sizeof(T), std::ios::beg);
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
551
|
for (unsigned long long i = 0; i < R[2]; i++){
|
63fc1df8
David Mayerich
added an inverse ...
|
552
|
if(PROGRESS) progress = (double)i / R[2] * 100;
|
724ec347
David Mayerich
simplified the EN...
|
553
554
555
556
557
|
file.read((char *)(p + i * R[0]), L);
file.seekg( jump, std::ios::cur);
std::cout<<i<<" ";
}
|
63fc1df8
David Mayerich
added an inverse ...
|
558
|
if(PROGRESS) progress = 100;
|
724ec347
David Mayerich
simplified the EN...
|
559
560
561
|
return true;
}
|
27b826a8
David Mayerich
added a spherical...
|
562
563
564
565
|
/// Reads a plane given a coordinate along the 2-axis (XY plane)
/// @param p is a pointer to pre-allocated memory of size R[0] * R[1] * sizeof(T)
/// @param n is the 2-axis coordinate used to retrieve the plane
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
566
|
bool read_plane_2(T* p, unsigned long long n, bool PROGRESS = false){
|
63fc1df8
David Mayerich
added an inverse ...
|
567
|
return read_page(p, n, PROGRESS);
|
724ec347
David Mayerich
simplified the EN...
|
568
569
|
}
|
27b826a8
David Mayerich
added a spherical...
|
570
571
572
573
|
/// Reads a single pixel, treating the entire data set as a linear array
/// @param p is a pointer to pre-allocated memory of size sizeof(T)
/// @param i is the index to the pixel using linear indexing
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
574
|
bool read_pixel(T* p, unsigned long long i){
|
724ec347
David Mayerich
simplified the EN...
|
575
576
577
578
579
580
581
582
583
584
|
if(i >= R[0] * R[1] * R[2]){
std::cout<<"ERROR read_pixel: n is out of range"<<std::endl;
return false;
}
file.seekg(i * sizeof(T), std::ios::cur);
file.read((char*)p, sizeof(T));
}
|
27b826a8
David Mayerich
added a spherical...
|
585
586
587
588
589
590
|
/// Reads a single pixel, given an x, y, z coordinate
/// @param p is a pointer to pre-allocated memory of size sizeof(T)
/// @param x is the x (0) axis coordinate
/// @param y is the y (1) axis coordinate
/// @param z is the z (2) axis coordinate
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
591
|
bool read_pixel(T* p, unsigned long long x, unsigned long long y, unsigned long long z){
|
724ec347
David Mayerich
simplified the EN...
|
592
593
594
595
596
597
|
if(x < 0 || x >= R[0] || y < 0 || y >= R[1] || z < 0 || z > R[2]){
std::cout<<"ERROR read_pixel: (x,y,z) is out of range"<<std::endl;
return false;
}
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
598
|
unsigned long long i = z * R[0] * R[1] + y * R[0] + z;
|
724ec347
David Mayerich
simplified the EN...
|
599
600
601
|
return read_pixel(p, i);
}
|
8b899c24
David Mayerich
fixed asynchronou...
|
602
|
/// Reads a block specified by an (x, y, z) position and size using the largest possible contiguous reads
|
6db250f5
David Mayerich
adaptive streamin...
|
603
604
|
size_t read(T* dest, size_t x, size_t y, size_t z, size_t sx, size_t sy, size_t sz){
auto t0 = std::chrono::high_resolution_clock::now();
|
8b899c24
David Mayerich
fixed asynchronou...
|
605
606
607
608
609
610
611
612
613
|
size_t size_bytes = sx * sy * sz * sizeof(T); //size of the block to read in bytes
size_t start = z * R[0] * R[1] + y * R[0] + x; //calculate the start postion
size_t start_bytes = start * sizeof(T); //start position in bytes
file.seekg(start * sizeof(T), std::ios::beg); //seek to the start position
if(sx == R[0] && sy == R[1]){ //if sx and sy result in a contiguous volume along z
file.read((char*)dest, size_bytes); //read the block in one pass
|
8b899c24
David Mayerich
fixed asynchronou...
|
614
|
}
|
6db250f5
David Mayerich
adaptive streamin...
|
615
|
else if(sx == R[0]){ //if sx is contiguous, read each z-axis slice can be read in one pass
|
8b899c24
David Mayerich
fixed asynchronou...
|
616
617
618
619
620
621
622
|
size_t jump_bytes = (R[1] - sy) * R[0] * sizeof(T); //jump between each slice
size_t slice_bytes = sx * sy * sizeof(T); //size of the slice to be read
for(size_t zi = 0; zi < sz; zi++){ //for each z-axis slice
file.read((char*)dest, slice_bytes); //read the slice
dest += sx * sy; //move the destination pointer to the next slice
file.seekg(jump_bytes, std::ios::cur); //skip to the next slice in the file
}
|
8b899c24
David Mayerich
fixed asynchronou...
|
623
|
}
|
6db250f5
David Mayerich
adaptive streamin...
|
624
625
626
627
628
629
630
631
632
633
634
635
636
637
|
else{
//in this case, x is not contiguous so the volume must be read line-by-line
size_t jump_x_bytes = (R[0] - sx) * sizeof(T); //number of bytes skipped in the x direction
size_t jump_y_bytes = (R[1] - sy) * R[0] * sizeof(T) + jump_x_bytes; //number of bytes skipped between slices
size_t line_bytes = sx * sizeof(T); //size of the line to be read
size_t zi, yi;
for(zi = 0; zi < sz; zi++){ //for each slice
file.read((char*)dest, line_bytes); //read the first line
for(yi = 1; yi < sy; yi++){ //read each additional line
dest += sx; //move the pointer in the destination block to the next line
file.seekg(jump_x_bytes, std::ios::cur); //skip to the next line in the file
file.read((char*)dest, line_bytes); //read the line to the destination block
}
file.seekg(jump_y_bytes, std::ios::cur); //skip to the beginning of the next slice
|
8b899c24
David Mayerich
fixed asynchronou...
|
638
|
}
|
8b899c24
David Mayerich
fixed asynchronou...
|
639
|
}
|
6db250f5
David Mayerich
adaptive streamin...
|
640
641
|
auto t1 = std::chrono::high_resolution_clock::now();
return std::chrono::duration_cast<std::chrono::milliseconds>(t1-t0).count();
|
8b899c24
David Mayerich
fixed asynchronou...
|
642
643
|
}
|
cf5b4c92
David Mayerich
simplified asynch...
|
644
645
|
// permutes a block of data from the current interleave to the interleave specified (re-arranged dimensions to the order specified by [d0, d1, d2])
|
6db250f5
David Mayerich
adaptive streamin...
|
646
|
size_t permute(T* dest, T* src, size_t sx, size_t sy, size_t sz, size_t d0, size_t d1, size_t d2){
|
eae75592
David Mayerich
added GPU support...
|
647
648
649
|
std::chrono::high_resolution_clock::time_point t0, t1;
t0 = std::chrono::high_resolution_clock::now();
|
683f216a
David Mayerich
fixed? Linux comp...
|
650
|
#ifdef USE_CUDA
|
eae75592
David Mayerich
added GPU support...
|
651
652
653
654
655
656
657
658
659
660
661
662
663
664
|
T* gpu_src;
HANDLE_ERROR( cudaMalloc(&gpu_src, sx*sy*sz*sizeof(T)) );
HANDLE_ERROR( cudaMemcpy(gpu_src, src, sx*sy*sz*sizeof(T), cudaMemcpyHostToDevice) );
T* gpu_dest;
HANDLE_ERROR( cudaMalloc(&gpu_dest, sx*sy*sz*sizeof(T)) );
gpu_permute((char*)gpu_dest, (char*)gpu_src, sx, sy, sz, d0, d1, d2, sizeof(T));
HANDLE_ERROR( cudaMemcpy(dest, gpu_dest, sx*sy*sz*sizeof(T), cudaMemcpyDeviceToHost) );
HANDLE_ERROR( cudaFree(gpu_src) );
HANDLE_ERROR( cudaFree(gpu_dest) );
t1 = std::chrono::high_resolution_clock::now();
return std::chrono::duration_cast<std::chrono::milliseconds>(t1-t0).count();
#endif
|
cf5b4c92
David Mayerich
simplified asynch...
|
665
666
|
size_t d[3] = {d0, d1, d2};
size_t s[3] = {sx, sy, sz};
|
eae75592
David Mayerich
added GPU support...
|
667
|
size_t p[3];
|
cf5b4c92
David Mayerich
simplified asynch...
|
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
|
if(d[0] == 0 && d[1] == 1 && d[2] == 2){
//this isn't actually a permute - just copy the data
memcpy(dest, src, sizeof(T) * sx * sy * sz);
}
else if(d[0] == 0){ //the individual lines are contiguous, so you can memcpy line-by-line
size_t y, z;
size_t src_idx, dest_idx;
size_t x_bytes = sizeof(T) * sx;
for(z = 0; z < sz; z++){
p[2] = z;
for(y = 0; y < sy; y++){
p[1] = y;
src_idx = z * sx * sy + y * sx;
dest_idx = p[d[2]] * s[d[0]] * s[d[1]] + p[d[1]] * s[d[0]];
|
cf5b4c92
David Mayerich
simplified asynch...
|
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
|
memcpy(dest + dest_idx, src + src_idx, x_bytes);
}
}
}
else{ //loop through every damn point
size_t x, y, z;
size_t src_idx, dest_idx;
size_t src_z, src_y;
for(z = 0; z < sz; z++){
p[2] = z;
src_z = z * sx * sy;
for(y = 0; y < sy; y++){
p[1] = y;
src_y = src_z + y * sx;
for(x = 0; x < sx; x++){
p[0] = x;
src_idx = src_y + x;
dest_idx = p[d[2]] * s[d[0]] * s[d[1]] + p[d[1]] * s[d[0]] + p[d[0]];
dest[dest_idx] = src[src_idx];
}
}
}
}
|
eae75592
David Mayerich
added GPU support...
|
706
|
t1 = std::chrono::high_resolution_clock::now();
|
6db250f5
David Mayerich
adaptive streamin...
|
707
|
return std::chrono::duration_cast<std::chrono::milliseconds>(t1-t0).count();
|
cf5b4c92
David Mayerich
simplified asynch...
|
708
709
|
}
|
180d7f3a
David Mayerich
added binary file...
|
710
711
712
713
714
|
};
}
#endif
|