1f2ca8ec
David Mayerich
added files for b...
|
1
2
|
//make sure that this header file is only loaded once
|
180d7f3a
David Mayerich
added binary file...
|
3
4
5
|
#ifndef RTS_BINARY_H
#define RTS_BINARY_H
|
e8eb202f
David Mayerich
added a new ENVI ...
|
6
|
#include "../envi/envi_header.h"
|
4dcc9392
heziqi
Ziqi's first chan...
|
7
|
#include "../math/vector.h"
|
180d7f3a
David Mayerich
added binary file...
|
8
9
|
#include <fstream>
#include <sys/stat.h>
|
15a31763
David Mayerich
set up buffer est...
|
10
|
#include <cstring>
|
5435830b
David Mayerich
linux chrono fixes
|
11
|
#include <chrono>
|
008ae990
David Mayerich
implemented Windo...
|
12
13
|
#ifdef _WIN32
#include <Windows.h>
|
15a31763
David Mayerich
set up buffer est...
|
14
15
|
#else
#include <unistd.h>
|
008ae990
David Mayerich
implemented Windo...
|
16
17
|
#endif
|
8a86bd56
David Mayerich
changed rts names...
|
18
|
namespace stim{
|
180d7f3a
David Mayerich
added binary file...
|
19
|
|
6db250f5
David Mayerich
adaptive streamin...
|
20
21
22
23
|
/// This class calculates the optimal setting for independent parameter b (batch size) for
/// minimizing the dependent parameter bps (bytes per second)
class stream_optimizer{
protected:
|
0a575bb1
David Mayerich
updated optimizat...
|
24
|
size_t Bps[2]; //bytes per second for the previous batch
|
6db250f5
David Mayerich
adaptive streamin...
|
25
26
27
|
size_t interval_B; //number of bytes processed this interval
size_t interval_ms; //number of milliseconds spent in the current interval
size_t n[2]; //current batch size (in bytes)
|
0a575bb1
David Mayerich
updated optimizat...
|
28
|
size_t h; //spacing used for finite difference calculations
|
6db250f5
David Mayerich
adaptive streamin...
|
29
30
31
|
size_t dn; //delta value (in bytes) for setting the batch size (minimum change in batch parameter)
size_t maxn; //maximum value for the batch size
|
0a575bb1
David Mayerich
updated optimizat...
|
32
33
|
double alpha; //alpha value controls the factor of the gradient that is used to calculate the next point (speed of convergence)
|
6db250f5
David Mayerich
adaptive streamin...
|
34
35
36
37
38
39
|
bool sample_step; //calculating the derivative (this class alternates between calculating dBps and B)
bool forward_diff; //evaluate the derivative using forward differences
size_t window_ms; //size of the interval (in milliseconds) integrated to get a reliable bps value
// This function rounds x to the nearest value within dB
|
0a575bb1
David Mayerich
updated optimizat...
|
40
41
42
43
44
|
size_t round_limit(double n0){
if(n0 < 0) return dn; //if n0 is less than zero, return the lowest possible n
size_t new_n = (size_t)(n0 + 0.5); //now n0 must be positive, so round it to the nearest integer
if(new_n > maxn) new_n = maxn; //limit the returned size of x to within the specified bounds
|
6db250f5
David Mayerich
adaptive streamin...
|
45
|
|
0a575bb1
David Mayerich
updated optimizat...
|
46
|
size_t lowest = new_n / dn;
|
6db250f5
David Mayerich
adaptive streamin...
|
47
|
size_t highest = lowest + dn;
|
0a575bb1
David Mayerich
updated optimizat...
|
48
|
size_t diff[2] = {new_n - lowest, highest - new_n}; //calculate the two differences
|
6db250f5
David Mayerich
adaptive streamin...
|
49
50
51
52
53
54
55
56
|
if(diff[0] < diff[1])
return lowest;
return highest;
}
public:
//constructor initializes a stream optimizer
|
c6251f8b
David Mayerich
allows user to de...
|
57
|
stream_optimizer(size_t min_batch_size, size_t max_batch_size, double a = 0.003, size_t probe_step = 5, size_t window = 2000){
|
0a575bb1
David Mayerich
updated optimizat...
|
58
59
|
//Bps = 0; //initialize to zero bytes per second processed
Bps[0] = Bps[1] = 0; //initialize the bits per second to 0
|
6db250f5
David Mayerich
adaptive streamin...
|
60
61
62
63
|
interval_B = 0; //zero bytes have been processed at initialization
interval_ms = 0; //no time has been spent on the batch so far
dn = min_batch_size; //set the minimum batch size as the minimum change in batch size
maxn = max_batch_size; //set the maximum batch size
|
0a575bb1
David Mayerich
updated optimizat...
|
64
|
n[0] = max_batch_size; //set B
|
8933935f
David Mayerich
updated VERBOSE i...
|
65
|
h = (max_batch_size / min_batch_size) / probe_step * dn;
|
0a575bb1
David Mayerich
updated optimizat...
|
66
67
68
69
|
std::cout<<"h = "<<h<<std::endl;
if(h < dn) h = dn;
alpha = a;
//n[0] = round_limit( (max_batch_size - min_batch_size)/2 );
|
6db250f5
David Mayerich
adaptive streamin...
|
70
71
72
73
74
|
window_ms = window; //minimum integration interval (for getting a reliable bps measure)
sample_step = true; //the first step is to calculate the derivative
forward_diff = true; //start with the forward difference (since we start at the maximum batch size)
}
|
c6251f8b
David Mayerich
allows user to de...
|
75
|
size_t update(size_t bytes_processed, size_t ms_spent, size_t& data_rate, bool VERBOSE = false){
|
0a575bb1
David Mayerich
updated optimizat...
|
76
77
|
interval_B += bytes_processed; //increment the number of bytes processed
interval_ms += ms_spent; //increment the number of milliseconds spent processing
|
c6251f8b
David Mayerich
allows user to de...
|
78
|
data_rate = interval_B / interval_ms;
|
0a575bb1
David Mayerich
updated optimizat...
|
79
80
81
82
83
84
|
//if we have sufficient information to evaluate the optimization function at this point
if(interval_ms < window_ms){ //if insufficient time has passed to get a reliable Bps measurement
return n[0];
}
else{ //if we have collected enough information for a reliable Bps estimate
|
c6251f8b
David Mayerich
allows user to de...
|
85
|
|
0a575bb1
David Mayerich
updated optimizat...
|
86
|
if(Bps[0] == 0){ //if n[0] hasn't been evaluated yet, this is the first step
|
c6251f8b
David Mayerich
allows user to de...
|
87
|
Bps[0] = data_rate; //set the initial Bps value
|
0a575bb1
David Mayerich
updated optimizat...
|
88
|
n[1] = n[0] - h; //set the position of the next sample point
|
c6251f8b
David Mayerich
allows user to de...
|
89
90
|
if(VERBOSE)
std::cout<<"Bps value at n = "<<n[0]<<" is "<<Bps[0]<<" Bps, probing n = "<<n[1]<<std::endl;
|
0a575bb1
David Mayerich
updated optimizat...
|
91
92
93
|
return n[1]; //return the probe point
}
else{
|
c6251f8b
David Mayerich
allows user to de...
|
94
|
Bps[1] = data_rate; //set the Bps for the current point (n[1])
|
0a575bb1
David Mayerich
updated optimizat...
|
95
96
97
98
99
100
101
102
103
|
double Bps_p; //allocate a variable for the derivative
//calculate the derivative
if(n[0] < n[1]){ //if the current point is less than the previous one (probably the most common)
Bps_p = ((double)Bps[1] - (double)Bps[0]) / (double)h; //calculate the derivative using the forward finite difference
}
else{
Bps_p = ((double)Bps[0] - (double)Bps[1]) / (double)h; //calculate the derivative using the backward finite difference
}
|
c6251f8b
David Mayerich
allows user to de...
|
104
105
|
if(VERBOSE)
std::cout<<" probed n = "<<n[1]<<" with "<<Bps[1]<<" Bps, gradient = "<<Bps_p<<" Bps"<<std::endl;
|
0a575bb1
David Mayerich
updated optimizat...
|
106
107
108
109
110
111
112
113
|
double new_n_precise = n[0] + alpha * Bps_p; //calculate the next point (snap to closest integer)
size_t new_n_nearest = round_limit(new_n_precise); //calculate the next point (given batch parameters)
if(new_n_nearest == n[0]){ //if the newest point is the same as the original point
Bps[0] = Bps[1]; //update the Bps
//if(n[0] == dn) n[1] = n[0] + h; //if we're on the left edge, probe forward
//else n[1] = n[0] - h; //otherwise probe backwards
|
c6251f8b
David Mayerich
allows user to de...
|
114
115
|
if(VERBOSE)
std::cout<<" staying at n = "<<n[0]<<" for now"<<std::endl;
|
0a575bb1
David Mayerich
updated optimizat...
|
116
117
118
119
120
121
122
123
|
//return n[1]; //return the probe point
Bps[0] = 0; //reset the Bps for the current point
return n[0]; //return the current point for a re-calculation
}
else{ //if the newest point is different from the original point
n[0] = new_n_nearest; //move to the new point
Bps[0] = 0; //set the Bps to zero (point hasn't been tested)
|
c6251f8b
David Mayerich
allows user to de...
|
124
125
|
if(VERBOSE)
std::cout<<" moving to n = "<<n[0]<<std::endl;
|
0a575bb1
David Mayerich
updated optimizat...
|
126
127
128
129
130
131
132
|
return n[0]; //return the new point
}
}
}
}
/*// this function updates the optimizer, given the number of bytes processed in an interval and time spent processing
|
6db250f5
David Mayerich
adaptive streamin...
|
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
|
size_t update(size_t bytes_processed, size_t ms_spent){
interval_B += bytes_processed; //increment the number of bytes processed
interval_ms += ms_spent; //increment the number of milliseconds spent processing
//if we have sufficient information to evaluate the optimization function at this point
if(interval_ms >= window_ms){ //if sufficient time has passed to get a reliable Bps measurement
size_t new_Bps = interval_B / interval_ms; //calculate the current Bps
if(sample_step){ //if this is a sample step, collect the information for Bps = f(n0)
Bps = new_Bps; //set the Bps to the evaluated value
n[1] = n[0] - dn; //reduce the batch size by one delta to take a second sample
if(n[1] == 0){ //if the resulting batch size is zero
n[1] = 2*dn; //we're at the left edge: set the new sample point to 2*dn
}
interval_B = interval_ms = 0; //start a new interval at the new sample point
sample_step = false; //next step will calculate the new batch size via optimization
return n[1]; //return the new batch size
}
else{ //if we have sufficient information to evaluate the derivative and optimize
double f = (double)new_Bps; //we have evaluated the function at this location
double fprime;
if(n[1] < n[0] ){ //if the new point is less than the previous point (usually the case)
fprime = (double)(Bps - new_Bps) / (double)dn; //calculate the forward difference
}
else{ //if the new point is larger (only happens at the minimum limit)
fprime = (double)(new_Bps - Bps) / (double)dn; //calculate the backward difference
}
size_t bestn = n[1] - (size_t)(f / fprime); //calculate the best value for B using Newton's method
n[0] = round_limit( (size_t)bestn ); //set the new dependent point
sample_step = true; //the next step will be a sample step
}
}
if(sample_step) return n[0];
return n[1]; //insufficient information, keep the same batch size
|
0a575bb1
David Mayerich
updated optimizat...
|
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
|
}*/
/*size_t update(size_t bytes_processed, size_t ms_spent){
interval_B += bytes_processed; //increment the number of bytes processed
interval_ms += ms_spent; //increment the number of milliseconds spent processing
//if( Bps[0] == 0 ){ //if the left boundary hasn't been processed
//if we have sufficient information to evaluate the optimization function at this point
if(interval_ms >= window_ms){
size_t new_Bps = interval_B / interval_ms; //calculate the current Bps
if(Bps[0] == 0) //if the left interval Bps hasn't been calculated
Bps[0] = interval_B / interval_ms; //that is the interval being processed
else
Bps[1] = interval_B / interval_ms; //otherwise the right interval is being processed
if(Bps[0] != 0 && Bps[1] != 0){ //if both intervals have been processed
}
}*/
|
6db250f5
David Mayerich
adaptive streamin...
|
192
|
|
c6251f8b
David Mayerich
allows user to de...
|
193
194
|
/*size_t update(size_t bytes_processed, size_t ms_spent, size_t& data_rate, bool VERBOSE){
size_t time = update(bytes_processed, ms_spent, VERBOSE);
|
0a575bb1
David Mayerich
updated optimizat...
|
195
|
data_rate = Bps[0];
|
6db250f5
David Mayerich
adaptive streamin...
|
196
|
return time;
|
c6251f8b
David Mayerich
allows user to de...
|
197
|
}*/
|
6db250f5
David Mayerich
adaptive streamin...
|
198
199
|
};
|
33d7d3cf
David Mayerich
documentation for...
|
200
201
202
203
204
205
206
|
/** This class manages the streaming of large multidimensional binary files.
* Generally these are hyperspectral files with 2 spatial and 1 spectral dimension. However, this class supports
* other dimensions via the template parameter D.
*
* @param T is the data type used to store data to disk (generally float or double)
* @param D is the dimension of the data (default 3)
*/
|
180d7f3a
David Mayerich
added binary file...
|
207
208
209
210
|
template< typename T, unsigned int D = 3 >
class binary{
protected:
|
180d7f3a
David Mayerich
added binary file...
|
211
212
213
|
std::fstream file; //file stream used for reading and writing
std::string name; //file name
|
63fc1df8
David Mayerich
added an inverse ...
|
214
|
unsigned long long R[D]; //resolution
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
215
|
unsigned long long header; //header size (in bytes)
|
f4c5d71b
David Mayerich
started working w...
|
216
|
unsigned char* mask; //pointer to a character array: 0 = background, 1 = foreground (or valid data)
|
180d7f3a
David Mayerich
added binary file...
|
217
|
|
63fc1df8
David Mayerich
added an inverse ...
|
218
|
double progress; //stores the progress on the current operation (accessible using a thread)
|
c6251f8b
David Mayerich
allows user to de...
|
219
|
size_t data_rate; //data rate (currently in Bps)
|
ee4dea28
David Mayerich
fixed errors in c...
|
220
|
|
008ae990
David Mayerich
implemented Windo...
|
221
|
size_t buffer_size; //available memory for processing large files
|
ee4dea28
David Mayerich
fixed errors in c...
|
222
|
|
33d7d3cf
David Mayerich
documentation for...
|
223
|
/// Private initialization function used to set default parameters in the data structure.
|
180d7f3a
David Mayerich
added binary file...
|
224
|
void init(){
|
15a31763
David Mayerich
set up buffer est...
|
225
|
std::memset(R, 0, sizeof(unsigned long long) * D); //initialize the resolution to zero
|
008ae990
David Mayerich
implemented Windo...
|
226
|
header = 0; //initialize the header size to zero
|
f4c5d71b
David Mayerich
started working w...
|
227
|
mask = NULL;
|
cbce396e
David Mayerich
added support for...
|
228
|
|
c6251f8b
David Mayerich
allows user to de...
|
229
230
231
|
progress = 0; //initialize the progress for any algorithm to zero
data_rate = 0; //initialize the data rate to zero
set_buffer_frac(); //set the maximum buffer size to the default
|
63fc1df8
David Mayerich
added an inverse ...
|
232
233
|
}
|
33d7d3cf
David Mayerich
documentation for...
|
234
|
/// Private helper function that returns the size of the file on disk using system functions.
|
b0fee590
heziqi
convert long int ...
|
235
|
long long int get_file_size(){
|
193ff00e
David Mayerich
dealth with preci...
|
236
|
#ifdef _WIN32
|
b0fee590
heziqi
convert long int ...
|
237
238
|
struct _stat64 results;
if(_stat64(name.c_str(), &results) == 0)
|
180d7f3a
David Mayerich
added binary file...
|
239
|
return results.st_size;
|
193ff00e
David Mayerich
dealth with preci...
|
240
241
242
243
244
|
#else
struct stat results;
if(stat(name.c_str(), &results) == 0)
return results.st_size;
#endif
|
180d7f3a
David Mayerich
added binary file...
|
245
246
247
|
else return 0;
}
|
33d7d3cf
David Mayerich
documentation for...
|
248
|
/// Private helper function that tests to make sure that the calculated data size specified by the structure is the same as the data size on disk.
|
180d7f3a
David Mayerich
added binary file...
|
249
|
bool test_file_size(){
|
b0fee590
heziqi
convert long int ...
|
250
|
long long int npts = 1; //initialize the number of data points to 1
|
87da9adc
David Mayerich
fixed issues with...
|
251
252
|
for(unsigned int i = 0; i<D; i++) //iterate over each dimension
npts *= R[i]; //compute the total number of data points in the file
|
b0fee590
heziqi
convert long int ...
|
253
|
long long int datasize = npts * sizeof(T);//multiply the sum by the size of the template parameter
|
87da9adc
David Mayerich
fixed issues with...
|
254
255
|
if(datasize + header == get_file_size()) return true; //if the byte size matches the file size, we're golden
|
180d7f3a
David Mayerich
added binary file...
|
256
257
258
259
|
else return false; //otherwise return an error
}
|
f92397d2
David Mayerich
fixed a Win32 bug...
|
260
261
262
263
264
265
|
/// Private helper function that resets the file pointer to the beginning of the data
void reset(){
file.seekg(header, std::ios_base::beg);
}
|
33d7d3cf
David Mayerich
documentation for...
|
266
267
268
|
/// Private helper file that opens a specified binary file.
/// @param filename is the name of the binary file to stream
|
180d7f3a
David Mayerich
added binary file...
|
269
270
271
272
|
bool open_file(std::string filename){
//open the file as binary for reading and writing
file.open(filename.c_str(), std::ios::in | std::ios::out | std::ios::binary);
|
ba72c5b0
David Mayerich
added OBJ class
|
273
274
|
//if the file isn't open, the user may only have read access
if(!file.is_open()){
|
3d0b6243
David Mayerich
fixed hsiproc bug...
|
275
|
std::cout<<"class STIM::BINARY - failed to open file, trying for read only"<<std::endl;
|
ba72c5b0
David Mayerich
added OBJ class
|
276
277
278
279
280
281
282
|
file.open(filename.c_str(), std::ios::in | std::ios::binary);
if(!file.is_open()){
std::cout<<" still unable to load the file"<<std::endl;
return false;
}
}
|
180d7f3a
David Mayerich
added binary file...
|
283
284
285
286
287
288
|
//if the file is successful
if(file){
name = filename; //set the name
if(test_file_size()) //test the file size
return true;
}
|
f92397d2
David Mayerich
fixed a Win32 bug...
|
289
|
|
180d7f3a
David Mayerich
added binary file...
|
290
291
292
|
return false;
}
|
724ec347
David Mayerich
simplified the EN...
|
293
294
|
|
f92397d2
David Mayerich
fixed a Win32 bug...
|
295
296
|
|
180d7f3a
David Mayerich
added binary file...
|
297
298
|
public:
|
008ae990
David Mayerich
implemented Windo...
|
299
300
301
302
303
|
//default constructor
binary(){
init();
}
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
304
|
double get_progress(){
|
63fc1df8
David Mayerich
added an inverse ...
|
305
|
return progress;
|
cbce396e
David Mayerich
added support for...
|
306
307
|
}
|
63fc1df8
David Mayerich
added an inverse ...
|
308
309
|
void reset_progress(){
progress = 0;
|
cbce396e
David Mayerich
added support for...
|
310
311
|
}
|
c6251f8b
David Mayerich
allows user to de...
|
312
313
314
315
|
size_t get_data_rate(){
return data_rate;
}
|
008ae990
David Mayerich
implemented Windo...
|
316
|
//specify the maximum fraction of available memory that this class will use for buffering
|
c6251f8b
David Mayerich
allows user to de...
|
317
|
void set_buffer_frac(double mem_frac = 0.5){ //default to 50%
|
008ae990
David Mayerich
implemented Windo...
|
318
319
320
321
322
|
#ifdef _WIN32
MEMORYSTATUSEX statex;
statex.dwLength = sizeof (statex);
GlobalMemoryStatusEx (&statex);
buffer_size = (size_t)(statex.ullAvailPhys * mem_frac);
|
15a31763
David Mayerich
set up buffer est...
|
323
324
325
326
|
#else
size_t pages = sysconf(_SC_PHYS_PAGES);
size_t page_size = sysconf(_SC_PAGE_SIZE);
buffer_size = (size_t)(pages * page_size * mem_frac);
|
008ae990
David Mayerich
implemented Windo...
|
327
328
329
|
#endif
}
|
c6251f8b
David Mayerich
allows user to de...
|
330
331
332
333
|
void set_buffer_raw(size_t bytes){
buffer_size = bytes;
}
|
33d7d3cf
David Mayerich
documentation for...
|
334
335
336
337
338
|
/// Open a binary file for streaming.
/// @param filename is the name of the binary file
/// @param r is a STIM vector specifying the size of the binary file along each dimension
/// @param h is the length (in bytes) of any header file (default zero)
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
339
|
bool open(std::string filename, vec<unsigned long long> r, unsigned long long h = 0){
|
180d7f3a
David Mayerich
added binary file...
|
340
|
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
341
|
for(unsigned long long i = 0; i < D; i++) //set the dimensions of the binary file object
|
180d7f3a
David Mayerich
added binary file...
|
342
343
|
R[i] = r[i];
|
87da9adc
David Mayerich
fixed issues with...
|
344
345
346
|
header = h; //save the header size
if(!open_file(filename)) return false; //open the binary file
|
180d7f3a
David Mayerich
added binary file...
|
347
|
|
f92397d2
David Mayerich
fixed a Win32 bug...
|
348
349
|
//reset();
|
180d7f3a
David Mayerich
added binary file...
|
350
351
352
|
return test_file_size();
}
|
33d7d3cf
David Mayerich
documentation for...
|
353
354
355
356
357
|
/// Creates a new binary file for streaming
/// @param filename is the name of the binary file to be created
/// @param r is a STIM vector specifying the size of the file along each dimension
/// @offset specifies how many bytes to offset the file (used to leave room for a header)
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
358
|
bool create(std::string filename, vec<unsigned long long> r, unsigned long long offset = 0){
|
f47168a2
heziqi
Ziqi added create...
|
359
360
361
362
363
|
std::ofstream target(filename.c_str(), std::ios::binary);
//initialize binary file
T p = 0;
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
364
|
for(unsigned long long i =0; i < r[0] * r[1] * r[2]; i++){
|
f47168a2
heziqi
Ziqi added create...
|
365
366
367
|
target.write((char*)(&p), sizeof(T));
}
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
368
|
for(unsigned long long i = 0; i < D; i++) //set the dimensions of the binary file object
|
f47168a2
heziqi
Ziqi added create...
|
369
370
371
372
373
374
375
376
377
|
R[i] = r[i];
header = offset; //save the header size
if(!open_file(filename)) return false; //open the binary file
return test_file_size();
}
|
33d7d3cf
David Mayerich
documentation for...
|
378
379
380
381
|
/// Writes a single page of data to disk. A page consists of a sequence of data of size R[0] * R[1] * ... * R[D-1].
/// @param p is a pointer to the data to be written
/// @param page is the page number (index of the highest-numbered dimension)
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
382
|
bool write_page( T * p, unsigned long long page){
|
f47168a2
heziqi
Ziqi added create...
|
383
384
385
386
387
388
|
if(p == NULL){
std::cout<<"ERROR: unable to write into file, empty pointer"<<std::endl;
exit(1);
}
|
f92397d2
David Mayerich
fixed a Win32 bug...
|
389
|
file.seekg(R[1] * R[0] * page * sizeof(T) + header, std::ios::beg); //seek to the desired location on disk
|
8157c392
David Mayerich
added parser and ...
|
390
|
file.write((char *)p, R[0] * R[1] * sizeof(T)); //write binary data
|
f47168a2
heziqi
Ziqi added create...
|
391
392
393
394
|
return true;
}
|
33d7d3cf
David Mayerich
documentation for...
|
395
396
397
398
|
/// Reads a page from disk. A page consists of a sequence of data of size R[0] * R[1] * ... * R[D-1].
/// @param p is a pointer to pre-allocated memory equal to the page size
/// @param page is the index of the page
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
399
|
bool read_page( T * p, unsigned long long page, bool PROGRESS = false){
|
63fc1df8
David Mayerich
added an inverse ...
|
400
401
|
if(PROGRESS) progress = 0;
|
c7da85e3
heziqi
he,ziqi add metho...
|
402
|
|
5d5ce39a
heziqi
Ziqi convert the ...
|
403
|
if (page >= R[2]){ //make sure the bank number is right
|
2ad0ce4f
David Mayerich
syntactic edits b...
|
404
|
std::cout<<"ERROR: page out of range"<<std::endl;
|
5d5ce39a
heziqi
Ziqi convert the ...
|
405
|
return false;
|
c7da85e3
heziqi
he,ziqi add metho...
|
406
407
|
}
|
f92397d2
David Mayerich
fixed a Win32 bug...
|
408
|
file.seekg(R[1] * R[0] * page * sizeof(T) + header, std::ios::beg); //write into memory from the binary file
|
2ad0ce4f
David Mayerich
syntactic edits b...
|
409
|
file.read((char *)p, R[0] * R[1] * sizeof(T));
|
63fc1df8
David Mayerich
added an inverse ...
|
410
|
if(PROGRESS) progress = 100;
|
5d5ce39a
heziqi
Ziqi convert the ...
|
411
|
return true;
|
c7da85e3
heziqi
he,ziqi add metho...
|
412
413
|
}
|
724ec347
David Mayerich
simplified the EN...
|
414
415
416
417
418
419
420
|
///Reads a line Z (slowest dimension) for a given XY value
/// @param p is a pointer to pre-allocated memory equal to the line size R[2]
/// @param x is the x coordinate
/// @param y is the y coordinate
|
735a2a24
David Mayerich
started testing o...
|
421
|
void read_line_2( T* p, size_t n, bool PROGRESS = false){
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
422
|
unsigned long long i;
|
724ec347
David Mayerich
simplified the EN...
|
423
|
|
63fc1df8
David Mayerich
added an inverse ...
|
424
425
|
if(PROGRESS) progress = 0;
|
735a2a24
David Mayerich
started testing o...
|
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
|
if ( n > R[0] * R[1]){ //make sure the sample and line number is right
std::cout<<"ERROR: sample or line out of range in "<<__FILE__<<" (line "<<__LINE__<<")"<<std::endl;
exit(1);
}
file.seekg(n * sizeof(T), std::ios::beg); //point to the certain sample and line
for (i = 0; i < R[2]; i++){ //for each band
file.read((char *)(p + i), sizeof(T));
file.seekg((R[1] * R[0] - 1) * sizeof(T), std::ios::cur); //go to the next band
if(PROGRESS) progress = (double)i / (double)R[2] * 100;
}
if(PROGRESS) progress = 100;
}
void read_line_2( T * p, unsigned long long x, unsigned long long y, bool PROGRESS = false){
read_line_2(p, y * R[0] + x, PROGRESS);
/*unsigned long long i;
if(PROGRESS) progress = 0;
|
724ec347
David Mayerich
simplified the EN...
|
445
446
447
448
449
450
451
452
453
454
|
if ( x >= R[0] || y >= R[1]){ //make sure the sample and line number is right
std::cout<<"ERROR: sample or line out of range"<<std::endl;
return false;
}
file.seekg((x + y * R[0]) * sizeof(T), std::ios::beg); //point to the certain sample and line
for (i = 0; i < R[2]; i++)
{
file.read((char *)(p + i), sizeof(T));
file.seekg((R[1] * R[0] - 1) * sizeof(T), std::ios::cur); //go to the next band
|
63fc1df8
David Mayerich
added an inverse ...
|
455
|
if(PROGRESS) progress = (double)i / (double)R[2] * 100;
|
724ec347
David Mayerich
simplified the EN...
|
456
|
}
|
63fc1df8
David Mayerich
added an inverse ...
|
457
|
if(PROGRESS) progress = 100;
|
724ec347
David Mayerich
simplified the EN...
|
458
|
|
735a2a24
David Mayerich
started testing o...
|
459
|
return true;*/
|
724ec347
David Mayerich
simplified the EN...
|
460
461
462
463
464
465
466
|
}
///Reads a line X (fastest dimension) for a given YZ value
/// @param p is a pointer to pre-allocated memory equal to the line size R[2]
/// @param x is the y coordinate
/// @param y is the z coordinate
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
467
|
bool read_line_0(T * p, unsigned long long y, unsigned long long z, bool PROGRESS = false){
|
724ec347
David Mayerich
simplified the EN...
|
468
469
|
//test to make sure the specified value is within range
if( y >= R[1] || z >= R[2] ){
|
735a2a24
David Mayerich
started testing o...
|
470
|
std::cout<<"ERROR: sample ("<<y<<", "<<z<<") out of range in "<<__FILE__<<" (line "<<__LINE__<<")"<<std::endl;
|
724ec347
David Mayerich
simplified the EN...
|
471
472
473
474
475
|
return false;
}
file.seekg((z * R[0] * R[1] + y * R[0]) * sizeof(T), std::ios::beg); //seek to the start of the line
file.read((char *)p, sizeof(T) * R[0]); //read the line
|
63fc1df8
David Mayerich
added an inverse ...
|
476
|
if(PROGRESS) progress = 100;
|
724ec347
David Mayerich
simplified the EN...
|
477
478
479
480
481
482
483
484
|
return true;
}
///Reads a line Y (second fastest dimension) for a given XZ value
/// @param p is a pointer to pre-allocated memory equal to the line size R[2]
/// @param x is the y coordinate
/// @param z is the z coordinate
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
485
|
bool read_line_1(T * p, unsigned long long x, unsigned long long z, bool PROGRESS = false){
|
63fc1df8
David Mayerich
added an inverse ...
|
486
|
if(PROGRESS) progress = 0;
|
724ec347
David Mayerich
simplified the EN...
|
487
488
|
//test to make sure the specified value is within range
if( x >= R[0] || z >= R[2] ){
|
735a2a24
David Mayerich
started testing o...
|
489
|
std::cout<<"ERROR: sample or line out of range in "<<__FILE__<<" (line "<<__LINE__<<")"<<std::endl;
|
724ec347
David Mayerich
simplified the EN...
|
490
491
492
493
|
return false;
}
file.seekg((z * R[0] * R[1] + x) * sizeof(T), std::ios::beg); //seek to the start of the line
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
494
|
for (unsigned long long i = 0; i < R[1]; i++){ //for each pixel in the line
|
724ec347
David Mayerich
simplified the EN...
|
495
496
|
file.read((char *)(p + i), sizeof(T)); //read the pixel
file.seekg((R[0] - 1) * sizeof(T), std::ios::cur); //seek to the next pixel in the line
|
63fc1df8
David Mayerich
added an inverse ...
|
497
|
if(PROGRESS) progress = (double)i / (double)R[1] * 100;
|
724ec347
David Mayerich
simplified the EN...
|
498
|
}
|
63fc1df8
David Mayerich
added an inverse ...
|
499
|
if(PROGRESS) progress = 100;
|
724ec347
David Mayerich
simplified the EN...
|
500
501
502
|
return true;
}
|
27b826a8
David Mayerich
added a spherical...
|
503
504
505
506
|
/// Reads a plane given a coordinate along the 0-axis (YZ plane)
/// @param p is a pointer to pre-allocated memory of size R[1] * R[2] * sizeof(T)
/// @param n is the 0-axis coordinate used to retrieve the plane
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
507
|
bool read_plane_0(T* p, unsigned long long n, bool PROGRESS = false){
|
63fc1df8
David Mayerich
added an inverse ...
|
508
|
if(PROGRESS) progress = 0;
|
724ec347
David Mayerich
simplified the EN...
|
509
|
if (n >= R[0]){ //make sure the number is within the possible range
|
735a2a24
David Mayerich
started testing o...
|
510
|
std::cout<<"ERROR: sample or line out of range in "<<__FILE__<<" (line "<<__LINE__<<")"<<std::endl;
|
724ec347
David Mayerich
simplified the EN...
|
511
512
|
return false;
}
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
513
|
unsigned long long jump = (R[0] - 1) * sizeof(T); //number of bytes to skip between samples
|
724ec347
David Mayerich
simplified the EN...
|
514
515
516
517
|
//seek to the start of the plane
file.seekg(n * sizeof(T), std::ios::beg);
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
518
519
|
unsigned long long N = R[1] * R[2];
for(unsigned long long i = 0; i<N; i++){
|
724ec347
David Mayerich
simplified the EN...
|
520
521
|
file.read((char*)(p+i), sizeof(T));
file.seekg(jump, std::ios::cur);
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
522
|
if(PROGRESS) progress = (double)(i+1) / N * 100;
|
724ec347
David Mayerich
simplified the EN...
|
523
524
|
}
|
724ec347
David Mayerich
simplified the EN...
|
525
526
527
528
529
|
return true;
}
|
27b826a8
David Mayerich
added a spherical...
|
530
531
532
533
|
/// Reads a plane given a coordinate along the 1-axis (XZ plane)
/// @param p is a pointer to pre-allocated memory of size R[0] * R[2] * sizeof(T)
/// @param n is the 1-axis coordinate used to retrieve the plane
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
534
|
bool read_plane_1(T* p, unsigned long long n, bool PROGRESS = false){
|
63fc1df8
David Mayerich
added an inverse ...
|
535
|
if(PROGRESS) progress = 0;
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
536
537
|
unsigned long long L = R[0] * sizeof(T); //caculate the number of bytes in a sample line
unsigned long long jump = R[0] * (R[1] - 1) * sizeof(T);
|
724ec347
David Mayerich
simplified the EN...
|
538
539
540
541
542
543
544
|
if (n >= R[1]){ //make sure the bank number is right
std::cout<<"ERROR read_plane_1: page out of range"<<std::endl;
return false;
}
file.seekg(R[0] * n * sizeof(T), std::ios::beg);
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
545
|
for (unsigned long long i = 0; i < R[2]; i++){
|
63fc1df8
David Mayerich
added an inverse ...
|
546
|
if(PROGRESS) progress = (double)i / R[2] * 100;
|
724ec347
David Mayerich
simplified the EN...
|
547
548
549
550
551
|
file.read((char *)(p + i * R[0]), L);
file.seekg( jump, std::ios::cur);
std::cout<<i<<" ";
}
|
63fc1df8
David Mayerich
added an inverse ...
|
552
|
if(PROGRESS) progress = 100;
|
724ec347
David Mayerich
simplified the EN...
|
553
554
555
|
return true;
}
|
27b826a8
David Mayerich
added a spherical...
|
556
557
558
559
|
/// Reads a plane given a coordinate along the 2-axis (XY plane)
/// @param p is a pointer to pre-allocated memory of size R[0] * R[1] * sizeof(T)
/// @param n is the 2-axis coordinate used to retrieve the plane
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
560
|
bool read_plane_2(T* p, unsigned long long n, bool PROGRESS = false){
|
63fc1df8
David Mayerich
added an inverse ...
|
561
|
return read_page(p, n, PROGRESS);
|
724ec347
David Mayerich
simplified the EN...
|
562
563
|
}
|
27b826a8
David Mayerich
added a spherical...
|
564
565
566
567
|
/// Reads a single pixel, treating the entire data set as a linear array
/// @param p is a pointer to pre-allocated memory of size sizeof(T)
/// @param i is the index to the pixel using linear indexing
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
568
|
bool read_pixel(T* p, unsigned long long i){
|
724ec347
David Mayerich
simplified the EN...
|
569
570
571
572
573
574
575
576
577
578
|
if(i >= R[0] * R[1] * R[2]){
std::cout<<"ERROR read_pixel: n is out of range"<<std::endl;
return false;
}
file.seekg(i * sizeof(T), std::ios::cur);
file.read((char*)p, sizeof(T));
}
|
27b826a8
David Mayerich
added a spherical...
|
579
580
581
582
583
584
|
/// Reads a single pixel, given an x, y, z coordinate
/// @param p is a pointer to pre-allocated memory of size sizeof(T)
/// @param x is the x (0) axis coordinate
/// @param y is the y (1) axis coordinate
/// @param z is the z (2) axis coordinate
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
585
|
bool read_pixel(T* p, unsigned long long x, unsigned long long y, unsigned long long z){
|
724ec347
David Mayerich
simplified the EN...
|
586
587
588
589
590
591
|
if(x < 0 || x >= R[0] || y < 0 || y >= R[1] || z < 0 || z > R[2]){
std::cout<<"ERROR read_pixel: (x,y,z) is out of range"<<std::endl;
return false;
}
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
592
|
unsigned long long i = z * R[0] * R[1] + y * R[0] + z;
|
724ec347
David Mayerich
simplified the EN...
|
593
594
595
|
return read_pixel(p, i);
}
|
8b899c24
David Mayerich
fixed asynchronou...
|
596
|
/// Reads a block specified by an (x, y, z) position and size using the largest possible contiguous reads
|
6db250f5
David Mayerich
adaptive streamin...
|
597
598
|
size_t read(T* dest, size_t x, size_t y, size_t z, size_t sx, size_t sy, size_t sz){
auto t0 = std::chrono::high_resolution_clock::now();
|
8b899c24
David Mayerich
fixed asynchronou...
|
599
600
601
602
603
604
605
606
607
|
size_t size_bytes = sx * sy * sz * sizeof(T); //size of the block to read in bytes
size_t start = z * R[0] * R[1] + y * R[0] + x; //calculate the start postion
size_t start_bytes = start * sizeof(T); //start position in bytes
file.seekg(start * sizeof(T), std::ios::beg); //seek to the start position
if(sx == R[0] && sy == R[1]){ //if sx and sy result in a contiguous volume along z
file.read((char*)dest, size_bytes); //read the block in one pass
|
8b899c24
David Mayerich
fixed asynchronou...
|
608
|
}
|
6db250f5
David Mayerich
adaptive streamin...
|
609
|
else if(sx == R[0]){ //if sx is contiguous, read each z-axis slice can be read in one pass
|
8b899c24
David Mayerich
fixed asynchronou...
|
610
611
612
613
614
615
616
|
size_t jump_bytes = (R[1] - sy) * R[0] * sizeof(T); //jump between each slice
size_t slice_bytes = sx * sy * sizeof(T); //size of the slice to be read
for(size_t zi = 0; zi < sz; zi++){ //for each z-axis slice
file.read((char*)dest, slice_bytes); //read the slice
dest += sx * sy; //move the destination pointer to the next slice
file.seekg(jump_bytes, std::ios::cur); //skip to the next slice in the file
}
|
8b899c24
David Mayerich
fixed asynchronou...
|
617
|
}
|
6db250f5
David Mayerich
adaptive streamin...
|
618
619
620
621
622
623
624
625
626
627
628
629
630
631
|
else{
//in this case, x is not contiguous so the volume must be read line-by-line
size_t jump_x_bytes = (R[0] - sx) * sizeof(T); //number of bytes skipped in the x direction
size_t jump_y_bytes = (R[1] - sy) * R[0] * sizeof(T) + jump_x_bytes; //number of bytes skipped between slices
size_t line_bytes = sx * sizeof(T); //size of the line to be read
size_t zi, yi;
for(zi = 0; zi < sz; zi++){ //for each slice
file.read((char*)dest, line_bytes); //read the first line
for(yi = 1; yi < sy; yi++){ //read each additional line
dest += sx; //move the pointer in the destination block to the next line
file.seekg(jump_x_bytes, std::ios::cur); //skip to the next line in the file
file.read((char*)dest, line_bytes); //read the line to the destination block
}
file.seekg(jump_y_bytes, std::ios::cur); //skip to the beginning of the next slice
|
8b899c24
David Mayerich
fixed asynchronou...
|
632
|
}
|
8b899c24
David Mayerich
fixed asynchronou...
|
633
|
}
|
6db250f5
David Mayerich
adaptive streamin...
|
634
635
|
auto t1 = std::chrono::high_resolution_clock::now();
return std::chrono::duration_cast<std::chrono::milliseconds>(t1-t0).count();
|
8b899c24
David Mayerich
fixed asynchronou...
|
636
637
|
}
|
cf5b4c92
David Mayerich
simplified asynch...
|
638
639
|
// permutes a block of data from the current interleave to the interleave specified (re-arranged dimensions to the order specified by [d0, d1, d2])
|
6db250f5
David Mayerich
adaptive streamin...
|
640
641
|
size_t permute(T* dest, T* src, size_t sx, size_t sy, size_t sz, size_t d0, size_t d1, size_t d2){
auto t0 = std::chrono::high_resolution_clock::now();
|
cf5b4c92
David Mayerich
simplified asynch...
|
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
|
size_t d[3] = {d0, d1, d2};
size_t s[3] = {sx, sy, sz};
size_t p[3];// = {x, y, z};
if(d[0] == 0 && d[1] == 1 && d[2] == 2){
//this isn't actually a permute - just copy the data
memcpy(dest, src, sizeof(T) * sx * sy * sz);
}
else if(d[0] == 0){ //the individual lines are contiguous, so you can memcpy line-by-line
size_t y, z;
size_t src_idx, dest_idx;
size_t x_bytes = sizeof(T) * sx;
for(z = 0; z < sz; z++){
p[2] = z;
for(y = 0; y < sy; y++){
p[1] = y;
src_idx = z * sx * sy + y * sx;
dest_idx = p[d[2]] * s[d[0]] * s[d[1]] + p[d[1]] * s[d[0]];
|
cf5b4c92
David Mayerich
simplified asynch...
|
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
|
memcpy(dest + dest_idx, src + src_idx, x_bytes);
}
}
}
else{ //loop through every damn point
size_t x, y, z;
size_t src_idx, dest_idx;
size_t src_z, src_y;
for(z = 0; z < sz; z++){
p[2] = z;
src_z = z * sx * sy;
for(y = 0; y < sy; y++){
p[1] = y;
src_y = src_z + y * sx;
for(x = 0; x < sx; x++){
p[0] = x;
src_idx = src_y + x;
dest_idx = p[d[2]] * s[d[0]] * s[d[1]] + p[d[1]] * s[d[0]] + p[d[0]];
dest[dest_idx] = src[src_idx];
}
}
}
}
|
6db250f5
David Mayerich
adaptive streamin...
|
683
684
|
auto t1 = std::chrono::high_resolution_clock::now();
return std::chrono::duration_cast<std::chrono::milliseconds>(t1-t0).count();
|
cf5b4c92
David Mayerich
simplified asynch...
|
685
686
|
}
|
180d7f3a
David Mayerich
added binary file...
|
687
688
689
690
691
|
};
}
#endif
|