1f2ca8ec
David Mayerich
added files for b...
|
1
2
|
//make sure that this header file is only loaded once
|
180d7f3a
David Mayerich
added binary file...
|
3
4
5
|
#ifndef RTS_BINARY_H
#define RTS_BINARY_H
|
e8eb202f
David Mayerich
added a new ENVI ...
|
6
|
#include "../envi/envi_header.h"
|
4dcc9392
heziqi
Ziqi's first chan...
|
7
|
#include "../math/vector.h"
|
180d7f3a
David Mayerich
added binary file...
|
8
9
|
#include <fstream>
#include <sys/stat.h>
|
15a31763
David Mayerich
set up buffer est...
|
10
|
#include <cstring>
|
5435830b
David Mayerich
linux chrono fixes
|
11
|
#include <chrono>
|
008ae990
David Mayerich
implemented Windo...
|
12
13
|
#ifdef _WIN32
#include <Windows.h>
|
15a31763
David Mayerich
set up buffer est...
|
14
15
|
#else
#include <unistd.h>
|
008ae990
David Mayerich
implemented Windo...
|
16
17
|
#endif
|
8a86bd56
David Mayerich
changed rts names...
|
18
|
namespace stim{
|
180d7f3a
David Mayerich
added binary file...
|
19
|
|
6db250f5
David Mayerich
adaptive streamin...
|
20
21
22
23
|
/// This class calculates the optimal setting for independent parameter b (batch size) for
/// minimizing the dependent parameter bps (bytes per second)
class stream_optimizer{
protected:
|
0a575bb1
David Mayerich
updated optimizat...
|
24
|
size_t Bps[2]; //bytes per second for the previous batch
|
6db250f5
David Mayerich
adaptive streamin...
|
25
26
27
|
size_t interval_B; //number of bytes processed this interval
size_t interval_ms; //number of milliseconds spent in the current interval
size_t n[2]; //current batch size (in bytes)
|
0a575bb1
David Mayerich
updated optimizat...
|
28
|
size_t h; //spacing used for finite difference calculations
|
6db250f5
David Mayerich
adaptive streamin...
|
29
30
31
|
size_t dn; //delta value (in bytes) for setting the batch size (minimum change in batch parameter)
size_t maxn; //maximum value for the batch size
|
0a575bb1
David Mayerich
updated optimizat...
|
32
33
|
double alpha; //alpha value controls the factor of the gradient that is used to calculate the next point (speed of convergence)
|
6db250f5
David Mayerich
adaptive streamin...
|
34
35
36
37
38
39
|
bool sample_step; //calculating the derivative (this class alternates between calculating dBps and B)
bool forward_diff; //evaluate the derivative using forward differences
size_t window_ms; //size of the interval (in milliseconds) integrated to get a reliable bps value
// This function rounds x to the nearest value within dB
|
0a575bb1
David Mayerich
updated optimizat...
|
40
41
42
43
44
|
size_t round_limit(double n0){
if(n0 < 0) return dn; //if n0 is less than zero, return the lowest possible n
size_t new_n = (size_t)(n0 + 0.5); //now n0 must be positive, so round it to the nearest integer
if(new_n > maxn) new_n = maxn; //limit the returned size of x to within the specified bounds
|
6db250f5
David Mayerich
adaptive streamin...
|
45
|
|
0a575bb1
David Mayerich
updated optimizat...
|
46
|
size_t lowest = new_n / dn;
|
6db250f5
David Mayerich
adaptive streamin...
|
47
|
size_t highest = lowest + dn;
|
0a575bb1
David Mayerich
updated optimizat...
|
48
|
size_t diff[2] = {new_n - lowest, highest - new_n}; //calculate the two differences
|
6db250f5
David Mayerich
adaptive streamin...
|
49
50
51
52
53
54
55
56
|
if(diff[0] < diff[1])
return lowest;
return highest;
}
public:
//constructor initializes a stream optimizer
|
0a575bb1
David Mayerich
updated optimizat...
|
57
58
59
|
stream_optimizer(size_t min_batch_size, size_t max_batch_size, double a = 0.0001, size_t window = 1000){
//Bps = 0; //initialize to zero bytes per second processed
Bps[0] = Bps[1] = 0; //initialize the bits per second to 0
|
6db250f5
David Mayerich
adaptive streamin...
|
60
61
62
63
|
interval_B = 0; //zero bytes have been processed at initialization
interval_ms = 0; //no time has been spent on the batch so far
dn = min_batch_size; //set the minimum batch size as the minimum change in batch size
maxn = max_batch_size; //set the maximum batch size
|
0a575bb1
David Mayerich
updated optimizat...
|
64
65
66
67
68
69
|
n[0] = max_batch_size; //set B
h = (max_batch_size / min_batch_size) / 10 * dn;
std::cout<<"h = "<<h<<std::endl;
if(h < dn) h = dn;
alpha = a;
//n[0] = round_limit( (max_batch_size - min_batch_size)/2 );
|
6db250f5
David Mayerich
adaptive streamin...
|
70
71
72
73
74
|
window_ms = window; //minimum integration interval (for getting a reliable bps measure)
sample_step = true; //the first step is to calculate the derivative
forward_diff = true; //start with the forward difference (since we start at the maximum batch size)
}
|
0a575bb1
David Mayerich
updated optimizat...
|
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
|
size_t update(size_t bytes_processed, size_t ms_spent){
interval_B += bytes_processed; //increment the number of bytes processed
interval_ms += ms_spent; //increment the number of milliseconds spent processing
//if we have sufficient information to evaluate the optimization function at this point
if(interval_ms < window_ms){ //if insufficient time has passed to get a reliable Bps measurement
return n[0];
}
else{ //if we have collected enough information for a reliable Bps estimate
size_t new_Bps = interval_B / interval_ms; //calculate the current Bps
if(Bps[0] == 0){ //if n[0] hasn't been evaluated yet, this is the first step
Bps[0] = new_Bps; //set the initial Bps value
n[1] = n[0] - h; //set the position of the next sample point
std::cout<<"Bps value at n = "<<n[0]<<" is "<<Bps[0]<<" Bps, probing n = "<<n[1]<<std::endl;
return n[1]; //return the probe point
}
else{
Bps[1] = new_Bps; //set the Bps for the current point (n[1])
double Bps_p; //allocate a variable for the derivative
//calculate the derivative
if(n[0] < n[1]){ //if the current point is less than the previous one (probably the most common)
Bps_p = ((double)Bps[1] - (double)Bps[0]) / (double)h; //calculate the derivative using the forward finite difference
}
else{
Bps_p = ((double)Bps[0] - (double)Bps[1]) / (double)h; //calculate the derivative using the backward finite difference
}
std::cout<<" probed n = "<<n[1]<<" with "<<Bps[1]<<" Bps, gradient = "<<Bps_p<<" Bps"<<std::endl;
double new_n_precise = n[0] + alpha * Bps_p; //calculate the next point (snap to closest integer)
size_t new_n_nearest = round_limit(new_n_precise); //calculate the next point (given batch parameters)
if(new_n_nearest == n[0]){ //if the newest point is the same as the original point
Bps[0] = Bps[1]; //update the Bps
//if(n[0] == dn) n[1] = n[0] + h; //if we're on the left edge, probe forward
//else n[1] = n[0] - h; //otherwise probe backwards
std::cout<<" staying at n = "<<n[0]<<" for now"<<std::endl;
//return n[1]; //return the probe point
Bps[0] = 0; //reset the Bps for the current point
return n[0]; //return the current point for a re-calculation
}
else{ //if the newest point is different from the original point
n[0] = new_n_nearest; //move to the new point
Bps[0] = 0; //set the Bps to zero (point hasn't been tested)
std::cout<<" moving to n = "<<n[0]<<std::endl;
return n[0]; //return the new point
}
}
}
}
/*// this function updates the optimizer, given the number of bytes processed in an interval and time spent processing
|
6db250f5
David Mayerich
adaptive streamin...
|
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
|
size_t update(size_t bytes_processed, size_t ms_spent){
interval_B += bytes_processed; //increment the number of bytes processed
interval_ms += ms_spent; //increment the number of milliseconds spent processing
//if we have sufficient information to evaluate the optimization function at this point
if(interval_ms >= window_ms){ //if sufficient time has passed to get a reliable Bps measurement
size_t new_Bps = interval_B / interval_ms; //calculate the current Bps
if(sample_step){ //if this is a sample step, collect the information for Bps = f(n0)
Bps = new_Bps; //set the Bps to the evaluated value
n[1] = n[0] - dn; //reduce the batch size by one delta to take a second sample
if(n[1] == 0){ //if the resulting batch size is zero
n[1] = 2*dn; //we're at the left edge: set the new sample point to 2*dn
}
interval_B = interval_ms = 0; //start a new interval at the new sample point
sample_step = false; //next step will calculate the new batch size via optimization
return n[1]; //return the new batch size
}
else{ //if we have sufficient information to evaluate the derivative and optimize
double f = (double)new_Bps; //we have evaluated the function at this location
double fprime;
if(n[1] < n[0] ){ //if the new point is less than the previous point (usually the case)
fprime = (double)(Bps - new_Bps) / (double)dn; //calculate the forward difference
}
else{ //if the new point is larger (only happens at the minimum limit)
fprime = (double)(new_Bps - Bps) / (double)dn; //calculate the backward difference
}
size_t bestn = n[1] - (size_t)(f / fprime); //calculate the best value for B using Newton's method
n[0] = round_limit( (size_t)bestn ); //set the new dependent point
sample_step = true; //the next step will be a sample step
}
}
if(sample_step) return n[0];
return n[1]; //insufficient information, keep the same batch size
|
0a575bb1
David Mayerich
updated optimizat...
|
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
|
}*/
/*size_t update(size_t bytes_processed, size_t ms_spent){
interval_B += bytes_processed; //increment the number of bytes processed
interval_ms += ms_spent; //increment the number of milliseconds spent processing
//if( Bps[0] == 0 ){ //if the left boundary hasn't been processed
//if we have sufficient information to evaluate the optimization function at this point
if(interval_ms >= window_ms){
size_t new_Bps = interval_B / interval_ms; //calculate the current Bps
if(Bps[0] == 0) //if the left interval Bps hasn't been calculated
Bps[0] = interval_B / interval_ms; //that is the interval being processed
else
Bps[1] = interval_B / interval_ms; //otherwise the right interval is being processed
if(Bps[0] != 0 && Bps[1] != 0){ //if both intervals have been processed
}
}*/
|
6db250f5
David Mayerich
adaptive streamin...
|
189
190
191
|
size_t update(size_t bytes_processed, size_t ms_spent, size_t& data_rate){
size_t time = update(bytes_processed, ms_spent);
|
0a575bb1
David Mayerich
updated optimizat...
|
192
|
data_rate = Bps[0];
|
6db250f5
David Mayerich
adaptive streamin...
|
193
194
195
196
|
return time;
}
};
|
33d7d3cf
David Mayerich
documentation for...
|
197
198
199
200
201
202
203
|
/** This class manages the streaming of large multidimensional binary files.
* Generally these are hyperspectral files with 2 spatial and 1 spectral dimension. However, this class supports
* other dimensions via the template parameter D.
*
* @param T is the data type used to store data to disk (generally float or double)
* @param D is the dimension of the data (default 3)
*/
|
180d7f3a
David Mayerich
added binary file...
|
204
205
206
207
|
template< typename T, unsigned int D = 3 >
class binary{
protected:
|
180d7f3a
David Mayerich
added binary file...
|
208
209
210
|
std::fstream file; //file stream used for reading and writing
std::string name; //file name
|
63fc1df8
David Mayerich
added an inverse ...
|
211
|
unsigned long long R[D]; //resolution
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
212
|
unsigned long long header; //header size (in bytes)
|
f4c5d71b
David Mayerich
started working w...
|
213
|
unsigned char* mask; //pointer to a character array: 0 = background, 1 = foreground (or valid data)
|
180d7f3a
David Mayerich
added binary file...
|
214
|
|
63fc1df8
David Mayerich
added an inverse ...
|
215
|
double progress; //stores the progress on the current operation (accessible using a thread)
|
ee4dea28
David Mayerich
fixed errors in c...
|
216
|
|
008ae990
David Mayerich
implemented Windo...
|
217
|
size_t buffer_size; //available memory for processing large files
|
ee4dea28
David Mayerich
fixed errors in c...
|
218
|
|
33d7d3cf
David Mayerich
documentation for...
|
219
|
/// Private initialization function used to set default parameters in the data structure.
|
180d7f3a
David Mayerich
added binary file...
|
220
|
void init(){
|
15a31763
David Mayerich
set up buffer est...
|
221
|
std::memset(R, 0, sizeof(unsigned long long) * D); //initialize the resolution to zero
|
008ae990
David Mayerich
implemented Windo...
|
222
|
header = 0; //initialize the header size to zero
|
f4c5d71b
David Mayerich
started working w...
|
223
|
mask = NULL;
|
cbce396e
David Mayerich
added support for...
|
224
|
|
63fc1df8
David Mayerich
added an inverse ...
|
225
|
progress = 0;
|
008ae990
David Mayerich
implemented Windo...
|
226
|
set_buffer(); //set the maximum buffer size to the default
|
63fc1df8
David Mayerich
added an inverse ...
|
227
228
|
}
|
33d7d3cf
David Mayerich
documentation for...
|
229
|
/// Private helper function that returns the size of the file on disk using system functions.
|
b0fee590
heziqi
convert long int ...
|
230
|
long long int get_file_size(){
|
193ff00e
David Mayerich
dealth with preci...
|
231
|
#ifdef _WIN32
|
b0fee590
heziqi
convert long int ...
|
232
233
|
struct _stat64 results;
if(_stat64(name.c_str(), &results) == 0)
|
180d7f3a
David Mayerich
added binary file...
|
234
|
return results.st_size;
|
193ff00e
David Mayerich
dealth with preci...
|
235
236
237
238
239
|
#else
struct stat results;
if(stat(name.c_str(), &results) == 0)
return results.st_size;
#endif
|
180d7f3a
David Mayerich
added binary file...
|
240
241
242
|
else return 0;
}
|
33d7d3cf
David Mayerich
documentation for...
|
243
|
/// Private helper function that tests to make sure that the calculated data size specified by the structure is the same as the data size on disk.
|
180d7f3a
David Mayerich
added binary file...
|
244
|
bool test_file_size(){
|
b0fee590
heziqi
convert long int ...
|
245
|
long long int npts = 1; //initialize the number of data points to 1
|
87da9adc
David Mayerich
fixed issues with...
|
246
247
|
for(unsigned int i = 0; i<D; i++) //iterate over each dimension
npts *= R[i]; //compute the total number of data points in the file
|
b0fee590
heziqi
convert long int ...
|
248
|
long long int datasize = npts * sizeof(T);//multiply the sum by the size of the template parameter
|
87da9adc
David Mayerich
fixed issues with...
|
249
250
|
if(datasize + header == get_file_size()) return true; //if the byte size matches the file size, we're golden
|
180d7f3a
David Mayerich
added binary file...
|
251
252
253
254
|
else return false; //otherwise return an error
}
|
f92397d2
David Mayerich
fixed a Win32 bug...
|
255
256
257
258
259
260
|
/// Private helper function that resets the file pointer to the beginning of the data
void reset(){
file.seekg(header, std::ios_base::beg);
}
|
33d7d3cf
David Mayerich
documentation for...
|
261
262
263
|
/// Private helper file that opens a specified binary file.
/// @param filename is the name of the binary file to stream
|
180d7f3a
David Mayerich
added binary file...
|
264
265
266
267
|
bool open_file(std::string filename){
//open the file as binary for reading and writing
file.open(filename.c_str(), std::ios::in | std::ios::out | std::ios::binary);
|
ba72c5b0
David Mayerich
added OBJ class
|
268
269
|
//if the file isn't open, the user may only have read access
if(!file.is_open()){
|
3d0b6243
David Mayerich
fixed hsiproc bug...
|
270
|
std::cout<<"class STIM::BINARY - failed to open file, trying for read only"<<std::endl;
|
ba72c5b0
David Mayerich
added OBJ class
|
271
272
273
274
275
276
277
|
file.open(filename.c_str(), std::ios::in | std::ios::binary);
if(!file.is_open()){
std::cout<<" still unable to load the file"<<std::endl;
return false;
}
}
|
180d7f3a
David Mayerich
added binary file...
|
278
279
280
281
282
283
|
//if the file is successful
if(file){
name = filename; //set the name
if(test_file_size()) //test the file size
return true;
}
|
f92397d2
David Mayerich
fixed a Win32 bug...
|
284
|
|
180d7f3a
David Mayerich
added binary file...
|
285
286
287
|
return false;
}
|
724ec347
David Mayerich
simplified the EN...
|
288
289
|
|
f92397d2
David Mayerich
fixed a Win32 bug...
|
290
291
|
|
180d7f3a
David Mayerich
added binary file...
|
292
293
|
public:
|
008ae990
David Mayerich
implemented Windo...
|
294
295
296
297
298
|
//default constructor
binary(){
init();
}
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
299
|
double get_progress(){
|
63fc1df8
David Mayerich
added an inverse ...
|
300
|
return progress;
|
cbce396e
David Mayerich
added support for...
|
301
302
|
}
|
63fc1df8
David Mayerich
added an inverse ...
|
303
304
|
void reset_progress(){
progress = 0;
|
cbce396e
David Mayerich
added support for...
|
305
306
|
}
|
008ae990
David Mayerich
implemented Windo...
|
307
308
309
310
311
312
313
|
//specify the maximum fraction of available memory that this class will use for buffering
void set_buffer(double mem_frac = 0.5){ //default to 50%
#ifdef _WIN32
MEMORYSTATUSEX statex;
statex.dwLength = sizeof (statex);
GlobalMemoryStatusEx (&statex);
buffer_size = (size_t)(statex.ullAvailPhys * mem_frac);
|
15a31763
David Mayerich
set up buffer est...
|
314
315
316
317
|
#else
size_t pages = sysconf(_SC_PHYS_PAGES);
size_t page_size = sysconf(_SC_PAGE_SIZE);
buffer_size = (size_t)(pages * page_size * mem_frac);
|
008ae990
David Mayerich
implemented Windo...
|
318
319
320
|
#endif
}
|
33d7d3cf
David Mayerich
documentation for...
|
321
322
323
324
325
|
/// Open a binary file for streaming.
/// @param filename is the name of the binary file
/// @param r is a STIM vector specifying the size of the binary file along each dimension
/// @param h is the length (in bytes) of any header file (default zero)
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
326
|
bool open(std::string filename, vec<unsigned long long> r, unsigned long long h = 0){
|
180d7f3a
David Mayerich
added binary file...
|
327
|
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
328
|
for(unsigned long long i = 0; i < D; i++) //set the dimensions of the binary file object
|
180d7f3a
David Mayerich
added binary file...
|
329
330
|
R[i] = r[i];
|
87da9adc
David Mayerich
fixed issues with...
|
331
332
333
|
header = h; //save the header size
if(!open_file(filename)) return false; //open the binary file
|
180d7f3a
David Mayerich
added binary file...
|
334
|
|
f92397d2
David Mayerich
fixed a Win32 bug...
|
335
336
|
//reset();
|
180d7f3a
David Mayerich
added binary file...
|
337
338
339
|
return test_file_size();
}
|
33d7d3cf
David Mayerich
documentation for...
|
340
341
342
343
344
|
/// Creates a new binary file for streaming
/// @param filename is the name of the binary file to be created
/// @param r is a STIM vector specifying the size of the file along each dimension
/// @offset specifies how many bytes to offset the file (used to leave room for a header)
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
345
|
bool create(std::string filename, vec<unsigned long long> r, unsigned long long offset = 0){
|
f47168a2
heziqi
Ziqi added create...
|
346
347
348
349
350
|
std::ofstream target(filename.c_str(), std::ios::binary);
//initialize binary file
T p = 0;
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
351
|
for(unsigned long long i =0; i < r[0] * r[1] * r[2]; i++){
|
f47168a2
heziqi
Ziqi added create...
|
352
353
354
|
target.write((char*)(&p), sizeof(T));
}
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
355
|
for(unsigned long long i = 0; i < D; i++) //set the dimensions of the binary file object
|
f47168a2
heziqi
Ziqi added create...
|
356
357
358
359
360
361
362
363
364
|
R[i] = r[i];
header = offset; //save the header size
if(!open_file(filename)) return false; //open the binary file
return test_file_size();
}
|
33d7d3cf
David Mayerich
documentation for...
|
365
366
367
368
|
/// Writes a single page of data to disk. A page consists of a sequence of data of size R[0] * R[1] * ... * R[D-1].
/// @param p is a pointer to the data to be written
/// @param page is the page number (index of the highest-numbered dimension)
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
369
|
bool write_page( T * p, unsigned long long page){
|
f47168a2
heziqi
Ziqi added create...
|
370
371
372
373
374
375
|
if(p == NULL){
std::cout<<"ERROR: unable to write into file, empty pointer"<<std::endl;
exit(1);
}
|
f92397d2
David Mayerich
fixed a Win32 bug...
|
376
|
file.seekg(R[1] * R[0] * page * sizeof(T) + header, std::ios::beg); //seek to the desired location on disk
|
8157c392
David Mayerich
added parser and ...
|
377
|
file.write((char *)p, R[0] * R[1] * sizeof(T)); //write binary data
|
f47168a2
heziqi
Ziqi added create...
|
378
379
380
381
|
return true;
}
|
33d7d3cf
David Mayerich
documentation for...
|
382
383
384
385
|
/// Reads a page from disk. A page consists of a sequence of data of size R[0] * R[1] * ... * R[D-1].
/// @param p is a pointer to pre-allocated memory equal to the page size
/// @param page is the index of the page
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
386
|
bool read_page( T * p, unsigned long long page, bool PROGRESS = false){
|
63fc1df8
David Mayerich
added an inverse ...
|
387
388
|
if(PROGRESS) progress = 0;
|
c7da85e3
heziqi
he,ziqi add metho...
|
389
|
|
5d5ce39a
heziqi
Ziqi convert the ...
|
390
|
if (page >= R[2]){ //make sure the bank number is right
|
2ad0ce4f
David Mayerich
syntactic edits b...
|
391
|
std::cout<<"ERROR: page out of range"<<std::endl;
|
5d5ce39a
heziqi
Ziqi convert the ...
|
392
|
return false;
|
c7da85e3
heziqi
he,ziqi add metho...
|
393
394
|
}
|
f92397d2
David Mayerich
fixed a Win32 bug...
|
395
|
file.seekg(R[1] * R[0] * page * sizeof(T) + header, std::ios::beg); //write into memory from the binary file
|
2ad0ce4f
David Mayerich
syntactic edits b...
|
396
|
file.read((char *)p, R[0] * R[1] * sizeof(T));
|
63fc1df8
David Mayerich
added an inverse ...
|
397
|
if(PROGRESS) progress = 100;
|
5d5ce39a
heziqi
Ziqi convert the ...
|
398
|
return true;
|
c7da85e3
heziqi
he,ziqi add metho...
|
399
400
|
}
|
724ec347
David Mayerich
simplified the EN...
|
401
402
403
404
405
406
407
|
///Reads a line Z (slowest dimension) for a given XY value
/// @param p is a pointer to pre-allocated memory equal to the line size R[2]
/// @param x is the x coordinate
/// @param y is the y coordinate
|
735a2a24
David Mayerich
started testing o...
|
408
|
void read_line_2( T* p, size_t n, bool PROGRESS = false){
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
409
|
unsigned long long i;
|
724ec347
David Mayerich
simplified the EN...
|
410
|
|
63fc1df8
David Mayerich
added an inverse ...
|
411
412
|
if(PROGRESS) progress = 0;
|
735a2a24
David Mayerich
started testing o...
|
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
|
if ( n > R[0] * R[1]){ //make sure the sample and line number is right
std::cout<<"ERROR: sample or line out of range in "<<__FILE__<<" (line "<<__LINE__<<")"<<std::endl;
exit(1);
}
file.seekg(n * sizeof(T), std::ios::beg); //point to the certain sample and line
for (i = 0; i < R[2]; i++){ //for each band
file.read((char *)(p + i), sizeof(T));
file.seekg((R[1] * R[0] - 1) * sizeof(T), std::ios::cur); //go to the next band
if(PROGRESS) progress = (double)i / (double)R[2] * 100;
}
if(PROGRESS) progress = 100;
}
void read_line_2( T * p, unsigned long long x, unsigned long long y, bool PROGRESS = false){
read_line_2(p, y * R[0] + x, PROGRESS);
/*unsigned long long i;
if(PROGRESS) progress = 0;
|
724ec347
David Mayerich
simplified the EN...
|
432
433
434
435
436
437
438
439
440
441
|
if ( x >= R[0] || y >= R[1]){ //make sure the sample and line number is right
std::cout<<"ERROR: sample or line out of range"<<std::endl;
return false;
}
file.seekg((x + y * R[0]) * sizeof(T), std::ios::beg); //point to the certain sample and line
for (i = 0; i < R[2]; i++)
{
file.read((char *)(p + i), sizeof(T));
file.seekg((R[1] * R[0] - 1) * sizeof(T), std::ios::cur); //go to the next band
|
63fc1df8
David Mayerich
added an inverse ...
|
442
|
if(PROGRESS) progress = (double)i / (double)R[2] * 100;
|
724ec347
David Mayerich
simplified the EN...
|
443
|
}
|
63fc1df8
David Mayerich
added an inverse ...
|
444
|
if(PROGRESS) progress = 100;
|
724ec347
David Mayerich
simplified the EN...
|
445
|
|
735a2a24
David Mayerich
started testing o...
|
446
|
return true;*/
|
724ec347
David Mayerich
simplified the EN...
|
447
448
449
450
451
452
453
|
}
///Reads a line X (fastest dimension) for a given YZ value
/// @param p is a pointer to pre-allocated memory equal to the line size R[2]
/// @param x is the y coordinate
/// @param y is the z coordinate
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
454
|
bool read_line_0(T * p, unsigned long long y, unsigned long long z, bool PROGRESS = false){
|
724ec347
David Mayerich
simplified the EN...
|
455
456
|
//test to make sure the specified value is within range
if( y >= R[1] || z >= R[2] ){
|
735a2a24
David Mayerich
started testing o...
|
457
|
std::cout<<"ERROR: sample ("<<y<<", "<<z<<") out of range in "<<__FILE__<<" (line "<<__LINE__<<")"<<std::endl;
|
724ec347
David Mayerich
simplified the EN...
|
458
459
460
461
462
|
return false;
}
file.seekg((z * R[0] * R[1] + y * R[0]) * sizeof(T), std::ios::beg); //seek to the start of the line
file.read((char *)p, sizeof(T) * R[0]); //read the line
|
63fc1df8
David Mayerich
added an inverse ...
|
463
|
if(PROGRESS) progress = 100;
|
724ec347
David Mayerich
simplified the EN...
|
464
465
466
467
468
469
470
471
|
return true;
}
///Reads a line Y (second fastest dimension) for a given XZ value
/// @param p is a pointer to pre-allocated memory equal to the line size R[2]
/// @param x is the y coordinate
/// @param z is the z coordinate
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
472
|
bool read_line_1(T * p, unsigned long long x, unsigned long long z, bool PROGRESS = false){
|
63fc1df8
David Mayerich
added an inverse ...
|
473
|
if(PROGRESS) progress = 0;
|
724ec347
David Mayerich
simplified the EN...
|
474
475
|
//test to make sure the specified value is within range
if( x >= R[0] || z >= R[2] ){
|
735a2a24
David Mayerich
started testing o...
|
476
|
std::cout<<"ERROR: sample or line out of range in "<<__FILE__<<" (line "<<__LINE__<<")"<<std::endl;
|
724ec347
David Mayerich
simplified the EN...
|
477
478
479
480
|
return false;
}
file.seekg((z * R[0] * R[1] + x) * sizeof(T), std::ios::beg); //seek to the start of the line
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
481
|
for (unsigned long long i = 0; i < R[1]; i++){ //for each pixel in the line
|
724ec347
David Mayerich
simplified the EN...
|
482
483
|
file.read((char *)(p + i), sizeof(T)); //read the pixel
file.seekg((R[0] - 1) * sizeof(T), std::ios::cur); //seek to the next pixel in the line
|
63fc1df8
David Mayerich
added an inverse ...
|
484
|
if(PROGRESS) progress = (double)i / (double)R[1] * 100;
|
724ec347
David Mayerich
simplified the EN...
|
485
|
}
|
63fc1df8
David Mayerich
added an inverse ...
|
486
|
if(PROGRESS) progress = 100;
|
724ec347
David Mayerich
simplified the EN...
|
487
488
489
|
return true;
}
|
27b826a8
David Mayerich
added a spherical...
|
490
491
492
493
|
/// Reads a plane given a coordinate along the 0-axis (YZ plane)
/// @param p is a pointer to pre-allocated memory of size R[1] * R[2] * sizeof(T)
/// @param n is the 0-axis coordinate used to retrieve the plane
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
494
|
bool read_plane_0(T* p, unsigned long long n, bool PROGRESS = false){
|
63fc1df8
David Mayerich
added an inverse ...
|
495
|
if(PROGRESS) progress = 0;
|
724ec347
David Mayerich
simplified the EN...
|
496
|
if (n >= R[0]){ //make sure the number is within the possible range
|
735a2a24
David Mayerich
started testing o...
|
497
|
std::cout<<"ERROR: sample or line out of range in "<<__FILE__<<" (line "<<__LINE__<<")"<<std::endl;
|
724ec347
David Mayerich
simplified the EN...
|
498
499
|
return false;
}
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
500
|
unsigned long long jump = (R[0] - 1) * sizeof(T); //number of bytes to skip between samples
|
724ec347
David Mayerich
simplified the EN...
|
501
502
503
504
|
//seek to the start of the plane
file.seekg(n * sizeof(T), std::ios::beg);
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
505
506
|
unsigned long long N = R[1] * R[2];
for(unsigned long long i = 0; i<N; i++){
|
724ec347
David Mayerich
simplified the EN...
|
507
508
|
file.read((char*)(p+i), sizeof(T));
file.seekg(jump, std::ios::cur);
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
509
|
if(PROGRESS) progress = (double)(i+1) / N * 100;
|
724ec347
David Mayerich
simplified the EN...
|
510
511
|
}
|
724ec347
David Mayerich
simplified the EN...
|
512
513
514
515
516
|
return true;
}
|
27b826a8
David Mayerich
added a spherical...
|
517
518
519
520
|
/// Reads a plane given a coordinate along the 1-axis (XZ plane)
/// @param p is a pointer to pre-allocated memory of size R[0] * R[2] * sizeof(T)
/// @param n is the 1-axis coordinate used to retrieve the plane
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
521
|
bool read_plane_1(T* p, unsigned long long n, bool PROGRESS = false){
|
63fc1df8
David Mayerich
added an inverse ...
|
522
|
if(PROGRESS) progress = 0;
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
523
524
|
unsigned long long L = R[0] * sizeof(T); //caculate the number of bytes in a sample line
unsigned long long jump = R[0] * (R[1] - 1) * sizeof(T);
|
724ec347
David Mayerich
simplified the EN...
|
525
526
527
528
529
530
531
|
if (n >= R[1]){ //make sure the bank number is right
std::cout<<"ERROR read_plane_1: page out of range"<<std::endl;
return false;
}
file.seekg(R[0] * n * sizeof(T), std::ios::beg);
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
532
|
for (unsigned long long i = 0; i < R[2]; i++){
|
63fc1df8
David Mayerich
added an inverse ...
|
533
|
if(PROGRESS) progress = (double)i / R[2] * 100;
|
724ec347
David Mayerich
simplified the EN...
|
534
535
536
537
538
|
file.read((char *)(p + i * R[0]), L);
file.seekg( jump, std::ios::cur);
std::cout<<i<<" ";
}
|
63fc1df8
David Mayerich
added an inverse ...
|
539
|
if(PROGRESS) progress = 100;
|
724ec347
David Mayerich
simplified the EN...
|
540
541
542
|
return true;
}
|
27b826a8
David Mayerich
added a spherical...
|
543
544
545
546
|
/// Reads a plane given a coordinate along the 2-axis (XY plane)
/// @param p is a pointer to pre-allocated memory of size R[0] * R[1] * sizeof(T)
/// @param n is the 2-axis coordinate used to retrieve the plane
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
547
|
bool read_plane_2(T* p, unsigned long long n, bool PROGRESS = false){
|
63fc1df8
David Mayerich
added an inverse ...
|
548
|
return read_page(p, n, PROGRESS);
|
724ec347
David Mayerich
simplified the EN...
|
549
550
|
}
|
27b826a8
David Mayerich
added a spherical...
|
551
552
553
554
|
/// Reads a single pixel, treating the entire data set as a linear array
/// @param p is a pointer to pre-allocated memory of size sizeof(T)
/// @param i is the index to the pixel using linear indexing
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
555
|
bool read_pixel(T* p, unsigned long long i){
|
724ec347
David Mayerich
simplified the EN...
|
556
557
558
559
560
561
562
563
564
565
|
if(i >= R[0] * R[1] * R[2]){
std::cout<<"ERROR read_pixel: n is out of range"<<std::endl;
return false;
}
file.seekg(i * sizeof(T), std::ios::cur);
file.read((char*)p, sizeof(T));
}
|
27b826a8
David Mayerich
added a spherical...
|
566
567
568
569
570
571
|
/// Reads a single pixel, given an x, y, z coordinate
/// @param p is a pointer to pre-allocated memory of size sizeof(T)
/// @param x is the x (0) axis coordinate
/// @param y is the y (1) axis coordinate
/// @param z is the z (2) axis coordinate
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
572
|
bool read_pixel(T* p, unsigned long long x, unsigned long long y, unsigned long long z){
|
724ec347
David Mayerich
simplified the EN...
|
573
574
575
576
577
578
|
if(x < 0 || x >= R[0] || y < 0 || y >= R[1] || z < 0 || z > R[2]){
std::cout<<"ERROR read_pixel: (x,y,z) is out of range"<<std::endl;
return false;
}
|
9d3ba0b1
David Mayerich
added stim::hsi a...
|
579
|
unsigned long long i = z * R[0] * R[1] + y * R[0] + z;
|
724ec347
David Mayerich
simplified the EN...
|
580
581
582
|
return read_pixel(p, i);
}
|
8b899c24
David Mayerich
fixed asynchronou...
|
583
|
/// Reads a block specified by an (x, y, z) position and size using the largest possible contiguous reads
|
6db250f5
David Mayerich
adaptive streamin...
|
584
585
|
size_t read(T* dest, size_t x, size_t y, size_t z, size_t sx, size_t sy, size_t sz){
auto t0 = std::chrono::high_resolution_clock::now();
|
8b899c24
David Mayerich
fixed asynchronou...
|
586
587
588
589
590
591
592
593
594
|
size_t size_bytes = sx * sy * sz * sizeof(T); //size of the block to read in bytes
size_t start = z * R[0] * R[1] + y * R[0] + x; //calculate the start postion
size_t start_bytes = start * sizeof(T); //start position in bytes
file.seekg(start * sizeof(T), std::ios::beg); //seek to the start position
if(sx == R[0] && sy == R[1]){ //if sx and sy result in a contiguous volume along z
file.read((char*)dest, size_bytes); //read the block in one pass
|
8b899c24
David Mayerich
fixed asynchronou...
|
595
|
}
|
6db250f5
David Mayerich
adaptive streamin...
|
596
|
else if(sx == R[0]){ //if sx is contiguous, read each z-axis slice can be read in one pass
|
8b899c24
David Mayerich
fixed asynchronou...
|
597
598
599
600
601
602
603
|
size_t jump_bytes = (R[1] - sy) * R[0] * sizeof(T); //jump between each slice
size_t slice_bytes = sx * sy * sizeof(T); //size of the slice to be read
for(size_t zi = 0; zi < sz; zi++){ //for each z-axis slice
file.read((char*)dest, slice_bytes); //read the slice
dest += sx * sy; //move the destination pointer to the next slice
file.seekg(jump_bytes, std::ios::cur); //skip to the next slice in the file
}
|
8b899c24
David Mayerich
fixed asynchronou...
|
604
|
}
|
6db250f5
David Mayerich
adaptive streamin...
|
605
606
607
608
609
610
611
612
613
614
615
616
617
618
|
else{
//in this case, x is not contiguous so the volume must be read line-by-line
size_t jump_x_bytes = (R[0] - sx) * sizeof(T); //number of bytes skipped in the x direction
size_t jump_y_bytes = (R[1] - sy) * R[0] * sizeof(T) + jump_x_bytes; //number of bytes skipped between slices
size_t line_bytes = sx * sizeof(T); //size of the line to be read
size_t zi, yi;
for(zi = 0; zi < sz; zi++){ //for each slice
file.read((char*)dest, line_bytes); //read the first line
for(yi = 1; yi < sy; yi++){ //read each additional line
dest += sx; //move the pointer in the destination block to the next line
file.seekg(jump_x_bytes, std::ios::cur); //skip to the next line in the file
file.read((char*)dest, line_bytes); //read the line to the destination block
}
file.seekg(jump_y_bytes, std::ios::cur); //skip to the beginning of the next slice
|
8b899c24
David Mayerich
fixed asynchronou...
|
619
|
}
|
8b899c24
David Mayerich
fixed asynchronou...
|
620
|
}
|
6db250f5
David Mayerich
adaptive streamin...
|
621
622
|
auto t1 = std::chrono::high_resolution_clock::now();
return std::chrono::duration_cast<std::chrono::milliseconds>(t1-t0).count();
|
8b899c24
David Mayerich
fixed asynchronou...
|
623
624
|
}
|
cf5b4c92
David Mayerich
simplified asynch...
|
625
626
|
// permutes a block of data from the current interleave to the interleave specified (re-arranged dimensions to the order specified by [d0, d1, d2])
|
6db250f5
David Mayerich
adaptive streamin...
|
627
628
|
size_t permute(T* dest, T* src, size_t sx, size_t sy, size_t sz, size_t d0, size_t d1, size_t d2){
auto t0 = std::chrono::high_resolution_clock::now();
|
cf5b4c92
David Mayerich
simplified asynch...
|
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
|
size_t d[3] = {d0, d1, d2};
size_t s[3] = {sx, sy, sz};
size_t p[3];// = {x, y, z};
if(d[0] == 0 && d[1] == 1 && d[2] == 2){
//this isn't actually a permute - just copy the data
memcpy(dest, src, sizeof(T) * sx * sy * sz);
}
else if(d[0] == 0){ //the individual lines are contiguous, so you can memcpy line-by-line
size_t y, z;
size_t src_idx, dest_idx;
size_t x_bytes = sizeof(T) * sx;
for(z = 0; z < sz; z++){
p[2] = z;
for(y = 0; y < sy; y++){
p[1] = y;
src_idx = z * sx * sy + y * sx;
dest_idx = p[d[2]] * s[d[0]] * s[d[1]] + p[d[1]] * s[d[0]];
|
cf5b4c92
David Mayerich
simplified asynch...
|
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
|
memcpy(dest + dest_idx, src + src_idx, x_bytes);
}
}
}
else{ //loop through every damn point
size_t x, y, z;
size_t src_idx, dest_idx;
size_t src_z, src_y;
for(z = 0; z < sz; z++){
p[2] = z;
src_z = z * sx * sy;
for(y = 0; y < sy; y++){
p[1] = y;
src_y = src_z + y * sx;
for(x = 0; x < sx; x++){
p[0] = x;
src_idx = src_y + x;
dest_idx = p[d[2]] * s[d[0]] * s[d[1]] + p[d[1]] * s[d[0]] + p[d[0]];
dest[dest_idx] = src[src_idx];
}
}
}
}
|
6db250f5
David Mayerich
adaptive streamin...
|
670
671
|
auto t1 = std::chrono::high_resolution_clock::now();
return std::chrono::duration_cast<std::chrono::milliseconds>(t1-t0).count();
|
cf5b4c92
David Mayerich
simplified asynch...
|
672
673
|
}
|
180d7f3a
David Mayerich
added binary file...
|
674
675
676
677
678
|
};
}
#endif
|