Blame view

stim/envi/binary.h 23.8 KB
1f2ca8ec   David Mayerich   added files for b...
1
2
  
  //make sure that this header file is only loaded once
180d7f3a   David Mayerich   added binary file...
3
4
5
  #ifndef RTS_BINARY_H
  #define RTS_BINARY_H
  
37e61850   David Mayerich   added an is_open(...
6
7
  #include <stim/envi/envi_header.h>
  #include <stim/math/vector.h>
180d7f3a   David Mayerich   added binary file...
8
9
  #include <fstream>
  #include <sys/stat.h>
15a31763   David Mayerich   set up buffer est...
10
  #include <cstring>
5435830b   David Mayerich   linux chrono fixes
11
  #include <chrono>
008ae990   David Mayerich   implemented Windo...
12
13
  #ifdef _WIN32
  #include <Windows.h>
15a31763   David Mayerich   set up buffer est...
14
15
  #else
  #include <unistd.h>
008ae990   David Mayerich   implemented Windo...
16
17
  #endif
  
683f216a   David Mayerich   fixed? Linux comp...
18
  #ifdef USE_CUDA
eae75592   David Mayerich   added GPU support...
19
20
21
22
23
  //CUDA externs
  void gpu_permute(char* dest, char* src, size_t sx, size_t sy, size_t sz, size_t d0, size_t d1, size_t d2, size_t typesize);
  #include <stim/cuda/cudatools/error.h>
  #endif
  
8a86bd56   David Mayerich   changed rts names...
24
  namespace stim{
180d7f3a   David Mayerich   added binary file...
25
  
6db250f5   David Mayerich   adaptive streamin...
26
27
28
29
  /// This class calculates the optimal setting for independent parameter b (batch size) for
  ///		minimizing the dependent parameter bps (bytes per second)
  class stream_optimizer{
  protected:
0a575bb1   David Mayerich   updated optimizat...
30
  	size_t Bps[2];							//bytes per second for the previous batch
6db250f5   David Mayerich   adaptive streamin...
31
32
33
  	size_t interval_B;					//number of bytes processed this interval
  	size_t interval_ms;					//number of milliseconds spent in the current interval
  	size_t n[2];							//current batch size (in bytes)
0a575bb1   David Mayerich   updated optimizat...
34
  	size_t h;							//spacing used for finite difference calculations
6db250f5   David Mayerich   adaptive streamin...
35
36
37
  	size_t dn;							//delta value (in bytes) for setting the batch size (minimum change in batch parameter)
  	size_t maxn;						//maximum value for the batch size
  
0a575bb1   David Mayerich   updated optimizat...
38
39
  	double alpha;						//alpha value controls the factor of the gradient that is used to calculate the next point (speed of convergence)
  
6db250f5   David Mayerich   adaptive streamin...
40
41
42
43
44
45
  	bool sample_step;					//calculating the derivative (this class alternates between calculating dBps and B)
  	bool forward_diff;					//evaluate the derivative using forward differences
  
  	size_t window_ms;					//size of the interval (in milliseconds) integrated to get a reliable bps value
  
  	// This function rounds x to the nearest value within dB
0a575bb1   David Mayerich   updated optimizat...
46
47
48
49
50
  	size_t round_limit(double n0){
  		if(n0 < 0) return dn;					//if n0 is less than zero, return the lowest possible n
  
  		size_t new_n = (size_t)(n0 + 0.5);		//now n0 must be positive, so round it to the nearest integer
  		if(new_n > maxn) new_n = maxn;			//limit the returned size of x to within the specified bounds
6db250f5   David Mayerich   adaptive streamin...
51
  
0a575bb1   David Mayerich   updated optimizat...
52
  		size_t lowest = new_n / dn;
6db250f5   David Mayerich   adaptive streamin...
53
  		size_t highest = lowest + dn;
0a575bb1   David Mayerich   updated optimizat...
54
  		size_t diff[2] = {new_n - lowest, highest - new_n};	//calculate the two differences
6db250f5   David Mayerich   adaptive streamin...
55
56
57
58
59
60
61
62
  		if(diff[0] < diff[1])
  			return lowest;
  		return highest;
  	}
  
  public:
  
  	//constructor initializes a stream optimizer
c6251f8b   David Mayerich   allows user to de...
63
  	stream_optimizer(size_t min_batch_size, size_t max_batch_size, double a = 0.003, size_t probe_step = 5, size_t window = 2000){
0a575bb1   David Mayerich   updated optimizat...
64
65
  		//Bps = 0;						//initialize to zero bytes per second processed
  		Bps[0] = Bps[1] = 0;			//initialize the bits per second to 0
6db250f5   David Mayerich   adaptive streamin...
66
67
68
69
  		interval_B = 0;					//zero bytes have been processed at initialization
  		interval_ms = 0;				//no time has been spent on the batch so far
  		dn = min_batch_size;			//set the minimum batch size as the minimum change in batch size
  		maxn = max_batch_size;			//set the maximum batch size
0a575bb1   David Mayerich   updated optimizat...
70
  		n[0] = max_batch_size;			//set B
8933935f   David Mayerich   updated VERBOSE i...
71
  		h = (max_batch_size / min_batch_size) / probe_step * dn;
0a575bb1   David Mayerich   updated optimizat...
72
73
74
75
  		std::cout<<"h = "<<h<<std::endl;
  		if(h < dn) h = dn;
  		alpha = a;
  		//n[0] = round_limit( (max_batch_size - min_batch_size)/2 );
6db250f5   David Mayerich   adaptive streamin...
76
77
78
79
80
  		window_ms = window;		//minimum integration interval (for getting a reliable bps measure)
  		sample_step = true;					//the first step is to calculate the derivative
  		forward_diff = true;			//start with the forward difference (since we start at the maximum batch size)
  	}
  
c6251f8b   David Mayerich   allows user to de...
81
  	size_t update(size_t bytes_processed, size_t ms_spent, size_t& data_rate, bool VERBOSE = false){
0a575bb1   David Mayerich   updated optimizat...
82
83
  		interval_B += bytes_processed;		//increment the number of bytes processed
  		interval_ms += ms_spent;			//increment the number of milliseconds spent processing
c6251f8b   David Mayerich   allows user to de...
84
  		data_rate = interval_B / interval_ms;
0a575bb1   David Mayerich   updated optimizat...
85
86
87
88
89
90
  
  		//if we have sufficient information to evaluate the optimization function at this point
  		if(interval_ms < window_ms){					//if insufficient time has passed to get a reliable Bps measurement
  			return n[0];
  		}
  		else{											//if we have collected enough information for a reliable Bps estimate
c6251f8b   David Mayerich   allows user to de...
91
  			
0a575bb1   David Mayerich   updated optimizat...
92
  			if(Bps[0] == 0){							//if n[0] hasn't been evaluated yet, this is the first step
c6251f8b   David Mayerich   allows user to de...
93
  				Bps[0] = data_rate;						//set the initial Bps value
0a575bb1   David Mayerich   updated optimizat...
94
  				n[1] = n[0] - h;						//set the position of the next sample point
c6251f8b   David Mayerich   allows user to de...
95
96
  				if(VERBOSE)
  					std::cout<<"Bps value at n = "<<n[0]<<" is "<<Bps[0]<<" Bps, probing n = "<<n[1]<<std::endl;
0a575bb1   David Mayerich   updated optimizat...
97
98
99
  				return n[1];							//return the probe point
  			}
  			else{
c6251f8b   David Mayerich   allows user to de...
100
  				Bps[1] = data_rate;						//set the Bps for the current point (n[1])
0a575bb1   David Mayerich   updated optimizat...
101
102
103
104
105
106
107
108
109
  
  				double Bps_p;							//allocate a variable for the derivative
  				//calculate the derivative
  				if(n[0] < n[1]){						//if the current point is less than the previous one (probably the most common)
  					Bps_p = ((double)Bps[1] - (double)Bps[0]) / (double)h;		//calculate the derivative using the forward finite difference
  				}
  				else{
  					Bps_p = ((double)Bps[0] - (double)Bps[1]) / (double)h;		//calculate the derivative using the backward finite difference
  				}
c6251f8b   David Mayerich   allows user to de...
110
111
  				if(VERBOSE)
  					std::cout<<"     probed n = "<<n[1]<<" with "<<Bps[1]<<" Bps, gradient = "<<Bps_p<<" Bps"<<std::endl;
0a575bb1   David Mayerich   updated optimizat...
112
113
114
115
116
117
118
119
  
  				double new_n_precise = n[0] + alpha * Bps_p;			//calculate the next point (snap to closest integer)
  				size_t new_n_nearest = round_limit(new_n_precise);		//calculate the next point (given batch parameters)
  
  				if(new_n_nearest == n[0]){								//if the newest point is the same as the original point
  					Bps[0] = Bps[1];									//update the Bps
  					//if(n[0] == dn) n[1] = n[0] + h;					//if we're on the left edge, probe forward
  					//else n[1] = n[0] - h;								//otherwise probe backwards
c6251f8b   David Mayerich   allows user to de...
120
121
  					if(VERBOSE)
  						std::cout<<"     staying at n = "<<n[0]<<" for now"<<std::endl;
0a575bb1   David Mayerich   updated optimizat...
122
123
124
125
126
127
128
129
  					//return n[1];										//return the probe point
  
  					Bps[0] = 0;											//reset the Bps for the current point
  					return n[0];										//return the current point for a re-calculation
  				}
  				else{													//if the newest point is different from the original point
  					n[0] = new_n_nearest;								//move to the new point
  					Bps[0] = 0;											//set the Bps to zero (point hasn't been tested)
c6251f8b   David Mayerich   allows user to de...
130
131
  					if(VERBOSE)
  						std::cout<<"     moving to n = "<<n[0]<<std::endl;
0a575bb1   David Mayerich   updated optimizat...
132
133
134
135
136
  					return n[0];										//return the new point
  				}
  			}
  		}
  	}
6db250f5   David Mayerich   adaptive streamin...
137
138
  };
  
33d7d3cf   David Mayerich   documentation for...
139
140
141
142
143
144
145
  /** This class manages the streaming of large multidimensional binary files.
   *  Generally these are hyperspectral files with 2 spatial and 1 spectral dimension. However, this class supports
   *  other dimensions via the template parameter D.
   *
   *  @param T is the data type used to store data to disk (generally float or double)
   *  @param D is the dimension of the data (default 3)
   */
180d7f3a   David Mayerich   added binary file...
146
147
148
149
  template< typename T, unsigned int D = 3 >
  class binary{
  
  protected:
180d7f3a   David Mayerich   added binary file...
150
151
152
  	std::fstream file;		//file stream used for reading and writing
  	std::string name;		//file name
  
63fc1df8   David Mayerich   added an inverse ...
153
  	unsigned long long R[D];		//resolution
9d3ba0b1   David Mayerich   added stim::hsi a...
154
  	unsigned long long header;	//header size (in bytes)
f4c5d71b   David Mayerich   started working w...
155
  	unsigned char* mask;	//pointer to a character array: 0 = background, 1 = foreground (or valid data)
180d7f3a   David Mayerich   added binary file...
156
  
63fc1df8   David Mayerich   added an inverse ...
157
  	double progress;		//stores the progress on the current operation (accessible using a thread)
c6251f8b   David Mayerich   allows user to de...
158
  	size_t data_rate;		//data rate (currently in Bps)
ee4dea28   David Mayerich   fixed errors in c...
159
  
008ae990   David Mayerich   implemented Windo...
160
  	size_t buffer_size;		//available memory for processing large files
ee4dea28   David Mayerich   fixed errors in c...
161
  
33d7d3cf   David Mayerich   documentation for...
162
  	/// Private initialization function used to set default parameters in the data structure.
180d7f3a   David Mayerich   added binary file...
163
  	void init(){
15a31763   David Mayerich   set up buffer est...
164
  		std::memset(R, 0, sizeof(unsigned long long) * D);		//initialize the resolution to zero
008ae990   David Mayerich   implemented Windo...
165
  		header = 0;											//initialize the header size to zero
f4c5d71b   David Mayerich   started working w...
166
  		mask = NULL;
cbce396e   David Mayerich   added support for...
167
  
c6251f8b   David Mayerich   allows user to de...
168
169
170
  		progress = 0;										//initialize the progress for any algorithm to zero
  		data_rate = 0;										//initialize the data rate to zero
  		set_buffer_frac();										//set the maximum buffer size to the default
63fc1df8   David Mayerich   added an inverse ...
171
172
  	}
  
33d7d3cf   David Mayerich   documentation for...
173
  	/// Private helper function that returns the size of the file on disk using system functions.
b0fee590   heziqi   convert long int ...
174
  	long long int get_file_size(){
193ff00e   David Mayerich   dealth with preci...
175
  #ifdef _WIN32
b0fee590   heziqi   convert long int ...
176
177
  		struct _stat64 results;
  		if(_stat64(name.c_str(), &results) == 0)
180d7f3a   David Mayerich   added binary file...
178
  			return results.st_size;
193ff00e   David Mayerich   dealth with preci...
179
180
181
182
183
  #else
  		struct stat results;
  		if(stat(name.c_str(), &results) == 0)
  			return results.st_size;
  #endif
180d7f3a   David Mayerich   added binary file...
184
185
186
  		else return 0;
  	}
  
33d7d3cf   David Mayerich   documentation for...
187
  	/// Private helper function that tests to make sure that the calculated data size specified by the structure is the same as the data size on disk.
180d7f3a   David Mayerich   added binary file...
188
  	bool test_file_size(){
b0fee590   heziqi   convert long int ...
189
  		long long int npts = 1;				//initialize the number of data points to 1
87da9adc   David Mayerich   fixed issues with...
190
191
  		for(unsigned int i = 0; i<D; i++)	//iterate over each dimension
  			npts *= R[i];					//compute the total number of data points in the file
b0fee590   heziqi   convert long int ...
192
  		long long int datasize = npts * sizeof(T);//multiply the sum by the size of the template parameter
87da9adc   David Mayerich   fixed issues with...
193
194
  
  		if(datasize + header == get_file_size()) return true;	//if the byte size matches the file size, we're golden
180d7f3a   David Mayerich   added binary file...
195
196
197
198
  		else return false;					//otherwise return an error
  
  	}
  
f92397d2   David Mayerich   fixed a Win32 bug...
199
200
201
202
203
204
  	/// Private helper function that resets the file pointer to the beginning of the data
  
  	void reset(){
  		file.seekg(header, std::ios_base::beg);
  	}
  
33d7d3cf   David Mayerich   documentation for...
205
206
207
  	/// Private helper file that opens a specified binary file.
  
  	/// @param filename is the name of the binary file to stream
180d7f3a   David Mayerich   added binary file...
208
209
210
211
  	bool open_file(std::string filename){
  		//open the file as binary for reading and writing
  		file.open(filename.c_str(), std::ios::in | std::ios::out | std::ios::binary);
  
ba72c5b0   David Mayerich   added OBJ class
212
213
  		//if the file isn't open, the user may only have read access
  		if(!file.is_open()){
3d0b6243   David Mayerich   fixed hsiproc bug...
214
  			std::cout<<"class STIM::BINARY - failed to open file, trying for read only"<<std::endl;
ba72c5b0   David Mayerich   added OBJ class
215
216
217
218
219
220
221
  			file.open(filename.c_str(), std::ios::in | std::ios::binary);
  			if(!file.is_open()){
  				std::cout<<"               still unable to load the file"<<std::endl;
  				return false;
  			}
  		}
  
180d7f3a   David Mayerich   added binary file...
222
223
224
225
226
227
  		//if the file is successful
  		if(file){
  			name = filename;		//set the name
  			if(test_file_size())	//test the file size
  				return true;
  		}
f92397d2   David Mayerich   fixed a Win32 bug...
228
  
180d7f3a   David Mayerich   added binary file...
229
230
231
  		return false;
  	}
  
724ec347   David Mayerich   simplified the EN...
232
233
  
  
f92397d2   David Mayerich   fixed a Win32 bug...
234
235
  
  
180d7f3a   David Mayerich   added binary file...
236
237
  public:
  
008ae990   David Mayerich   implemented Windo...
238
239
240
241
242
  	//default constructor
  	binary(){
  		init();
  	}
  
9d3ba0b1   David Mayerich   added stim::hsi a...
243
  	double get_progress(){
63fc1df8   David Mayerich   added an inverse ...
244
  		return progress;
cbce396e   David Mayerich   added support for...
245
246
  	}
  
63fc1df8   David Mayerich   added an inverse ...
247
248
  	void reset_progress(){
  		progress = 0;
cbce396e   David Mayerich   added support for...
249
250
  	}
  
c6251f8b   David Mayerich   allows user to de...
251
252
253
254
  	size_t get_data_rate(){
  		return data_rate;
  	}
  
008ae990   David Mayerich   implemented Windo...
255
  	//specify the maximum fraction of available memory that this class will use for buffering
c6251f8b   David Mayerich   allows user to de...
256
  	void set_buffer_frac(double mem_frac = 0.5){				//default to 50%
008ae990   David Mayerich   implemented Windo...
257
258
259
260
261
  #ifdef _WIN32
  		MEMORYSTATUSEX statex;
  		statex.dwLength = sizeof (statex);
  		GlobalMemoryStatusEx (&statex);
  		buffer_size = (size_t)(statex.ullAvailPhys * mem_frac);
15a31763   David Mayerich   set up buffer est...
262
263
264
265
  #else
  		size_t pages = sysconf(_SC_PHYS_PAGES);
  		size_t page_size = sysconf(_SC_PAGE_SIZE);
  		buffer_size = (size_t)(pages * page_size *  mem_frac);
008ae990   David Mayerich   implemented Windo...
266
267
268
  #endif
  	}
  
c6251f8b   David Mayerich   allows user to de...
269
270
271
272
  	void set_buffer_raw(size_t bytes){
  		buffer_size = bytes;
  	}
  
33d7d3cf   David Mayerich   documentation for...
273
274
275
276
277
  	/// Open a binary file for streaming.
  
  	/// @param filename is the name of the binary file
  	/// @param r is a STIM vector specifying the size of the binary file along each dimension
  	/// @param h is the length (in bytes) of any header file (default zero)
9d3ba0b1   David Mayerich   added stim::hsi a...
278
  	bool open(std::string filename, vec<unsigned long long> r, unsigned long long h = 0){
180d7f3a   David Mayerich   added binary file...
279
  
9d3ba0b1   David Mayerich   added stim::hsi a...
280
  		for(unsigned long long i = 0; i < D; i++)		//set the dimensions of the binary file object
180d7f3a   David Mayerich   added binary file...
281
282
  			R[i] = r[i];
  
87da9adc   David Mayerich   fixed issues with...
283
284
285
  		header = h;				//save the header size
  
  		if(!open_file(filename)) return false;	//open the binary file
180d7f3a   David Mayerich   added binary file...
286
  
f92397d2   David Mayerich   fixed a Win32 bug...
287
288
  		//reset();
  
180d7f3a   David Mayerich   added binary file...
289
290
291
  		return test_file_size();
  	}
  
37e61850   David Mayerich   added an is_open(...
292
293
294
295
  	bool is_open() {
  		return file.is_open();
  	}
  
33d7d3cf   David Mayerich   documentation for...
296
297
298
299
300
  	/// Creates a new binary file for streaming
  
  	/// @param filename is the name of the binary file to be created
  	/// @param r is a STIM vector specifying the size of the file along each dimension
  	/// @offset specifies how many bytes to offset the file (used to leave room for a header)
9d3ba0b1   David Mayerich   added stim::hsi a...
301
  	bool create(std::string filename, vec<unsigned long long> r, unsigned long long offset = 0){
f47168a2   heziqi   Ziqi added create...
302
303
304
305
306
  
  		std::ofstream target(filename.c_str(), std::ios::binary);
  
  		//initialize binary file
  		T p = 0;
9d3ba0b1   David Mayerich   added stim::hsi a...
307
  		for(unsigned long long i =0; i < r[0] * r[1] * r[2]; i++){
f47168a2   heziqi   Ziqi added create...
308
309
310
  			target.write((char*)(&p), sizeof(T));
  		}
  
9d3ba0b1   David Mayerich   added stim::hsi a...
311
  		for(unsigned long long i = 0; i < D; i++)		//set the dimensions of the binary file object
f47168a2   heziqi   Ziqi added create...
312
313
314
315
316
317
318
319
320
  			R[i] = r[i];
  
  		header = offset;				//save the header size
  
  		if(!open_file(filename)) return false;	//open the binary file
  
  		return test_file_size();
  	}
  
33d7d3cf   David Mayerich   documentation for...
321
322
323
324
  	/// Writes a single page of data to disk. A page consists of a sequence of data of size R[0] * R[1] * ... * R[D-1].
  
  	/// @param p is a pointer to the data to be written
  	/// @param page is the page number (index of the highest-numbered dimension)
9d3ba0b1   David Mayerich   added stim::hsi a...
325
  	bool write_page( T * p, unsigned long long page){
f47168a2   heziqi   Ziqi added create...
326
327
328
329
330
331
  
  		if(p == NULL){
  			std::cout<<"ERROR: unable to write into file, empty pointer"<<std::endl;
  			exit(1);
  		}
  
f92397d2   David Mayerich   fixed a Win32 bug...
332
  		file.seekg(R[1] * R[0] * page * sizeof(T) + header, std::ios::beg);   //seek to the desired location on disk
8157c392   David Mayerich   added parser and ...
333
  		file.write((char *)p, R[0] * R[1] * sizeof(T));				 //write binary data
f47168a2   heziqi   Ziqi added create...
334
335
336
337
  
  		return true;
  	}
  
33d7d3cf   David Mayerich   documentation for...
338
339
340
341
  	/// Reads a page from disk. A page consists of a sequence of data of size R[0] * R[1] * ... * R[D-1].
  
  	/// @param p is a pointer to pre-allocated memory equal to the page size
  	/// @param page is the index of the page
9d3ba0b1   David Mayerich   added stim::hsi a...
342
  	bool read_page( T * p, unsigned long long page, bool PROGRESS = false){
63fc1df8   David Mayerich   added an inverse ...
343
344
  
  		if(PROGRESS) progress = 0;
c7da85e3   heziqi   he,ziqi add metho...
345
  
5d5ce39a   heziqi   Ziqi convert the ...
346
  		if (page >= R[2]){										//make sure the bank number is right
2ad0ce4f   David Mayerich   syntactic edits b...
347
  			std::cout<<"ERROR: page out of range"<<std::endl;
5d5ce39a   heziqi   Ziqi convert the ...
348
  			return false;
c7da85e3   heziqi   he,ziqi add metho...
349
350
  		}
  
f92397d2   David Mayerich   fixed a Win32 bug...
351
  		file.seekg(R[1] * R[0] * page * sizeof(T) + header, std::ios::beg);   //write into memory from the binary file
2ad0ce4f   David Mayerich   syntactic edits b...
352
  		file.read((char *)p, R[0] * R[1] * sizeof(T));
63fc1df8   David Mayerich   added an inverse ...
353
  		if(PROGRESS) progress = 100;
5d5ce39a   heziqi   Ziqi convert the ...
354
  		return true;
c7da85e3   heziqi   he,ziqi add metho...
355
356
  	}
  
724ec347   David Mayerich   simplified the EN...
357
358
359
360
361
362
363
  
  
  	///Reads a line Z (slowest dimension) for a given XY value
  
  	/// @param p is a pointer to pre-allocated memory equal to the line size R[2]
  	/// @param x is the x coordinate
  	/// @param y is the y coordinate
735a2a24   David Mayerich   started testing o...
364
  	void read_line_2( T* p, size_t n, bool PROGRESS = false){
9d3ba0b1   David Mayerich   added stim::hsi a...
365
  		unsigned long long i;
724ec347   David Mayerich   simplified the EN...
366
  
63fc1df8   David Mayerich   added an inverse ...
367
368
  		if(PROGRESS) progress = 0;
  
735a2a24   David Mayerich   started testing o...
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
  		if ( n > R[0] * R[1]){							//make sure the sample and line number is right
  			std::cout<<"ERROR: sample or line out of range in "<<__FILE__<<" (line "<<__LINE__<<")"<<std::endl;
  			exit(1);
  		}
  
  		file.seekg(n * sizeof(T), std::ios::beg);           //point to the certain sample and line
  		for (i = 0; i < R[2]; i++){							//for each band
  			file.read((char *)(p + i), sizeof(T));
  			file.seekg((R[1] * R[0] - 1) * sizeof(T), std::ios::cur);    //go to the next band
  			if(PROGRESS) progress = (double)i / (double)R[2] * 100;
  		}
  		if(PROGRESS) progress = 100;
  	}
  	void read_line_2( T * p, unsigned long long x, unsigned long long y, bool PROGRESS = false){
  		read_line_2(p, y * R[0] + x, PROGRESS);
  		/*unsigned long long i;
  
  		if(PROGRESS) progress = 0;
  
724ec347   David Mayerich   simplified the EN...
388
389
390
391
392
393
394
395
396
397
  		if ( x >= R[0] || y >= R[1]){							//make sure the sample and line number is right
  			std::cout<<"ERROR: sample or line out of range"<<std::endl;
  			return false;
  		}
  
  		file.seekg((x + y * R[0]) * sizeof(T), std::ios::beg);           //point to the certain sample and line
  		for (i = 0; i < R[2]; i++)
  		{
  			file.read((char *)(p + i), sizeof(T));
  			file.seekg((R[1] * R[0] - 1) * sizeof(T), std::ios::cur);    //go to the next band
63fc1df8   David Mayerich   added an inverse ...
398
  			if(PROGRESS) progress = (double)i / (double)R[2] * 100;
724ec347   David Mayerich   simplified the EN...
399
  		}
63fc1df8   David Mayerich   added an inverse ...
400
  		if(PROGRESS) progress = 100;
724ec347   David Mayerich   simplified the EN...
401
  
735a2a24   David Mayerich   started testing o...
402
  		return true;*/
724ec347   David Mayerich   simplified the EN...
403
404
405
406
407
408
409
  	}
  
  	///Reads a line X (fastest dimension) for a given YZ value
  
  	/// @param p is a pointer to pre-allocated memory equal to the line size R[2]
  	/// @param x is the y coordinate
  	/// @param y is the z coordinate
9d3ba0b1   David Mayerich   added stim::hsi a...
410
  	bool read_line_0(T * p, unsigned long long y, unsigned long long z, bool PROGRESS = false){
724ec347   David Mayerich   simplified the EN...
411
412
  		//test to make sure the specified value is within range
  		if( y >= R[1] || z >= R[2] ){
735a2a24   David Mayerich   started testing o...
413
  			std::cout<<"ERROR: sample ("<<y<<", "<<z<<") out of range in "<<__FILE__<<" (line "<<__LINE__<<")"<<std::endl;
724ec347   David Mayerich   simplified the EN...
414
415
416
417
418
  			return false;
  		}
  
  		file.seekg((z * R[0] * R[1] + y * R[0]) * sizeof(T), std::ios::beg);	//seek to the start of the line
  		file.read((char *)p, sizeof(T) * R[0]);									//read the line
63fc1df8   David Mayerich   added an inverse ...
419
  		if(PROGRESS) progress = 100;
724ec347   David Mayerich   simplified the EN...
420
421
422
423
424
425
426
427
  		return true;
  	}
  
  	///Reads a line Y (second fastest dimension) for a given XZ value
  
  	/// @param p is a pointer to pre-allocated memory equal to the line size R[2]
  	/// @param x is the y coordinate
  	/// @param z is the z coordinate
9d3ba0b1   David Mayerich   added stim::hsi a...
428
  	bool read_line_1(T * p, unsigned long long x, unsigned long long z, bool PROGRESS = false){
63fc1df8   David Mayerich   added an inverse ...
429
  		if(PROGRESS) progress = 0;
724ec347   David Mayerich   simplified the EN...
430
431
  		//test to make sure the specified value is within range
  		if( x >= R[0] || z >= R[2] ){
735a2a24   David Mayerich   started testing o...
432
  			std::cout<<"ERROR: sample or line out of range in "<<__FILE__<<" (line "<<__LINE__<<")"<<std::endl;
724ec347   David Mayerich   simplified the EN...
433
434
435
436
  			return false;
  		}
  
  		file.seekg((z * R[0] * R[1] + x) * sizeof(T), std::ios::beg);           //seek to the start of the line
9d3ba0b1   David Mayerich   added stim::hsi a...
437
  		for (unsigned long long i = 0; i < R[1]; i++){									//for each pixel in the line
724ec347   David Mayerich   simplified the EN...
438
439
  			file.read((char *)(p + i), sizeof(T));					//read the pixel
  			file.seekg((R[0] - 1) * sizeof(T), std::ios::cur);		//seek to the next pixel in the line
63fc1df8   David Mayerich   added an inverse ...
440
  			if(PROGRESS) progress = (double)i / (double)R[1] * 100;
724ec347   David Mayerich   simplified the EN...
441
  		}
63fc1df8   David Mayerich   added an inverse ...
442
  		if(PROGRESS) progress = 100;
724ec347   David Mayerich   simplified the EN...
443
444
445
  		return true;
  	}
  
27b826a8   David Mayerich   added a spherical...
446
447
448
449
  	/// Reads a plane given a coordinate along the 0-axis (YZ plane)
  
  	/// @param p is a pointer to pre-allocated memory of size R[1] * R[2] * sizeof(T)
  	/// @param n is the 0-axis coordinate used to retrieve the plane
9d3ba0b1   David Mayerich   added stim::hsi a...
450
  	bool read_plane_0(T* p, unsigned long long n, bool PROGRESS = false){
63fc1df8   David Mayerich   added an inverse ...
451
  		if(PROGRESS) progress = 0;
724ec347   David Mayerich   simplified the EN...
452
  		if (n >= R[0]){										//make sure the number is within the possible range
735a2a24   David Mayerich   started testing o...
453
  			std::cout<<"ERROR: sample or line out of range in "<<__FILE__<<" (line "<<__LINE__<<")"<<std::endl;
724ec347   David Mayerich   simplified the EN...
454
455
  			return false;
  		}
9d3ba0b1   David Mayerich   added stim::hsi a...
456
  		unsigned long long jump = (R[0] - 1) * sizeof(T);		//number of bytes to skip between samples
724ec347   David Mayerich   simplified the EN...
457
458
459
460
  
  		//seek to the start of the plane
  		file.seekg(n * sizeof(T), std::ios::beg);
  
9d3ba0b1   David Mayerich   added stim::hsi a...
461
462
  		unsigned long long N = R[1] * R[2];
  		for(unsigned long long i = 0; i<N; i++){
724ec347   David Mayerich   simplified the EN...
463
464
  			file.read((char*)(p+i), sizeof(T));
  			file.seekg(jump, std::ios::cur);
9d3ba0b1   David Mayerich   added stim::hsi a...
465
  			if(PROGRESS) progress = (double)(i+1) / N * 100;
724ec347   David Mayerich   simplified the EN...
466
467
  		}
  
724ec347   David Mayerich   simplified the EN...
468
469
470
471
472
  		return true;
  
  
  	}
  
27b826a8   David Mayerich   added a spherical...
473
474
475
476
  	/// Reads a plane given a coordinate along the 1-axis (XZ plane)
  
  	/// @param p is a pointer to pre-allocated memory of size R[0] * R[2] * sizeof(T)
  	/// @param n is the 1-axis coordinate used to retrieve the plane
9d3ba0b1   David Mayerich   added stim::hsi a...
477
  	bool read_plane_1(T* p, unsigned long long n, bool PROGRESS = false){
63fc1df8   David Mayerich   added an inverse ...
478
  		if(PROGRESS) progress = 0;
9d3ba0b1   David Mayerich   added stim::hsi a...
479
480
  		unsigned long long L = R[0] * sizeof(T);		//caculate the number of bytes in a sample line
  		unsigned long long jump = R[0] * (R[1] - 1) * sizeof(T);
724ec347   David Mayerich   simplified the EN...
481
482
483
484
485
486
487
  
  		if (n >= R[1]){										//make sure the bank number is right
  			std::cout<<"ERROR read_plane_1: page out of range"<<std::endl;
  			return false;
  		}
  
  		file.seekg(R[0] * n * sizeof(T), std::ios::beg);
9d3ba0b1   David Mayerich   added stim::hsi a...
488
  		for (unsigned long long i = 0; i < R[2]; i++){
63fc1df8   David Mayerich   added an inverse ...
489
  			if(PROGRESS) progress = (double)i / R[2] * 100;
724ec347   David Mayerich   simplified the EN...
490
491
492
493
494
  			file.read((char *)(p + i * R[0]), L);
  			file.seekg( jump, std::ios::cur);
  			std::cout<<i<<"    ";
  		}
  
63fc1df8   David Mayerich   added an inverse ...
495
  		if(PROGRESS) progress = 100;
724ec347   David Mayerich   simplified the EN...
496
497
498
  		return true;
  	}
  
27b826a8   David Mayerich   added a spherical...
499
500
501
502
  	/// Reads a plane given a coordinate along the 2-axis (XY plane)
  
  	/// @param p is a pointer to pre-allocated memory of size R[0] * R[1] * sizeof(T)
  	/// @param n is the 2-axis coordinate used to retrieve the plane
9d3ba0b1   David Mayerich   added stim::hsi a...
503
  	bool read_plane_2(T* p, unsigned long long n, bool PROGRESS = false){
63fc1df8   David Mayerich   added an inverse ...
504
  		return read_page(p, n, PROGRESS);
724ec347   David Mayerich   simplified the EN...
505
506
  	}
  
27b826a8   David Mayerich   added a spherical...
507
508
509
510
  	/// Reads a single pixel, treating the entire data set as a linear array
  
  	/// @param p is a pointer to pre-allocated memory of size sizeof(T)
  	/// @param i is the index to the pixel using linear indexing
9d3ba0b1   David Mayerich   added stim::hsi a...
511
  	bool read_pixel(T* p, unsigned long long i){
724ec347   David Mayerich   simplified the EN...
512
513
514
515
516
517
518
519
520
521
  		if(i >= R[0] * R[1] * R[2]){
  			std::cout<<"ERROR read_pixel: n is out of range"<<std::endl;
  			return false;
  		}
  
  		file.seekg(i * sizeof(T), std::ios::cur);
  		file.read((char*)p, sizeof(T));
  
  	}
  
27b826a8   David Mayerich   added a spherical...
522
523
524
525
526
527
  	/// Reads a single pixel, given an x, y, z coordinate
  
  	/// @param p is a pointer to pre-allocated memory of size sizeof(T)
  	/// @param x is the x (0) axis coordinate
  	/// @param y is the y (1) axis coordinate
  	/// @param z is the z (2) axis coordinate
9d3ba0b1   David Mayerich   added stim::hsi a...
528
  	bool read_pixel(T* p, unsigned long long x, unsigned long long y, unsigned long long z){
724ec347   David Mayerich   simplified the EN...
529
530
531
532
533
534
  
  		if(x < 0 || x >= R[0] || y < 0 || y >= R[1] || z < 0 || z > R[2]){
  			std::cout<<"ERROR read_pixel: (x,y,z) is out of range"<<std::endl;
  			return false;
  		}
  
9d3ba0b1   David Mayerich   added stim::hsi a...
535
  		unsigned long long i = z * R[0] * R[1] + y * R[0] + z;
724ec347   David Mayerich   simplified the EN...
536
537
538
  		return read_pixel(p, i);
  	}
  
8b899c24   David Mayerich   fixed asynchronou...
539
  	/// Reads a block specified by an (x, y, z) position and size using the largest possible contiguous reads
6db250f5   David Mayerich   adaptive streamin...
540
541
  	size_t read(T* dest, size_t x, size_t y, size_t z, size_t sx, size_t sy, size_t sz){
  		auto t0 = std::chrono::high_resolution_clock::now();
8b899c24   David Mayerich   fixed asynchronou...
542
543
544
  		size_t size_bytes = sx * sy * sz * sizeof(T);					//size of the block to read in bytes
  
  		size_t start = z * R[0] * R[1] + y * R[0] + x;						//calculate the start postion
f1365506   David Mayerich   modifications to ...
545
  		//size_t start_bytes = start * sizeof(T);							//start position in bytes
8b899c24   David Mayerich   fixed asynchronou...
546
547
548
549
550
  		file.seekg(start * sizeof(T), std::ios::beg);					//seek to the start position
  
  		
  		if(sx == R[0] && sy == R[1]){				//if sx and sy result in a contiguous volume along z
  			file.read((char*)dest, size_bytes);			//read the block in one pass
8b899c24   David Mayerich   fixed asynchronou...
551
  		}
6db250f5   David Mayerich   adaptive streamin...
552
  		else if(sx == R[0]){												//if sx is contiguous, read each z-axis slice can be read in one pass
8b899c24   David Mayerich   fixed asynchronou...
553
554
555
556
557
558
559
  			size_t jump_bytes = (R[1] - sy) * R[0] * sizeof(T);		//jump between each slice
  			size_t slice_bytes = sx * sy * sizeof(T);				//size of the slice to be read
  			for(size_t zi = 0; zi < sz; zi++){						//for each z-axis slice
  				file.read((char*)dest, slice_bytes);						//read the slice
  				dest += sx * sy;									//move the destination pointer to the next slice
  				file.seekg(jump_bytes, std::ios::cur);				//skip to the next slice in the file
  			}
8b899c24   David Mayerich   fixed asynchronou...
560
  		}
6db250f5   David Mayerich   adaptive streamin...
561
562
563
564
565
566
567
568
569
570
571
572
573
574
  		else{
  			//in this case, x is not contiguous so the volume must be read line-by-line
  			size_t jump_x_bytes = (R[0] - sx) * sizeof(T);				//number of bytes skipped in the x direction
  			size_t jump_y_bytes = (R[1] - sy) * R[0] * sizeof(T) + jump_x_bytes;	//number of bytes skipped between slices
  			size_t line_bytes = sx * sizeof(T);							//size of the line to be read
  			size_t zi, yi;
  			for(zi = 0; zi < sz; zi++){									//for each slice
  				file.read((char*)dest, line_bytes);							//read the first line
  				for(yi = 1; yi < sy; yi++){								//read each additional line
  					dest += sx;											//move the pointer in the destination block to the next line
  					file.seekg(jump_x_bytes, std::ios::cur);			//skip to the next line in the file
  					file.read((char*)dest, line_bytes);						//read the line to the destination block
  				}
  				file.seekg(jump_y_bytes, std::ios::cur);				//skip to the beginning of the next slice
8b899c24   David Mayerich   fixed asynchronou...
575
  			}
8b899c24   David Mayerich   fixed asynchronou...
576
  		}
6db250f5   David Mayerich   adaptive streamin...
577
578
  		auto t1 = std::chrono::high_resolution_clock::now();
  		return std::chrono::duration_cast<std::chrono::milliseconds>(t1-t0).count();
8b899c24   David Mayerich   fixed asynchronou...
579
580
  	}
  
cf5b4c92   David Mayerich   simplified asynch...
581
582
  	// permutes a block of data from the current interleave to the interleave specified (re-arranged dimensions to the order specified by [d0, d1, d2])
  
6db250f5   David Mayerich   adaptive streamin...
583
  	size_t permute(T* dest, T* src, size_t sx, size_t sy, size_t sz, size_t d0, size_t d1, size_t d2){
eae75592   David Mayerich   added GPU support...
584
585
586
  		std::chrono::high_resolution_clock::time_point t0, t1;
  		t0 = std::chrono::high_resolution_clock::now();
  
683f216a   David Mayerich   fixed? Linux comp...
587
  #ifdef USE_CUDA
eae75592   David Mayerich   added GPU support...
588
589
590
591
592
593
594
595
596
597
598
599
600
601
  		T* gpu_src;
  		HANDLE_ERROR( cudaMalloc(&gpu_src, sx*sy*sz*sizeof(T)) );
  		HANDLE_ERROR( cudaMemcpy(gpu_src, src, sx*sy*sz*sizeof(T), cudaMemcpyHostToDevice) );
  		T* gpu_dest;
  		HANDLE_ERROR( cudaMalloc(&gpu_dest, sx*sy*sz*sizeof(T)) );		
  		gpu_permute((char*)gpu_dest, (char*)gpu_src, sx, sy, sz, d0, d1, d2, sizeof(T));
  		HANDLE_ERROR( cudaMemcpy(dest, gpu_dest, sx*sy*sz*sizeof(T), cudaMemcpyDeviceToHost) );
  		HANDLE_ERROR( cudaFree(gpu_src) );
  		HANDLE_ERROR( cudaFree(gpu_dest) );
  		t1 = std::chrono::high_resolution_clock::now();
  		return std::chrono::duration_cast<std::chrono::milliseconds>(t1-t0).count();
  		
  #endif
  		
cf5b4c92   David Mayerich   simplified asynch...
602
603
  		size_t d[3] = {d0, d1, d2};
  		size_t s[3] = {sx, sy, sz};
eae75592   David Mayerich   added GPU support...
604
  		size_t p[3];
cf5b4c92   David Mayerich   simplified asynch...
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
  		
  		if(d[0] == 0 && d[1] == 1 && d[2] == 2){
  			//this isn't actually a permute - just copy the data
  			memcpy(dest, src, sizeof(T) * sx * sy * sz);
  		}
  		else if(d[0] == 0){						//the individual lines are contiguous, so you can memcpy line-by-line
  			size_t y, z;
  			size_t src_idx, dest_idx;
  			size_t x_bytes = sizeof(T) * sx;
  			for(z = 0; z < sz; z++){
  				p[2] = z;
  				for(y = 0; y < sy; y++){
  					p[1] = y;
  					src_idx = z * sx * sy + y * sx;
  					dest_idx = p[d[2]] * s[d[0]] * s[d[1]] + p[d[1]] * s[d[0]];
cf5b4c92   David Mayerich   simplified asynch...
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
  					memcpy(dest + dest_idx, src + src_idx, x_bytes);
  				}
  			}
  		}
  		else{									//loop through every damn point
  			size_t x, y, z;
  			size_t src_idx, dest_idx;
  			size_t src_z, src_y;
  			for(z = 0; z < sz; z++){
  				p[2] = z;
  				src_z = z * sx * sy;
  				for(y = 0; y < sy; y++){
  					p[1] = y;
  					src_y = src_z + y * sx;
  					for(x = 0; x < sx; x++){
  						p[0] = x;
  						src_idx = src_y + x;
  						dest_idx = p[d[2]] * s[d[0]] * s[d[1]] + p[d[1]] * s[d[0]] + p[d[0]];
  						dest[dest_idx] = src[src_idx];
  					}
  				}
  			}
  		}
eae75592   David Mayerich   added GPU support...
643
  		t1 = std::chrono::high_resolution_clock::now();
6db250f5   David Mayerich   adaptive streamin...
644
  		return std::chrono::duration_cast<std::chrono::milliseconds>(t1-t0).count();
cf5b4c92   David Mayerich   simplified asynch...
645
646
  	}
  
180d7f3a   David Mayerich   added binary file...
647
648
649
650
651
  };
  
  }
  
  #endif