allows user to deactivate optimization for profiling

David Mayerich
1 parent 8933935f
Showing 3 changed files with 130 additions and 35 deletions Show diff stats
stim/envi/binary.h
stim/envi/bsq.h
stim/envi/envi.h
@@ -54,7 +54,7 @@ protected:
 public:
 	//constructor initializes a stream optimizer
-	stream_optimizer(size_t min_batch_size, size_t max_batch_size, double a = 0.001, double probe_step = 5, size_t window = 2000){
+	stream_optimizer(size_t min_batch_size, size_t max_batch_size, double a = 0.003, size_t probe_step = 5, size_t window = 2000){
 		//Bps = 0;						//initialize to zero bytes per second processed
 		Bps[0] = Bps[1] = 0;			//initialize the bits per second to 0
 		interval_B = 0;					//zero bytes have been processed at initialization
@@ -72,25 +72,26 @@ public:
 		forward_diff = true;			//start with the forward difference (since we start at the maximum batch size)
 	}
-	size_t update(size_t bytes_processed, size_t ms_spent){
+	size_t update(size_t bytes_processed, size_t ms_spent, size_t& data_rate, bool VERBOSE = false){
 		interval_B += bytes_processed;		//increment the number of bytes processed
 		interval_ms += ms_spent;			//increment the number of milliseconds spent processing
+		data_rate = interval_B / interval_ms;
 		//if we have sufficient information to evaluate the optimization function at this point
 		if(interval_ms < window_ms){					//if insufficient time has passed to get a reliable Bps measurement
 			return n[0];
 		}
 		else{											//if we have collected enough information for a reliable Bps estimate
-			size_t new_Bps = interval_B / interval_ms;	//calculate the current Bps
-
+			
 			if(Bps[0] == 0){							//if n[0] hasn't been evaluated yet, this is the first step
-				Bps[0] = new_Bps;						//set the initial Bps value
+				Bps[0] = data_rate;						//set the initial Bps value
 				n[1] = n[0] - h;						//set the position of the next sample point
-				std::cout<<"Bps value at n = "<<n[0]<<" is "<<Bps[0]<<" Bps, probing n = "<<n[1]<<std::endl;
+				if(VERBOSE)
+					std::cout<<"Bps value at n = "<<n[0]<<" is "<<Bps[0]<<" Bps, probing n = "<<n[1]<<std::endl;
 				return n[1];							//return the probe point
 			}
 			else{
-				Bps[1] = new_Bps;						//set the Bps for the current point (n[1])
+				Bps[1] = data_rate;						//set the Bps for the current point (n[1])
 				double Bps_p;							//allocate a variable for the derivative
 				//calculate the derivative
@@ -100,8 +101,8 @@ public:
 				else{
 					Bps_p = ((double)Bps[0] - (double)Bps[1]) / (double)h;		//calculate the derivative using the backward finite difference
 				}
-
-				std::cout<<"     probed n = "<<n[1]<<" with "<<Bps[1]<<" Bps, gradient = "<<Bps_p<<" Bps"<<std::endl;
+				if(VERBOSE)
+					std::cout<<"     probed n = "<<n[1]<<" with "<<Bps[1]<<" Bps, gradient = "<<Bps_p<<" Bps"<<std::endl;
 				double new_n_precise = n[0] + alpha * Bps_p;			//calculate the next point (snap to closest integer)
 				size_t new_n_nearest = round_limit(new_n_precise);		//calculate the next point (given batch parameters)
@@ -110,7 +111,8 @@ public:
 					Bps[0] = Bps[1];									//update the Bps
 					//if(n[0] == dn) n[1] = n[0] + h;					//if we're on the left edge, probe forward
 					//else n[1] = n[0] - h;								//otherwise probe backwards
-					std::cout<<"     staying at n = "<<n[0]<<" for now"<<std::endl;
+					if(VERBOSE)
+						std::cout<<"     staying at n = "<<n[0]<<" for now"<<std::endl;
 					//return n[1];										//return the probe point
 					Bps[0] = 0;											//reset the Bps for the current point
@@ -119,7 +121,8 @@ public:
 				else{													//if the newest point is different from the original point
 					n[0] = new_n_nearest;								//move to the new point
 					Bps[0] = 0;											//set the Bps to zero (point hasn't been tested)
-					std::cout<<"     moving to n = "<<n[0]<<std::endl;
+					if(VERBOSE)
+						std::cout<<"     moving to n = "<<n[0]<<std::endl;
 					return n[0];										//return the new point
 				}
 			}
@@ -187,11 +190,11 @@ public:
 		}
 	}*/
-	size_t update(size_t bytes_processed, size_t ms_spent, size_t& data_rate){
-		size_t time = update(bytes_processed, ms_spent);
+	/*size_t update(size_t bytes_processed, size_t ms_spent, size_t& data_rate, bool VERBOSE){
+		size_t time = update(bytes_processed, ms_spent, VERBOSE);
 		data_rate = Bps[0];
 		return time;
-	}
+	}*/
 };
 /** This class manages the streaming of large multidimensional binary files.
@@ -213,6 +216,7 @@ protected:
 	unsigned char* mask;	//pointer to a character array: 0 = background, 1 = foreground (or valid data)
 	double progress;		//stores the progress on the current operation (accessible using a thread)
+	size_t data_rate;		//data rate (currently in Bps)
 	size_t buffer_size;		//available memory for processing large files
@@ -222,8 +226,9 @@ protected:
 		header = 0;											//initialize the header size to zero
 		mask = NULL;
-		progress = 0;
-		set_buffer();										//set the maximum buffer size to the default
+		progress = 0;										//initialize the progress for any algorithm to zero
+		data_rate = 0;										//initialize the data rate to zero
+		set_buffer_frac();										//set the maximum buffer size to the default
 	}
 	/// Private helper function that returns the size of the file on disk using system functions.
@@ -304,8 +309,12 @@ public:
 		progress = 0;
 	}
+	size_t get_data_rate(){
+		return data_rate;
+	}
+
 	//specify the maximum fraction of available memory that this class will use for buffering
-	void set_buffer(double mem_frac = 0.5){				//default to 50%
+	void set_buffer_frac(double mem_frac = 0.5){				//default to 50%
 #ifdef _WIN32
 		MEMORYSTATUSEX statex;
 		statex.dwLength = sizeof (statex);
@@ -318,6 +327,10 @@ public:
 #endif
 	}
+	void set_buffer_raw(size_t bytes){
+		buffer_size = bytes;
+	}
+
 	/// Open a binary file for streaming.
 	/// @param filename is the name of the binary file
@@ -389,7 +389,7 @@ public:
 	}
 	/// Convert this BSQ file to a BIL
-	bool bil(std::string outname, bool PROGRESS = false, bool VERBOSE = false){
+	bool bil(std::string outname, bool PROGRESS = false, bool VERBOSE = false, bool OPTIMIZATION = true){
 		const size_t buffers = 4;													//number of buffers required for this algorithm
@@ -398,10 +398,10 @@ public:
 		size_t slice_bytes = X() * Z() * sizeof(T);									//number of bytes in an input batch slice (Y-slice in this case)
 		size_t max_slices_per_batch = mem_per_batch / slice_bytes;					//maximum number of slices we can process in one batch given memory constraints
-		if(VERBOSE){
+		//if(VERBOSE){
 			std::cout<<"maximum memory available for processing: "<<(double)binary<T>::buffer_size/(double)1000000<<" MB"<<std::endl;
-			std::cout<<"     this supports a batch size of "<<max_slices_per_batch<<" Y-axis slices ("<<X()<<" x "<<Z()<<")"<<std::endl;
-		}
+			std::cout<<"     this supports a batch size of "<<max_slices_per_batch<<" Y-axis slices ("<<X()<<" x "<<Z()<<") = "<<X() * Z() * sizeof(T) * max_slices_per_batch/1000000<<" MB"<<std::endl;
+		//}
 		if(max_slices_per_batch == 0){														//if there is insufficient memory for a single slice, throw an error
 			std::cout<<"error, insufficient memory for stim::bsq::bil()"<<std::endl;
@@ -435,7 +435,7 @@ public:
 		size_t pt_total = 0;													//total time spent processing data
 		size_t rt_total = 0;													//total time spent reading data
 		size_t wt_total = 0;
-		size_t data_rate;
+		size_t dr = 0;
 		rt_total += readlines(src[0], 0, N[0]);					//read the first batch into the 0 source buffer
 		y_load += N[0];											//increment the loaded slice counter
@@ -458,11 +458,13 @@ public:
 			wt_total += writeblock(&target, dst[b], N[b] * slice_bytes);			//write the permuted data to the output file
 			y_proc += N[b];														//increment the counter of processed pixels
 			if(PROGRESS) progress = (double)( y_proc + 1 ) / Y() * 100;			//increment the progress counter based on the number of processed pixels
+			if(y_load < Y()) rt_total += rthread.get();					//if a new batch was set to load, make sure it loads after calculations
 			t_end = std::chrono::high_resolution_clock::now();
 			t_batch = std::chrono::duration_cast<std::chrono::milliseconds>(t_end-t_start).count();
 			t_total += t_batch;
-			if(y_load < Y()) rt_total += rthread.get();					//if a new batch was set to load, make sure it loads after calculations
-			N[b] = O.update(N[!b] * slice_bytes, t_batch, data_rate);					//set the batch size based on optimization
+			if(OPTIMIZATION)
+				N[b] = O.update(N[!b] * slice_bytes, t_batch, binary<T>::data_rate, VERBOSE);					//set the batch size based on optimization
+			//binary<T>::data_rate = dr;
 			//std::cout<<"New N = "<<N[!b]<<" selected with "<<(double)data_rate / 1000000<<" MB/s"<<std::endl;
 		}
@@ -79,30 +79,64 @@ public:
 		return alloc_array(header.samples * header.lines);
 	}
-	void set_buffer(double memfrac = 0.5){
+	void set_buffer_frac(double memfrac = 0.5){
 		if(header.interleave == envi_header::BSQ){		//if the infile is bsq file
 			if(header.data_type ==envi_header::float32)
-				((bsq<float>*)file)->set_buffer(memfrac);
+				((bsq<float>*)file)->set_buffer_frac(memfrac);
 			else if(header.data_type == envi_header::float64)
-				((bsq<double>*)file)->set_buffer(memfrac);
+				((bsq<double>*)file)->set_buffer_frac(memfrac);
 			else
 				std::cout<<"ERROR: unidentified data type"<<std::endl;
 		}
 		else if(header.interleave == envi_header::BIL){		//if the infile is bil file
 			if(header.data_type ==envi_header::float32)
-				((bil<float>*)file)->set_buffer(memfrac);
+				((bil<float>*)file)->set_buffer_frac(memfrac);
 			else if(header.data_type == envi_header::float64)
-				((bil<double>*)file)->set_buffer(memfrac);
+				((bil<double>*)file)->set_buffer_frac(memfrac);
 			else
 				std::cout<<"ERROR: unidentified data type"<<std::endl;
 		}
 		else if(header.interleave == envi_header::BIP){		//if the infile is bip file
 			if(header.data_type ==envi_header::float32)
-				((bip<float>*)file)->set_buffer(memfrac);
+				((bip<float>*)file)->set_buffer_frac(memfrac);
 			else if(header.data_type == envi_header::float64)
-				((bip<double>*)file)->set_buffer(memfrac);
+				((bip<double>*)file)->set_buffer_frac(memfrac);
+			else
+				std::cout<<"ERROR: unidentified data type"<<std::endl;
+		}
+
+		else{
+			std::cout<<"ERROR: unidentified file type"<<std::endl;
+			exit(1);
+		}
+	}
+
+	void set_buffer_raw(size_t bytes){
+		if(header.interleave == envi_header::BSQ){		//if the infile is bsq file
+			if(header.data_type ==envi_header::float32)
+				((bsq<float>*)file)->set_buffer_raw(bytes);
+			else if(header.data_type == envi_header::float64)
+				((bsq<double>*)file)->set_buffer_raw(bytes);
+			else
+				std::cout<<"ERROR: unidentified data type"<<std::endl;
+		}
+
+		else if(header.interleave == envi_header::BIL){		//if the infile is bil file
+			if(header.data_type ==envi_header::float32)
+				((bil<float>*)file)->set_buffer_raw(bytes);
+			else if(header.data_type == envi_header::float64)
+				((bil<double>*)file)->set_buffer_raw(bytes);
+			else
+				std::cout<<"ERROR: unidentified data type"<<std::endl;
+		}
+
+		else if(header.interleave == envi_header::BIP){		//if the infile is bip file
+			if(header.data_type ==envi_header::float32)
+				((bip<float>*)file)->set_buffer_raw(bytes);
+			else if(header.data_type == envi_header::float64)
+				((bip<double>*)file)->set_buffer_raw(bytes);
 			else
 				std::cout<<"ERROR: unidentified data type"<<std::endl;
 		}
@@ -121,6 +155,16 @@ public:
 		exit(1);
 	}
+	size_t X(){ return header.samples; }
+	size_t Y(){ return header.lines; }
+	size_t Z(){ return header.bands; }
+	size_t B(){ return Z();	}
+
+	/// Return the size of the data set in bytes
+	size_t bytes(){
+		return X() * Y() * Z() * type_size();
+	}
+
 	/// Returns the progress of the current processing operation as a percentage
 	void reset_progress(){
@@ -193,6 +237,42 @@ public:
 		return 0;
 	}
+	/// Returns the progress of the current processing operation as a percentage
+	size_t data_rate(){
+
+		if(header.interleave == envi_header::BSQ){		//if the infile is bsq file
+			if(header.data_type ==envi_header::float32)
+				return ((bsq<float>*)file)->get_data_rate();
+			else if(header.data_type == envi_header::float64)
+				return ((bsq<double>*)file)->get_data_rate();
+			else
+				std::cout<<"ERROR: unidentified data type"<<std::endl;
+		}
+
+		else if(header.interleave == envi_header::BIL){		//if the infile is bil file
+			if(header.data_type ==envi_header::float32)
+				return ((bil<float>*)file)->get_data_rate();
+			else if(header.data_type == envi_header::float64)
+				return ((bil<double>*)file)->get_data_rate();
+			else
+				std::cout<<"ERROR: unidentified data type"<<std::endl;
+		}
+
+		else if(header.interleave == envi_header::BIP){		//if the infile is bip file
+			if(header.data_type ==envi_header::float32)
+				return ((bip<float>*)file)->get_data_rate();
+			else if(header.data_type == envi_header::float64)
+				return ((bip<double>*)file)->get_data_rate();
+			else
+				std::cout<<"ERROR: unidentified data type"<<std::endl;
+		}
+
+		else{
+			std::cout<<"ERROR: unidentified file type"<<std::endl;
+		}
+		return 0;
+	}
+
 	/// Allocate memory for a new ENVI file based on the current interleave format (BIP, BIL, BSQ) and data type.
 	void allocate(){
@@ -509,7 +589,7 @@ public:
 	/// @param outfile is the file name for the converted output
 	/// @param interleave is the interleave format for the destination file
-	bool convert(std::string outfile, stim::envi_header::interleaveType interleave, bool PROGRESS = false, bool VERBOSE = false){
+	bool convert(std::string outfile, stim::envi_header::interleaveType interleave, bool PROGRESS = false, bool VERBOSE = false, bool OPTIMIZATION = true){
 		if(header.interleave == envi_header::BSQ){			//if the infile is bsq file
@@ -519,7 +599,7 @@ public:
 					exit(1);
 				}
 				else if(interleave == envi_header::BIL)			//convert BSQ -> BIL
-					((bsq<float>*)file)->bil(outfile, PROGRESS, VERBOSE);
+					((bsq<float>*)file)->bil(outfile, PROGRESS, VERBOSE, OPTIMIZATION);
 				else if(interleave == envi_header::BIP){			//ERROR
 					//std::cout<<"ERROR: conversion from BSQ to BIP isn't practical; use BSQ->BIL->BIP instead"<<std::endl;
 					((bsq<float>*)file)->bip(outfile, PROGRESS, VERBOSE);
@@ -533,10 +613,10 @@ public:
 					exit(1);
 				}
 				else if(interleave == envi_header::BIL)					//convert BSQ -> BIL
-					((bsq<double>*)file)->bil(outfile, PROGRESS);
+					((bsq<double>*)file)->bil(outfile, PROGRESS, OPTIMIZATION);
 				else if(interleave == envi_header::BIP){					//ERROR
 					//std::cout<<"ERROR: conversion from BSQ to BIP isn't practical; use BSQ->BIL->BIP instead"<<std::endl;
-					((bsq<float>*)file)->bip(outfile, PROGRESS);
+					((bsq<float>*)file)->bip(outfile, PROGRESS, OPTIMIZATION);
 					//exit(1);
 				}
 			}