output timing data after optimization

David Mayerich
1 parent c6251f8b
Showing 2 changed files with 44 additions and 29 deletions Show diff stats
stim/envi/bsq.h
stim/envi/envi.h
@@ -398,10 +398,8 @@ public:
 		size_t slice_bytes = X() * Z() * sizeof(T);									//number of bytes in an input batch slice (Y-slice in this case)
 		size_t max_slices_per_batch = mem_per_batch / slice_bytes;					//maximum number of slices we can process in one batch given memory constraints
-		//if(VERBOSE){
-			std::cout<<"maximum memory available for processing: "<<(double)binary<T>::buffer_size/(double)1000000<<" MB"<<std::endl;
-			std::cout<<"     this supports a batch size of "<<max_slices_per_batch<<" Y-axis slices ("<<X()<<" x "<<Z()<<") = "<<X() * Z() * sizeof(T) * max_slices_per_batch/1000000<<" MB"<<std::endl;
-		//}
+		std::cout<<"maximum memory available for processing: "<<(double)binary<T>::buffer_size/(double)1000000<<" MB"<<std::endl;
+		std::cout<<"     this supports a batch size of "<<max_slices_per_batch<<" Y-axis slices ("<<X()<<" x "<<Z()<<") = "<<X() * Z() * sizeof(T) * max_slices_per_batch/1000000<<" MB"<<std::endl;
 		if(max_slices_per_batch == 0){														//if there is insufficient memory for a single slice, throw an error
 			std::cout<<"error, insufficient memory for stim::bsq::bil()"<<std::endl;
@@ -458,7 +456,7 @@ public:
 			wt_total += writeblock(&target, dst[b], N[b] * slice_bytes);			//write the permuted data to the output file
 			y_proc += N[b];														//increment the counter of processed pixels
 			if(PROGRESS) progress = (double)( y_proc + 1 ) / Y() * 100;			//increment the progress counter based on the number of processed pixels
-			if(y_load < Y()) rt_total += rthread.get();					//if a new batch was set to load, make sure it loads after calculations
+			if(y_proc < Y()) rt_total += rthread.get();					//if a new batch was set to load, make sure it loads after calculations
 			t_end = std::chrono::high_resolution_clock::now();
 			t_batch = std::chrono::duration_cast<std::chrono::milliseconds>(t_end-t_start).count();
 			t_total += t_batch;
@@ -472,29 +470,36 @@ public:
 		free(src[1]);
 		free(dst[0]);
 		free(dst[1]);
-		if(VERBOSE){
+		//if(VERBOSE){
 			std::cout<<"total time to execute bsq::bil(): "<<t_total<<" ms"<<std::endl;
 			std::cout<<"     total time spent processing: "<<pt_total<<" ms"<<std::endl;
 			std::cout<<"        total time spent reading: "<<rt_total<<" ms"<<std::endl;
 			std::cout<<"        total time spent writing: "<<wt_total<<" ms"<<std::endl;
-		}
+		//}
 		return true;															//return true
 	}
 	/// Convert this BSQ file to a BIP
-	bool bip(std::string outname, bool PROGRESS = false, bool VERBOSE = false){
+	bool bip(std::string outname, bool PROGRESS = false, bool VERBOSE = false, bool OPTIMIZATION = true){
 		const size_t buffers = 4;													//number of buffers required for this algorithm
+		
 		size_t mem_per_batch = binary<T>::buffer_size / buffers;					//calculate the maximum memory available for a batch
 		size_t slice_bytes = X() * Z() * sizeof(T);									//number of bytes in an input batch slice (Y-slice in this case)
 		size_t max_slices_per_batch = mem_per_batch / slice_bytes;					//maximum number of slices we can process in one batch given memory constraints
+
+		std::cout<<"maximum memory available for processing: "<<(double)binary<T>::buffer_size/(double)1000000<<" MB"<<std::endl;
+		std::cout<<"     this supports a batch size of "<<max_slices_per_batch<<" Y-axis slices ("<<X()<<" x "<<Z()<<") = "<<X() * Z() * sizeof(T) * max_slices_per_batch/1000000<<" MB"<<std::endl;
+
 		if(max_slices_per_batch == 0){														//if there is insufficient memory for a single slice, throw an error
 			std::cout<<"error, insufficient memory for stim::bsq::bil()"<<std::endl;
 			exit(1);
 		}
 		size_t max_batch_bytes = max_slices_per_batch * slice_bytes;				//calculate the amount of memory that will be allocated for all four buffers
+		stream_optimizer O(1, max_slices_per_batch);
+
 		T* src[2];																	//source double-buffer for asynchronous batching
 		src[0] = (T*) malloc(max_batch_bytes);
 		src[1] = (T*) malloc(max_batch_bytes);
@@ -512,46 +517,56 @@ public:
 		std::future<size_t> rthread;
 		std::future<std::ostream&> wthread;										//create asynchronous threads for reading and writing
-		readlines(src[0], 0, N[0]);												//read the first batch into the 0 source buffer
-		y_load += N[0];															//increment the loaded slice counter
-		int b = 1;
-
-		std::chrono::high_resolution_clock::time_point t_start;						//high-resolution timers
-		std::chrono::high_resolution_clock::time_point t_end;
+		std::chrono::high_resolution_clock::time_point t_start, pt_start;						//high-resolution timers
+		std::chrono::high_resolution_clock::time_point t_end, pt_end;
 		size_t t_batch;																//number of milliseconds to process a batch
-		size_t t_total = 0;
-		size_t pt_total = 0;
-		size_t rt_total = 0;
+		size_t t_total = 0;														//total time for operation
+		size_t pt_total = 0;													//total time spent processing data
+		size_t rt_total = 0;													//total time spent reading data
+		size_t wt_total = 0;
+		size_t dr = 0;
+		
+		rt_total += readlines(src[0], 0, N[0]);					//read the first batch into the 0 source buffer
+		y_load += N[0];											//increment the loaded slice counter
+		int b = 1;												//initialize the double buffer to 0
 		while(y_proc < Y()){													//while there are still slices to be processed
 			t_start = std::chrono::high_resolution_clock::now();					//start the timer for this batch
 			if(y_load < Y()){													//if there are still slices to be loaded, load them
+				//if(y_proc > 0){
+					
+					
+				//}
 				if(y_load + N[b] > Y()) N[b] = Y() - y_load;					//if the next batch would process more than the total slices, adjust the batch size
 				rthread = std::async(std::launch::async, &stim::bsq<T>::readlines, this, src[b], y_load, N[b]);
-				
+				//rt_total += rthread.get();
 				y_load += N[b];													//increment the number of loaded slices
 			}
 			b = !b;																//swap the double-buffer
-
-			pt_total += binary<T>::permute(dst[b], src[b], X(), N[b], Z(), 2, 0, 1);		//permute the batch to a BIP file
-			target.write((char*)dst[b], N[b] * slice_bytes);					//write the permuted data to the output file
+			pt_total += binary<T>::permute(dst[b], src[b], X(), N[b], Z(), 0, 2, 1);		//permute the batch to a BIL file
+			wt_total += writeblock(&target, dst[b], N[b] * slice_bytes);			//write the permuted data to the output file
 			y_proc += N[b];														//increment the counter of processed pixels
 			if(PROGRESS) progress = (double)( y_proc + 1 ) / Y() * 100;			//increment the progress counter based on the number of processed pixels
+			if(y_proc < Y()) rt_total += rthread.get();					//if a new batch was set to load, make sure it loads after calculations
 			t_end = std::chrono::high_resolution_clock::now();
 			t_batch = std::chrono::duration_cast<std::chrono::milliseconds>(t_end-t_start).count();
 			t_total += t_batch;
-			if(y_load < Y()) rt_total += rthread.get();					//if a batch was threaded to load, make sure it finishes
-		}
-
-		if(VERBOSE){
-			std::cout<<"total time to execute bsq::bil(): "<<t_total<<" ms"<<std::endl;
-			std::cout<<"     total time spent processing: "<<pt_total<<" ms"<<std::endl;
-			std::cout<<"        total time spent reading: "<<rt_total<<" ms"<<std::endl;
+			if(OPTIMIZATION)
+				N[b] = O.update(N[!b] * slice_bytes, t_batch, binary<T>::data_rate, VERBOSE);					//set the batch size based on optimization
+			//binary<T>::data_rate = dr;
+			//std::cout<<"New N = "<<N[!b]<<" selected with "<<(double)data_rate / 1000000<<" MB/s"<<std::endl;
 		}
+		
 		free(src[0]);															//free buffer resources
 		free(src[1]);
 		free(dst[0]);
 		free(dst[1]);
+		//if(VERBOSE){
+			std::cout<<"total time to execute bsq::bip(): "<<t_total<<" ms"<<std::endl;
+			std::cout<<"     total time spent processing: "<<pt_total<<" ms"<<std::endl;
+			std::cout<<"        total time spent reading: "<<rt_total<<" ms"<<std::endl;
+			std::cout<<"        total time spent writing: "<<wt_total<<" ms"<<std::endl;
+		//}
 		return true;															//return true
 	}
@@ -602,7 +602,7 @@ public:
 					((bsq<float>*)file)->bil(outfile, PROGRESS, VERBOSE, OPTIMIZATION);
 				else if(interleave == envi_header::BIP){			//ERROR
 					//std::cout<<"ERROR: conversion from BSQ to BIP isn't practical; use BSQ->BIL->BIP instead"<<std::endl;
-					((bsq<float>*)file)->bip(outfile, PROGRESS, VERBOSE);
+					((bsq<float>*)file)->bip(outfile, PROGRESS, VERBOSE, OPTIMIZATION);
 					//exit(1);
 				}
 			}