implemented Windows code for automatically estimating buffer sizes

David Mayerich
1 parent 3c25631c
Showing 3 changed files with 74 additions and 49 deletions Show diff stats
stim/envi/binary.h
stim/envi/bsq.h
stim/envi/envi.h
@@ -8,6 +8,10 @@
 #include <fstream>
 #include <sys/stat.h>
+#ifdef _WIN32
+#include <Windows.h>
+#endif
+
 namespace stim{
 /** This class manages the streaming of large multidimensional binary files.
@@ -30,16 +34,16 @@ protected:
 	double progress;		//stores the progress on the current operation (accessible using a thread)
-	static const size_t bufferSize = 1000000000;
-
+	size_t buffer_size;		//available memory for processing large files
 	/// Private initialization function used to set default parameters in the data structure.
 	void init(){
 		memset(R, 0, sizeof(unsigned long long) * D);		//initialize the resolution to zero
-		header = 0;									//initialize the header size to zero
+		header = 0;											//initialize the header size to zero
 		mask = NULL;
 		progress = 0;
+		set_buffer();										//set the maximum buffer size to the default
 	}
 	/// Private helper function that returns the size of the file on disk using system functions.
@@ -107,6 +111,11 @@ protected:
 public:
+	//default constructor
+	binary(){
+		init();
+	}
+
 	double get_progress(){
 		return progress;
 	}
@@ -115,6 +124,16 @@ public:
 		progress = 0;
 	}
+	//specify the maximum fraction of available memory that this class will use for buffering
+	void set_buffer(double mem_frac = 0.5){				//default to 50%
+#ifdef _WIN32
+		MEMORYSTATUSEX statex;
+		statex.dwLength = sizeof (statex);
+		GlobalMemoryStatusEx (&statex);
+		buffer_size = (size_t)(statex.ullAvailPhys * mem_frac);
+#endif
+	}
+
 	/// Open a binary file for streaming.
 	/// @param filename is the name of the binary file
@@ -378,80 +378,52 @@ public:
 	/// Convert the current BSQ file to a BIL file with the specified file name.
 	bool bil(std::string outname, bool PROGRESS = false){
-		size_t XY = X() * Y();												//number of elements in a band
-		size_t XYbytes = XY * sizeof(T);									//number of bytes in a band
-		size_t batchB = binary<T>::bufferSize / (XYbytes);					//calculate the number of slices that can fit in memory
-		size_t batchXB = X() * batchB;										//number of elements in a batch
+		size_t XY = X() * Y();											//number of elements in a band
+		size_t XYbytes = XY * sizeof(T);								//number of bytes in a band
+		size_t batchB = binary<T>::buffer_size / (XYbytes);				//calculate the number of slices that can fit in memory
+		if(Z() < batchB) batchB = Z();									//if the entire data set will fit in memory, do it
+		size_t batchXB = X() * batchB;									//number of elements in a batch
+		T* ptrBatch = (T*) malloc(batchB * XYbytes);					//allocate a large buffer storing the read data
+		T* ptrSlice = (T*) malloc(sizeof(T) * batchB * X());			//allocate space for storing an output buffer
-		T* ptrBatch = (T*) malloc(batchB * XYbytes);						//allocate a large buffer storing the read data
-		T* ptrSlice = (T*) malloc(sizeof(T) * batchB * X());					//allocate space for storing an output buffer
-
-		size_t jump = (Z() - batchB) * X() * sizeof(T);			//jump between writes in the output file
+		size_t jump = (Z() - batchB) * X() * sizeof(T);					//jump between writes in the output file
 		std::ofstream target(outname.c_str(), std::ios::binary);
 		std::string headername = outname + ".hdr";
-		size_t batches = ceil((double)(Z()) / (double)batchB);				//calculate the number of batches
+		size_t batches = ceil((double)(Z()) / (double)batchB);			//calculate the number of batches
 		T* ptrDst;
 		T* ptrSrc;
 		for(size_t c = 0; c < batches; c++){
 			target.seekp(c * batchB * sizeof(T) * X(), std::ios::beg);	//seek to the start of the current batch in the output file
-			file.read((char*)ptrBatch, sizeof(T) * X() * Y() * batchB);		//read a batch
+			file.read((char*)ptrBatch, sizeof(T) * X() * Y() * batchB);	//read a batch
+
+			auto pbegin = std::chrono::high_resolution_clock::now();
 			if(c == (batches - 1)){
 				batchB = Z() - (batches - 1) * batchB;					//if this is the last batch, calculate the remaining # of bands
 				jump = (Z() - batchB) * X() * sizeof(T);
 			}
-			for(size_t y = 0; y < Y(); y++){								//for each line, store an XB slice in ptrDest
+			for(size_t y = 0; y < Y(); y++){							//for each line, store an XB slice in ptrDest
 				ptrDst = ptrSlice;										//initialize ptrDst to the start of the XB output slice
 				ptrSrc = ptrBatch + (y * X());
 				for(size_t b = 0; b < batchB; b++){						//for each band in the current line
 					memcpy(ptrDst, ptrSrc, X() * sizeof(T));			//copy the band line from the source to the destination
-					ptrDst += X();
-					ptrSrc += X() * Y();
+					ptrDst += X();										//increment the pointer within the XB slice (to be output)
+					ptrSrc += X() * Y();								//increment the pointer within the current buffer array (batch)					
 				}
+				
 				target.write((char*)ptrSlice, sizeof(T) * X() * batchB);	//write the XB slice to disk
-				target.seekp(jump, std::ios::cur);						//seek to the beginning of the next XB slice in the batch
+				target.seekp(jump, std::ios::cur);							//seek to the beginning of the next XB slice in the batch
+				if(PROGRESS) progress = (double)( c * Y() + y + 1 ) / (Y() * batches) * 100;
 			}
-			if(PROGRESS) progress = (double)(c+1)/(double)(batches) * 100;
 		}
-		//if(PROGRESS) progress = 100;
-
 		free(ptrBatch);
 		free(ptrSlice);
 		target.close();
 		return true;
 	}
-	/// @param outname is the name of the output BIL file to be saved to disk.
-	/*bool bil(std::string outname, bool PROGRESS = false){
-
-		//simplify image resolution
-		unsigned long long jump = (Y() - 1) * X() * sizeof(T);
-
-		std::ofstream target(outname.c_str(), std::ios::binary);
-		std::string headername = outname + ".hdr";
-
-		unsigned long long L = X();											//calculate the number of pixels in a line
-		T* line = (T*)malloc(sizeof(T) * L);								//allocate space for a line
-
-		for ( unsigned long long y = 0; y < Y(); y++)									//for each y position
-		{
-			file.seekg(y * X() * sizeof(T), std::ios::beg);					//seek to the beginning of the xz slice
-			for ( unsigned long long z = 0; z < Z(); z++ )							//for each band
-			{
-				file.read((char *)line, sizeof(T) * X());					//read a line
-				target.write((char*)line, sizeof(T) * X());					//write the line to the output file
-				file.seekg(jump, std::ios::cur);							//seek to the next band
-				if(PROGRESS) progress = (double)((y+1) * Z() + z + 1) / (Z() * Y()) * 100;	//update the progress counter
-			}
-		}
-
-		free(line);
-		target.close();
-
-		return true;
-	}*/
 	/// Return a baseline corrected band given two adjacent baseline points and their bands. The result is stored in a pre-allocated array.
@@ -77,6 +77,40 @@ public:
 		return alloc_array(header.samples * header.lines);
 	}
+	void set_buffer(double memfrac = 0.5){
+		if(header.interleave == envi_header::BSQ){		//if the infile is bsq file
+			if(header.data_type ==envi_header::float32)
+				((bsq<float>*)file)->set_buffer(memfrac);
+			else if(header.data_type == envi_header::float64)
+				((bsq<double>*)file)->set_buffer(memfrac);
+			else
+				std::cout<<"ERROR: unidentified data type"<<std::endl;
+		}
+
+		else if(header.interleave == envi_header::BIL){		//if the infile is bil file
+			if(header.data_type ==envi_header::float32)
+				((bil<float>*)file)->set_buffer(memfrac);
+			else if(header.data_type == envi_header::float64)
+				((bil<double>*)file)->set_buffer(memfrac);
+			else
+				std::cout<<"ERROR: unidentified data type"<<std::endl;
+		}
+
+		else if(header.interleave == envi_header::BIP){		//if the infile is bip file
+			if(header.data_type ==envi_header::float32)
+				((bip<float>*)file)->set_buffer(memfrac);
+			else if(header.data_type == envi_header::float64)
+				((bip<double>*)file)->set_buffer(memfrac);
+			else
+				std::cout<<"ERROR: unidentified data type"<<std::endl;
+		}
+
+		else{
+			std::cout<<"ERROR: unidentified file type"<<std::endl;
+			exit(1);
+		}
+	}
+
 	/// Returns the size of the data type in bytes
 	unsigned int type_size(){
 		if(header.data_type == envi_header::float32) return 4;