diff --git a/stim/envi/binary.h b/stim/envi/binary.h index 48810ab..399b418 100644 --- a/stim/envi/binary.h +++ b/stim/envi/binary.h @@ -8,6 +8,10 @@ #include #include +#ifdef _WIN32 +#include +#endif + namespace stim{ /** This class manages the streaming of large multidimensional binary files. @@ -30,16 +34,16 @@ protected: double progress; //stores the progress on the current operation (accessible using a thread) - static const size_t bufferSize = 1000000000; - + size_t buffer_size; //available memory for processing large files /// Private initialization function used to set default parameters in the data structure. void init(){ memset(R, 0, sizeof(unsigned long long) * D); //initialize the resolution to zero - header = 0; //initialize the header size to zero + header = 0; //initialize the header size to zero mask = NULL; progress = 0; + set_buffer(); //set the maximum buffer size to the default } /// Private helper function that returns the size of the file on disk using system functions. @@ -107,6 +111,11 @@ protected: public: + //default constructor + binary(){ + init(); + } + double get_progress(){ return progress; } @@ -115,6 +124,16 @@ public: progress = 0; } + //specify the maximum fraction of available memory that this class will use for buffering + void set_buffer(double mem_frac = 0.5){ //default to 50% +#ifdef _WIN32 + MEMORYSTATUSEX statex; + statex.dwLength = sizeof (statex); + GlobalMemoryStatusEx (&statex); + buffer_size = (size_t)(statex.ullAvailPhys * mem_frac); +#endif + } + /// Open a binary file for streaming. /// @param filename is the name of the binary file diff --git a/stim/envi/bsq.h b/stim/envi/bsq.h index 69a3f6e..ab37595 100644 --- a/stim/envi/bsq.h +++ b/stim/envi/bsq.h @@ -378,80 +378,52 @@ public: /// Convert the current BSQ file to a BIL file with the specified file name. bool bil(std::string outname, bool PROGRESS = false){ - size_t XY = X() * Y(); //number of elements in a band - size_t XYbytes = XY * sizeof(T); //number of bytes in a band - size_t batchB = binary::bufferSize / (XYbytes); //calculate the number of slices that can fit in memory - size_t batchXB = X() * batchB; //number of elements in a batch + size_t XY = X() * Y(); //number of elements in a band + size_t XYbytes = XY * sizeof(T); //number of bytes in a band + size_t batchB = binary::buffer_size / (XYbytes); //calculate the number of slices that can fit in memory + if(Z() < batchB) batchB = Z(); //if the entire data set will fit in memory, do it + size_t batchXB = X() * batchB; //number of elements in a batch + T* ptrBatch = (T*) malloc(batchB * XYbytes); //allocate a large buffer storing the read data + T* ptrSlice = (T*) malloc(sizeof(T) * batchB * X()); //allocate space for storing an output buffer - T* ptrBatch = (T*) malloc(batchB * XYbytes); //allocate a large buffer storing the read data - T* ptrSlice = (T*) malloc(sizeof(T) * batchB * X()); //allocate space for storing an output buffer - - size_t jump = (Z() - batchB) * X() * sizeof(T); //jump between writes in the output file + size_t jump = (Z() - batchB) * X() * sizeof(T); //jump between writes in the output file std::ofstream target(outname.c_str(), std::ios::binary); std::string headername = outname + ".hdr"; - size_t batches = ceil((double)(Z()) / (double)batchB); //calculate the number of batches + size_t batches = ceil((double)(Z()) / (double)batchB); //calculate the number of batches T* ptrDst; T* ptrSrc; for(size_t c = 0; c < batches; c++){ target.seekp(c * batchB * sizeof(T) * X(), std::ios::beg); //seek to the start of the current batch in the output file - file.read((char*)ptrBatch, sizeof(T) * X() * Y() * batchB); //read a batch + file.read((char*)ptrBatch, sizeof(T) * X() * Y() * batchB); //read a batch + + auto pbegin = std::chrono::high_resolution_clock::now(); if(c == (batches - 1)){ batchB = Z() - (batches - 1) * batchB; //if this is the last batch, calculate the remaining # of bands jump = (Z() - batchB) * X() * sizeof(T); } - for(size_t y = 0; y < Y(); y++){ //for each line, store an XB slice in ptrDest + for(size_t y = 0; y < Y(); y++){ //for each line, store an XB slice in ptrDest ptrDst = ptrSlice; //initialize ptrDst to the start of the XB output slice ptrSrc = ptrBatch + (y * X()); for(size_t b = 0; b < batchB; b++){ //for each band in the current line memcpy(ptrDst, ptrSrc, X() * sizeof(T)); //copy the band line from the source to the destination - ptrDst += X(); - ptrSrc += X() * Y(); + ptrDst += X(); //increment the pointer within the XB slice (to be output) + ptrSrc += X() * Y(); //increment the pointer within the current buffer array (batch) } + target.write((char*)ptrSlice, sizeof(T) * X() * batchB); //write the XB slice to disk - target.seekp(jump, std::ios::cur); //seek to the beginning of the next XB slice in the batch + target.seekp(jump, std::ios::cur); //seek to the beginning of the next XB slice in the batch + if(PROGRESS) progress = (double)( c * Y() + y + 1 ) / (Y() * batches) * 100; } - if(PROGRESS) progress = (double)(c+1)/(double)(batches) * 100; } - //if(PROGRESS) progress = 100; - free(ptrBatch); free(ptrSlice); target.close(); return true; } - /// @param outname is the name of the output BIL file to be saved to disk. - /*bool bil(std::string outname, bool PROGRESS = false){ - - //simplify image resolution - unsigned long long jump = (Y() - 1) * X() * sizeof(T); - - std::ofstream target(outname.c_str(), std::ios::binary); - std::string headername = outname + ".hdr"; - - unsigned long long L = X(); //calculate the number of pixels in a line - T* line = (T*)malloc(sizeof(T) * L); //allocate space for a line - - for ( unsigned long long y = 0; y < Y(); y++) //for each y position - { - file.seekg(y * X() * sizeof(T), std::ios::beg); //seek to the beginning of the xz slice - for ( unsigned long long z = 0; z < Z(); z++ ) //for each band - { - file.read((char *)line, sizeof(T) * X()); //read a line - target.write((char*)line, sizeof(T) * X()); //write the line to the output file - file.seekg(jump, std::ios::cur); //seek to the next band - if(PROGRESS) progress = (double)((y+1) * Z() + z + 1) / (Z() * Y()) * 100; //update the progress counter - } - } - - free(line); - target.close(); - - return true; - }*/ /// Return a baseline corrected band given two adjacent baseline points and their bands. The result is stored in a pre-allocated array. diff --git a/stim/envi/envi.h b/stim/envi/envi.h index f859fd2..0d365c3 100644 --- a/stim/envi/envi.h +++ b/stim/envi/envi.h @@ -77,6 +77,40 @@ public: return alloc_array(header.samples * header.lines); } + void set_buffer(double memfrac = 0.5){ + if(header.interleave == envi_header::BSQ){ //if the infile is bsq file + if(header.data_type ==envi_header::float32) + ((bsq*)file)->set_buffer(memfrac); + else if(header.data_type == envi_header::float64) + ((bsq*)file)->set_buffer(memfrac); + else + std::cout<<"ERROR: unidentified data type"<*)file)->set_buffer(memfrac); + else if(header.data_type == envi_header::float64) + ((bil*)file)->set_buffer(memfrac); + else + std::cout<<"ERROR: unidentified data type"<*)file)->set_buffer(memfrac); + else if(header.data_type == envi_header::float64) + ((bip*)file)->set_buffer(memfrac); + else + std::cout<<"ERROR: unidentified data type"<