From 5e1b4869d074e1b17683a3b9d1a7f0d40c033375 Mon Sep 17 00:00:00 2001 From: David Date: Tue, 6 Sep 2016 02:17:21 -0500 Subject: [PATCH] BSQ->BIL performance increased for multiple drives in Windows --- stim/envi/bsq.h | 161 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------------ 1 file changed, 137 insertions(+), 24 deletions(-) diff --git a/stim/envi/bsq.h b/stim/envi/bsq.h index ab37595..697edc2 100644 --- a/stim/envi/bsq.h +++ b/stim/envi/bsq.h @@ -376,54 +376,167 @@ public: } - /// Convert the current BSQ file to a BIL file with the specified file name. bool bil(std::string outname, bool PROGRESS = false){ - size_t XY = X() * Y(); //number of elements in a band - size_t XYbytes = XY * sizeof(T); //number of bytes in a band - size_t batchB = binary::buffer_size / (XYbytes); //calculate the number of slices that can fit in memory - if(Z() < batchB) batchB = Z(); //if the entire data set will fit in memory, do it - size_t batchXB = X() * batchB; //number of elements in a batch - T* ptrBatch = (T*) malloc(batchB * XYbytes); //allocate a large buffer storing the read data - T* ptrSlice = (T*) malloc(sizeof(T) * batchB * X()); //allocate space for storing an output buffer + size_t XY = X() * Y(); //number of elements in an input slice + size_t XB = X() * Z(); //number of elements in an output slice + size_t XYbytes = XY * sizeof(T); //number of bytes in an input slice + size_t XBbytes = XB * sizeof(T); //number of bytes in an output slice + size_t batch_slices = binary::buffer_size / (2*XBbytes); //calculate the number of slices that can fit in memory + if(Y() < batch_slices) batch_slices = Y(); //if the entire data set will fit in memory, do it + size_t batchN = XB * batch_slices; //number of elements in a batch + size_t batch_bytes = batchN * sizeof(T); //calculate the number of bytes in a batch - size_t jump = (Z() - batchB) * X() * sizeof(T); //jump between writes in the output file + T* ptrIn = (T*) malloc(batch_bytes); //allocate a large buffer storing the read data + T* ptrOut = (T*) malloc(batch_bytes); //allocate space for storing an output buffer + + size_t jump = (Y() - batch_slices) * X() * sizeof(T); //jump between reads in the input file + + std::ofstream target(outname.c_str(), std::ios::binary); + std::string headername = outname + ".hdr"; + + size_t batches = ceil((double)(Y()) / (double)batch_slices); //calculate the number of batches + T* ptrDst; + T* ptrSrc; + for(size_t c = 0; c < batches; c++){ + file.seekg(c * X() * batch_slices * sizeof(T), std::ios::beg); + + if(c == (batches - 1)){ + batch_slices = Y() - (batches - 1) * batch_slices; //if this is the last batch, calculate the remaining # of bands + jump = (Y() - batch_slices) * X() * sizeof(T); + batchN = XB * batch_slices; + batch_bytes = batchN * sizeof(T); + } + + auto in_begin = std::chrono::high_resolution_clock::now(); + for(size_t b = 0; b < Z(); b++){ + file.read((char*)(ptrIn + b * X() * batch_slices), sizeof(T) * X() * batch_slices); //read a number of lines equal to "batch_slices" + file.seekg(jump, std::ios::cur); //jump to the next band + } + auto in_end = std::chrono::high_resolution_clock::now(); + std::cout << std::chrono::duration_cast(in_end-in_begin).count() << "ms" << std::endl; + + auto calc_begin = std::chrono::high_resolution_clock::now(); + + for(size_t b = 0; b < Z(); b++){ //for each line, store an XB slice in ptrDest + ptrSrc = ptrIn + (b * X() * batch_slices); + ptrDst = ptrOut + (b * X()); //initialize ptrDst to the start of the XB output slice + + for(size_t y = 0; y < batch_slices; y++){ //for each band in the current line + memcpy(ptrDst, ptrSrc, X() * sizeof(T)); //copy the band line from the source to the destination + ptrSrc += X(); //increment the pointer within the current buffer array (batch) + ptrDst += X() * Z(); //increment the pointer within the XB slice (to be output) + } + } + auto calc_end = std::chrono::high_resolution_clock::now(); + std::cout << std::chrono::duration_cast(calc_end-calc_begin).count() << "ms" << std::endl; + + auto out_begin = std::chrono::high_resolution_clock::now(); + target.write((char*)ptrOut, batch_bytes); //write the batch to disk + auto out_end = std::chrono::high_resolution_clock::now(); + std::cout << std::chrono::duration_cast(out_end-out_begin).count() << "ms" << std::endl; + if(PROGRESS) progress = (double)( c + 1 ) / (batches) * 100; + } + + free(ptrIn); + free(ptrOut); + target.close(); + + return true; + } + + /*/// Convert the current BSQ file to a BIL file with the specified file name. + bool bil(std::string outname, bool PROGRESS = false){ + size_t XY = X() * Y(); //number of elements in an input slice + size_t XB = X() * Z(); //number of elements in an output slice + size_t XYbytes = XY * sizeof(T); //number of bytes in an input slice + size_t XBbytes = XB * sizeof(T); //number of bytes in an output slice + size_t batch_bands = binary::buffer_size / (2*XYbytes); //calculate the number of slices that can fit in memory + if(Z() < batch_bands) batch_bands = Z(); //if the entire data set will fit in memory, do it + size_t batchXB = X() * batch_bands; //number of elements in a batch + + size_t batch_bytes = batch_bands * XYbytes; //calculate the number of bytes in a batch + T* ptrIn = (T*) malloc(batch_bytes); //allocate a large buffer storing the read data + T* ptrOut = (T*) malloc(batch_bytes); //allocate space for storing an output buffer + + size_t jump = (Z() - batch_bands) * X() * sizeof(T); //jump between writes in the output file std::ofstream target(outname.c_str(), std::ios::binary); std::string headername = outname + ".hdr"; - size_t batches = ceil((double)(Z()) / (double)batchB); //calculate the number of batches + size_t batches = ceil((double)(Z()) / (double)batch_bands); //calculate the number of batches T* ptrDst; T* ptrSrc; for(size_t c = 0; c < batches; c++){ - target.seekp(c * batchB * sizeof(T) * X(), std::ios::beg); //seek to the start of the current batch in the output file - file.read((char*)ptrBatch, sizeof(T) * X() * Y() * batchB); //read a batch + auto in_begin = std::chrono::high_resolution_clock::now(); + target.seekp(c * batch_bands * sizeof(T) * X(), std::ios::beg); //seek to the start of the current batch in the output file + file.read((char*)ptrIn, sizeof(T) * X() * Y() * batch_bands); //read a batch + auto in_end = std::chrono::high_resolution_clock::now(); + std::cout << std::chrono::duration_cast(in_end-in_begin).count() << "ms" << std::endl; - auto pbegin = std::chrono::high_resolution_clock::now(); + auto calc_begin = std::chrono::high_resolution_clock::now(); if(c == (batches - 1)){ - batchB = Z() - (batches - 1) * batchB; //if this is the last batch, calculate the remaining # of bands - jump = (Z() - batchB) * X() * sizeof(T); + batch_bands = Z() - (batches - 1) * batch_bands; //if this is the last batch, calculate the remaining # of bands + jump = (Z() - batch_bands) * X() * sizeof(T); } for(size_t y = 0; y < Y(); y++){ //for each line, store an XB slice in ptrDest - ptrDst = ptrSlice; //initialize ptrDst to the start of the XB output slice - ptrSrc = ptrBatch + (y * X()); - for(size_t b = 0; b < batchB; b++){ //for each band in the current line + ptrDst = ptrOut + (y * X() * batch_bands); //initialize ptrDst to the start of the XB output slice + ptrSrc = ptrIn + (y * X()); + for(size_t b = 0; b < batch_bands; b++){ //for each band in the current line memcpy(ptrDst, ptrSrc, X() * sizeof(T)); //copy the band line from the source to the destination ptrDst += X(); //increment the pointer within the XB slice (to be output) ptrSrc += X() * Y(); //increment the pointer within the current buffer array (batch) } - - target.write((char*)ptrSlice, sizeof(T) * X() * batchB); //write the XB slice to disk + } + auto calc_end = std::chrono::high_resolution_clock::now(); + std::cout << std::chrono::duration_cast(calc_end-calc_begin).count() << "ms" << std::endl; + + auto out_begin = std::chrono::high_resolution_clock::now(); + target.seekp(0, std::ios::beg); + for(size_t y = 0; y < Y(); y++){ //for each y-slice + target.write((char*)(ptrOut + y * X() * batch_bands), sizeof(T) * X() * batch_bands); //write the XB slice to disk target.seekp(jump, std::ios::cur); //seek to the beginning of the next XB slice in the batch - if(PROGRESS) progress = (double)( c * Y() + y + 1 ) / (Y() * batches) * 100; } + auto out_end = std::chrono::high_resolution_clock::now(); + std::cout << std::chrono::duration_cast(out_end-out_begin).count() << "ms" << std::endl; + if(PROGRESS) progress = (double)( c + 1 ) / (batches) * 100; } - free(ptrBatch); - free(ptrSlice); + free(ptrIn); + free(ptrOut); target.close(); return true; - } + }*/ + /* + /// @param outname is the name of the output BIL file to be saved to disk. + bool bil(std::string outname, bool PROGRESS = false) + { + //simplify image resolution + unsigned long long jump = (Y() - 1) * X() * sizeof(T); + + std::ofstream target(outname.c_str(), std::ios::binary); + std::string headername = outname + ".hdr"; + + unsigned long long L = X(); + T* line = (T*)malloc(sizeof(T) * L); + + for ( unsigned long long y = 0; y < Y(); y++) //for each y position + { + file.seekg(y * X() * sizeof(T), std::ios::beg); //seek to the beginning of the xz slice + for ( unsigned long long z = 0; z < Z(); z++ ) //for each band + { + file.read((char *)line, sizeof(T) * X()); //read a line + target.write((char*)line, sizeof(T) * X()); //write the line to the output file + file.seekg(jump, std::ios::cur); //seek to the next band + if(PROGRESS) progress = (double)((y+1) * Z() + z + 1) / (Z() * Y()) * 100; //update the progress counter + } + } + + free(line); + target.close(); + + return true; + }*/ /// Return a baseline corrected band given two adjacent baseline points and their bands. The result is stored in a pre-allocated array. -- libgit2 0.21.4