codebase / stimlib

Commit 5e1b4869d074e1b17683a3b9d1a7f0d40c033375

Authored by David Mayerich
1 parent 008ae990

BSQ->BIL performance increased for multiple drives in Windows

Showing 1 changed file with 137 additions and 24 deletions   Show diff stats
Inline Side-by-side
stim/envi/bsq.h
  Show/Hide comments   View file @5e1b486
... ... @@ -376,54 +376,167 @@ public:
376 376  
377 377 }
378 378  
379   - /// Convert the current BSQ file to a BIL file with the specified file name.
380 379 bool bil(std::string outname, bool PROGRESS = false){
381   - size_t XY = X() * Y(); //number of elements in a band
382   - size_t XYbytes = XY * sizeof(T); //number of bytes in a band
383   - size_t batchB = binary<T>::buffer_size / (XYbytes); //calculate the number of slices that can fit in memory
384   - if(Z() < batchB) batchB = Z(); //if the entire data set will fit in memory, do it
385   - size_t batchXB = X() * batchB; //number of elements in a batch
386   - T* ptrBatch = (T*) malloc(batchB * XYbytes); //allocate a large buffer storing the read data
387   - T* ptrSlice = (T*) malloc(sizeof(T) * batchB * X()); //allocate space for storing an output buffer
  380 + size_t XY = X() * Y(); //number of elements in an input slice
  381 + size_t XB = X() * Z(); //number of elements in an output slice
  382 + size_t XYbytes = XY * sizeof(T); //number of bytes in an input slice
  383 + size_t XBbytes = XB * sizeof(T); //number of bytes in an output slice
  384 + size_t batch_slices = binary<T>::buffer_size / (2*XBbytes); //calculate the number of slices that can fit in memory
  385 + if(Y() < batch_slices) batch_slices = Y(); //if the entire data set will fit in memory, do it
  386 + size_t batchN = XB * batch_slices; //number of elements in a batch
  387 + size_t batch_bytes = batchN * sizeof(T); //calculate the number of bytes in a batch
388 388  
389   - size_t jump = (Z() - batchB) * X() * sizeof(T); //jump between writes in the output file
  389 + T* ptrIn = (T*) malloc(batch_bytes); //allocate a large buffer storing the read data
  390 + T* ptrOut = (T*) malloc(batch_bytes); //allocate space for storing an output buffer
  391 +
  392 + size_t jump = (Y() - batch_slices) * X() * sizeof(T); //jump between reads in the input file
  393 +
  394 + std::ofstream target(outname.c_str(), std::ios::binary);
  395 + std::string headername = outname + ".hdr";
  396 +
  397 + size_t batches = ceil((double)(Y()) / (double)batch_slices); //calculate the number of batches
  398 + T* ptrDst;
  399 + T* ptrSrc;
  400 + for(size_t c = 0; c < batches; c++){
  401 + file.seekg(c * X() * batch_slices * sizeof(T), std::ios::beg);
  402 +
  403 + if(c == (batches - 1)){
  404 + batch_slices = Y() - (batches - 1) * batch_slices; //if this is the last batch, calculate the remaining # of bands
  405 + jump = (Y() - batch_slices) * X() * sizeof(T);
  406 + batchN = XB * batch_slices;
  407 + batch_bytes = batchN * sizeof(T);
  408 + }
  409 +
  410 + auto in_begin = std::chrono::high_resolution_clock::now();
  411 + for(size_t b = 0; b < Z(); b++){
  412 + file.read((char*)(ptrIn + b * X() * batch_slices), sizeof(T) * X() * batch_slices); //read a number of lines equal to "batch_slices"
  413 + file.seekg(jump, std::ios::cur); //jump to the next band
  414 + }
  415 + auto in_end = std::chrono::high_resolution_clock::now();
  416 + std::cout << std::chrono::duration_cast<std::chrono::milliseconds>(in_end-in_begin).count() << "ms" << std::endl;
  417 +
  418 + auto calc_begin = std::chrono::high_resolution_clock::now();
  419 +
  420 + for(size_t b = 0; b < Z(); b++){ //for each line, store an XB slice in ptrDest
  421 + ptrSrc = ptrIn + (b * X() * batch_slices);
  422 + ptrDst = ptrOut + (b * X()); //initialize ptrDst to the start of the XB output slice
  423 +
  424 + for(size_t y = 0; y < batch_slices; y++){ //for each band in the current line
  425 + memcpy(ptrDst, ptrSrc, X() * sizeof(T)); //copy the band line from the source to the destination
  426 + ptrSrc += X(); //increment the pointer within the current buffer array (batch)
  427 + ptrDst += X() * Z(); //increment the pointer within the XB slice (to be output)
  428 + }
  429 + }
  430 + auto calc_end = std::chrono::high_resolution_clock::now();
  431 + std::cout << std::chrono::duration_cast<std::chrono::milliseconds>(calc_end-calc_begin).count() << "ms" << std::endl;
  432 +
  433 + auto out_begin = std::chrono::high_resolution_clock::now();
  434 + target.write((char*)ptrOut, batch_bytes); //write the batch to disk
  435 + auto out_end = std::chrono::high_resolution_clock::now();
  436 + std::cout << std::chrono::duration_cast<std::chrono::milliseconds>(out_end-out_begin).count() << "ms" << std::endl;
  437 + if(PROGRESS) progress = (double)( c + 1 ) / (batches) * 100;
  438 + }
  439 +
  440 + free(ptrIn);
  441 + free(ptrOut);
  442 + target.close();
  443 +
  444 + return true;
  445 + }
  446 +
  447 + /*/// Convert the current BSQ file to a BIL file with the specified file name.
  448 + bool bil(std::string outname, bool PROGRESS = false){
  449 + size_t XY = X() * Y(); //number of elements in an input slice
  450 + size_t XB = X() * Z(); //number of elements in an output slice
  451 + size_t XYbytes = XY * sizeof(T); //number of bytes in an input slice
  452 + size_t XBbytes = XB * sizeof(T); //number of bytes in an output slice
  453 + size_t batch_bands = binary<T>::buffer_size / (2*XYbytes); //calculate the number of slices that can fit in memory
  454 + if(Z() < batch_bands) batch_bands = Z(); //if the entire data set will fit in memory, do it
  455 + size_t batchXB = X() * batch_bands; //number of elements in a batch
  456 +
  457 + size_t batch_bytes = batch_bands * XYbytes; //calculate the number of bytes in a batch
  458 + T* ptrIn = (T*) malloc(batch_bytes); //allocate a large buffer storing the read data
  459 + T* ptrOut = (T*) malloc(batch_bytes); //allocate space for storing an output buffer
  460 +
  461 + size_t jump = (Z() - batch_bands) * X() * sizeof(T); //jump between writes in the output file
390 462  
391 463 std::ofstream target(outname.c_str(), std::ios::binary);
392 464 std::string headername = outname + ".hdr";
393 465  
394   - size_t batches = ceil((double)(Z()) / (double)batchB); //calculate the number of batches
  466 + size_t batches = ceil((double)(Z()) / (double)batch_bands); //calculate the number of batches
395 467 T* ptrDst;
396 468 T* ptrSrc;
397 469 for(size_t c = 0; c < batches; c++){
398   - target.seekp(c * batchB * sizeof(T) * X(), std::ios::beg); //seek to the start of the current batch in the output file
399   - file.read((char*)ptrBatch, sizeof(T) * X() * Y() * batchB); //read a batch
  470 + auto in_begin = std::chrono::high_resolution_clock::now();
  471 + target.seekp(c * batch_bands * sizeof(T) * X(), std::ios::beg); //seek to the start of the current batch in the output file
  472 + file.read((char*)ptrIn, sizeof(T) * X() * Y() * batch_bands); //read a batch
  473 + auto in_end = std::chrono::high_resolution_clock::now();
  474 + std::cout << std::chrono::duration_cast<std::chrono::milliseconds>(in_end-in_begin).count() << "ms" << std::endl;
400 475  
401   - auto pbegin = std::chrono::high_resolution_clock::now();
  476 + auto calc_begin = std::chrono::high_resolution_clock::now();
402 477 if(c == (batches - 1)){
403   - batchB = Z() - (batches - 1) * batchB; //if this is the last batch, calculate the remaining # of bands
404   - jump = (Z() - batchB) * X() * sizeof(T);
  478 + batch_bands = Z() - (batches - 1) * batch_bands; //if this is the last batch, calculate the remaining # of bands
  479 + jump = (Z() - batch_bands) * X() * sizeof(T);
405 480 }
406 481 for(size_t y = 0; y < Y(); y++){ //for each line, store an XB slice in ptrDest
407   - ptrDst = ptrSlice; //initialize ptrDst to the start of the XB output slice
408   - ptrSrc = ptrBatch + (y * X());
409   - for(size_t b = 0; b < batchB; b++){ //for each band in the current line
  482 + ptrDst = ptrOut + (y * X() * batch_bands); //initialize ptrDst to the start of the XB output slice
  483 + ptrSrc = ptrIn + (y * X());
  484 + for(size_t b = 0; b < batch_bands; b++){ //for each band in the current line
410 485 memcpy(ptrDst, ptrSrc, X() * sizeof(T)); //copy the band line from the source to the destination
411 486 ptrDst += X(); //increment the pointer within the XB slice (to be output)
412 487 ptrSrc += X() * Y(); //increment the pointer within the current buffer array (batch)
413 488 }
414   -
415   - target.write((char*)ptrSlice, sizeof(T) * X() * batchB); //write the XB slice to disk
  489 + }
  490 + auto calc_end = std::chrono::high_resolution_clock::now();
  491 + std::cout << std::chrono::duration_cast<std::chrono::milliseconds>(calc_end-calc_begin).count() << "ms" << std::endl;
  492 +
  493 + auto out_begin = std::chrono::high_resolution_clock::now();
  494 + target.seekp(0, std::ios::beg);
  495 + for(size_t y = 0; y < Y(); y++){ //for each y-slice
  496 + target.write((char*)(ptrOut + y * X() * batch_bands), sizeof(T) * X() * batch_bands); //write the XB slice to disk
416 497 target.seekp(jump, std::ios::cur); //seek to the beginning of the next XB slice in the batch
417   - if(PROGRESS) progress = (double)( c * Y() + y + 1 ) / (Y() * batches) * 100;
418 498 }
  499 + auto out_end = std::chrono::high_resolution_clock::now();
  500 + std::cout << std::chrono::duration_cast<std::chrono::milliseconds>(out_end-out_begin).count() << "ms" << std::endl;
  501 + if(PROGRESS) progress = (double)( c + 1 ) / (batches) * 100;
419 502 }
420 503  
421   - free(ptrBatch);
422   - free(ptrSlice);
  504 + free(ptrIn);
  505 + free(ptrOut);
423 506 target.close();
424 507  
425 508 return true;
426   - }
  509 + }*/
  510 + /*
  511 + /// @param outname is the name of the output BIL file to be saved to disk.
  512 + bool bil(std::string outname, bool PROGRESS = false)
  513 + {
  514 + //simplify image resolution
  515 + unsigned long long jump = (Y() - 1) * X() * sizeof(T);
  516 +
  517 + std::ofstream target(outname.c_str(), std::ios::binary);
  518 + std::string headername = outname + ".hdr";
  519 +
  520 + unsigned long long L = X();
  521 + T* line = (T*)malloc(sizeof(T) * L);
  522 +
  523 + for ( unsigned long long y = 0; y < Y(); y++) //for each y position
  524 + {
  525 + file.seekg(y * X() * sizeof(T), std::ios::beg); //seek to the beginning of the xz slice
  526 + for ( unsigned long long z = 0; z < Z(); z++ ) //for each band
  527 + {
  528 + file.read((char *)line, sizeof(T) * X()); //read a line
  529 + target.write((char*)line, sizeof(T) * X()); //write the line to the output file
  530 + file.seekg(jump, std::ios::cur); //seek to the next band
  531 + if(PROGRESS) progress = (double)((y+1) * Z() + z + 1) / (Z() * Y()) * 100; //update the progress counter
  532 + }
  533 + }
  534 +
  535 + free(line);
  536 + target.close();
  537 +
  538 + return true;
  539 + }*/
427 540  
428 541 /// Return a baseline corrected band given two adjacent baseline points and their bands. The result is stored in a pre-allocated array.
429 542  
... ...