Commit 5e1b4869d074e1b17683a3b9d1a7f0d40c033375
1 parent
008ae990
BSQ->BIL performance increased for multiple drives in Windows
Showing
1 changed file
with
137 additions
and
24 deletions
Show diff stats
stim/envi/bsq.h
@@ -376,54 +376,167 @@ public: | @@ -376,54 +376,167 @@ public: | ||
376 | 376 | ||
377 | } | 377 | } |
378 | 378 | ||
379 | - /// Convert the current BSQ file to a BIL file with the specified file name. | ||
380 | bool bil(std::string outname, bool PROGRESS = false){ | 379 | bool bil(std::string outname, bool PROGRESS = false){ |
381 | - size_t XY = X() * Y(); //number of elements in a band | ||
382 | - size_t XYbytes = XY * sizeof(T); //number of bytes in a band | ||
383 | - size_t batchB = binary<T>::buffer_size / (XYbytes); //calculate the number of slices that can fit in memory | ||
384 | - if(Z() < batchB) batchB = Z(); //if the entire data set will fit in memory, do it | ||
385 | - size_t batchXB = X() * batchB; //number of elements in a batch | ||
386 | - T* ptrBatch = (T*) malloc(batchB * XYbytes); //allocate a large buffer storing the read data | ||
387 | - T* ptrSlice = (T*) malloc(sizeof(T) * batchB * X()); //allocate space for storing an output buffer | 380 | + size_t XY = X() * Y(); //number of elements in an input slice |
381 | + size_t XB = X() * Z(); //number of elements in an output slice | ||
382 | + size_t XYbytes = XY * sizeof(T); //number of bytes in an input slice | ||
383 | + size_t XBbytes = XB * sizeof(T); //number of bytes in an output slice | ||
384 | + size_t batch_slices = binary<T>::buffer_size / (2*XBbytes); //calculate the number of slices that can fit in memory | ||
385 | + if(Y() < batch_slices) batch_slices = Y(); //if the entire data set will fit in memory, do it | ||
386 | + size_t batchN = XB * batch_slices; //number of elements in a batch | ||
387 | + size_t batch_bytes = batchN * sizeof(T); //calculate the number of bytes in a batch | ||
388 | 388 | ||
389 | - size_t jump = (Z() - batchB) * X() * sizeof(T); //jump between writes in the output file | 389 | + T* ptrIn = (T*) malloc(batch_bytes); //allocate a large buffer storing the read data |
390 | + T* ptrOut = (T*) malloc(batch_bytes); //allocate space for storing an output buffer | ||
391 | + | ||
392 | + size_t jump = (Y() - batch_slices) * X() * sizeof(T); //jump between reads in the input file | ||
393 | + | ||
394 | + std::ofstream target(outname.c_str(), std::ios::binary); | ||
395 | + std::string headername = outname + ".hdr"; | ||
396 | + | ||
397 | + size_t batches = ceil((double)(Y()) / (double)batch_slices); //calculate the number of batches | ||
398 | + T* ptrDst; | ||
399 | + T* ptrSrc; | ||
400 | + for(size_t c = 0; c < batches; c++){ | ||
401 | + file.seekg(c * X() * batch_slices * sizeof(T), std::ios::beg); | ||
402 | + | ||
403 | + if(c == (batches - 1)){ | ||
404 | + batch_slices = Y() - (batches - 1) * batch_slices; //if this is the last batch, calculate the remaining # of bands | ||
405 | + jump = (Y() - batch_slices) * X() * sizeof(T); | ||
406 | + batchN = XB * batch_slices; | ||
407 | + batch_bytes = batchN * sizeof(T); | ||
408 | + } | ||
409 | + | ||
410 | + auto in_begin = std::chrono::high_resolution_clock::now(); | ||
411 | + for(size_t b = 0; b < Z(); b++){ | ||
412 | + file.read((char*)(ptrIn + b * X() * batch_slices), sizeof(T) * X() * batch_slices); //read a number of lines equal to "batch_slices" | ||
413 | + file.seekg(jump, std::ios::cur); //jump to the next band | ||
414 | + } | ||
415 | + auto in_end = std::chrono::high_resolution_clock::now(); | ||
416 | + std::cout << std::chrono::duration_cast<std::chrono::milliseconds>(in_end-in_begin).count() << "ms" << std::endl; | ||
417 | + | ||
418 | + auto calc_begin = std::chrono::high_resolution_clock::now(); | ||
419 | + | ||
420 | + for(size_t b = 0; b < Z(); b++){ //for each line, store an XB slice in ptrDest | ||
421 | + ptrSrc = ptrIn + (b * X() * batch_slices); | ||
422 | + ptrDst = ptrOut + (b * X()); //initialize ptrDst to the start of the XB output slice | ||
423 | + | ||
424 | + for(size_t y = 0; y < batch_slices; y++){ //for each band in the current line | ||
425 | + memcpy(ptrDst, ptrSrc, X() * sizeof(T)); //copy the band line from the source to the destination | ||
426 | + ptrSrc += X(); //increment the pointer within the current buffer array (batch) | ||
427 | + ptrDst += X() * Z(); //increment the pointer within the XB slice (to be output) | ||
428 | + } | ||
429 | + } | ||
430 | + auto calc_end = std::chrono::high_resolution_clock::now(); | ||
431 | + std::cout << std::chrono::duration_cast<std::chrono::milliseconds>(calc_end-calc_begin).count() << "ms" << std::endl; | ||
432 | + | ||
433 | + auto out_begin = std::chrono::high_resolution_clock::now(); | ||
434 | + target.write((char*)ptrOut, batch_bytes); //write the batch to disk | ||
435 | + auto out_end = std::chrono::high_resolution_clock::now(); | ||
436 | + std::cout << std::chrono::duration_cast<std::chrono::milliseconds>(out_end-out_begin).count() << "ms" << std::endl; | ||
437 | + if(PROGRESS) progress = (double)( c + 1 ) / (batches) * 100; | ||
438 | + } | ||
439 | + | ||
440 | + free(ptrIn); | ||
441 | + free(ptrOut); | ||
442 | + target.close(); | ||
443 | + | ||
444 | + return true; | ||
445 | + } | ||
446 | + | ||
447 | + /*/// Convert the current BSQ file to a BIL file with the specified file name. | ||
448 | + bool bil(std::string outname, bool PROGRESS = false){ | ||
449 | + size_t XY = X() * Y(); //number of elements in an input slice | ||
450 | + size_t XB = X() * Z(); //number of elements in an output slice | ||
451 | + size_t XYbytes = XY * sizeof(T); //number of bytes in an input slice | ||
452 | + size_t XBbytes = XB * sizeof(T); //number of bytes in an output slice | ||
453 | + size_t batch_bands = binary<T>::buffer_size / (2*XYbytes); //calculate the number of slices that can fit in memory | ||
454 | + if(Z() < batch_bands) batch_bands = Z(); //if the entire data set will fit in memory, do it | ||
455 | + size_t batchXB = X() * batch_bands; //number of elements in a batch | ||
456 | + | ||
457 | + size_t batch_bytes = batch_bands * XYbytes; //calculate the number of bytes in a batch | ||
458 | + T* ptrIn = (T*) malloc(batch_bytes); //allocate a large buffer storing the read data | ||
459 | + T* ptrOut = (T*) malloc(batch_bytes); //allocate space for storing an output buffer | ||
460 | + | ||
461 | + size_t jump = (Z() - batch_bands) * X() * sizeof(T); //jump between writes in the output file | ||
390 | 462 | ||
391 | std::ofstream target(outname.c_str(), std::ios::binary); | 463 | std::ofstream target(outname.c_str(), std::ios::binary); |
392 | std::string headername = outname + ".hdr"; | 464 | std::string headername = outname + ".hdr"; |
393 | 465 | ||
394 | - size_t batches = ceil((double)(Z()) / (double)batchB); //calculate the number of batches | 466 | + size_t batches = ceil((double)(Z()) / (double)batch_bands); //calculate the number of batches |
395 | T* ptrDst; | 467 | T* ptrDst; |
396 | T* ptrSrc; | 468 | T* ptrSrc; |
397 | for(size_t c = 0; c < batches; c++){ | 469 | for(size_t c = 0; c < batches; c++){ |
398 | - target.seekp(c * batchB * sizeof(T) * X(), std::ios::beg); //seek to the start of the current batch in the output file | ||
399 | - file.read((char*)ptrBatch, sizeof(T) * X() * Y() * batchB); //read a batch | 470 | + auto in_begin = std::chrono::high_resolution_clock::now(); |
471 | + target.seekp(c * batch_bands * sizeof(T) * X(), std::ios::beg); //seek to the start of the current batch in the output file | ||
472 | + file.read((char*)ptrIn, sizeof(T) * X() * Y() * batch_bands); //read a batch | ||
473 | + auto in_end = std::chrono::high_resolution_clock::now(); | ||
474 | + std::cout << std::chrono::duration_cast<std::chrono::milliseconds>(in_end-in_begin).count() << "ms" << std::endl; | ||
400 | 475 | ||
401 | - auto pbegin = std::chrono::high_resolution_clock::now(); | 476 | + auto calc_begin = std::chrono::high_resolution_clock::now(); |
402 | if(c == (batches - 1)){ | 477 | if(c == (batches - 1)){ |
403 | - batchB = Z() - (batches - 1) * batchB; //if this is the last batch, calculate the remaining # of bands | ||
404 | - jump = (Z() - batchB) * X() * sizeof(T); | 478 | + batch_bands = Z() - (batches - 1) * batch_bands; //if this is the last batch, calculate the remaining # of bands |
479 | + jump = (Z() - batch_bands) * X() * sizeof(T); | ||
405 | } | 480 | } |
406 | for(size_t y = 0; y < Y(); y++){ //for each line, store an XB slice in ptrDest | 481 | for(size_t y = 0; y < Y(); y++){ //for each line, store an XB slice in ptrDest |
407 | - ptrDst = ptrSlice; //initialize ptrDst to the start of the XB output slice | ||
408 | - ptrSrc = ptrBatch + (y * X()); | ||
409 | - for(size_t b = 0; b < batchB; b++){ //for each band in the current line | 482 | + ptrDst = ptrOut + (y * X() * batch_bands); //initialize ptrDst to the start of the XB output slice |
483 | + ptrSrc = ptrIn + (y * X()); | ||
484 | + for(size_t b = 0; b < batch_bands; b++){ //for each band in the current line | ||
410 | memcpy(ptrDst, ptrSrc, X() * sizeof(T)); //copy the band line from the source to the destination | 485 | memcpy(ptrDst, ptrSrc, X() * sizeof(T)); //copy the band line from the source to the destination |
411 | ptrDst += X(); //increment the pointer within the XB slice (to be output) | 486 | ptrDst += X(); //increment the pointer within the XB slice (to be output) |
412 | ptrSrc += X() * Y(); //increment the pointer within the current buffer array (batch) | 487 | ptrSrc += X() * Y(); //increment the pointer within the current buffer array (batch) |
413 | } | 488 | } |
414 | - | ||
415 | - target.write((char*)ptrSlice, sizeof(T) * X() * batchB); //write the XB slice to disk | 489 | + } |
490 | + auto calc_end = std::chrono::high_resolution_clock::now(); | ||
491 | + std::cout << std::chrono::duration_cast<std::chrono::milliseconds>(calc_end-calc_begin).count() << "ms" << std::endl; | ||
492 | + | ||
493 | + auto out_begin = std::chrono::high_resolution_clock::now(); | ||
494 | + target.seekp(0, std::ios::beg); | ||
495 | + for(size_t y = 0; y < Y(); y++){ //for each y-slice | ||
496 | + target.write((char*)(ptrOut + y * X() * batch_bands), sizeof(T) * X() * batch_bands); //write the XB slice to disk | ||
416 | target.seekp(jump, std::ios::cur); //seek to the beginning of the next XB slice in the batch | 497 | target.seekp(jump, std::ios::cur); //seek to the beginning of the next XB slice in the batch |
417 | - if(PROGRESS) progress = (double)( c * Y() + y + 1 ) / (Y() * batches) * 100; | ||
418 | } | 498 | } |
499 | + auto out_end = std::chrono::high_resolution_clock::now(); | ||
500 | + std::cout << std::chrono::duration_cast<std::chrono::milliseconds>(out_end-out_begin).count() << "ms" << std::endl; | ||
501 | + if(PROGRESS) progress = (double)( c + 1 ) / (batches) * 100; | ||
419 | } | 502 | } |
420 | 503 | ||
421 | - free(ptrBatch); | ||
422 | - free(ptrSlice); | 504 | + free(ptrIn); |
505 | + free(ptrOut); | ||
423 | target.close(); | 506 | target.close(); |
424 | 507 | ||
425 | return true; | 508 | return true; |
426 | - } | 509 | + }*/ |
510 | + /* | ||
511 | + /// @param outname is the name of the output BIL file to be saved to disk. | ||
512 | + bool bil(std::string outname, bool PROGRESS = false) | ||
513 | + { | ||
514 | + //simplify image resolution | ||
515 | + unsigned long long jump = (Y() - 1) * X() * sizeof(T); | ||
516 | + | ||
517 | + std::ofstream target(outname.c_str(), std::ios::binary); | ||
518 | + std::string headername = outname + ".hdr"; | ||
519 | + | ||
520 | + unsigned long long L = X(); | ||
521 | + T* line = (T*)malloc(sizeof(T) * L); | ||
522 | + | ||
523 | + for ( unsigned long long y = 0; y < Y(); y++) //for each y position | ||
524 | + { | ||
525 | + file.seekg(y * X() * sizeof(T), std::ios::beg); //seek to the beginning of the xz slice | ||
526 | + for ( unsigned long long z = 0; z < Z(); z++ ) //for each band | ||
527 | + { | ||
528 | + file.read((char *)line, sizeof(T) * X()); //read a line | ||
529 | + target.write((char*)line, sizeof(T) * X()); //write the line to the output file | ||
530 | + file.seekg(jump, std::ios::cur); //seek to the next band | ||
531 | + if(PROGRESS) progress = (double)((y+1) * Z() + z + 1) / (Z() * Y()) * 100; //update the progress counter | ||
532 | + } | ||
533 | + } | ||
534 | + | ||
535 | + free(line); | ||
536 | + target.close(); | ||
537 | + | ||
538 | + return true; | ||
539 | + }*/ | ||
427 | 540 | ||
428 | /// Return a baseline corrected band given two adjacent baseline points and their bands. The result is stored in a pre-allocated array. | 541 | /// Return a baseline corrected band given two adjacent baseline points and their bands. The result is stored in a pre-allocated array. |
429 | 542 |