Commit cf5b4c92000edfba0d9a10f99bb46a0d7f0689ae
1 parent
846b3f18
simplified asynchronous code for BIL and BIP conversion
Showing
3 changed files
with
154 additions
and
242 deletions
Show diff stats
stim/envi/binary.h
@@ -446,6 +446,53 @@ public: | @@ -446,6 +446,53 @@ public: | ||
446 | return false; | 446 | return false; |
447 | } | 447 | } |
448 | 448 | ||
449 | + // permutes a block of data from the current interleave to the interleave specified (re-arranged dimensions to the order specified by [d0, d1, d2]) | ||
450 | + | ||
451 | + void permute(T* dest, T* src, size_t sx, size_t sy, size_t sz, size_t d0, size_t d1, size_t d2){ | ||
452 | + size_t d[3] = {d0, d1, d2}; | ||
453 | + size_t s[3] = {sx, sy, sz}; | ||
454 | + size_t p[3];// = {x, y, z}; | ||
455 | + | ||
456 | + if(d[0] == 0 && d[1] == 1 && d[2] == 2){ | ||
457 | + //this isn't actually a permute - just copy the data | ||
458 | + memcpy(dest, src, sizeof(T) * sx * sy * sz); | ||
459 | + } | ||
460 | + else if(d[0] == 0){ //the individual lines are contiguous, so you can memcpy line-by-line | ||
461 | + size_t y, z; | ||
462 | + size_t src_idx, dest_idx; | ||
463 | + size_t x_bytes = sizeof(T) * sx; | ||
464 | + for(z = 0; z < sz; z++){ | ||
465 | + p[2] = z; | ||
466 | + for(y = 0; y < sy; y++){ | ||
467 | + p[1] = y; | ||
468 | + src_idx = z * sx * sy + y * sx; | ||
469 | + dest_idx = p[d[2]] * s[d[0]] * s[d[1]] + p[d[1]] * s[d[0]]; | ||
470 | + //std::cout<<z<<", "<<y<<" ------- "<<p[d[2]]<<" * "<<s[d[0]]<<" * "<<s[d[1]]<<" + "<<p[d[1]]<<" * "<<s[d[0]]<<std::endl; | ||
471 | + memcpy(dest + dest_idx, src + src_idx, x_bytes); | ||
472 | + } | ||
473 | + } | ||
474 | + } | ||
475 | + else{ //loop through every damn point | ||
476 | + size_t x, y, z; | ||
477 | + size_t src_idx, dest_idx; | ||
478 | + size_t src_z, src_y; | ||
479 | + for(z = 0; z < sz; z++){ | ||
480 | + p[2] = z; | ||
481 | + src_z = z * sx * sy; | ||
482 | + for(y = 0; y < sy; y++){ | ||
483 | + p[1] = y; | ||
484 | + src_y = src_z + y * sx; | ||
485 | + for(x = 0; x < sx; x++){ | ||
486 | + p[0] = x; | ||
487 | + src_idx = src_y + x; | ||
488 | + dest_idx = p[d[2]] * s[d[0]] * s[d[1]] + p[d[1]] * s[d[0]] + p[d[0]]; | ||
489 | + dest[dest_idx] = src[src_idx]; | ||
490 | + } | ||
491 | + } | ||
492 | + } | ||
493 | + } | ||
494 | + } | ||
495 | + | ||
449 | }; | 496 | }; |
450 | 497 | ||
451 | } | 498 | } |
stim/envi/bsq.h
@@ -381,276 +381,141 @@ public: | @@ -381,276 +381,141 @@ public: | ||
381 | hsi<T>::read(dest, 0, start, 0, X(), n, Z()); | 381 | hsi<T>::read(dest, 0, start, 0, X(), n, Z()); |
382 | } | 382 | } |
383 | 383 | ||
384 | + /// Convert this BSQ file to a BIL | ||
384 | bool bil(std::string outname, bool PROGRESS = false){ | 385 | bool bil(std::string outname, bool PROGRESS = false){ |
385 | 386 | ||
386 | - size_t in_time, out_time, calc_time; //initialize the timing variables | ||
387 | - in_time = out_time = calc_time = 0; | ||
388 | - | ||
389 | - size_t XY = X() * Y(); //number of elements in an input slice | ||
390 | - size_t XB = X() * Z(); //number of elements in an output slice | ||
391 | - size_t XYbytes = XY * sizeof(T); //number of bytes in an input slice | ||
392 | - size_t XBbytes = XB * sizeof(T); //number of bytes in an output slice | ||
393 | - size_t batch_slices[2]; | ||
394 | - batch_slices[0] = binary<T>::buffer_size / (4*XBbytes); //calculate the number of slices that can fit in memory | ||
395 | - batch_slices[1] = batch_slices[0]; | ||
396 | - if(batch_slices == 0){ | 387 | + const size_t buffers = 4; //number of buffers required for this algorithm |
388 | + size_t mem_per_batch = binary<T>::buffer_size / buffers; //calculate the maximum memory available for a batch | ||
389 | + | ||
390 | + size_t slice_bytes = X() * Z() * sizeof(T); //number of bytes in an input batch slice (Y-slice in this case) | ||
391 | + size_t max_slices_per_batch = mem_per_batch / slice_bytes; //maximum number of slices we can process in one batch given memory constraints | ||
392 | + if(max_slices_per_batch == 0){ //if there is insufficient memory for a single slice, throw an error | ||
397 | std::cout<<"error, insufficient memory for stim::bsq::bil()"<<std::endl; | 393 | std::cout<<"error, insufficient memory for stim::bsq::bil()"<<std::endl; |
398 | exit(1); | 394 | exit(1); |
399 | } | 395 | } |
400 | - if(Y() < batch_slices[0]) batch_slices[0] = Y(); //if the entire data set will fit in memory, do it | ||
401 | - size_t batchN = XB * batch_slices[0]; //number of elements in a batch | ||
402 | - size_t batch_bytes = batchN * sizeof(T); //calculate the number of bytes in a batch | ||
403 | - | ||
404 | - //T* ptrIn = (T*) malloc(batch_bytes); //allocate a large buffer storing the read data | ||
405 | - //T* ptrOut = (T*) malloc(batch_bytes); //allocate space for storing an output buffer | ||
406 | - T* ptrIn[2]; //input double-buffer for asynchronous batching | ||
407 | - ptrIn[0] = (T*) malloc(batch_bytes); | ||
408 | - ptrIn[1] = (T*) malloc(batch_bytes); | ||
409 | - T* ptrOut[2]; //output double-buffer for asynchronous batching | ||
410 | - ptrOut[0] = (T*) malloc(batch_bytes); | ||
411 | - ptrOut[1] = (T*) malloc(batch_bytes); | ||
412 | - | ||
413 | - size_t jump = (Y() - batch_slices[0]) * X() * sizeof(T); //jump between reads in the input file | ||
414 | - | ||
415 | - std::ofstream target(outname.c_str(), std::ios::binary); | ||
416 | - std::string headername = outname + ".hdr"; | ||
417 | - | ||
418 | - size_t batches = (size_t)ceil((double)(Y()) / (double)batch_slices[0]); //calculate the number of batches | ||
419 | - T* ptrDst; | ||
420 | - T* ptrSrc; | ||
421 | - size_t y = 0; //initialize the current y-slice position | ||
422 | - int i = 0; | 396 | + size_t max_batch_bytes = max_slices_per_batch * slice_bytes; //calculate the amount of memory that will be allocated for all four buffers |
397 | + | ||
398 | + T* src[2]; //source double-buffer for asynchronous batching | ||
399 | + src[0] = (T*) malloc(max_batch_bytes); | ||
400 | + src[1] = (T*) malloc(max_batch_bytes); | ||
401 | + T* dst[2]; //destination double-buffer for asynchronous batching | ||
402 | + dst[0] = (T*) malloc(max_batch_bytes); | ||
403 | + dst[1] = (T*) malloc(max_batch_bytes); | ||
404 | + | ||
405 | + size_t N[2]; //number of slices stored in buffers 0 and 1 | ||
406 | + N[0] = N[1] = min(Y(), max_slices_per_batch); //start with the maximum number of slices that can be stored (may be the entire data set) | ||
407 | + | ||
408 | + std::ofstream target(outname.c_str(), std::ios::binary); //open an output file for writing | ||
409 | + //initialize with buffer 0 (used for double buffering) | ||
410 | + size_t y_load = 0; | ||
411 | + size_t y_proc = 0; | ||
423 | std::future<void> rthread; | 412 | std::future<void> rthread; |
413 | + std::future<std::ostream&> wthread; //create asynchronous threads for reading and writing | ||
424 | 414 | ||
425 | - readlines(ptrIn[0], 0, batch_slices[0]); | ||
426 | - y += batch_slices[i]; | ||
427 | - | ||
428 | - std::future<std::ostream&> wthread; | 415 | + readlines(src[0], 0, N[0]); //read the first batch into the 0 source buffer |
416 | + y_load += N[0]; //increment the loaded slice counter | ||
417 | + int b = 1; | ||
429 | 418 | ||
430 | std::chrono::high_resolution_clock::time_point t_start; //high-resolution timers | 419 | std::chrono::high_resolution_clock::time_point t_start; //high-resolution timers |
431 | std::chrono::high_resolution_clock::time_point t_end; | 420 | std::chrono::high_resolution_clock::time_point t_end; |
432 | size_t t_batch; //number of milliseconds to process a batch | 421 | size_t t_batch; //number of milliseconds to process a batch |
433 | size_t t_total = 0; | 422 | size_t t_total = 0; |
434 | - | ||
435 | - for(size_t c = 0; c < batches; c++){ | 423 | + while(y_proc < Y()){ //while there are still slices to be processed |
436 | t_start = std::chrono::high_resolution_clock::now(); //start the timer for this batch | 424 | t_start = std::chrono::high_resolution_clock::now(); //start the timer for this batch |
437 | - if(c == (batches - 2)){ | ||
438 | - batch_slices[!i] = Y() - (batches - 1) * batch_slices[!i]; //if this is the last batch, calculate the remaining # of bands | ||
439 | - } | ||
440 | - jump = (Y() - batch_slices[i]) * X() * sizeof(T); | ||
441 | - batchN = XB * batch_slices[i]; | ||
442 | - batch_bytes = batchN * sizeof(T); | ||
443 | - | ||
444 | - rthread = std::async(&stim::bsq<T>::readlines, this, ptrIn[!i], y, batch_slices[!i]); //start reading the next batch | ||
445 | - y += batch_slices[i]; | ||
446 | - | ||
447 | - for(size_t b = 0; b < Z(); b++){ //for each line, store an XB slice in ptrDest | ||
448 | - ptrSrc = ptrIn[i] + (b * X() * batch_slices[i]); | ||
449 | - ptrDst = ptrOut[i] + (b * X()); //initialize ptrDst to the start of the XB output slice | 425 | + if(y_load < Y()){ //if there are still slices to be loaded, load them |
426 | + if(y_load + N[b] > Y()) N[b] = Y() - y_load; //if the next batch would process more than the total slices, adjust the batch size | ||
427 | + rthread = std::async(std::launch::async, &stim::bsq<T>::readlines, this, src[b], y_load, N[b]); | ||
450 | 428 | ||
451 | - for(size_t y = 0; y < batch_slices[i]; y++){ //for each band in the current line | ||
452 | - memcpy(ptrDst, ptrSrc, X() * sizeof(T)); //copy the band line from the source to the destination | ||
453 | - ptrSrc += X(); //increment the pointer within the current buffer array (batch) | ||
454 | - ptrDst += X() * Z(); //increment the pointer within the XB slice (to be output) | ||
455 | - } | 429 | + y_load += N[b]; //increment the number of loaded slices |
456 | } | 430 | } |
457 | - | ||
458 | - wthread = std::async( &std::fstream::write, &target, (char*)ptrOut[i], batch_bytes); | ||
459 | 431 | ||
460 | - if(PROGRESS) progress = (double)( c + 1 ) / (batches) * 100; | ||
461 | - i = !i; | 432 | + b = !b; //swap the double-buffer |
462 | 433 | ||
463 | - rthread.wait(); | ||
464 | - wthread.wait(); | 434 | + binary<T>::permute(dst[b], src[b], X(), N[b], Z(), 0, 2, 1); //permute the batch to a BIL file |
435 | + target.write((char*)dst[b], N[b] * slice_bytes); //write the permuted data to the output file | ||
436 | + y_proc += N[b]; //increment the counter of processed pixels | ||
437 | + if(PROGRESS) progress = (double)( y_proc + 1 ) / Y() * 100; //increment the progress counter based on the number of processed pixels | ||
465 | t_end = std::chrono::high_resolution_clock::now(); | 438 | t_end = std::chrono::high_resolution_clock::now(); |
466 | t_batch = std::chrono::duration_cast<std::chrono::milliseconds>(t_end-t_start).count(); | 439 | t_batch = std::chrono::duration_cast<std::chrono::milliseconds>(t_end-t_start).count(); |
467 | t_total += t_batch; | 440 | t_total += t_batch; |
441 | + rthread.wait(); | ||
468 | } | 442 | } |
469 | - | ||
470 | - std::cout<<"Total time to execute: "<<t_total<<" ms"<<std::endl; | ||
471 | 443 | ||
472 | - free(ptrIn[0]); | ||
473 | - free(ptrIn[1]); | ||
474 | - free(ptrOut[0]); | ||
475 | - free(ptrOut[1]); | ||
476 | - target.close(); | ||
477 | - | ||
478 | - //std::cout<<"BSQ->BIL reads: "<<(double)in_time / (1000 * 60)<<" min"<<std::endl; | ||
479 | - //std::cout<<"BSQ->BIL calculations: "<<(double)calc_time / (1000 * 60)<<" min"<<std::endl; | ||
480 | - //std::cout<<"BSQ->BIL writes: "<<(double)out_time / (1000 * 60)<<" min"<<std::endl; | ||
481 | - | ||
482 | - return true; | 444 | + std::cout<<"Total time to execute: "<<t_total<<" ms"<<std::endl; |
445 | + free(src[0]); //free buffer resources | ||
446 | + free(src[1]); | ||
447 | + free(dst[0]); | ||
448 | + free(dst[1]); | ||
449 | + return true; //return true | ||
483 | } | 450 | } |
484 | 451 | ||
485 | - /*bool bil(std::string outname, bool PROGRESS = false){ | ||
486 | - | ||
487 | - size_t in_time, out_time, calc_time; //initialize the timing variables | ||
488 | - in_time = out_time = calc_time = 0; | ||
489 | - | ||
490 | - size_t XY = X() * Y(); //number of elements in an input slice | ||
491 | - size_t XB = X() * Z(); //number of elements in an output slice | ||
492 | - size_t XYbytes = XY * sizeof(T); //number of bytes in an input slice | ||
493 | - size_t XBbytes = XB * sizeof(T); //number of bytes in an output slice | ||
494 | - size_t batch_slices = binary<T>::buffer_size / (2*XBbytes); //calculate the number of slices that can fit in memory | ||
495 | - if(Y() < batch_slices) batch_slices = Y(); //if the entire data set will fit in memory, do it | ||
496 | - size_t batchN = XB * batch_slices; //number of elements in a batch | ||
497 | - size_t batch_bytes = batchN * sizeof(T); //calculate the number of bytes in a batch | ||
498 | - | ||
499 | - T* ptrIn = (T*) malloc(batch_bytes); //allocate a large buffer storing the read data | ||
500 | - T* ptrOut = (T*) malloc(batch_bytes); //allocate space for storing an output buffer | 452 | + /// Convert this BSQ file to a BIP |
453 | + bool bip(std::string outname, bool PROGRESS = false){ | ||
501 | 454 | ||
502 | - size_t jump = (Y() - batch_slices) * X() * sizeof(T); //jump between reads in the input file | 455 | + const size_t buffers = 4; //number of buffers required for this algorithm |
456 | + size_t mem_per_batch = binary<T>::buffer_size / buffers; //calculate the maximum memory available for a batch | ||
503 | 457 | ||
504 | - std::ofstream target(outname.c_str(), std::ios::binary); | ||
505 | - std::string headername = outname + ".hdr"; | ||
506 | - | ||
507 | - size_t batches = (size_t)ceil((double)(Y()) / (double)batch_slices); //calculate the number of batches | ||
508 | - T* ptrDst; | ||
509 | - T* ptrSrc; | ||
510 | - for(size_t c = 0; c < batches; c++){ | ||
511 | - file.seekg(c * X() * batch_slices * sizeof(T), std::ios::beg); | ||
512 | - | ||
513 | - if(c == (batches - 1)){ | ||
514 | - batch_slices = Y() - (batches - 1) * batch_slices; //if this is the last batch, calculate the remaining # of bands | ||
515 | - jump = (Y() - batch_slices) * X() * sizeof(T); | ||
516 | - batchN = XB * batch_slices; | ||
517 | - batch_bytes = batchN * sizeof(T); | ||
518 | - } | ||
519 | - | ||
520 | - auto in_begin = std::chrono::high_resolution_clock::now(); | ||
521 | - for(size_t b = 0; b < Z(); b++){ | ||
522 | - file.read((char*)(ptrIn + b * X() * batch_slices), sizeof(T) * X() * batch_slices); //read a number of lines equal to "batch_slices" | ||
523 | - file.seekg(jump, std::ios::cur); //jump to the next band | ||
524 | - } | ||
525 | - auto in_end = std::chrono::high_resolution_clock::now(); | ||
526 | - in_time += std::chrono::duration_cast<std::chrono::milliseconds>(in_end-in_begin).count(); | ||
527 | - | ||
528 | - auto calc_begin = std::chrono::high_resolution_clock::now(); | ||
529 | - | ||
530 | - for(size_t b = 0; b < Z(); b++){ //for each line, store an XB slice in ptrDest | ||
531 | - ptrSrc = ptrIn + (b * X() * batch_slices); | ||
532 | - ptrDst = ptrOut + (b * X()); //initialize ptrDst to the start of the XB output slice | ||
533 | - | ||
534 | - for(size_t y = 0; y < batch_slices; y++){ //for each band in the current line | ||
535 | - memcpy(ptrDst, ptrSrc, X() * sizeof(T)); //copy the band line from the source to the destination | ||
536 | - ptrSrc += X(); //increment the pointer within the current buffer array (batch) | ||
537 | - ptrDst += X() * Z(); //increment the pointer within the XB slice (to be output) | ||
538 | - } | ||
539 | - } | ||
540 | - auto calc_end = std::chrono::high_resolution_clock::now(); | ||
541 | - calc_time += std::chrono::duration_cast<std::chrono::milliseconds>(calc_end-calc_begin).count(); | ||
542 | - | ||
543 | - auto out_begin = std::chrono::high_resolution_clock::now(); | ||
544 | - target.write((char*)ptrOut, batch_bytes); //write the batch to disk | ||
545 | - auto out_end = std::chrono::high_resolution_clock::now(); | ||
546 | - out_time += std::chrono::duration_cast<std::chrono::milliseconds>(out_end-out_begin).count(); | ||
547 | - if(PROGRESS) progress = (double)( c + 1 ) / (batches) * 100; | 458 | + size_t slice_bytes = X() * Z() * sizeof(T); //number of bytes in an input batch slice (Y-slice in this case) |
459 | + size_t max_slices_per_batch = mem_per_batch / slice_bytes; //maximum number of slices we can process in one batch given memory constraints | ||
460 | + if(max_slices_per_batch == 0){ //if there is insufficient memory for a single slice, throw an error | ||
461 | + std::cout<<"error, insufficient memory for stim::bsq::bil()"<<std::endl; | ||
462 | + exit(1); | ||
548 | } | 463 | } |
464 | + size_t max_batch_bytes = max_slices_per_batch * slice_bytes; //calculate the amount of memory that will be allocated for all four buffers | ||
465 | + | ||
466 | + T* src[2]; //source double-buffer for asynchronous batching | ||
467 | + src[0] = (T*) malloc(max_batch_bytes); | ||
468 | + src[1] = (T*) malloc(max_batch_bytes); | ||
469 | + T* dst[2]; //destination double-buffer for asynchronous batching | ||
470 | + dst[0] = (T*) malloc(max_batch_bytes); | ||
471 | + dst[1] = (T*) malloc(max_batch_bytes); | ||
472 | + | ||
473 | + size_t N[2]; //number of slices stored in buffers 0 and 1 | ||
474 | + N[0] = N[1] = min(Y(), max_slices_per_batch); //start with the maximum number of slices that can be stored (may be the entire data set) | ||
475 | + | ||
476 | + std::ofstream target(outname.c_str(), std::ios::binary); //open an output file for writing | ||
477 | + //initialize with buffer 0 (used for double buffering) | ||
478 | + size_t y_load = 0; | ||
479 | + size_t y_proc = 0; | ||
480 | + std::future<void> rthread; | ||
481 | + std::future<std::ostream&> wthread; //create asynchronous threads for reading and writing | ||
549 | 482 | ||
550 | - free(ptrIn); | ||
551 | - free(ptrOut); | ||
552 | - target.close(); | ||
553 | - | ||
554 | - std::cout<<"BSQ->BIL reads: "<<(double)in_time / (1000 * 60)<<" min"<<std::endl; | ||
555 | - std::cout<<"BSQ->BIL calculations: "<<(double)calc_time / (1000 * 60)<<" min"<<std::endl; | ||
556 | - std::cout<<"BSQ->BIL writes: "<<(double)out_time / (1000 * 60)<<" min"<<std::endl; | 483 | + readlines(src[0], 0, N[0]); //read the first batch into the 0 source buffer |
484 | + y_load += N[0]; //increment the loaded slice counter | ||
485 | + int b = 1; | ||
557 | 486 | ||
558 | - return true; | ||
559 | - }*/ | ||
560 | - | ||
561 | - /*/// Convert the current BSQ file to a BIL file with the specified file name. | ||
562 | - bool bil(std::string outname, bool PROGRESS = false){ | ||
563 | - size_t XY = X() * Y(); //number of elements in an input slice | ||
564 | - size_t XB = X() * Z(); //number of elements in an output slice | ||
565 | - size_t XYbytes = XY * sizeof(T); //number of bytes in an input slice | ||
566 | - size_t XBbytes = XB * sizeof(T); //number of bytes in an output slice | ||
567 | - size_t batch_bands = binary<T>::buffer_size / (2*XYbytes); //calculate the number of slices that can fit in memory | ||
568 | - if(Z() < batch_bands) batch_bands = Z(); //if the entire data set will fit in memory, do it | ||
569 | - size_t batchXB = X() * batch_bands; //number of elements in a batch | ||
570 | - | ||
571 | - size_t batch_bytes = batch_bands * XYbytes; //calculate the number of bytes in a batch | ||
572 | - T* ptrIn = (T*) malloc(batch_bytes); //allocate a large buffer storing the read data | ||
573 | - T* ptrOut = (T*) malloc(batch_bytes); //allocate space for storing an output buffer | ||
574 | - | ||
575 | - size_t jump = (Z() - batch_bands) * X() * sizeof(T); //jump between writes in the output file | ||
576 | - | ||
577 | - std::ofstream target(outname.c_str(), std::ios::binary); | ||
578 | - std::string headername = outname + ".hdr"; | ||
579 | - | ||
580 | - size_t batches = ceil((double)(Z()) / (double)batch_bands); //calculate the number of batches | ||
581 | - T* ptrDst; | ||
582 | - T* ptrSrc; | ||
583 | - for(size_t c = 0; c < batches; c++){ | ||
584 | - auto in_begin = std::chrono::high_resolution_clock::now(); | ||
585 | - target.seekp(c * batch_bands * sizeof(T) * X(), std::ios::beg); //seek to the start of the current batch in the output file | ||
586 | - file.read((char*)ptrIn, sizeof(T) * X() * Y() * batch_bands); //read a batch | ||
587 | - auto in_end = std::chrono::high_resolution_clock::now(); | ||
588 | - std::cout << std::chrono::duration_cast<std::chrono::milliseconds>(in_end-in_begin).count() << "ms" << std::endl; | ||
589 | - | ||
590 | - auto calc_begin = std::chrono::high_resolution_clock::now(); | ||
591 | - if(c == (batches - 1)){ | ||
592 | - batch_bands = Z() - (batches - 1) * batch_bands; //if this is the last batch, calculate the remaining # of bands | ||
593 | - jump = (Z() - batch_bands) * X() * sizeof(T); | ||
594 | - } | ||
595 | - for(size_t y = 0; y < Y(); y++){ //for each line, store an XB slice in ptrDest | ||
596 | - ptrDst = ptrOut + (y * X() * batch_bands); //initialize ptrDst to the start of the XB output slice | ||
597 | - ptrSrc = ptrIn + (y * X()); | ||
598 | - for(size_t b = 0; b < batch_bands; b++){ //for each band in the current line | ||
599 | - memcpy(ptrDst, ptrSrc, X() * sizeof(T)); //copy the band line from the source to the destination | ||
600 | - ptrDst += X(); //increment the pointer within the XB slice (to be output) | ||
601 | - ptrSrc += X() * Y(); //increment the pointer within the current buffer array (batch) | ||
602 | - } | ||
603 | - } | ||
604 | - auto calc_end = std::chrono::high_resolution_clock::now(); | ||
605 | - std::cout << std::chrono::duration_cast<std::chrono::milliseconds>(calc_end-calc_begin).count() << "ms" << std::endl; | ||
606 | - | ||
607 | - auto out_begin = std::chrono::high_resolution_clock::now(); | ||
608 | - target.seekp(0, std::ios::beg); | ||
609 | - for(size_t y = 0; y < Y(); y++){ //for each y-slice | ||
610 | - target.write((char*)(ptrOut + y * X() * batch_bands), sizeof(T) * X() * batch_bands); //write the XB slice to disk | ||
611 | - target.seekp(jump, std::ios::cur); //seek to the beginning of the next XB slice in the batch | 487 | + std::chrono::high_resolution_clock::time_point t_start; //high-resolution timers |
488 | + std::chrono::high_resolution_clock::time_point t_end; | ||
489 | + size_t t_batch; //number of milliseconds to process a batch | ||
490 | + size_t t_total = 0; | ||
491 | + while(y_proc < Y()){ //while there are still slices to be processed | ||
492 | + t_start = std::chrono::high_resolution_clock::now(); //start the timer for this batch | ||
493 | + if(y_load < Y()){ //if there are still slices to be loaded, load them | ||
494 | + if(y_load + N[b] > Y()) N[b] = Y() - y_load; //if the next batch would process more than the total slices, adjust the batch size | ||
495 | + rthread = std::async(std::launch::async, &stim::bsq<T>::readlines, this, src[b], y_load, N[b]); | ||
496 | + | ||
497 | + y_load += N[b]; //increment the number of loaded slices | ||
612 | } | 498 | } |
613 | - auto out_end = std::chrono::high_resolution_clock::now(); | ||
614 | - std::cout << std::chrono::duration_cast<std::chrono::milliseconds>(out_end-out_begin).count() << "ms" << std::endl; | ||
615 | - if(PROGRESS) progress = (double)( c + 1 ) / (batches) * 100; | ||
616 | - } | ||
617 | - | ||
618 | - free(ptrIn); | ||
619 | - free(ptrOut); | ||
620 | - target.close(); | ||
621 | 499 | ||
622 | - return true; | ||
623 | - }*/ | ||
624 | - /* | ||
625 | - /// @param outname is the name of the output BIL file to be saved to disk. | ||
626 | - bool bil(std::string outname, bool PROGRESS = false) | ||
627 | - { | ||
628 | - //simplify image resolution | ||
629 | - unsigned long long jump = (Y() - 1) * X() * sizeof(T); | 500 | + b = !b; //swap the double-buffer |
630 | 501 | ||
631 | - std::ofstream target(outname.c_str(), std::ios::binary); | ||
632 | - std::string headername = outname + ".hdr"; | ||
633 | - | ||
634 | - unsigned long long L = X(); | ||
635 | - T* line = (T*)malloc(sizeof(T) * L); | ||
636 | - | ||
637 | - for ( unsigned long long y = 0; y < Y(); y++) //for each y position | ||
638 | - { | ||
639 | - file.seekg(y * X() * sizeof(T), std::ios::beg); //seek to the beginning of the xz slice | ||
640 | - for ( unsigned long long z = 0; z < Z(); z++ ) //for each band | ||
641 | - { | ||
642 | - file.read((char *)line, sizeof(T) * X()); //read a line | ||
643 | - target.write((char*)line, sizeof(T) * X()); //write the line to the output file | ||
644 | - file.seekg(jump, std::ios::cur); //seek to the next band | ||
645 | - if(PROGRESS) progress = (double)((y+1) * Z() + z + 1) / (Z() * Y()) * 100; //update the progress counter | ||
646 | - } | 502 | + binary<T>::permute(dst[b], src[b], X(), N[b], Z(), 2, 0, 1); //permute the batch to a BIP file |
503 | + target.write((char*)dst[b], N[b] * slice_bytes); //write the permuted data to the output file | ||
504 | + y_proc += N[b]; //increment the counter of processed pixels | ||
505 | + if(PROGRESS) progress = (double)( y_proc + 1 ) / Y() * 100; //increment the progress counter based on the number of processed pixels | ||
506 | + t_end = std::chrono::high_resolution_clock::now(); | ||
507 | + t_batch = std::chrono::duration_cast<std::chrono::milliseconds>(t_end-t_start).count(); | ||
508 | + t_total += t_batch; | ||
509 | + rthread.wait(); | ||
647 | } | 510 | } |
648 | 511 | ||
649 | - free(line); | ||
650 | - target.close(); | ||
651 | - | ||
652 | - return true; | ||
653 | - }*/ | 512 | + std::cout<<"Total time to execute: "<<t_total<<" ms"<<std::endl; |
513 | + free(src[0]); //free buffer resources | ||
514 | + free(src[1]); | ||
515 | + free(dst[0]); | ||
516 | + free(dst[1]); | ||
517 | + return true; //return true | ||
518 | + } | ||
654 | 519 | ||
655 | /// Return a baseline corrected band given two adjacent baseline points and their bands. The result is stored in a pre-allocated array. | 520 | /// Return a baseline corrected band given two adjacent baseline points and their bands. The result is stored in a pre-allocated array. |
656 | 521 |
stim/envi/envi.h
@@ -521,9 +521,9 @@ public: | @@ -521,9 +521,9 @@ public: | ||
521 | else if(interleave == envi_header::BIL) //convert BSQ -> BIL | 521 | else if(interleave == envi_header::BIL) //convert BSQ -> BIL |
522 | ((bsq<float>*)file)->bil(outfile, PROGRESS); | 522 | ((bsq<float>*)file)->bil(outfile, PROGRESS); |
523 | else if(interleave == envi_header::BIP){ //ERROR | 523 | else if(interleave == envi_header::BIP){ //ERROR |
524 | - std::cout<<"ERROR: conversion from BSQ to BIP isn't practical; use BSQ->BIL->BIP instead"<<std::endl; | ||
525 | - //return ((bsq<float>*)file)->bip(outfile, PROGRESS); | ||
526 | - exit(1); | 524 | + //std::cout<<"ERROR: conversion from BSQ to BIP isn't practical; use BSQ->BIL->BIP instead"<<std::endl; |
525 | + ((bsq<float>*)file)->bip(outfile, PROGRESS); | ||
526 | + //exit(1); | ||
527 | } | 527 | } |
528 | } | 528 | } |
529 | 529 | ||
@@ -535,9 +535,9 @@ public: | @@ -535,9 +535,9 @@ public: | ||
535 | else if(interleave == envi_header::BIL) //convert BSQ -> BIL | 535 | else if(interleave == envi_header::BIL) //convert BSQ -> BIL |
536 | ((bsq<double>*)file)->bil(outfile, PROGRESS); | 536 | ((bsq<double>*)file)->bil(outfile, PROGRESS); |
537 | else if(interleave == envi_header::BIP){ //ERROR | 537 | else if(interleave == envi_header::BIP){ //ERROR |
538 | - std::cout<<"ERROR: conversion from BSQ to BIP isn't practical; use BSQ->BIL->BIP instead"<<std::endl; | ||
539 | - //return ((bsq<float>*)file)->bip(outfile, PROGRESS); | ||
540 | - exit(1); | 538 | + //std::cout<<"ERROR: conversion from BSQ to BIP isn't practical; use BSQ->BIL->BIP instead"<<std::endl; |
539 | + ((bsq<float>*)file)->bip(outfile, PROGRESS); | ||
540 | + //exit(1); | ||
541 | } | 541 | } |
542 | } | 542 | } |
543 | 543 |