Commit 8b899c24248604acddbaefeebd1a6a8e0bdd9dcf
1 parent
8cfba32f
fixed asynchronous read bugs
Showing
4 changed files
with
142 additions
and
1 deletions
Show diff stats
stim/biomodels/cellset.h
@@ -15,6 +15,7 @@ protected: | @@ -15,6 +15,7 @@ protected: | ||
15 | std::vector<double*> cells; //vector storing field data for each cell | 15 | std::vector<double*> cells; //vector storing field data for each cell |
16 | std::unordered_map<std::string, size_t> fields; //unordered map storing field->index information for each field | 16 | std::unordered_map<std::string, size_t> fields; //unordered map storing field->index information for each field |
17 | size_t ip[3]; //hard code to position indices (for speed) | 17 | size_t ip[3]; //hard code to position indices (for speed) |
18 | + | ||
18 | void init(){ | 19 | void init(){ |
19 | 20 | ||
20 | } | 21 | } |
@@ -80,6 +81,13 @@ public: | @@ -80,6 +81,13 @@ public: | ||
80 | return cells[c][idx]; | 81 | return cells[c][idx]; |
81 | } | 82 | } |
82 | 83 | ||
84 | + /// returns an ID used to look up a field | ||
85 | + bool exists(std::string f){ | ||
86 | + std::unordered_map<std::string, size_t>::iterator iter = fields.find(f); | ||
87 | + if(iter == fields.end()) return false; | ||
88 | + else return true; | ||
89 | + } | ||
90 | + | ||
83 | /// Return the position of cell [i] | 91 | /// Return the position of cell [i] |
84 | stim::vec3<double> p(size_t i){ | 92 | stim::vec3<double> p(size_t i){ |
85 | stim::vec3<double> pos(cells[i][ip[0]], cells[i][ip[1]], cells[i][ip[2]]); | 93 | stim::vec3<double> pos(cells[i][ip[0]], cells[i][ip[1]], cells[i][ip[2]]); |
stim/envi/binary.h
@@ -403,6 +403,48 @@ public: | @@ -403,6 +403,48 @@ public: | ||
403 | return read_pixel(p, i); | 403 | return read_pixel(p, i); |
404 | } | 404 | } |
405 | 405 | ||
406 | + /// Reads a block specified by an (x, y, z) position and size using the largest possible contiguous reads | ||
407 | + bool read_block(T* dest, size_t x, size_t y, size_t z, size_t sx, size_t sy, size_t sz){ | ||
408 | + | ||
409 | + size_t size_bytes = sx * sy * sz * sizeof(T); //size of the block to read in bytes | ||
410 | + | ||
411 | + size_t start = z * R[0] * R[1] + y * R[0] + x; //calculate the start postion | ||
412 | + size_t start_bytes = start * sizeof(T); //start position in bytes | ||
413 | + file.seekg(start * sizeof(T), std::ios::beg); //seek to the start position | ||
414 | + | ||
415 | + | ||
416 | + if(sx == R[0] && sy == R[1]){ //if sx and sy result in a contiguous volume along z | ||
417 | + file.read((char*)dest, size_bytes); //read the block in one pass | ||
418 | + return true; | ||
419 | + } | ||
420 | + | ||
421 | + if(sx == R[0]){ //if sx is contiguous, read each z-axis slice can be read in one pass | ||
422 | + size_t jump_bytes = (R[1] - sy) * R[0] * sizeof(T); //jump between each slice | ||
423 | + size_t slice_bytes = sx * sy * sizeof(T); //size of the slice to be read | ||
424 | + for(size_t zi = 0; zi < sz; zi++){ //for each z-axis slice | ||
425 | + file.read((char*)dest, slice_bytes); //read the slice | ||
426 | + dest += sx * sy; //move the destination pointer to the next slice | ||
427 | + file.seekg(jump_bytes, std::ios::cur); //skip to the next slice in the file | ||
428 | + } | ||
429 | + return true; | ||
430 | + } | ||
431 | + | ||
432 | + //in this case, x is not contiguous so the volume must be read line-by-line | ||
433 | + size_t jump_x_bytes = (R[0] - sx) * sizeof(T); //number of bytes skipped in the x direction | ||
434 | + size_t jump_y_bytes = (R[1] - sy) * R[0] * sizeof(T) + jump_x_bytes; //number of bytes skipped between slices | ||
435 | + size_t line_bytes = sx * sizeof(T); //size of the line to be read | ||
436 | + size_t zi, yi; | ||
437 | + for(zi = 0; zi < sz; zi++){ //for each slice | ||
438 | + file.read((char*)dest, line_bytes); //read the first line | ||
439 | + for(yi = 1; yi < sy; yi++){ //read each additional line | ||
440 | + dest += sx; //move the pointer in the destination block to the next line | ||
441 | + file.seekg(jump_x_bytes, std::ios::cur); //skip to the next line in the file | ||
442 | + file.read((char*)dest, line_bytes); //read the line to the destination block | ||
443 | + } | ||
444 | + file.seekg(jump_y_bytes, std::ios::cur); //skip to the beginning of the next slice | ||
445 | + } | ||
446 | + } | ||
447 | + | ||
406 | }; | 448 | }; |
407 | 449 | ||
408 | } | 450 | } |
stim/envi/bsq.h
@@ -401,6 +401,79 @@ public: | @@ -401,6 +401,79 @@ public: | ||
401 | size_t batches = (size_t)ceil((double)(Y()) / (double)batch_slices); //calculate the number of batches | 401 | size_t batches = (size_t)ceil((double)(Y()) / (double)batch_slices); //calculate the number of batches |
402 | T* ptrDst; | 402 | T* ptrDst; |
403 | T* ptrSrc; | 403 | T* ptrSrc; |
404 | + size_t y = 0; //initialize the current y-slice position | ||
405 | + for(size_t c = 0; c < batches; c++){ | ||
406 | + if(c == (batches - 1)){ | ||
407 | + batch_slices = Y() - (batches - 1) * batch_slices; //if this is the last batch, calculate the remaining # of bands | ||
408 | + jump = (Y() - batch_slices) * X() * sizeof(T); | ||
409 | + batchN = XB * batch_slices; | ||
410 | + batch_bytes = batchN * sizeof(T); | ||
411 | + } | ||
412 | + | ||
413 | + auto in_begin = std::chrono::high_resolution_clock::now(); | ||
414 | + hsi<T>::read_block(ptrIn, 0, y, 0, X(), batch_slices, Z()); //read the input block | ||
415 | + y += batch_slices; | ||
416 | + auto in_end = std::chrono::high_resolution_clock::now(); | ||
417 | + in_time += std::chrono::duration_cast<std::chrono::milliseconds>(in_end-in_begin).count(); | ||
418 | + | ||
419 | + auto calc_begin = std::chrono::high_resolution_clock::now(); | ||
420 | + | ||
421 | + for(size_t b = 0; b < Z(); b++){ //for each line, store an XB slice in ptrDest | ||
422 | + ptrSrc = ptrIn + (b * X() * batch_slices); | ||
423 | + ptrDst = ptrOut + (b * X()); //initialize ptrDst to the start of the XB output slice | ||
424 | + | ||
425 | + for(size_t y = 0; y < batch_slices; y++){ //for each band in the current line | ||
426 | + memcpy(ptrDst, ptrSrc, X() * sizeof(T)); //copy the band line from the source to the destination | ||
427 | + ptrSrc += X(); //increment the pointer within the current buffer array (batch) | ||
428 | + ptrDst += X() * Z(); //increment the pointer within the XB slice (to be output) | ||
429 | + } | ||
430 | + } | ||
431 | + auto calc_end = std::chrono::high_resolution_clock::now(); | ||
432 | + calc_time += std::chrono::duration_cast<std::chrono::milliseconds>(calc_end-calc_begin).count(); | ||
433 | + | ||
434 | + auto out_begin = std::chrono::high_resolution_clock::now(); | ||
435 | + target.write((char*)ptrOut, batch_bytes); //write the batch to disk | ||
436 | + auto out_end = std::chrono::high_resolution_clock::now(); | ||
437 | + out_time += std::chrono::duration_cast<std::chrono::milliseconds>(out_end-out_begin).count(); | ||
438 | + if(PROGRESS) progress = (double)( c + 1 ) / (batches) * 100; | ||
439 | + } | ||
440 | + | ||
441 | + free(ptrIn); | ||
442 | + free(ptrOut); | ||
443 | + target.close(); | ||
444 | + | ||
445 | + std::cout<<"BSQ->BIL reads: "<<(double)in_time / (1000 * 60)<<" min"<<std::endl; | ||
446 | + std::cout<<"BSQ->BIL calculations: "<<(double)calc_time / (1000 * 60)<<" min"<<std::endl; | ||
447 | + std::cout<<"BSQ->BIL writes: "<<(double)out_time / (1000 * 60)<<" min"<<std::endl; | ||
448 | + | ||
449 | + return true; | ||
450 | + } | ||
451 | + | ||
452 | + /*bool bil(std::string outname, bool PROGRESS = false){ | ||
453 | + | ||
454 | + size_t in_time, out_time, calc_time; //initialize the timing variables | ||
455 | + in_time = out_time = calc_time = 0; | ||
456 | + | ||
457 | + size_t XY = X() * Y(); //number of elements in an input slice | ||
458 | + size_t XB = X() * Z(); //number of elements in an output slice | ||
459 | + size_t XYbytes = XY * sizeof(T); //number of bytes in an input slice | ||
460 | + size_t XBbytes = XB * sizeof(T); //number of bytes in an output slice | ||
461 | + size_t batch_slices = binary<T>::buffer_size / (2*XBbytes); //calculate the number of slices that can fit in memory | ||
462 | + if(Y() < batch_slices) batch_slices = Y(); //if the entire data set will fit in memory, do it | ||
463 | + size_t batchN = XB * batch_slices; //number of elements in a batch | ||
464 | + size_t batch_bytes = batchN * sizeof(T); //calculate the number of bytes in a batch | ||
465 | + | ||
466 | + T* ptrIn = (T*) malloc(batch_bytes); //allocate a large buffer storing the read data | ||
467 | + T* ptrOut = (T*) malloc(batch_bytes); //allocate space for storing an output buffer | ||
468 | + | ||
469 | + size_t jump = (Y() - batch_slices) * X() * sizeof(T); //jump between reads in the input file | ||
470 | + | ||
471 | + std::ofstream target(outname.c_str(), std::ios::binary); | ||
472 | + std::string headername = outname + ".hdr"; | ||
473 | + | ||
474 | + size_t batches = (size_t)ceil((double)(Y()) / (double)batch_slices); //calculate the number of batches | ||
475 | + T* ptrDst; | ||
476 | + T* ptrSrc; | ||
404 | for(size_t c = 0; c < batches; c++){ | 477 | for(size_t c = 0; c < batches; c++){ |
405 | file.seekg(c * X() * batch_slices * sizeof(T), std::ios::beg); | 478 | file.seekg(c * X() * batch_slices * sizeof(T), std::ios::beg); |
406 | 479 | ||
@@ -450,7 +523,7 @@ public: | @@ -450,7 +523,7 @@ public: | ||
450 | std::cout<<"BSQ->BIL writes: "<<(double)out_time / (1000 * 60)<<" min"<<std::endl; | 523 | std::cout<<"BSQ->BIL writes: "<<(double)out_time / (1000 * 60)<<" min"<<std::endl; |
451 | 524 | ||
452 | return true; | 525 | return true; |
453 | - } | 526 | + }*/ |
454 | 527 | ||
455 | /*/// Convert the current BSQ file to a BIL file with the specified file name. | 528 | /*/// Convert the current BSQ file to a BIL file with the specified file name. |
456 | bool bil(std::string outname, bool PROGRESS = false){ | 529 | bool bil(std::string outname, bool PROGRESS = false){ |
stim/envi/hsi.h
@@ -202,6 +202,24 @@ public: | @@ -202,6 +202,24 @@ public: | ||
202 | } | 202 | } |
203 | } | 203 | } |
204 | 204 | ||
205 | + void read_block(T* dest, size_t x, size_t y, size_t z, size_t sx, size_t sy, size_t sz){ | ||
206 | + size_t d[3]; //position in the binary coordinate system | ||
207 | + size_t sd[3]; //size in the binary coordinate system | ||
208 | + | ||
209 | + d[O[0]] = x; //set the point in the binary coordinate system | ||
210 | + d[O[1]] = y; | ||
211 | + d[O[2]] = z; | ||
212 | + | ||
213 | + sd[O[0]] = sx; //set the size in the binary coordinate system | ||
214 | + sd[O[1]] = sy; | ||
215 | + sd[O[2]] = sz; | ||
216 | + | ||
217 | + if(!binary<T>::read_block(dest, d[0], d[1], d[2], sd[0], sd[1], sd[2])){ | ||
218 | + std::cout<<"error reading block in stim::hsi: ("<<d[0]<<", "<<d[1]<<", "<<d[2]<<") - ["<<sd[0]<<", "<<sd[1]<<", "<<sd[2]<<"]"<<std::endl; | ||
219 | + exit(1); | ||
220 | + } | ||
221 | + } | ||
222 | + | ||
205 | }; | 223 | }; |
206 | 224 | ||
207 | } //end namespace STIM | 225 | } //end namespace STIM |