18368aa9
David Mayerich
added a new set o...
|
1
2
3
4
5
6
7
8
9
10
11
|
# -*- coding: utf-8 -*-
"""
Created on Fri Jul 21 20:18:01 2017
@author: david
"""
import os
import numpy
import scipy
import matplotlib.pyplot as plt
|
59f31ea4
David Mayerich
fixed bugs in env...
|
12
|
import sys
|
35534291
sberisha
envi.py: added fu...
|
13
14
|
from math import floor
import progressbar
|
18368aa9
David Mayerich
added a new set o...
|
15
16
17
18
|
class envi_header:
def __init__(self, filename = ""):
if filename != "":
|
6691c66c
David Mayerich
added a save func...
|
19
|
self.initialize()
|
18368aa9
David Mayerich
added a new set o...
|
20
21
22
23
24
25
26
27
28
29
30
31
|
self.load(filename)
else:
self.initialize()
#initialization function
def initialize(self):
self.samples = int(0)
self.lines = int(0)
self.bands = int(0)
self.header_offset = int(0)
self.data_type = int(4)
self.interleave = "bsq"
|
403139ae
David Mayerich
added the ability...
|
32
|
self.sensor_type = "Unknown"
|
18368aa9
David Mayerich
added a new set o...
|
33
34
35
|
self.byte_order = int(0)
self.x_start = int(0)
self.y_start = int(0)
|
403139ae
David Mayerich
added the ability...
|
36
|
self.z_plot_titles = "Unknown, Unknown"
|
18368aa9
David Mayerich
added a new set o...
|
37
38
39
|
self.pixel_size = [float(0), float(0)]
self.pixel_size_units = "Meters"
self.wavelength_units = "Wavenumber"
|
403139ae
David Mayerich
added the ability...
|
40
|
self.description = "no description"
|
18368aa9
David Mayerich
added a new set o...
|
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
|
self.band_names = []
self.wavelength = []
#convert an ENVI data_type value to a numpy data type
def get_numpy_type(self, val):
if val == 1:
return numpy.byte
elif val == 2:
return numpy.int16
elif val == 3:
return numpy.int32
elif val == 4:
return numpy.float32
elif val == 5:
return numpy.float64
elif val == 6:
return numpy.complex64
elif val == 9:
return numpy.complex128
elif val == 12:
return numpy.uint16
elif val == 13:
return numpy.uint32
elif val == 14:
return numpy.int64
elif val == 15:
return numpy.uint64
def get_envi_type(self, val):
if val == numpy.byte:
return 1
elif val == numpy.int16:
return 2
elif val == numpy.int32:
return 3
elif val == numpy.float32:
return 4
elif val == numpy.float64:
return 5
elif val == numpy.complex64:
return 6
elif val == numpy.complex128:
return 9
elif val == numpy.uint16:
return 12
elif val == numpy.uint32:
return 13
elif val == numpy.int64:
return 14
elif val == numpy.uint64:
return 15
def load(self, fname):
f = open(fname)
l = f.readlines()
if l[0].strip() != "ENVI":
print("ERROR: not an ENVI file")
return
li = 1
while li < len(l):
#t = l[li].split() #split the line into tokens
#t = map(str.strip, t) #strip all of the tokens in the token list
#handle the simple conditions
|
9b3cbdda
David Mayerich
changed the name ...
|
105
106
107
108
109
|
#if l[li].startswith("file type"):
# if not l[li].strip().endswith("ENVI Standard"):
# print("ERROR: unsupported ENVI file format: " + l[li].strip())
# return
if l[li].startswith("samples"):
|
18368aa9
David Mayerich
added a new set o...
|
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
|
self.samples = int(l[li].split()[-1])
elif l[li].startswith("lines"):
self.lines = int(l[li].split()[-1])
elif l[li].startswith("bands"):
self.bands = int(l[li].split()[-1])
elif l[li].startswith("header offset"):
self.header_offset = int(l[li].split()[-1])
elif l[li].startswith("data type"):
self.data_type = self.get_numpy_type(int(l[li].split()[-1]))
elif l[li].startswith("interleave"):
self.interleave = l[li].split()[-1].strip()
elif l[li].startswith("sensor type"):
self.sensor_type = l[li].split()[-1].strip()
elif l[li].startswith("byte order"):
self.byte_order = int(l[li].split()[-1])
elif l[li].startswith("x start"):
self.x_start = int(l[li].split()[-1])
elif l[li].startswith("y start"):
self.y_start = int(l[li].split()[-1])
elif l[li].startswith("z plot titles"):
i0 = l[li].rindex('{')
i1 = l[li].rindex('}')
self.z_plot_titles = l[li][i0 + 1 : i1]
elif l[li].startswith("pixel size"):
i0 = l[li].rindex('{')
i1 = l[li].rindex('}')
s = l[li][i0 + 1 : i1].split(',')
self.pixel_size = [float(s[0]), float(s[1])]
self.pixel_size_units = s[2][s[2].rindex('=') + 1:].strip()
elif l[li].startswith("wavelength units"):
self.wavelength_units = l[li].split()[-1].strip()
#handle the complicated conditions
elif l[li].startswith("description"):
desc = [l[li]]
|
6b2be991
sberisha
added utility fun...
|
145
146
147
148
149
150
151
152
153
|
'''
while l[li].strip()[-1] != '}': #will fail if l[li].strip() is empty
li += 1
desc.append(l[li])
'''
while True:
if l[li].strip():
if l[li].strip()[-1] == '}':
break
|
18368aa9
David Mayerich
added a new set o...
|
154
155
|
li += 1
desc.append(l[li])
|
6b2be991
sberisha
added utility fun...
|
156
|
|
18368aa9
David Mayerich
added a new set o...
|
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
|
desc = ''.join(list(map(str.strip, desc))) #strip all white space from the string list
i0 = desc.rindex('{')
i1 = desc.rindex('}')
self.description = desc[i0 + 1 : i1]
elif l[li].startswith("band names"):
names = [l[li]]
while l[li].strip()[-1] != '}':
li += 1
names.append(l[li])
names = ''.join(list(map(str.strip, names))) #strip all white space from the string list
i0 = names.rindex('{')
i1 = names.rindex('}')
names = names[i0 + 1 : i1]
self.band_names = list(map(str.strip, names.split(',')))
elif l[li].startswith("wavelength"):
waves = [l[li]]
while l[li].strip()[-1] != '}':
li += 1
waves.append(l[li])
waves = ''.join(list(map(str.strip, waves))) #strip all white space from the string list
i0 = waves.rindex('{')
i1 = waves.rindex('}')
waves = waves[i0 + 1 : i1]
self.wavelength = list(map(float, waves.split(',')))
li += 1
f.close()
|
403139ae
David Mayerich
added the ability...
|
186
187
188
|
#save an ENVI header
def save(self, fname):
|
59b0e5f5
David Mayerich
updated envi file...
|
189
190
191
192
193
194
195
196
|
f = open(fname, "w")
f.write("ENVI\n")
f.write("description = {" + self.description + "}" + "\n")
f.write("samples = " + str(self.samples) + "\n")
f.write("lines = " + str(self.lines) + "\n")
f.write("bands = " + str(self.bands) + "\n")
f.write("header offset = " + str(self.header_offset) + "\n")
f.write("file type = ENVI Standard" + "\n")
|
6691c66c
David Mayerich
added a save func...
|
197
|
f.write("data type = " + str(self.get_envi_type(self.data_type)) + "\n")
|
59b0e5f5
David Mayerich
updated envi file...
|
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
|
f.write("interleave = " + self.interleave + "\n")
f.write("sensor type = " + self.sensor_type + "\n")
f.write("byte order = " + str(self.byte_order) + "\n")
f.write("x start = " + str(self.x_start) + "\n")
f.write("y start = " + str(self.y_start) + "\n")
f.write("wavelength units = " + self.wavelength_units + "\n")
f.write("z plot titles = {" + self.z_plot_titles + "}" + "\n")
# save the wavelength values
if self.wavelength != []:
if len(self.wavelength) == self.bands:
f.write("wavelength = {")
f.write(",".join(map(str, self.wavelength)))
f.write("}\n")
else:
raise Exception("ENVI HEADER ERROR: Number of wavelengths does not match number of bands")
|
403139ae
David Mayerich
added the ability...
|
214
|
|
59b0e5f5
David Mayerich
updated envi file...
|
215
|
f.close()
|
403139ae
David Mayerich
added the ability...
|
216
217
|
#sets the properties of the header to match those of the input array
|
59b0e5f5
David Mayerich
updated envi file...
|
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
|
def setprops(self, A, interleave="BSQ", wavelength=[]):
# determine the data type automatically
self.type = A.dtype
# determine the ordering based on the specified interleave
if interleave == "BSQ":
self.samples = A.shape[2]
self.lines = A.shape[1]
self.bands = A.shape[0]
elif interleave == "BIP":
self.samples = A.shape[1]
self.lines = A.shape[2]
self.bands = A.shape[0]
elif interleave == "BIL":
self.samples = A.shape[0]
self.lines = A.shape[2]
self.bands = A.shape[1]
else:
raise Exception("invalid interleave format (requires 'BSQ', 'BIP', or 'BIL') - interleave is set to {}".interleave)
# if wavelength units are given, make sure that they match the number of bands
if wavelength != []:
if len(wavelength) != self.bands:
raise Exception("invalid number of wavelengths specified")
else:
self.wavelength = wavelength
|
403139ae
David Mayerich
added the ability...
|
245
|
|
18368aa9
David Mayerich
added a new set o...
|
246
247
|
class envi:
|
71ee1c23
David Mayerich
added additional ...
|
248
|
def __init__(self, filename, headername = "", mask = []):
|
18368aa9
David Mayerich
added a new set o...
|
249
|
self.open(filename, headername)
|
71ee1c23
David Mayerich
added additional ...
|
250
|
if mask == []:
|
777fef6c
David Mayerich
added digital sta...
|
251
|
self.mask = numpy.ones((self.header.lines, self.header.samples), dtype=numpy.bool)
|
ed3d7d30
David Mayerich
updated NWT file ...
|
252
|
elif type(mask) == numpy.ndarray:
|
71ee1c23
David Mayerich
added additional ...
|
253
|
self.mask = mask
|
ed3d7d30
David Mayerich
updated NWT file ...
|
254
255
|
else:
print("ERROR: unrecognized mask format - expecting a boolean array")
|
71ee1c23
David Mayerich
added additional ...
|
256
|
self.idx = 0 #initialize the batch IDX to 0 for batch reading
|
18368aa9
David Mayerich
added a new set o...
|
257
|
|
18368aa9
David Mayerich
added a new set o...
|
258
259
260
261
262
263
264
265
266
267
268
269
270
271
|
def open(self, filename, headername = ""):
if headername == "":
headername = filename + ".hdr"
if not os.path.isfile(filename):
print("ERROR: " + filename + " not found")
return
if not os.path.isfile(headername):
print("ERROR: " + headername + " not found")
return
#open the file
self.header = envi_header(headername)
self.file = open(filename, "rb")
|
6691c66c
David Mayerich
added a save func...
|
272
273
|
# load the entire ENVI file into memory and return it as an array
|
18368aa9
David Mayerich
added a new set o...
|
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
|
def loadall(self):
X = self.header.samples
Y = self.header.lines
B = self.header.bands
#load the data
D = numpy.fromfile(self.file, dtype=self.header.data_type)
if self.header.interleave == "bsq":
return numpy.reshape(D, (B, Y, X))
#return numpy.swapaxes(D, 0, 2)
elif self.header.interleave == "bip":
D = numpy.reshape(D, (Y, X, B))
return numpy.rollaxis(D, 2)
elif self.header.interleave == "bil":
D = numpy.reshape(D, (Y, B, X))
return numpy.rollaxis(D, 1)
|
6691c66c
David Mayerich
added a save func...
|
291
292
293
294
295
296
297
298
299
|
#save an updated version of the file (all header information is assumed to be the same)
def saveall(self, D, filename):
new_header = self.header
new_header.interleave = "bsq"
new_header.save(filename + ".hdr")
D.tofile(filename)
|
18368aa9
David Mayerich
added a new set o...
|
300
301
302
303
304
305
306
307
308
309
310
|
#loads all of the pixels where mask != 0 and returns them as a matrix
def loadmask(self, mask):
X = self.header.samples
Y = self.header.lines
B = self.header.bands
P = numpy.count_nonzero(mask) #count the number of zeros in the mask file
M = numpy.zeros((B, P), dtype=self.header.data_type)
type_bytes = numpy.dtype(self.header.data_type).itemsize
|
71ee1c23
David Mayerich
added additional ...
|
311
|
prev_pos = self.file.tell()
|
18368aa9
David Mayerich
added a new set o...
|
312
313
314
315
316
317
|
self.file.seek(0)
if self.header.interleave == "bip":
spectrum = numpy.zeros(B, dtype=self.header.data_type)
flatmask = numpy.reshape(mask, (X * Y))
i = numpy.flatnonzero(flatmask)
bar = progressbar.ProgressBar(max_value = P)
|
35534291
sberisha
envi.py: added fu...
|
318
|
#bar = pyprind.ProgBar(P)
|
18368aa9
David Mayerich
added a new set o...
|
319
320
321
322
323
|
for p in range(0, P):
self.file.seek(i[p] * B * type_bytes)
self.file.readinto(spectrum)
M[:, p] = spectrum
bar.update(p+1)
|
35534291
sberisha
envi.py: added fu...
|
324
|
#bar.update()
|
59f31ea4
David Mayerich
fixed bugs in env...
|
325
|
elif self.header.interleave == "bsq":
|
18368aa9
David Mayerich
added a new set o...
|
326
327
328
|
band = numpy.zeros(mask.shape, dtype=self.header.data_type)
i = numpy.nonzero(mask)
bar = progressbar.ProgressBar(max_value=B)
|
35534291
sberisha
envi.py: added fu...
|
329
|
#bar = pyprind.ProgBar(P)
|
18368aa9
David Mayerich
added a new set o...
|
330
331
332
333
334
|
for b in range(0, B):
self.file.seek(b * X * Y * type_bytes)
self.file.readinto(band)
M[b, :] = band[i]
bar.update(b+1)
|
35534291
sberisha
envi.py: added fu...
|
335
|
#bar.update()
|
59f31ea4
David Mayerich
fixed bugs in env...
|
336
|
elif self.header.interleave == "bil":
|
18368aa9
David Mayerich
added a new set o...
|
337
338
339
|
plane = numpy.zeros((B, X), dtype=self.header.data_type)
p = 0
bar = progressbar.ProgressBar(max_value=Y)
|
35534291
sberisha
envi.py: added fu...
|
340
|
#bar = pyprind.ProgBar(P)
|
18368aa9
David Mayerich
added a new set o...
|
341
342
343
344
345
346
|
for l in range(0, Y):
i = numpy.flatnonzero(mask[l, :])
self.file.readinto(plane)
M[:, p:p+i.shape[0]] = plane[:, i]
p = p + i.shape[0]
bar.update(l+1)
|
35534291
sberisha
envi.py: added fu...
|
347
|
#bar.update()
|
71ee1c23
David Mayerich
added additional ...
|
348
|
self.file.seek(prev_pos)
|
18368aa9
David Mayerich
added a new set o...
|
349
|
return M
|
90c935e3
David Mayerich
updates
|
350
351
352
353
354
355
356
357
|
def loadband(self, n):
X = self.header.samples
Y = self.header.lines
B = self.header.bands
band = numpy.zeros((Y, X), dtype=self.header.data_type)
type_bytes = numpy.dtype(self.header.data_type).itemsize
|
71ee1c23
David Mayerich
added additional ...
|
358
359
|
prev_pos = self.file.tell()
|
90c935e3
David Mayerich
updates
|
360
361
362
|
if self.header.interleave == "bsq":
self.file.seek(n * X * Y * type_bytes)
self.file.readinto(band)
|
71ee1c23
David Mayerich
added additional ...
|
363
|
self.file.seek(prev_pos)
|
90c935e3
David Mayerich
updates
|
364
365
|
return band
|
f275c01c
David Mayerich
finalized the pre...
|
366
367
368
369
|
#create a set of feature/target pairs for classification
#input: envi file object, stack of class masks C x Y x X
#output: feature matrix (features x pixels), target matrix (1 x pixels)
#example: generate_training(("class_coll.bmp", "class_epith.bmp"), (1, 2))
|
ed3d7d30
David Mayerich
updated NWT file ...
|
370
371
|
# verify verify that there are no NaN or Inf values
def loadtrain(self, classimages, verify=True):
|
f275c01c
David Mayerich
finalized the pre...
|
372
373
374
375
376
377
378
|
# get number of classes
C = classimages.shape[0]
F = []
T = []
for c in range(0, C):
|
59f31ea4
David Mayerich
fixed bugs in env...
|
379
|
print("\nLoading class " + str(c+1) + "...")
|
f275c01c
David Mayerich
finalized the pre...
|
380
381
382
383
|
f = self.loadmask(classimages[c, :, :]) #load the feature matrix for class c
t = numpy.ones((f.shape[1])) * (c+1) #generate a target array
F.append(f)
T.append(t)
|
ed3d7d30
David Mayerich
updated NWT file ...
|
384
385
|
return numpy.nan_to_num(numpy.concatenate(F, 1).transpose()), numpy.concatenate(T)
|
35534291
sberisha
envi.py: added fu...
|
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
|
#create a set of feature/target pairs for classification with balanced data
#input: envi file object, stack of class masks C x Y x X, number of samples per class
#output: feature matrix (features x pixels), target matrix (1 x pixels)
#example: generate_training(("class_coll.bmp", "class_epith.bmp"), (1, 2))
# verify verify that there are no NaN or Inf values
def loadtrain_balance(self, classimages, num_samples=None):
# get number of classes
C = classimages.shape[0]
F = []
T = []
# get number of samples per class
samples_per_class = numpy.zeros(C, dtype=numpy.int32)
for c in range(0, C):
if num_samples is None:
samples_per_class[c] = numpy.count_nonzero(classimages[c, :, :])
else:
# if user has specified a max number of samples per class
if num_samples > numpy.count_nonzero(classimages[c, :, :]):
samples_per_class[c] = numpy.count_nonzero(classimages[c, :, :])
else:
samples_per_class[c] = num_samples
for c in range(0, C):
print("\nLoading class " + str(c+1) + "...")
# row, col index of valid pixels
temp = classimages[c,:]
flat_temp = numpy.reshape(temp, temp.shape[0]*temp.shape[1])
idx = numpy.flatnonzero(temp) # indices of nonzero values
if num_samples:
# use specific number of samples for training
numpy.random.shuffle(idx)
idx = idx[0:samples_per_class[c]]
# increase number of samples by copying them over multiple times
max_samples = numpy.amax(samples_per_class)
# num of times to copy for even division
copy_times = int(floor(max_samples / samples_per_class[c]))
rem = max_samples % samples_per_class[c] # remaining samples
for i in range(0, copy_times):
numpy.random.shuffle(idx)
shuffle_temp = numpy.zeros(flat_temp.shape, dtype=bool)
shuffle_temp[idx] = flat_temp[idx]
f = self.loadmask(numpy.reshape(shuffle_temp, (temp.shape[0], temp.shape[1]))) # load the feature matrix for class c
t = numpy.ones((f.shape[1])) * (c+1) # generate a target array
F.append(f)
T.append(t)
# copy the remaning samples so the total matches the max number of samples chosen by user
if rem > 0:
numpy.random.shuffle(idx)
idx = idx[0:rem]
shuffle_temp = numpy.zeros(flat_temp.shape, dtype=bool)
shuffle_temp[idx] = flat_temp[idx]
f = self.loadmask(numpy.reshape(shuffle_temp, (temp.shape[0], temp.shape[1]))) # load the feature matrix for class c
t = numpy.ones((f.shape[1])) * (c+1) # generate a target array
F.append(f)
T.append(t)
return numpy.nan_to_num(numpy.concatenate(F, 1).transpose()), numpy.concatenate(T)
|
71ee1c23
David Mayerich
added additional ...
|
453
454
455
456
|
#read a batch of data based on the mask
def loadbatch(self, npixels):
i = numpy.flatnonzero(self.mask) #get the indices of valid pixels
|
59b0e5f5
David Mayerich
updated envi file...
|
457
458
|
if len(i) == self.idx: #if all of the pixels have been read, return an empyt array
return []
|
777fef6c
David Mayerich
added digital sta...
|
459
|
npixels = min(npixels, len(i) - self.idx) #if there aren't enough pixels, change the batch size
|
71ee1c23
David Mayerich
added additional ...
|
460
461
462
463
464
465
466
467
468
469
470
|
B = self.header.bands
batch = numpy.zeros((B, npixels), dtype=self.header.data_type) #allocate space for the batch
pixel = numpy.zeros((B), dtype=self.header.data_type) #allocate space for a single pixel
type_bytes = numpy.dtype(self.header.data_type).itemsize #calculate the size of a single value
if self.header.interleave == "bip":
for n in range(0, npixels): #for each pixel in the batch
self.file.seek(i[self.idx] * B * type_bytes) #seek to the current pixel in the file
self.file.readinto(pixel) #read a single pixel
batch[:, n] = pixel #save the pixel into the batch matrix
self.idx = self.idx + 1
|
71ee1c23
David Mayerich
added additional ...
|
471
472
473
474
475
476
477
478
479
|
return batch
elif self.header.interleave == "bsq":
print("ERROR: BSQ batch loading isn't implemented yet!")
elif self.header.interleave == "bil":
print("ERROR: BIL batch loading isn't implemented yet!")
#returns the current batch index
def getidx(self):
return self.idx
|
777fef6c
David Mayerich
added digital sta...
|
480
481
482
|
#returns an image of the pixels that have been read using batch loading
def batchmask(self):
|
59b0e5f5
David Mayerich
updated envi file...
|
483
484
|
#allocate a new mask
outmask = numpy.zeros(self.mask.shape, dtype=numpy.bool)
|
777fef6c
David Mayerich
added digital sta...
|
485
|
|
59b0e5f5
David Mayerich
updated envi file...
|
486
487
488
489
490
|
#zero out any unclassified pixels
idx = self.getidx()
i = numpy.nonzero(self.mask)
outmask[i[0][0:idx], i[1][0:idx]] = self.mask[i[0][0:idx], i[1][0:idx]]
return outmask
|
777fef6c
David Mayerich
added digital sta...
|
491
|
|
9bef85ea
David Mayerich
Put parenthesis a...
|
492
493
|
def close(self):
self.file.close()
|
18368aa9
David Mayerich
added a new set o...
|
494
|
|
18368aa9
David Mayerich
added a new set o...
|
495
|
def __del__(self):
|
403139ae
David Mayerich
added the ability...
|
496
497
498
|
self.file.close()
#saves an array as an ENVI file
|
59b0e5f5
David Mayerich
updated envi file...
|
499
|
def save_envi(A, fname, interleave="BSQ", wavelength=[]):
|
403139ae
David Mayerich
added the ability...
|
500
501
502
|
#create and save a header file
header = envi_header();
|
59b0e5f5
David Mayerich
updated envi file...
|
503
|
header.setprops(A, interleave, wavelength)
|
403139ae
David Mayerich
added the ability...
|
504
505
506
507
508
|
header.save(fname + ".hdr")
#save the raw data
file = open(fname, "wb")
file.write(bytearray(A))
|
35534291
sberisha
envi.py: added fu...
|
509
|
file.close()
|