18368aa9
David Mayerich
added a new set o...
|
1
2
3
4
5
6
7
8
9
10
11
|
# -*- coding: utf-8 -*-
"""
Created on Fri Jul 21 20:18:01 2017
@author: david
"""
import os
import numpy
import scipy
import matplotlib.pyplot as plt
|
35534291
sberisha
envi.py: added fu...
|
12
|
#import pyprind
|
59f31ea4
David Mayerich
fixed bugs in env...
|
13
|
import sys
|
35534291
sberisha
envi.py: added fu...
|
14
15
|
from math import floor
import progressbar
|
18368aa9
David Mayerich
added a new set o...
|
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
|
class envi_header:
def __init__(self, filename = ""):
if filename != "":
self.load(filename)
else:
self.initialize()
#initialization function
def initialize(self):
self.samples = int(0)
self.lines = int(0)
self.bands = int(0)
self.header_offset = int(0)
self.data_type = int(4)
self.interleave = "bsq"
|
403139ae
David Mayerich
added the ability...
|
32
|
self.sensor_type = "Unknown"
|
18368aa9
David Mayerich
added a new set o...
|
33
34
35
|
self.byte_order = int(0)
self.x_start = int(0)
self.y_start = int(0)
|
403139ae
David Mayerich
added the ability...
|
36
|
self.z_plot_titles = "Unknown, Unknown"
|
18368aa9
David Mayerich
added a new set o...
|
37
38
39
|
self.pixel_size = [float(0), float(0)]
self.pixel_size_units = "Meters"
self.wavelength_units = "Wavenumber"
|
403139ae
David Mayerich
added the ability...
|
40
|
self.description = "no description"
|
18368aa9
David Mayerich
added a new set o...
|
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
|
self.band_names = []
self.wavelength = []
#convert an ENVI data_type value to a numpy data type
def get_numpy_type(self, val):
if val == 1:
return numpy.byte
elif val == 2:
return numpy.int16
elif val == 3:
return numpy.int32
elif val == 4:
return numpy.float32
elif val == 5:
return numpy.float64
elif val == 6:
return numpy.complex64
elif val == 9:
return numpy.complex128
elif val == 12:
return numpy.uint16
elif val == 13:
return numpy.uint32
elif val == 14:
return numpy.int64
elif val == 15:
return numpy.uint64
def get_envi_type(self, val):
if val == numpy.byte:
return 1
elif val == numpy.int16:
return 2
elif val == numpy.int32:
return 3
elif val == numpy.float32:
return 4
elif val == numpy.float64:
return 5
elif val == numpy.complex64:
return 6
elif val == numpy.complex128:
return 9
elif val == numpy.uint16:
return 12
elif val == numpy.uint32:
return 13
elif val == numpy.int64:
return 14
elif val == numpy.uint64:
return 15
def load(self, fname):
f = open(fname)
l = f.readlines()
if l[0].strip() != "ENVI":
print("ERROR: not an ENVI file")
return
li = 1
while li < len(l):
#t = l[li].split() #split the line into tokens
#t = map(str.strip, t) #strip all of the tokens in the token list
#handle the simple conditions
|
9b3cbdda
David Mayerich
changed the name ...
|
105
106
107
108
109
|
#if l[li].startswith("file type"):
# if not l[li].strip().endswith("ENVI Standard"):
# print("ERROR: unsupported ENVI file format: " + l[li].strip())
# return
if l[li].startswith("samples"):
|
18368aa9
David Mayerich
added a new set o...
|
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
|
self.samples = int(l[li].split()[-1])
elif l[li].startswith("lines"):
self.lines = int(l[li].split()[-1])
elif l[li].startswith("bands"):
self.bands = int(l[li].split()[-1])
elif l[li].startswith("header offset"):
self.header_offset = int(l[li].split()[-1])
elif l[li].startswith("data type"):
self.data_type = self.get_numpy_type(int(l[li].split()[-1]))
elif l[li].startswith("interleave"):
self.interleave = l[li].split()[-1].strip()
elif l[li].startswith("sensor type"):
self.sensor_type = l[li].split()[-1].strip()
elif l[li].startswith("byte order"):
self.byte_order = int(l[li].split()[-1])
elif l[li].startswith("x start"):
self.x_start = int(l[li].split()[-1])
elif l[li].startswith("y start"):
self.y_start = int(l[li].split()[-1])
elif l[li].startswith("z plot titles"):
i0 = l[li].rindex('{')
i1 = l[li].rindex('}')
self.z_plot_titles = l[li][i0 + 1 : i1]
elif l[li].startswith("pixel size"):
i0 = l[li].rindex('{')
i1 = l[li].rindex('}')
s = l[li][i0 + 1 : i1].split(',')
self.pixel_size = [float(s[0]), float(s[1])]
self.pixel_size_units = s[2][s[2].rindex('=') + 1:].strip()
elif l[li].startswith("wavelength units"):
self.wavelength_units = l[li].split()[-1].strip()
#handle the complicated conditions
elif l[li].startswith("description"):
desc = [l[li]]
|
6b2be991
sberisha
added utility fun...
|
145
146
147
148
149
150
151
152
153
|
'''
while l[li].strip()[-1] != '}': #will fail if l[li].strip() is empty
li += 1
desc.append(l[li])
'''
while True:
if l[li].strip():
if l[li].strip()[-1] == '}':
break
|
18368aa9
David Mayerich
added a new set o...
|
154
155
|
li += 1
desc.append(l[li])
|
6b2be991
sberisha
added utility fun...
|
156
|
|
18368aa9
David Mayerich
added a new set o...
|
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
|
desc = ''.join(list(map(str.strip, desc))) #strip all white space from the string list
i0 = desc.rindex('{')
i1 = desc.rindex('}')
self.description = desc[i0 + 1 : i1]
elif l[li].startswith("band names"):
names = [l[li]]
while l[li].strip()[-1] != '}':
li += 1
names.append(l[li])
names = ''.join(list(map(str.strip, names))) #strip all white space from the string list
i0 = names.rindex('{')
i1 = names.rindex('}')
names = names[i0 + 1 : i1]
self.band_names = list(map(str.strip, names.split(',')))
elif l[li].startswith("wavelength"):
waves = [l[li]]
while l[li].strip()[-1] != '}':
li += 1
waves.append(l[li])
waves = ''.join(list(map(str.strip, waves))) #strip all white space from the string list
i0 = waves.rindex('{')
i1 = waves.rindex('}')
waves = waves[i0 + 1 : i1]
self.wavelength = list(map(float, waves.split(',')))
li += 1
f.close()
|
403139ae
David Mayerich
added the ability...
|
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
|
#save an ENVI header
def save(self, fname):
f = open(fname, "w")
f.write("ENVI\n")
f.write("description = {" + self.description + "}" + "\n")
f.write("samples = " + str(self.samples) + "\n")
f.write("lines = " + str(self.lines) + "\n")
f.write("bands = " + str(self.bands) + "\n")
f.write("header offset = " + str(self.header_offset) + "\n")
f.write("file type = ENVI Standard" + "\n")
f.write("data type = " + str(self.get_envi_type(self.type)) + "\n")
f.write("interleave = " + self.interleave + "\n")
f.write("sensor type = " + self.sensor_type + "\n")
f.write("byte order = " + str(self.byte_order) + "\n")
f.write("x start = " + str(self.x_start) + "\n")
f.write("y start = " + str(self.y_start) + "\n")
f.write("wavelength units = " + self.wavelength_units + "\n")
f.write("z plot titles = {" + self.z_plot_titles + "}" + "\n")
f.close()
#sets the properties of the header to match those of the input array
def set(self, A):
self.type = A.dtype
self.samples = A.shape[2]
self.lines = A.shape[1]
self.bands = A.shape[0]
|
18368aa9
David Mayerich
added a new set o...
|
215
216
|
class envi:
|
71ee1c23
David Mayerich
added additional ...
|
217
|
def __init__(self, filename, headername = "", mask = []):
|
18368aa9
David Mayerich
added a new set o...
|
218
|
self.open(filename, headername)
|
71ee1c23
David Mayerich
added additional ...
|
219
|
if mask == []:
|
777fef6c
David Mayerich
added digital sta...
|
220
|
self.mask = numpy.ones((self.header.lines, self.header.samples), dtype=numpy.bool)
|
ed3d7d30
David Mayerich
updated NWT file ...
|
221
|
elif type(mask) == numpy.ndarray:
|
71ee1c23
David Mayerich
added additional ...
|
222
|
self.mask = mask
|
ed3d7d30
David Mayerich
updated NWT file ...
|
223
224
|
else:
print("ERROR: unrecognized mask format - expecting a boolean array")
|
71ee1c23
David Mayerich
added additional ...
|
225
|
self.idx = 0 #initialize the batch IDX to 0 for batch reading
|
18368aa9
David Mayerich
added a new set o...
|
226
|
|
18368aa9
David Mayerich
added a new set o...
|
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
|
def open(self, filename, headername = ""):
if headername == "":
headername = filename + ".hdr"
if not os.path.isfile(filename):
print("ERROR: " + filename + " not found")
return
if not os.path.isfile(headername):
print("ERROR: " + headername + " not found")
return
#open the file
self.header = envi_header(headername)
self.file = open(filename, "rb")
def loadall(self):
X = self.header.samples
Y = self.header.lines
B = self.header.bands
#load the data
D = numpy.fromfile(self.file, dtype=self.header.data_type)
if self.header.interleave == "bsq":
return numpy.reshape(D, (B, Y, X))
#return numpy.swapaxes(D, 0, 2)
elif self.header.interleave == "bip":
D = numpy.reshape(D, (Y, X, B))
return numpy.rollaxis(D, 2)
elif self.header.interleave == "bil":
D = numpy.reshape(D, (Y, B, X))
return numpy.rollaxis(D, 1)
#loads all of the pixels where mask != 0 and returns them as a matrix
def loadmask(self, mask):
X = self.header.samples
Y = self.header.lines
B = self.header.bands
P = numpy.count_nonzero(mask) #count the number of zeros in the mask file
M = numpy.zeros((B, P), dtype=self.header.data_type)
type_bytes = numpy.dtype(self.header.data_type).itemsize
|
71ee1c23
David Mayerich
added additional ...
|
270
|
prev_pos = self.file.tell()
|
18368aa9
David Mayerich
added a new set o...
|
271
272
273
274
275
276
|
self.file.seek(0)
if self.header.interleave == "bip":
spectrum = numpy.zeros(B, dtype=self.header.data_type)
flatmask = numpy.reshape(mask, (X * Y))
i = numpy.flatnonzero(flatmask)
bar = progressbar.ProgressBar(max_value = P)
|
35534291
sberisha
envi.py: added fu...
|
277
|
#bar = pyprind.ProgBar(P)
|
18368aa9
David Mayerich
added a new set o...
|
278
279
280
281
282
|
for p in range(0, P):
self.file.seek(i[p] * B * type_bytes)
self.file.readinto(spectrum)
M[:, p] = spectrum
bar.update(p+1)
|
35534291
sberisha
envi.py: added fu...
|
283
|
#bar.update()
|
59f31ea4
David Mayerich
fixed bugs in env...
|
284
|
elif self.header.interleave == "bsq":
|
18368aa9
David Mayerich
added a new set o...
|
285
286
287
|
band = numpy.zeros(mask.shape, dtype=self.header.data_type)
i = numpy.nonzero(mask)
bar = progressbar.ProgressBar(max_value=B)
|
35534291
sberisha
envi.py: added fu...
|
288
|
#bar = pyprind.ProgBar(P)
|
18368aa9
David Mayerich
added a new set o...
|
289
290
291
292
293
|
for b in range(0, B):
self.file.seek(b * X * Y * type_bytes)
self.file.readinto(band)
M[b, :] = band[i]
bar.update(b+1)
|
35534291
sberisha
envi.py: added fu...
|
294
|
#bar.update()
|
59f31ea4
David Mayerich
fixed bugs in env...
|
295
|
elif self.header.interleave == "bil":
|
18368aa9
David Mayerich
added a new set o...
|
296
297
298
|
plane = numpy.zeros((B, X), dtype=self.header.data_type)
p = 0
bar = progressbar.ProgressBar(max_value=Y)
|
35534291
sberisha
envi.py: added fu...
|
299
|
#bar = pyprind.ProgBar(P)
|
18368aa9
David Mayerich
added a new set o...
|
300
301
302
303
304
305
|
for l in range(0, Y):
i = numpy.flatnonzero(mask[l, :])
self.file.readinto(plane)
M[:, p:p+i.shape[0]] = plane[:, i]
p = p + i.shape[0]
bar.update(l+1)
|
35534291
sberisha
envi.py: added fu...
|
306
|
#bar.update()
|
71ee1c23
David Mayerich
added additional ...
|
307
|
self.file.seek(prev_pos)
|
18368aa9
David Mayerich
added a new set o...
|
308
|
return M
|
90c935e3
David Mayerich
updates
|
309
310
311
312
313
314
315
316
|
def loadband(self, n):
X = self.header.samples
Y = self.header.lines
B = self.header.bands
band = numpy.zeros((Y, X), dtype=self.header.data_type)
type_bytes = numpy.dtype(self.header.data_type).itemsize
|
71ee1c23
David Mayerich
added additional ...
|
317
318
|
prev_pos = self.file.tell()
|
90c935e3
David Mayerich
updates
|
319
320
321
|
if self.header.interleave == "bsq":
self.file.seek(n * X * Y * type_bytes)
self.file.readinto(band)
|
71ee1c23
David Mayerich
added additional ...
|
322
|
self.file.seek(prev_pos)
|
90c935e3
David Mayerich
updates
|
323
324
|
return band
|
f275c01c
David Mayerich
finalized the pre...
|
325
326
327
328
|
#create a set of feature/target pairs for classification
#input: envi file object, stack of class masks C x Y x X
#output: feature matrix (features x pixels), target matrix (1 x pixels)
#example: generate_training(("class_coll.bmp", "class_epith.bmp"), (1, 2))
|
ed3d7d30
David Mayerich
updated NWT file ...
|
329
330
|
# verify verify that there are no NaN or Inf values
def loadtrain(self, classimages, verify=True):
|
f275c01c
David Mayerich
finalized the pre...
|
331
332
333
334
335
336
337
|
# get number of classes
C = classimages.shape[0]
F = []
T = []
for c in range(0, C):
|
59f31ea4
David Mayerich
fixed bugs in env...
|
338
|
print("\nLoading class " + str(c+1) + "...")
|
f275c01c
David Mayerich
finalized the pre...
|
339
340
341
342
|
f = self.loadmask(classimages[c, :, :]) #load the feature matrix for class c
t = numpy.ones((f.shape[1])) * (c+1) #generate a target array
F.append(f)
T.append(t)
|
ed3d7d30
David Mayerich
updated NWT file ...
|
343
344
|
return numpy.nan_to_num(numpy.concatenate(F, 1).transpose()), numpy.concatenate(T)
|
35534291
sberisha
envi.py: added fu...
|
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
|
#create a set of feature/target pairs for classification with balanced data
#input: envi file object, stack of class masks C x Y x X, number of samples per class
#output: feature matrix (features x pixels), target matrix (1 x pixels)
#example: generate_training(("class_coll.bmp", "class_epith.bmp"), (1, 2))
# verify verify that there are no NaN or Inf values
def loadtrain_balance(self, classimages, num_samples=None):
# get number of classes
C = classimages.shape[0]
F = []
T = []
# get number of samples per class
samples_per_class = numpy.zeros(C, dtype=numpy.int32)
for c in range(0, C):
if num_samples is None:
samples_per_class[c] = numpy.count_nonzero(classimages[c, :, :])
else:
# if user has specified a max number of samples per class
if num_samples > numpy.count_nonzero(classimages[c, :, :]):
samples_per_class[c] = numpy.count_nonzero(classimages[c, :, :])
else:
samples_per_class[c] = num_samples
for c in range(0, C):
print("\nLoading class " + str(c+1) + "...")
# row, col index of valid pixels
temp = classimages[c,:]
flat_temp = numpy.reshape(temp, temp.shape[0]*temp.shape[1])
idx = numpy.flatnonzero(temp) # indices of nonzero values
if num_samples:
# use specific number of samples for training
numpy.random.shuffle(idx)
idx = idx[0:samples_per_class[c]]
# increase number of samples by copying them over multiple times
max_samples = numpy.amax(samples_per_class)
# num of times to copy for even division
copy_times = int(floor(max_samples / samples_per_class[c]))
rem = max_samples % samples_per_class[c] # remaining samples
for i in range(0, copy_times):
numpy.random.shuffle(idx)
shuffle_temp = numpy.zeros(flat_temp.shape, dtype=bool)
shuffle_temp[idx] = flat_temp[idx]
f = self.loadmask(numpy.reshape(shuffle_temp, (temp.shape[0], temp.shape[1]))) # load the feature matrix for class c
t = numpy.ones((f.shape[1])) * (c+1) # generate a target array
F.append(f)
T.append(t)
# copy the remaning samples so the total matches the max number of samples chosen by user
if rem > 0:
numpy.random.shuffle(idx)
idx = idx[0:rem]
shuffle_temp = numpy.zeros(flat_temp.shape, dtype=bool)
shuffle_temp[idx] = flat_temp[idx]
f = self.loadmask(numpy.reshape(shuffle_temp, (temp.shape[0], temp.shape[1]))) # load the feature matrix for class c
t = numpy.ones((f.shape[1])) * (c+1) # generate a target array
F.append(f)
T.append(t)
return numpy.nan_to_num(numpy.concatenate(F, 1).transpose()), numpy.concatenate(T)
|
71ee1c23
David Mayerich
added additional ...
|
412
413
414
415
|
#read a batch of data based on the mask
def loadbatch(self, npixels):
i = numpy.flatnonzero(self.mask) #get the indices of valid pixels
|
777fef6c
David Mayerich
added digital sta...
|
416
417
418
|
if len(i) == self.idx: #if all of the pixels have been read, return an empyt array
return []
npixels = min(npixels, len(i) - self.idx) #if there aren't enough pixels, change the batch size
|
71ee1c23
David Mayerich
added additional ...
|
419
420
421
422
423
424
425
426
427
428
429
|
B = self.header.bands
batch = numpy.zeros((B, npixels), dtype=self.header.data_type) #allocate space for the batch
pixel = numpy.zeros((B), dtype=self.header.data_type) #allocate space for a single pixel
type_bytes = numpy.dtype(self.header.data_type).itemsize #calculate the size of a single value
if self.header.interleave == "bip":
for n in range(0, npixels): #for each pixel in the batch
self.file.seek(i[self.idx] * B * type_bytes) #seek to the current pixel in the file
self.file.readinto(pixel) #read a single pixel
batch[:, n] = pixel #save the pixel into the batch matrix
self.idx = self.idx + 1
|
71ee1c23
David Mayerich
added additional ...
|
430
431
432
433
434
435
436
437
438
|
return batch
elif self.header.interleave == "bsq":
print("ERROR: BSQ batch loading isn't implemented yet!")
elif self.header.interleave == "bil":
print("ERROR: BIL batch loading isn't implemented yet!")
#returns the current batch index
def getidx(self):
return self.idx
|
777fef6c
David Mayerich
added digital sta...
|
439
440
441
442
443
444
445
446
447
448
449
450
|
#returns an image of the pixels that have been read using batch loading
def batchmask(self):
#allocate a new mask
outmask = numpy.zeros(self.mask.shape, dtype=numpy.bool)
#zero out any unclassified pixels
idx = self.getidx()
i = numpy.nonzero(self.mask)
outmask[i[0][0:idx], i[1][0:idx]] = self.mask[i[0][0:idx], i[1][0:idx]]
return outmask
|
9bef85ea
David Mayerich
Put parenthesis a...
|
451
452
|
def close(self):
self.file.close()
|
18368aa9
David Mayerich
added a new set o...
|
453
|
|
18368aa9
David Mayerich
added a new set o...
|
454
|
def __del__(self):
|
403139ae
David Mayerich
added the ability...
|
455
456
457
458
459
460
461
462
463
464
465
466
467
|
self.file.close()
#saves an array as an ENVI file
def save_envi(A, fname):
#create and save a header file
header = envi_header();
header.set(A)
header.save(fname + ".hdr")
#save the raw data
file = open(fname, "wb")
file.write(bytearray(A))
|
35534291
sberisha
envi.py: added fu...
|
468
|
file.close()
|