18368aa9
David Mayerich
added a new set o...
|
1
2
3
4
5
6
7
8
9
10
11
12
|
# -*- coding: utf-8 -*-
"""
Created on Fri Jul 21 20:18:01 2017
@author: david
"""
import os
import numpy
import scipy
import matplotlib.pyplot as plt
import progressbar
|
59f31ea4
David Mayerich
fixed bugs in env...
|
13
|
import sys
|
18368aa9
David Mayerich
added a new set o...
|
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
|
class envi_header:
def __init__(self, filename = ""):
if filename != "":
self.load(filename)
else:
self.initialize()
#initialization function
def initialize(self):
self.samples = int(0)
self.lines = int(0)
self.bands = int(0)
self.header_offset = int(0)
self.data_type = int(4)
self.interleave = "bsq"
|
403139ae
David Mayerich
added the ability...
|
30
|
self.sensor_type = "Unknown"
|
18368aa9
David Mayerich
added a new set o...
|
31
32
33
|
self.byte_order = int(0)
self.x_start = int(0)
self.y_start = int(0)
|
403139ae
David Mayerich
added the ability...
|
34
|
self.z_plot_titles = "Unknown, Unknown"
|
18368aa9
David Mayerich
added a new set o...
|
35
36
37
|
self.pixel_size = [float(0), float(0)]
self.pixel_size_units = "Meters"
self.wavelength_units = "Wavenumber"
|
403139ae
David Mayerich
added the ability...
|
38
|
self.description = "no description"
|
18368aa9
David Mayerich
added a new set o...
|
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
|
self.band_names = []
self.wavelength = []
#convert an ENVI data_type value to a numpy data type
def get_numpy_type(self, val):
if val == 1:
return numpy.byte
elif val == 2:
return numpy.int16
elif val == 3:
return numpy.int32
elif val == 4:
return numpy.float32
elif val == 5:
return numpy.float64
elif val == 6:
return numpy.complex64
elif val == 9:
return numpy.complex128
elif val == 12:
return numpy.uint16
elif val == 13:
return numpy.uint32
elif val == 14:
return numpy.int64
elif val == 15:
return numpy.uint64
def get_envi_type(self, val):
if val == numpy.byte:
return 1
elif val == numpy.int16:
return 2
elif val == numpy.int32:
return 3
elif val == numpy.float32:
return 4
elif val == numpy.float64:
return 5
elif val == numpy.complex64:
return 6
elif val == numpy.complex128:
return 9
elif val == numpy.uint16:
return 12
elif val == numpy.uint32:
return 13
elif val == numpy.int64:
return 14
elif val == numpy.uint64:
return 15
def load(self, fname):
f = open(fname)
l = f.readlines()
if l[0].strip() != "ENVI":
print("ERROR: not an ENVI file")
return
li = 1
while li < len(l):
#t = l[li].split() #split the line into tokens
#t = map(str.strip, t) #strip all of the tokens in the token list
#handle the simple conditions
|
9b3cbdda
David Mayerich
changed the name ...
|
103
104
105
106
107
|
#if l[li].startswith("file type"):
# if not l[li].strip().endswith("ENVI Standard"):
# print("ERROR: unsupported ENVI file format: " + l[li].strip())
# return
if l[li].startswith("samples"):
|
18368aa9
David Mayerich
added a new set o...
|
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
|
self.samples = int(l[li].split()[-1])
elif l[li].startswith("lines"):
self.lines = int(l[li].split()[-1])
elif l[li].startswith("bands"):
self.bands = int(l[li].split()[-1])
elif l[li].startswith("header offset"):
self.header_offset = int(l[li].split()[-1])
elif l[li].startswith("data type"):
self.data_type = self.get_numpy_type(int(l[li].split()[-1]))
elif l[li].startswith("interleave"):
self.interleave = l[li].split()[-1].strip()
elif l[li].startswith("sensor type"):
self.sensor_type = l[li].split()[-1].strip()
elif l[li].startswith("byte order"):
self.byte_order = int(l[li].split()[-1])
elif l[li].startswith("x start"):
self.x_start = int(l[li].split()[-1])
elif l[li].startswith("y start"):
self.y_start = int(l[li].split()[-1])
elif l[li].startswith("z plot titles"):
i0 = l[li].rindex('{')
i1 = l[li].rindex('}')
self.z_plot_titles = l[li][i0 + 1 : i1]
elif l[li].startswith("pixel size"):
i0 = l[li].rindex('{')
i1 = l[li].rindex('}')
s = l[li][i0 + 1 : i1].split(',')
self.pixel_size = [float(s[0]), float(s[1])]
self.pixel_size_units = s[2][s[2].rindex('=') + 1:].strip()
elif l[li].startswith("wavelength units"):
self.wavelength_units = l[li].split()[-1].strip()
#handle the complicated conditions
elif l[li].startswith("description"):
desc = [l[li]]
|
6b2be991
sberisha
added utility fun...
|
143
144
145
146
147
148
149
150
151
|
'''
while l[li].strip()[-1] != '}': #will fail if l[li].strip() is empty
li += 1
desc.append(l[li])
'''
while True:
if l[li].strip():
if l[li].strip()[-1] == '}':
break
|
18368aa9
David Mayerich
added a new set o...
|
152
153
|
li += 1
desc.append(l[li])
|
6b2be991
sberisha
added utility fun...
|
154
|
|
18368aa9
David Mayerich
added a new set o...
|
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
|
desc = ''.join(list(map(str.strip, desc))) #strip all white space from the string list
i0 = desc.rindex('{')
i1 = desc.rindex('}')
self.description = desc[i0 + 1 : i1]
elif l[li].startswith("band names"):
names = [l[li]]
while l[li].strip()[-1] != '}':
li += 1
names.append(l[li])
names = ''.join(list(map(str.strip, names))) #strip all white space from the string list
i0 = names.rindex('{')
i1 = names.rindex('}')
names = names[i0 + 1 : i1]
self.band_names = list(map(str.strip, names.split(',')))
elif l[li].startswith("wavelength"):
waves = [l[li]]
while l[li].strip()[-1] != '}':
li += 1
waves.append(l[li])
waves = ''.join(list(map(str.strip, waves))) #strip all white space from the string list
i0 = waves.rindex('{')
i1 = waves.rindex('}')
waves = waves[i0 + 1 : i1]
self.wavelength = list(map(float, waves.split(',')))
li += 1
f.close()
|
403139ae
David Mayerich
added the ability...
|
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
|
#save an ENVI header
def save(self, fname):
f = open(fname, "w")
f.write("ENVI\n")
f.write("description = {" + self.description + "}" + "\n")
f.write("samples = " + str(self.samples) + "\n")
f.write("lines = " + str(self.lines) + "\n")
f.write("bands = " + str(self.bands) + "\n")
f.write("header offset = " + str(self.header_offset) + "\n")
f.write("file type = ENVI Standard" + "\n")
f.write("data type = " + str(self.get_envi_type(self.type)) + "\n")
f.write("interleave = " + self.interleave + "\n")
f.write("sensor type = " + self.sensor_type + "\n")
f.write("byte order = " + str(self.byte_order) + "\n")
f.write("x start = " + str(self.x_start) + "\n")
f.write("y start = " + str(self.y_start) + "\n")
f.write("wavelength units = " + self.wavelength_units + "\n")
f.write("z plot titles = {" + self.z_plot_titles + "}" + "\n")
f.close()
#sets the properties of the header to match those of the input array
def set(self, A):
self.type = A.dtype
self.samples = A.shape[2]
self.lines = A.shape[1]
self.bands = A.shape[0]
|
18368aa9
David Mayerich
added a new set o...
|
213
214
|
class envi:
|
71ee1c23
David Mayerich
added additional ...
|
215
|
def __init__(self, filename, headername = "", mask = []):
|
18368aa9
David Mayerich
added a new set o...
|
216
|
self.open(filename, headername)
|
71ee1c23
David Mayerich
added additional ...
|
217
|
if mask == []:
|
777fef6c
David Mayerich
added digital sta...
|
218
|
self.mask = numpy.ones((self.header.lines, self.header.samples), dtype=numpy.bool)
|
ed3d7d30
David Mayerich
updated NWT file ...
|
219
|
elif type(mask) == numpy.ndarray:
|
71ee1c23
David Mayerich
added additional ...
|
220
|
self.mask = mask
|
ed3d7d30
David Mayerich
updated NWT file ...
|
221
222
|
else:
print("ERROR: unrecognized mask format - expecting a boolean array")
|
71ee1c23
David Mayerich
added additional ...
|
223
|
self.idx = 0 #initialize the batch IDX to 0 for batch reading
|
18368aa9
David Mayerich
added a new set o...
|
224
|
|
18368aa9
David Mayerich
added a new set o...
|
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
|
def open(self, filename, headername = ""):
if headername == "":
headername = filename + ".hdr"
if not os.path.isfile(filename):
print("ERROR: " + filename + " not found")
return
if not os.path.isfile(headername):
print("ERROR: " + headername + " not found")
return
#open the file
self.header = envi_header(headername)
self.file = open(filename, "rb")
def loadall(self):
X = self.header.samples
Y = self.header.lines
B = self.header.bands
#load the data
D = numpy.fromfile(self.file, dtype=self.header.data_type)
if self.header.interleave == "bsq":
return numpy.reshape(D, (B, Y, X))
#return numpy.swapaxes(D, 0, 2)
elif self.header.interleave == "bip":
D = numpy.reshape(D, (Y, X, B))
return numpy.rollaxis(D, 2)
elif self.header.interleave == "bil":
D = numpy.reshape(D, (Y, B, X))
return numpy.rollaxis(D, 1)
#loads all of the pixels where mask != 0 and returns them as a matrix
def loadmask(self, mask):
X = self.header.samples
Y = self.header.lines
B = self.header.bands
P = numpy.count_nonzero(mask) #count the number of zeros in the mask file
M = numpy.zeros((B, P), dtype=self.header.data_type)
type_bytes = numpy.dtype(self.header.data_type).itemsize
|
71ee1c23
David Mayerich
added additional ...
|
268
|
prev_pos = self.file.tell()
|
18368aa9
David Mayerich
added a new set o...
|
269
270
271
272
273
274
275
276
277
278
279
|
self.file.seek(0)
if self.header.interleave == "bip":
spectrum = numpy.zeros(B, dtype=self.header.data_type)
flatmask = numpy.reshape(mask, (X * Y))
i = numpy.flatnonzero(flatmask)
bar = progressbar.ProgressBar(max_value = P)
for p in range(0, P):
self.file.seek(i[p] * B * type_bytes)
self.file.readinto(spectrum)
M[:, p] = spectrum
bar.update(p+1)
|
59f31ea4
David Mayerich
fixed bugs in env...
|
280
|
elif self.header.interleave == "bsq":
|
18368aa9
David Mayerich
added a new set o...
|
281
282
283
284
285
286
287
288
|
band = numpy.zeros(mask.shape, dtype=self.header.data_type)
i = numpy.nonzero(mask)
bar = progressbar.ProgressBar(max_value=B)
for b in range(0, B):
self.file.seek(b * X * Y * type_bytes)
self.file.readinto(band)
M[b, :] = band[i]
bar.update(b+1)
|
59f31ea4
David Mayerich
fixed bugs in env...
|
289
|
elif self.header.interleave == "bil":
|
18368aa9
David Mayerich
added a new set o...
|
290
291
292
293
294
295
296
297
298
|
plane = numpy.zeros((B, X), dtype=self.header.data_type)
p = 0
bar = progressbar.ProgressBar(max_value=Y)
for l in range(0, Y):
i = numpy.flatnonzero(mask[l, :])
self.file.readinto(plane)
M[:, p:p+i.shape[0]] = plane[:, i]
p = p + i.shape[0]
bar.update(l+1)
|
71ee1c23
David Mayerich
added additional ...
|
299
|
self.file.seek(prev_pos)
|
18368aa9
David Mayerich
added a new set o...
|
300
|
return M
|
90c935e3
David Mayerich
updates
|
301
302
303
304
305
306
307
308
|
def loadband(self, n):
X = self.header.samples
Y = self.header.lines
B = self.header.bands
band = numpy.zeros((Y, X), dtype=self.header.data_type)
type_bytes = numpy.dtype(self.header.data_type).itemsize
|
71ee1c23
David Mayerich
added additional ...
|
309
310
|
prev_pos = self.file.tell()
|
90c935e3
David Mayerich
updates
|
311
312
313
|
if self.header.interleave == "bsq":
self.file.seek(n * X * Y * type_bytes)
self.file.readinto(band)
|
71ee1c23
David Mayerich
added additional ...
|
314
|
self.file.seek(prev_pos)
|
90c935e3
David Mayerich
updates
|
315
316
|
return band
|
f275c01c
David Mayerich
finalized the pre...
|
317
318
319
320
|
#create a set of feature/target pairs for classification
#input: envi file object, stack of class masks C x Y x X
#output: feature matrix (features x pixels), target matrix (1 x pixels)
#example: generate_training(("class_coll.bmp", "class_epith.bmp"), (1, 2))
|
ed3d7d30
David Mayerich
updated NWT file ...
|
321
322
|
# verify verify that there are no NaN or Inf values
def loadtrain(self, classimages, verify=True):
|
f275c01c
David Mayerich
finalized the pre...
|
323
324
325
326
327
328
329
|
# get number of classes
C = classimages.shape[0]
F = []
T = []
for c in range(0, C):
|
59f31ea4
David Mayerich
fixed bugs in env...
|
330
|
print("\nLoading class " + str(c+1) + "...")
|
f275c01c
David Mayerich
finalized the pre...
|
331
332
333
334
|
f = self.loadmask(classimages[c, :, :]) #load the feature matrix for class c
t = numpy.ones((f.shape[1])) * (c+1) #generate a target array
F.append(f)
T.append(t)
|
ed3d7d30
David Mayerich
updated NWT file ...
|
335
336
|
return numpy.nan_to_num(numpy.concatenate(F, 1).transpose()), numpy.concatenate(T)
|
71ee1c23
David Mayerich
added additional ...
|
337
338
339
340
|
#read a batch of data based on the mask
def loadbatch(self, npixels):
i = numpy.flatnonzero(self.mask) #get the indices of valid pixels
|
777fef6c
David Mayerich
added digital sta...
|
341
342
343
|
if len(i) == self.idx: #if all of the pixels have been read, return an empyt array
return []
npixels = min(npixels, len(i) - self.idx) #if there aren't enough pixels, change the batch size
|
71ee1c23
David Mayerich
added additional ...
|
344
345
346
347
348
349
350
351
352
353
354
|
B = self.header.bands
batch = numpy.zeros((B, npixels), dtype=self.header.data_type) #allocate space for the batch
pixel = numpy.zeros((B), dtype=self.header.data_type) #allocate space for a single pixel
type_bytes = numpy.dtype(self.header.data_type).itemsize #calculate the size of a single value
if self.header.interleave == "bip":
for n in range(0, npixels): #for each pixel in the batch
self.file.seek(i[self.idx] * B * type_bytes) #seek to the current pixel in the file
self.file.readinto(pixel) #read a single pixel
batch[:, n] = pixel #save the pixel into the batch matrix
self.idx = self.idx + 1
|
71ee1c23
David Mayerich
added additional ...
|
355
356
357
358
359
360
361
362
363
|
return batch
elif self.header.interleave == "bsq":
print("ERROR: BSQ batch loading isn't implemented yet!")
elif self.header.interleave == "bil":
print("ERROR: BIL batch loading isn't implemented yet!")
#returns the current batch index
def getidx(self):
return self.idx
|
777fef6c
David Mayerich
added digital sta...
|
364
365
366
367
368
369
370
371
372
373
374
375
|
#returns an image of the pixels that have been read using batch loading
def batchmask(self):
#allocate a new mask
outmask = numpy.zeros(self.mask.shape, dtype=numpy.bool)
#zero out any unclassified pixels
idx = self.getidx()
i = numpy.nonzero(self.mask)
outmask[i[0][0:idx], i[1][0:idx]] = self.mask[i[0][0:idx], i[1][0:idx]]
return outmask
|
9bef85ea
David Mayerich
Put parenthesis a...
|
376
377
|
def close(self):
self.file.close()
|
18368aa9
David Mayerich
added a new set o...
|
378
|
|
18368aa9
David Mayerich
added a new set o...
|
379
|
def __del__(self):
|
403139ae
David Mayerich
added the ability...
|
380
381
382
383
384
385
386
387
388
389
390
391
392
393
|
self.file.close()
#saves an array as an ENVI file
def save_envi(A, fname):
#create and save a header file
header = envi_header();
header.set(A)
header.save(fname + ".hdr")
#save the raw data
file = open(fname, "wb")
file.write(bytearray(A))
file.close()
|