18368aa9
David Mayerich
added a new set o...
|
1
2
3
4
5
6
7
8
9
10
11
12
|
# -*- coding: utf-8 -*-
"""
Created on Fri Jul 21 20:18:01 2017
@author: david
"""
import os
import numpy
import scipy
import matplotlib.pyplot as plt
import progressbar
|
59f31ea4
David Mayerich
fixed bugs in env...
|
13
|
import sys
|
18368aa9
David Mayerich
added a new set o...
|
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
|
class envi_header:
def __init__(self, filename = ""):
if filename != "":
self.load(filename)
else:
self.initialize()
#initialization function
def initialize(self):
self.samples = int(0)
self.lines = int(0)
self.bands = int(0)
self.header_offset = int(0)
self.data_type = int(4)
self.interleave = "bsq"
self.sensor_type = ""
self.byte_order = int(0)
self.x_start = int(0)
self.y_start = int(0)
self.z_plot_titles = ""
self.pixel_size = [float(0), float(0)]
self.pixel_size_units = "Meters"
self.wavelength_units = "Wavenumber"
self.description = ""
self.band_names = []
self.wavelength = []
#convert an ENVI data_type value to a numpy data type
def get_numpy_type(self, val):
if val == 1:
return numpy.byte
elif val == 2:
return numpy.int16
elif val == 3:
return numpy.int32
elif val == 4:
return numpy.float32
elif val == 5:
return numpy.float64
elif val == 6:
return numpy.complex64
elif val == 9:
return numpy.complex128
elif val == 12:
return numpy.uint16
elif val == 13:
return numpy.uint32
elif val == 14:
return numpy.int64
elif val == 15:
return numpy.uint64
def get_envi_type(self, val):
if val == numpy.byte:
return 1
elif val == numpy.int16:
return 2
elif val == numpy.int32:
return 3
elif val == numpy.float32:
return 4
elif val == numpy.float64:
return 5
elif val == numpy.complex64:
return 6
elif val == numpy.complex128:
return 9
elif val == numpy.uint16:
return 12
elif val == numpy.uint32:
return 13
elif val == numpy.int64:
return 14
elif val == numpy.uint64:
return 15
def load(self, fname):
f = open(fname)
l = f.readlines()
if l[0].strip() != "ENVI":
print("ERROR: not an ENVI file")
return
li = 1
while li < len(l):
#t = l[li].split() #split the line into tokens
#t = map(str.strip, t) #strip all of the tokens in the token list
#handle the simple conditions
if l[li].startswith("file type"):
if not l[li].strip().endswith("ENVI Standard"):
print("ERROR: unsupported ENVI file format: " + l[li].strip())
return
elif l[li].startswith("samples"):
self.samples = int(l[li].split()[-1])
elif l[li].startswith("lines"):
self.lines = int(l[li].split()[-1])
elif l[li].startswith("bands"):
self.bands = int(l[li].split()[-1])
elif l[li].startswith("header offset"):
self.header_offset = int(l[li].split()[-1])
elif l[li].startswith("data type"):
self.data_type = self.get_numpy_type(int(l[li].split()[-1]))
elif l[li].startswith("interleave"):
self.interleave = l[li].split()[-1].strip()
elif l[li].startswith("sensor type"):
self.sensor_type = l[li].split()[-1].strip()
elif l[li].startswith("byte order"):
self.byte_order = int(l[li].split()[-1])
elif l[li].startswith("x start"):
self.x_start = int(l[li].split()[-1])
elif l[li].startswith("y start"):
self.y_start = int(l[li].split()[-1])
elif l[li].startswith("z plot titles"):
i0 = l[li].rindex('{')
i1 = l[li].rindex('}')
self.z_plot_titles = l[li][i0 + 1 : i1]
elif l[li].startswith("pixel size"):
i0 = l[li].rindex('{')
i1 = l[li].rindex('}')
s = l[li][i0 + 1 : i1].split(',')
self.pixel_size = [float(s[0]), float(s[1])]
self.pixel_size_units = s[2][s[2].rindex('=') + 1:].strip()
elif l[li].startswith("wavelength units"):
self.wavelength_units = l[li].split()[-1].strip()
#handle the complicated conditions
elif l[li].startswith("description"):
desc = [l[li]]
|
6b2be991
sberisha
added utility fun...
|
143
144
145
146
147
148
149
150
151
|
'''
while l[li].strip()[-1] != '}': #will fail if l[li].strip() is empty
li += 1
desc.append(l[li])
'''
while True:
if l[li].strip():
if l[li].strip()[-1] == '}':
break
|
18368aa9
David Mayerich
added a new set o...
|
152
153
|
li += 1
desc.append(l[li])
|
6b2be991
sberisha
added utility fun...
|
154
|
|
18368aa9
David Mayerich
added a new set o...
|
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
|
desc = ''.join(list(map(str.strip, desc))) #strip all white space from the string list
i0 = desc.rindex('{')
i1 = desc.rindex('}')
self.description = desc[i0 + 1 : i1]
elif l[li].startswith("band names"):
names = [l[li]]
while l[li].strip()[-1] != '}':
li += 1
names.append(l[li])
names = ''.join(list(map(str.strip, names))) #strip all white space from the string list
i0 = names.rindex('{')
i1 = names.rindex('}')
names = names[i0 + 1 : i1]
self.band_names = list(map(str.strip, names.split(',')))
elif l[li].startswith("wavelength"):
waves = [l[li]]
while l[li].strip()[-1] != '}':
li += 1
waves.append(l[li])
waves = ''.join(list(map(str.strip, waves))) #strip all white space from the string list
i0 = waves.rindex('{')
i1 = waves.rindex('}')
waves = waves[i0 + 1 : i1]
self.wavelength = list(map(float, waves.split(',')))
li += 1
f.close()
class envi:
|
71ee1c23
David Mayerich
added additional ...
|
186
|
def __init__(self, filename, headername = "", mask = []):
|
18368aa9
David Mayerich
added a new set o...
|
187
|
self.open(filename, headername)
|
71ee1c23
David Mayerich
added additional ...
|
188
|
if mask == []:
|
777fef6c
David Mayerich
added digital sta...
|
189
|
self.mask = numpy.ones((self.header.lines, self.header.samples), dtype=numpy.bool)
|
18368aa9
David Mayerich
added a new set o...
|
190
|
else:
|
71ee1c23
David Mayerich
added additional ...
|
191
192
|
self.mask = mask
self.idx = 0 #initialize the batch IDX to 0 for batch reading
|
18368aa9
David Mayerich
added a new set o...
|
193
|
|
18368aa9
David Mayerich
added a new set o...
|
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
|
def open(self, filename, headername = ""):
if headername == "":
headername = filename + ".hdr"
if not os.path.isfile(filename):
print("ERROR: " + filename + " not found")
return
if not os.path.isfile(headername):
print("ERROR: " + headername + " not found")
return
#open the file
self.header = envi_header(headername)
self.file = open(filename, "rb")
def loadall(self):
X = self.header.samples
Y = self.header.lines
B = self.header.bands
#load the data
D = numpy.fromfile(self.file, dtype=self.header.data_type)
if self.header.interleave == "bsq":
return numpy.reshape(D, (B, Y, X))
#return numpy.swapaxes(D, 0, 2)
elif self.header.interleave == "bip":
D = numpy.reshape(D, (Y, X, B))
return numpy.rollaxis(D, 2)
elif self.header.interleave == "bil":
D = numpy.reshape(D, (Y, B, X))
return numpy.rollaxis(D, 1)
#loads all of the pixels where mask != 0 and returns them as a matrix
def loadmask(self, mask):
X = self.header.samples
Y = self.header.lines
B = self.header.bands
P = numpy.count_nonzero(mask) #count the number of zeros in the mask file
M = numpy.zeros((B, P), dtype=self.header.data_type)
type_bytes = numpy.dtype(self.header.data_type).itemsize
|
71ee1c23
David Mayerich
added additional ...
|
237
|
prev_pos = self.file.tell()
|
18368aa9
David Mayerich
added a new set o...
|
238
239
240
241
242
243
244
245
246
247
248
|
self.file.seek(0)
if self.header.interleave == "bip":
spectrum = numpy.zeros(B, dtype=self.header.data_type)
flatmask = numpy.reshape(mask, (X * Y))
i = numpy.flatnonzero(flatmask)
bar = progressbar.ProgressBar(max_value = P)
for p in range(0, P):
self.file.seek(i[p] * B * type_bytes)
self.file.readinto(spectrum)
M[:, p] = spectrum
bar.update(p+1)
|
59f31ea4
David Mayerich
fixed bugs in env...
|
249
|
elif self.header.interleave == "bsq":
|
18368aa9
David Mayerich
added a new set o...
|
250
251
252
253
254
255
256
257
|
band = numpy.zeros(mask.shape, dtype=self.header.data_type)
i = numpy.nonzero(mask)
bar = progressbar.ProgressBar(max_value=B)
for b in range(0, B):
self.file.seek(b * X * Y * type_bytes)
self.file.readinto(band)
M[b, :] = band[i]
bar.update(b+1)
|
59f31ea4
David Mayerich
fixed bugs in env...
|
258
|
elif self.header.interleave == "bil":
|
18368aa9
David Mayerich
added a new set o...
|
259
260
261
262
263
264
265
266
267
|
plane = numpy.zeros((B, X), dtype=self.header.data_type)
p = 0
bar = progressbar.ProgressBar(max_value=Y)
for l in range(0, Y):
i = numpy.flatnonzero(mask[l, :])
self.file.readinto(plane)
M[:, p:p+i.shape[0]] = plane[:, i]
p = p + i.shape[0]
bar.update(l+1)
|
71ee1c23
David Mayerich
added additional ...
|
268
|
self.file.seek(prev_pos)
|
18368aa9
David Mayerich
added a new set o...
|
269
|
return M
|
90c935e3
David Mayerich
updates
|
270
271
272
273
274
275
276
277
|
def loadband(self, n):
X = self.header.samples
Y = self.header.lines
B = self.header.bands
band = numpy.zeros((Y, X), dtype=self.header.data_type)
type_bytes = numpy.dtype(self.header.data_type).itemsize
|
71ee1c23
David Mayerich
added additional ...
|
278
279
|
prev_pos = self.file.tell()
|
90c935e3
David Mayerich
updates
|
280
281
282
|
if self.header.interleave == "bsq":
self.file.seek(n * X * Y * type_bytes)
self.file.readinto(band)
|
71ee1c23
David Mayerich
added additional ...
|
283
|
self.file.seek(prev_pos)
|
90c935e3
David Mayerich
updates
|
284
285
|
return band
|
f275c01c
David Mayerich
finalized the pre...
|
286
287
288
289
290
291
292
293
294
295
296
297
|
#create a set of feature/target pairs for classification
#input: envi file object, stack of class masks C x Y x X
#output: feature matrix (features x pixels), target matrix (1 x pixels)
#example: generate_training(("class_coll.bmp", "class_epith.bmp"), (1, 2))
def loadtrain(self, classimages):
# get number of classes
C = classimages.shape[0]
F = []
T = []
for c in range(0, C):
|
59f31ea4
David Mayerich
fixed bugs in env...
|
298
|
print("\nLoading class " + str(c+1) + "...")
|
f275c01c
David Mayerich
finalized the pre...
|
299
300
301
302
303
304
|
f = self.loadmask(classimages[c, :, :]) #load the feature matrix for class c
t = numpy.ones((f.shape[1])) * (c+1) #generate a target array
F.append(f)
T.append(t)
return numpy.concatenate(F, 1).transpose(), numpy.concatenate(T)
|
71ee1c23
David Mayerich
added additional ...
|
305
306
307
308
|
#read a batch of data based on the mask
def loadbatch(self, npixels):
i = numpy.flatnonzero(self.mask) #get the indices of valid pixels
|
777fef6c
David Mayerich
added digital sta...
|
309
310
311
|
if len(i) == self.idx: #if all of the pixels have been read, return an empyt array
return []
npixels = min(npixels, len(i) - self.idx) #if there aren't enough pixels, change the batch size
|
71ee1c23
David Mayerich
added additional ...
|
312
313
314
315
316
317
318
319
320
321
322
|
B = self.header.bands
batch = numpy.zeros((B, npixels), dtype=self.header.data_type) #allocate space for the batch
pixel = numpy.zeros((B), dtype=self.header.data_type) #allocate space for a single pixel
type_bytes = numpy.dtype(self.header.data_type).itemsize #calculate the size of a single value
if self.header.interleave == "bip":
for n in range(0, npixels): #for each pixel in the batch
self.file.seek(i[self.idx] * B * type_bytes) #seek to the current pixel in the file
self.file.readinto(pixel) #read a single pixel
batch[:, n] = pixel #save the pixel into the batch matrix
self.idx = self.idx + 1
|
71ee1c23
David Mayerich
added additional ...
|
323
324
325
326
327
328
329
330
331
|
return batch
elif self.header.interleave == "bsq":
print("ERROR: BSQ batch loading isn't implemented yet!")
elif self.header.interleave == "bil":
print("ERROR: BIL batch loading isn't implemented yet!")
#returns the current batch index
def getidx(self):
return self.idx
|
777fef6c
David Mayerich
added digital sta...
|
332
333
334
335
336
337
338
339
340
341
342
343
|
#returns an image of the pixels that have been read using batch loading
def batchmask(self):
#allocate a new mask
outmask = numpy.zeros(self.mask.shape, dtype=numpy.bool)
#zero out any unclassified pixels
idx = self.getidx()
i = numpy.nonzero(self.mask)
outmask[i[0][0:idx], i[1][0:idx]] = self.mask[i[0][0:idx], i[1][0:idx]]
return outmask
|
9bef85ea
David Mayerich
Put parenthesis a...
|
344
345
|
def close(self):
self.file.close()
|
18368aa9
David Mayerich
added a new set o...
|
346
|
|
18368aa9
David Mayerich
added a new set o...
|
347
348
|
def __del__(self):
self.file.close()
|