Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# SECUREAUTH LABS. Copyright 2018 SecureAuth Corporation. All rights reserved. 

2# 

3# This software is provided under under a slightly modified version 

4# of the Apache Software License. See the accompanying LICENSE file 

5# for more information. 

6# 

7from __future__ import division 

8from __future__ import print_function 

9from struct import pack, unpack, calcsize 

10from six import b, PY3 

11 

12class Structure: 

13 """ sublcasses can define commonHdr and/or structure. 

14 each of them is an tuple of either two: (fieldName, format) or three: (fieldName, ':', class) fields. 

15 [it can't be a dictionary, because order is important] 

16  

17 where format specifies how the data in the field will be converted to/from bytes (string) 

18 class is the class to use when unpacking ':' fields. 

19 

20 each field can only contain one value (or an array of values for *) 

21 i.e. struct.pack('Hl',1,2) is valid, but format specifier 'Hl' is not (you must use 2 dfferent fields) 

22 

23 format specifiers: 

24 specifiers from module pack can be used with the same format  

25 see struct.__doc__ (pack/unpack is finally called) 

26 x [padding byte] 

27 c [character] 

28 b [signed byte] 

29 B [unsigned byte] 

30 h [signed short] 

31 H [unsigned short] 

32 l [signed long] 

33 L [unsigned long] 

34 i [signed integer] 

35 I [unsigned integer] 

36 q [signed long long (quad)] 

37 Q [unsigned long long (quad)] 

38 s [string (array of chars), must be preceded with length in format specifier, padded with zeros] 

39 p [pascal string (includes byte count), must be preceded with length in format specifier, padded with zeros] 

40 f [float] 

41 d [double] 

42 = [native byte ordering, size and alignment] 

43 @ [native byte ordering, standard size and alignment] 

44 ! [network byte ordering] 

45 < [little endian] 

46 > [big endian] 

47 

48 usual printf like specifiers can be used (if started with %)  

49 [not recommended, there is no way to unpack this] 

50 

51 %08x will output an 8 bytes hex 

52 %s will output a string 

53 %s\\x00 will output a NUL terminated string 

54 %d%d will output 2 decimal digits (against the very same specification of Structure) 

55 ... 

56 

57 some additional format specifiers: 

58 : just copy the bytes from the field into the output string (input may be string, other structure, or anything responding to __str__()) (for unpacking, all what's left is returned) 

59 z same as :, but adds a NUL byte at the end (asciiz) (for unpacking the first NUL byte is used as terminator) [asciiz string] 

60 u same as z, but adds two NUL bytes at the end (after padding to an even size with NULs). (same for unpacking) [unicode string] 

61 w DCE-RPC/NDR string (it's a macro for [ '<L=(len(field)+1)/2','"\\x00\\x00\\x00\\x00','<L=(len(field)+1)/2',':' ] 

62 ?-field length of field named 'field', formatted as specified with ? ('?' may be '!H' for example). The input value overrides the real length 

63 ?1*?2 array of elements. Each formatted as '?2', the number of elements in the array is stored as specified by '?1' (?1 is optional, or can also be a constant (number), for unpacking) 

64 'xxxx literal xxxx (field's value doesn't change the output. quotes must not be closed or escaped) 

65 "xxxx literal xxxx (field's value doesn't change the output. quotes must not be closed or escaped) 

66 _ will not pack the field. Accepts a third argument, which is an unpack code. See _Test_UnpackCode for an example 

67 ?=packcode will evaluate packcode in the context of the structure, and pack the result as specified by ?. Unpacking is made plain 

68 ?&fieldname "Address of field fieldname". 

69 For packing it will simply pack the id() of fieldname. Or use 0 if fieldname doesn't exists. 

70 For unpacking, it's used to know weather fieldname has to be unpacked or not, i.e. by adding a & field you turn another field (fieldname) in an optional field. 

71  

72 """ 

73 commonHdr = () 

74 structure = () 

75 debug = 0 

76 

77 def __init__(self, data = None, alignment = 0): 

78 if not hasattr(self, 'alignment'): 78 ↛ 81line 78 didn't jump to line 81, because the condition on line 78 was never false

79 self.alignment = alignment 

80 

81 self.fields = {} 

82 self.rawData = data 

83 if data is not None: 

84 self.fromString(data) 

85 else: 

86 self.data = None 

87 

88 @classmethod 

89 def fromFile(self, file): 

90 answer = self() 

91 answer.fromString(file.read(len(answer))) 

92 return answer 

93 

94 def setAlignment(self, alignment): 

95 self.alignment = alignment 

96 

97 def setData(self, data): 

98 self.data = data 

99 

100 def packField(self, fieldName, format = None): 

101 if self.debug: 101 ↛ 102line 101 didn't jump to line 102, because the condition on line 101 was never true

102 print("packField( %s | %s )" % (fieldName, format)) 

103 

104 if format is None: 104 ↛ 105line 104 didn't jump to line 105, because the condition on line 104 was never true

105 format = self.formatForField(fieldName) 

106 

107 if fieldName in self.fields: 

108 ans = self.pack(format, self.fields[fieldName], field = fieldName) 

109 else: 

110 ans = self.pack(format, None, field = fieldName) 

111 

112 if self.debug: 112 ↛ 113line 112 didn't jump to line 113, because the condition on line 112 was never true

113 print("\tanswer %r" % ans) 

114 

115 return ans 

116 

117 def getData(self): 

118 if self.data is not None: 118 ↛ 119line 118 didn't jump to line 119, because the condition on line 118 was never true

119 return self.data 

120 data = bytes() 

121 for field in self.commonHdr+self.structure: 

122 try: 

123 data += self.packField(field[0], field[1]) 

124 except Exception as e: 

125 if field[0] in self.fields: 

126 e.args += ("When packing field '%s | %s | %r' in %s" % (field[0], field[1], self[field[0]], self.__class__),) 

127 else: 

128 e.args += ("When packing field '%s | %s' in %s" % (field[0], field[1], self.__class__),) 

129 raise 

130 if self.alignment: 

131 if len(data) % self.alignment: 

132 data += (b'\x00'*self.alignment)[:-(len(data) % self.alignment)] 

133 

134 #if len(data) % self.alignment: data += ('\x00'*self.alignment)[:-(len(data) % self.alignment)] 

135 return data 

136 

137 def fromString(self, data): 

138 self.rawData = data 

139 for field in self.commonHdr+self.structure: 

140 if self.debug: 140 ↛ 141line 140 didn't jump to line 141, because the condition on line 140 was never true

141 print("fromString( %s | %s | %r )" % (field[0], field[1], data)) 

142 size = self.calcUnpackSize(field[1], data, field[0]) 

143 if self.debug: 143 ↛ 144line 143 didn't jump to line 144, because the condition on line 143 was never true

144 print(" size = %d" % size) 

145 dataClassOrCode = b 

146 if len(field) > 2: 

147 dataClassOrCode = field[2] 

148 try: 

149 self[field[0]] = self.unpack(field[1], data[:size], dataClassOrCode = dataClassOrCode, field = field[0]) 

150 except Exception as e: 

151 e.args += ("When unpacking field '%s | %s | %r[:%d]'" % (field[0], field[1], data, size),) 

152 raise 

153 

154 size = self.calcPackSize(field[1], self[field[0]], field[0]) 

155 if self.alignment and size % self.alignment: 

156 size += self.alignment - (size % self.alignment) 

157 data = data[size:] 

158 

159 return self 

160 

161 def __setitem__(self, key, value): 

162 self.fields[key] = value 

163 self.data = None # force recompute 

164 

165 def __getitem__(self, key): 

166 return self.fields[key] 

167 

168 def __delitem__(self, key): 

169 del self.fields[key] 

170 

171 def __str__(self): 

172 return self.getData() 

173 

174 def __len__(self): 

175 # XXX: improve 

176 return len(self.getData()) 

177 

178 def pack(self, format, data, field = None): 

179 if self.debug: 179 ↛ 180line 179 didn't jump to line 180, because the condition on line 179 was never true

180 print(" pack( %s | %r | %s)" % (format, data, field)) 

181 

182 if field: 

183 addressField = self.findAddressFieldFor(field) 

184 if (addressField is not None) and (data is None): 

185 return b'' 

186 

187 # void specifier 

188 if format[:1] == '_': 

189 return b'' 

190 

191 # quote specifier 

192 if format[:1] == "'" or format[:1] == '"': 

193 return b(format[1:]) 

194 

195 # code specifier 

196 two = format.split('=') 

197 if len(two) >= 2: 

198 try: 

199 return self.pack(two[0], data) 

200 except: 

201 fields = {'self':self} 

202 fields.update(self.fields) 

203 return self.pack(two[0], eval(two[1], {}, fields)) 

204 

205 # address specifier 

206 two = format.split('&') 

207 if len(two) == 2: 

208 try: 

209 return self.pack(two[0], data) 

210 except: 

211 if (two[1] in self.fields) and (self[two[1]] is not None): 

212 return self.pack(two[0], id(self[two[1]]) & ((1<<(calcsize(two[0])*8))-1) ) 

213 else: 

214 return self.pack(two[0], 0) 

215 

216 # length specifier 

217 two = format.split('-') 

218 if len(two) == 2: 

219 try: 

220 return self.pack(two[0],data) 

221 except: 

222 return self.pack(two[0], self.calcPackFieldSize(two[1])) 

223 

224 # array specifier 

225 two = format.split('*') 

226 if len(two) == 2: 

227 answer = bytes() 

228 for each in data: 

229 answer += self.pack(two[1], each) 

230 if two[0]: 

231 if two[0].isdigit(): 231 ↛ 232line 231 didn't jump to line 232, because the condition on line 231 was never true

232 if int(two[0]) != len(data): 

233 raise Exception("Array field has a constant size, and it doesn't match the actual value") 

234 else: 

235 return self.pack(two[0], len(data))+answer 

236 return answer 

237 

238 # "printf" string specifier 

239 if format[:1] == '%': 239 ↛ 241line 239 didn't jump to line 241, because the condition on line 239 was never true

240 # format string like specifier 

241 return b(format % data) 

242 

243 # asciiz specifier 

244 if format[:1] == 'z': 

245 if isinstance(data,bytes): 245 ↛ 246line 245 didn't jump to line 246, because the condition on line 245 was never true

246 return data + b('\0') 

247 return bytes(b(data)+b('\0')) 

248 

249 # unicode specifier 

250 if format[:1] == 'u': 

251 return bytes(data+b('\0\0') + (len(data) & 1 and b('\0') or b'')) 

252 

253 # DCE-RPC/NDR string specifier 

254 if format[:1] == 'w': 

255 if len(data) == 0: 255 ↛ 256line 255 didn't jump to line 256, because the condition on line 255 was never true

256 data = b('\0\0') 

257 elif len(data) % 2: 

258 data = b(data) + b('\0') 

259 l = pack('<L', len(data)//2) 

260 return b''.join([l, l, b('\0\0\0\0'), data]) 

261 

262 if data is None: 

263 raise Exception("Trying to pack None") 

264 

265 # literal specifier 

266 if format[:1] == ':': 

267 if isinstance(data, Structure): 

268 return data.getData() 

269 # If we have an object that can serialize itself, go ahead 

270 elif hasattr(data, "getData"): 270 ↛ 271line 270 didn't jump to line 271, because the condition on line 270 was never true

271 return data.getData() 

272 elif isinstance(data, int): 272 ↛ 273line 272 didn't jump to line 273, because the condition on line 272 was never true

273 return bytes(data) 

274 elif isinstance(data, bytes) is not True: 

275 return bytes(b(data)) 

276 else: 

277 return data 

278 

279 if format[-1:] == 's': 

280 # Let's be sure we send the right type 

281 if isinstance(data, bytes) or isinstance(data, bytearray): 

282 return pack(format, data) 

283 else: 

284 return pack(format, b(data)) 

285 

286 # struct like specifier 

287 return pack(format, data) 

288 

289 def unpack(self, format, data, dataClassOrCode = b, field = None): 

290 if self.debug: 290 ↛ 291line 290 didn't jump to line 291, because the condition on line 290 was never true

291 print(" unpack( %s | %r )" % (format, data)) 

292 

293 if field: 

294 addressField = self.findAddressFieldFor(field) 

295 if addressField is not None: 

296 if not self[addressField]: 

297 return 

298 

299 # void specifier 

300 if format[:1] == '_': 

301 if dataClassOrCode != b: 

302 fields = {'self':self, 'inputDataLeft':data} 

303 fields.update(self.fields) 

304 return eval(dataClassOrCode, {}, fields) 

305 else: 

306 return None 

307 

308 # quote specifier 

309 if format[:1] == "'" or format[:1] == '"': 

310 answer = format[1:] 

311 if b(answer) != data: 

312 raise Exception("Unpacked data doesn't match constant value '%r' should be '%r'" % (data, answer)) 

313 return answer 

314 

315 # address specifier 

316 two = format.split('&') 

317 if len(two) == 2: 

318 return self.unpack(two[0],data) 

319 

320 # code specifier 

321 two = format.split('=') 

322 if len(two) >= 2: 

323 return self.unpack(two[0],data) 

324 

325 # length specifier 

326 two = format.split('-') 

327 if len(two) == 2: 

328 return self.unpack(two[0],data) 

329 

330 # array specifier 

331 two = format.split('*') 

332 if len(two) == 2: 

333 answer = [] 

334 sofar = 0 

335 if two[0].isdigit(): 335 ↛ 336line 335 didn't jump to line 336, because the condition on line 335 was never true

336 number = int(two[0]) 

337 elif two[0]: 

338 sofar += self.calcUnpackSize(two[0], data) 

339 number = self.unpack(two[0], data[:sofar]) 

340 else: 

341 number = -1 

342 

343 while number and sofar < len(data): 

344 nsofar = sofar + self.calcUnpackSize(two[1],data[sofar:]) 

345 answer.append(self.unpack(two[1], data[sofar:nsofar], dataClassOrCode)) 

346 number -= 1 

347 sofar = nsofar 

348 return answer 

349 

350 # "printf" string specifier 

351 if format[:1] == '%': 351 ↛ 353line 351 didn't jump to line 353, because the condition on line 351 was never true

352 # format string like specifier 

353 return format % data 

354 

355 # asciiz specifier 

356 if format == 'z': 

357 if data[-1:] != b('\x00'): 

358 raise Exception("%s 'z' field is not NUL terminated: %r" % (field, data)) 

359 if PY3: 359 ↛ 362line 359 didn't jump to line 362, because the condition on line 359 was never false

360 return data[:-1].decode('latin-1') 

361 else: 

362 return data[:-1] 

363 

364 # unicode specifier 

365 if format == 'u': 

366 if data[-2:] != b('\x00\x00'): 366 ↛ 367line 366 didn't jump to line 367, because the condition on line 366 was never true

367 raise Exception("%s 'u' field is not NUL-NUL terminated: %r" % (field, data)) 

368 return data[:-2] # remove trailing NUL 

369 

370 # DCE-RPC/NDR string specifier 

371 if format == 'w': 

372 l = unpack('<L', data[:4])[0] 

373 return data[12:12+l*2] 

374 

375 # literal specifier 

376 if format == ':': 

377 if isinstance(data, bytes) and dataClassOrCode is b: 

378 return data 

379 return dataClassOrCode(data) 

380 

381 # struct like specifier 

382 return unpack(format, data)[0] 

383 

384 def calcPackSize(self, format, data, field = None): 

385# # print " calcPackSize %s:%r" % (format, data) 

386 if field: 

387 addressField = self.findAddressFieldFor(field) 

388 if addressField is not None: 

389 if not self[addressField]: 

390 return 0 

391 

392 # void specifier 

393 if format[:1] == '_': 

394 return 0 

395 

396 # quote specifier 

397 if format[:1] == "'" or format[:1] == '"': 

398 return len(format)-1 

399 

400 # address specifier 

401 two = format.split('&') 

402 if len(two) == 2: 

403 return self.calcPackSize(two[0], data) 

404 

405 # code specifier 

406 two = format.split('=') 

407 if len(two) >= 2: 

408 return self.calcPackSize(two[0], data) 

409 

410 # length specifier 

411 two = format.split('-') 

412 if len(two) == 2: 

413 return self.calcPackSize(two[0], data) 

414 

415 # array specifier 

416 two = format.split('*') 

417 if len(two) == 2: 

418 answer = 0 

419 if two[0].isdigit(): 419 ↛ 420line 419 didn't jump to line 420, because the condition on line 419 was never true

420 if int(two[0]) != len(data): 

421 raise Exception("Array field has a constant size, and it doesn't match the actual value") 

422 elif two[0]: 

423 answer += self.calcPackSize(two[0], len(data)) 

424 

425 for each in data: 

426 answer += self.calcPackSize(two[1], each) 

427 return answer 

428 

429 # "printf" string specifier 

430 if format[:1] == '%': 430 ↛ 432line 430 didn't jump to line 432, because the condition on line 430 was never true

431 # format string like specifier 

432 return len(format % data) 

433 

434 # asciiz specifier 

435 if format[:1] == 'z': 

436 return len(data)+1 

437 

438 # asciiz specifier 

439 if format[:1] == 'u': 

440 l = len(data) 

441 return l + (l & 1 and 3 or 2) 

442 

443 # DCE-RPC/NDR string specifier 

444 if format[:1] == 'w': 

445 l = len(data) 

446 return 12+l+l % 2 

447 

448 # literal specifier 

449 if format[:1] == ':': 

450 return len(data) 

451 

452 # struct like specifier 

453 return calcsize(format) 

454 

455 def calcUnpackSize(self, format, data, field = None): 

456 if self.debug: 456 ↛ 457line 456 didn't jump to line 457, because the condition on line 456 was never true

457 print(" calcUnpackSize( %s | %s | %r)" % (field, format, data)) 

458 

459 # void specifier 

460 if format[:1] == '_': 

461 return 0 

462 

463 addressField = self.findAddressFieldFor(field) 

464 if addressField is not None: 

465 if not self[addressField]: 

466 return 0 

467 

468 try: 

469 lengthField = self.findLengthFieldFor(field) 

470 return int(self[lengthField]) 

471 except Exception: 

472 pass 

473 

474 # XXX: Try to match to actual values, raise if no match 

475 

476 # quote specifier 

477 if format[:1] == "'" or format[:1] == '"': 

478 return len(format)-1 

479 

480 # address specifier 

481 two = format.split('&') 

482 if len(two) == 2: 

483 return self.calcUnpackSize(two[0], data) 

484 

485 # code specifier 

486 two = format.split('=') 

487 if len(two) >= 2: 

488 return self.calcUnpackSize(two[0], data) 

489 

490 # length specifier 

491 two = format.split('-') 

492 if len(two) == 2: 

493 return self.calcUnpackSize(two[0], data) 

494 

495 # array specifier 

496 two = format.split('*') 

497 if len(two) == 2: 

498 answer = 0 

499 if two[0]: 

500 if two[0].isdigit(): 500 ↛ 501line 500 didn't jump to line 501, because the condition on line 500 was never true

501 number = int(two[0]) 

502 else: 

503 answer += self.calcUnpackSize(two[0], data) 

504 number = self.unpack(two[0], data[:answer]) 

505 

506 while number: 

507 number -= 1 

508 answer += self.calcUnpackSize(two[1], data[answer:]) 

509 else: 

510 while answer < len(data): 

511 answer += self.calcUnpackSize(two[1], data[answer:]) 

512 return answer 

513 

514 # "printf" string specifier 

515 if format[:1] == '%': 515 ↛ 516line 515 didn't jump to line 516, because the condition on line 515 was never true

516 raise Exception("Can't guess the size of a printf like specifier for unpacking") 

517 

518 # asciiz specifier 

519 if format[:1] == 'z': 

520 return data.index(b('\x00'))+1 

521 

522 # asciiz specifier 

523 if format[:1] == 'u': 

524 l = data.index(b('\x00\x00')) 

525 return l + (l & 1 and 3 or 2) 

526 

527 # DCE-RPC/NDR string specifier 

528 if format[:1] == 'w': 

529 l = unpack('<L', data[:4])[0] 

530 return 12+l*2 

531 

532 # literal specifier 

533 if format[:1] == ':': 

534 return len(data) 

535 

536 # struct like specifier 

537 return calcsize(format) 

538 

539 def calcPackFieldSize(self, fieldName, format = None): 

540 if format is None: 540 ↛ 543line 540 didn't jump to line 543, because the condition on line 540 was never false

541 format = self.formatForField(fieldName) 

542 

543 return self.calcPackSize(format, self[fieldName]) 

544 

545 def formatForField(self, fieldName): 

546 for field in self.commonHdr+self.structure: 546 ↛ 549line 546 didn't jump to line 549, because the loop on line 546 didn't complete

547 if field[0] == fieldName: 

548 return field[1] 

549 raise Exception("Field %s not found" % fieldName) 

550 

551 def findAddressFieldFor(self, fieldName): 

552 descriptor = '&%s' % fieldName 

553 l = len(descriptor) 

554 for field in self.commonHdr+self.structure: 

555 if field[1][-l:] == descriptor: 

556 return field[0] 

557 return None 

558 

559 def findLengthFieldFor(self, fieldName): 

560 descriptor = '-%s' % fieldName 

561 l = len(descriptor) 

562 for field in self.commonHdr+self.structure: 

563 if field[1][-l:] == descriptor: 

564 return field[0] 

565 return None 

566 

567 def zeroValue(self, format): 

568 two = format.split('*') 

569 if len(two) == 2: 

570 if two[0].isdigit(): 

571 return (self.zeroValue(two[1]),)*int(two[0]) 

572 

573 if not format.find('*') == -1: 

574 return () 

575 if 's' in format: 

576 return b'' 

577 if format in ['z',':','u']: 

578 return b'' 

579 if format == 'w': 

580 return b('\x00\x00') 

581 

582 return 0 

583 

584 def clear(self): 

585 for field in self.commonHdr + self.structure: 

586 self[field[0]] = self.zeroValue(field[1]) 

587 

588 def dump(self, msg = None, indent = 0): 

589 if msg is None: 

590 msg = self.__class__.__name__ 

591 ind = ' '*indent 

592 print("\n%s" % msg) 

593 fixedFields = [] 

594 for field in self.commonHdr+self.structure: 

595 i = field[0] 

596 if i in self.fields: 596 ↛ 594line 596 didn't jump to line 594, because the condition on line 596 was never false

597 fixedFields.append(i) 

598 if isinstance(self[i], Structure): 

599 self[i].dump('%s%s:{' % (ind,i), indent = indent + 4) 

600 print("%s}" % ind) 

601 else: 

602 print("%s%s: {%r}" % (ind,i,self[i])) 

603 # Do we have remaining fields not defined in the structures? let's  

604 # print them 

605 remainingFields = list(set(self.fields) - set(fixedFields)) 

606 for i in remainingFields: 

607 if isinstance(self[i], Structure): 607 ↛ 608line 607 didn't jump to line 608, because the condition on line 607 was never true

608 self[i].dump('%s%s:{' % (ind,i), indent = indent + 4) 

609 print("%s}" % ind) 

610 else: 

611 print("%s%s: {%r}" % (ind,i,self[i])) 

612 

613def pretty_print(x): 

614 if chr(x) in '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~ ': 

615 return chr(x) 

616 else: 

617 return u'.' 

618 

619def hexdump(data, indent = ''): 

620 if data is None: 620 ↛ 621line 620 didn't jump to line 621, because the condition on line 620 was never true

621 return 

622 if isinstance(data, int): 622 ↛ 623line 622 didn't jump to line 623, because the condition on line 622 was never true

623 data = str(data).encode('utf-8') 

624 x=bytearray(data) 

625 strLen = len(x) 

626 i = 0 

627 while i < strLen: 

628 line = " %s%04x " % (indent, i) 

629 for j in range(16): 

630 if i+j < strLen: 

631 line += "%02X " % x[i+j] 

632 else: 

633 line += u" " 

634 if j%16 == 7: 

635 line += " " 

636 line += " " 

637 line += ''.join(pretty_print(x) for x in x[i:i+16] ) 

638 print (line) 

639 i += 16 

640 

641def parse_bitmask(dict, value): 

642 ret = '' 

643 

644 for i in range(0, 31): 

645 flag = 1 << i 

646 

647 if value & flag == 0: 

648 continue 

649 

650 if flag in dict: 

651 ret += '%s | ' % dict[flag] 

652 else: 

653 ret += "0x%.8X | " % flag 

654 

655 if len(ret) == 0: 

656 return '0' 

657 else: 

658 return ret[:-3]