Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# SECUREAUTH LABS. Copyright 2018 SecureAuth Corporation. All rights reserved. 

2# 

3# This software is provided under under a slightly modified version 

4# of the Apache Software License. See the accompanying LICENSE file 

5# for more information. 

6# 

7# Description: 

8# Microsoft Extensive Storage Engine parser, just focused on trying  

9# to parse NTDS.dit files (not meant as a full parser, although it might work) 

10# 

11# Author: 

12# Alberto Solino (@agsolino) 

13# 

14# Reference for: 

15# Structure. 

16#  

17# Excellent reference done by Joachim Metz 

18# http://forensic-proof.com/wp-content/uploads/2011/07/Extensible-Storage-Engine-ESE-Database-File-EDB-format.pdf 

19# 

20# ToDo:  

21# [ ] Parse multi-values properly 

22# [ ] Support long values properly 

23from __future__ import division 

24from __future__ import print_function 

25from impacket import LOG 

26try: 

27 from collections import OrderedDict 

28except: 

29 try: 

30 from ordereddict.ordereddict import OrderedDict 

31 except: 

32 from ordereddict import OrderedDict 

33from impacket.structure import Structure, hexdump 

34from struct import unpack 

35from binascii import hexlify 

36from six import b 

37 

38# Constants 

39 

40FILE_TYPE_DATABASE = 0 

41FILE_TYPE_STREAMING_FILE = 1 

42 

43# Database state 

44JET_dbstateJustCreated = 1 

45JET_dbstateDirtyShutdown = 2 

46JET_dbstateCleanShutdown = 3 

47JET_dbstateBeingConverted = 4 

48JET_dbstateForceDetach = 5 

49 

50# Page Flags 

51FLAGS_ROOT = 1 

52FLAGS_LEAF = 2 

53FLAGS_PARENT = 4 

54FLAGS_EMPTY = 8 

55FLAGS_SPACE_TREE = 0x20 

56FLAGS_INDEX = 0x40 

57FLAGS_LONG_VALUE = 0x80 

58FLAGS_NEW_FORMAT = 0x2000 

59FLAGS_NEW_CHECKSUM = 0x2000 

60 

61# Tag Flags 

62TAG_UNKNOWN = 0x1 

63TAG_DEFUNCT = 0x2 

64TAG_COMMON = 0x4 

65 

66# Fixed Page Numbers 

67DATABASE_PAGE_NUMBER = 1 

68CATALOG_PAGE_NUMBER = 4 

69CATALOG_BACKUP_PAGE_NUMBER = 24 

70 

71# Fixed FatherDataPages 

72DATABASE_FDP = 1 

73CATALOG_FDP = 2 

74CATALOG_BACKUP_FDP = 3 

75 

76# Catalog Types 

77CATALOG_TYPE_TABLE = 1 

78CATALOG_TYPE_COLUMN = 2 

79CATALOG_TYPE_INDEX = 3 

80CATALOG_TYPE_LONG_VALUE = 4 

81CATALOG_TYPE_CALLBACK = 5 

82 

83# Column Types 

84JET_coltypNil = 0 

85JET_coltypBit = 1 

86JET_coltypUnsignedByte = 2 

87JET_coltypShort = 3 

88JET_coltypLong = 4 

89JET_coltypCurrency = 5 

90JET_coltypIEEESingle = 6 

91JET_coltypIEEEDouble = 7 

92JET_coltypDateTime = 8 

93JET_coltypBinary = 9 

94JET_coltypText = 10 

95JET_coltypLongBinary = 11 

96JET_coltypLongText = 12 

97JET_coltypSLV = 13 

98JET_coltypUnsignedLong = 14 

99JET_coltypLongLong = 15 

100JET_coltypGUID = 16 

101JET_coltypUnsignedShort= 17 

102JET_coltypMax = 18 

103 

104ColumnTypeToName = { 

105 JET_coltypNil : 'NULL', 

106 JET_coltypBit : 'Boolean', 

107 JET_coltypUnsignedByte : 'Signed byte', 

108 JET_coltypShort : 'Signed short', 

109 JET_coltypLong : 'Signed long', 

110 JET_coltypCurrency : 'Currency', 

111 JET_coltypIEEESingle : 'Single precision FP', 

112 JET_coltypIEEEDouble : 'Double precision FP', 

113 JET_coltypDateTime : 'DateTime', 

114 JET_coltypBinary : 'Binary', 

115 JET_coltypText : 'Text', 

116 JET_coltypLongBinary : 'Long Binary', 

117 JET_coltypLongText : 'Long Text', 

118 JET_coltypSLV : 'Obsolete', 

119 JET_coltypUnsignedLong : 'Unsigned long', 

120 JET_coltypLongLong : 'Long long', 

121 JET_coltypGUID : 'GUID', 

122 JET_coltypUnsignedShort: 'Unsigned short', 

123 JET_coltypMax : 'Max', 

124} 

125 

126ColumnTypeSize = { 

127 JET_coltypNil : None, 

128 JET_coltypBit : (1,'B'), 

129 JET_coltypUnsignedByte : (1,'B'), 

130 JET_coltypShort : (2,'<h'), 

131 JET_coltypLong : (4,'<l'), 

132 JET_coltypCurrency : (8,'<Q'), 

133 JET_coltypIEEESingle : (4,'<f'), 

134 JET_coltypIEEEDouble : (8,'<d'), 

135 JET_coltypDateTime : (8,'<Q'), 

136 JET_coltypBinary : None, 

137 JET_coltypText : None, 

138 JET_coltypLongBinary : None, 

139 JET_coltypLongText : None, 

140 JET_coltypSLV : None, 

141 JET_coltypUnsignedLong : (4,'<L'), 

142 JET_coltypLongLong : (8,'<Q'), 

143 JET_coltypGUID : (16,'16s'), 

144 JET_coltypUnsignedShort: (2,'<H'), 

145 JET_coltypMax : None, 

146} 

147 

148# Tagged Data Type Flags 

149TAGGED_DATA_TYPE_VARIABLE_SIZE = 1 

150TAGGED_DATA_TYPE_COMPRESSED = 2 

151TAGGED_DATA_TYPE_STORED = 4 

152TAGGED_DATA_TYPE_MULTI_VALUE = 8 

153TAGGED_DATA_TYPE_WHO_KNOWS = 10 

154 

155# Code pages 

156CODEPAGE_UNICODE = 1200 

157CODEPAGE_ASCII = 20127 

158CODEPAGE_WESTERN = 1252 

159 

160StringCodePages = { 

161 CODEPAGE_UNICODE : 'utf-16le', 

162 CODEPAGE_ASCII : 'ascii', 

163 CODEPAGE_WESTERN : 'cp1252', 

164} 

165 

166# Structures 

167 

168TABLE_CURSOR = { 

169 'TableData' : b'', 

170 'FatherDataPageNumber': 0, 

171 'CurrentPageData' : b'', 

172 'CurrentTag' : 0, 

173} 

174 

175class ESENT_JET_SIGNATURE(Structure): 

176 structure = ( 

177 ('Random','<L=0'), 

178 ('CreationTime','<Q=0'), 

179 ('NetBiosName','16s=b""'), 

180 ) 

181 

182class ESENT_DB_HEADER(Structure): 

183 structure = ( 

184 ('CheckSum','<L=0'), 

185 ('Signature','"\xef\xcd\xab\x89'), 

186 ('Version','<L=0'), 

187 ('FileType','<L=0'), 

188 ('DBTime','<Q=0'), 

189 ('DBSignature',':',ESENT_JET_SIGNATURE), 

190 ('DBState','<L=0'), 

191 ('ConsistentPosition','<Q=0'), 

192 ('ConsistentTime','<Q=0'), 

193 ('AttachTime','<Q=0'), 

194 ('AttachPosition','<Q=0'), 

195 ('DetachTime','<Q=0'), 

196 ('DetachPosition','<Q=0'), 

197 ('LogSignature',':',ESENT_JET_SIGNATURE), 

198 ('Unknown','<L=0'), 

199 ('PreviousBackup','24s=b""'), 

200 ('PreviousIncBackup','24s=b""'), 

201 ('CurrentFullBackup','24s=b""'), 

202 ('ShadowingDisables','<L=0'), 

203 ('LastObjectID','<L=0'), 

204 ('WindowsMajorVersion','<L=0'), 

205 ('WindowsMinorVersion','<L=0'), 

206 ('WindowsBuildNumber','<L=0'), 

207 ('WindowsServicePackNumber','<L=0'), 

208 ('FileFormatRevision','<L=0'), 

209 ('PageSize','<L=0'), 

210 ('RepairCount','<L=0'), 

211 ('RepairTime','<Q=0'), 

212 ('Unknown2','28s=b""'), 

213 ('ScrubTime','<Q=0'), 

214 ('RequiredLog','<Q=0'), 

215 ('UpgradeExchangeFormat','<L=0'), 

216 ('UpgradeFreePages','<L=0'), 

217 ('UpgradeSpaceMapPages','<L=0'), 

218 ('CurrentShadowBackup','24s=b""'), 

219 ('CreationFileFormatVersion','<L=0'), 

220 ('CreationFileFormatRevision','<L=0'), 

221 ('Unknown3','16s=b""'), 

222 ('OldRepairCount','<L=0'), 

223 ('ECCCount','<L=0'), 

224 ('LastECCTime','<Q=0'), 

225 ('OldECCFixSuccessCount','<L=0'), 

226 ('ECCFixErrorCount','<L=0'), 

227 ('LastECCFixErrorTime','<Q=0'), 

228 ('OldECCFixErrorCount','<L=0'), 

229 ('BadCheckSumErrorCount','<L=0'), 

230 ('LastBadCheckSumTime','<Q=0'), 

231 ('OldCheckSumErrorCount','<L=0'), 

232 ('CommittedLog','<L=0'), 

233 ('PreviousShadowCopy','24s=b""'), 

234 ('PreviousDifferentialBackup','24s=b""'), 

235 ('Unknown4','40s=b""'), 

236 ('NLSMajorVersion','<L=0'), 

237 ('NLSMinorVersion','<L=0'), 

238 ('Unknown5','148s=b""'), 

239 ('UnknownFlags','<L=0'), 

240 ) 

241 

242class ESENT_PAGE_HEADER(Structure): 

243 structure_2003_SP0 = ( 

244 ('CheckSum','<L=0'), 

245 ('PageNumber','<L=0'), 

246 ) 

247 structure_0x620_0x0b = ( 

248 ('CheckSum','<L=0'), 

249 ('ECCCheckSum','<L=0'), 

250 ) 

251 structure_win7 = ( 

252 ('CheckSum','<Q=0'), 

253 ) 

254 common = ( 

255 ('LastModificationTime','<Q=0'), 

256 ('PreviousPageNumber','<L=0'), 

257 ('NextPageNumber','<L=0'), 

258 ('FatherDataPage','<L=0'), 

259 ('AvailableDataSize','<H=0'), 

260 ('AvailableUncommittedDataSize','<H=0'), 

261 ('FirstAvailableDataOffset','<H=0'), 

262 ('FirstAvailablePageTag','<H=0'), 

263 ('PageFlags','<L=0'), 

264 ) 

265 extended_win7 = ( 

266 ('ExtendedCheckSum1','<Q=0'), 

267 ('ExtendedCheckSum2','<Q=0'), 

268 ('ExtendedCheckSum3','<Q=0'), 

269 ('PageNumber','<Q=0'), 

270 ('Unknown','<Q=0'), 

271 ) 

272 def __init__(self, version, revision, pageSize=8192, data=None): 

273 if (version < 0x620) or (version == 0x620 and revision < 0x0b): 273 ↛ 275line 273 didn't jump to line 275, because the condition on line 273 was never true

274 # For sure the old format 

275 self.structure = self.structure_2003_SP0 + self.common 

276 elif version == 0x620 and revision < 0x11: 276 ↛ 278line 276 didn't jump to line 278, because the condition on line 276 was never true

277 # Exchange 2003 SP1 and Windows Vista and later 

278 self.structure = self.structure_0x620_0x0b + self.common 

279 else: 

280 # Windows 7 and later 

281 self.structure = self.structure_win7 + self.common 

282 if pageSize > 8192: 282 ↛ 283line 282 didn't jump to line 283, because the condition on line 282 was never true

283 self.structure += self.extended_win7 

284 

285 Structure.__init__(self,data) 

286 

287class ESENT_ROOT_HEADER(Structure): 

288 structure = ( 

289 ('InitialNumberOfPages','<L=0'), 

290 ('ParentFatherDataPage','<L=0'), 

291 ('ExtentSpace','<L=0'), 

292 ('SpaceTreePageNumber','<L=0'), 

293 ) 

294 

295class ESENT_BRANCH_HEADER(Structure): 

296 structure = ( 

297 ('CommonPageKey',':'), 

298 ) 

299 

300class ESENT_BRANCH_ENTRY(Structure): 

301 common = ( 

302 ('CommonPageKeySize','<H=0'), 

303 ) 

304 structure = ( 

305 ('LocalPageKeySize','<H=0'), 

306 ('_LocalPageKey','_-LocalPageKey','self["LocalPageKeySize"]'), 

307 ('LocalPageKey',':'), 

308 ('ChildPageNumber','<L=0'), 

309 ) 

310 def __init__(self, flags, data=None): 

311 if flags & TAG_COMMON > 0: 

312 # Include the common header 

313 self.structure = self.common + self.structure 

314 Structure.__init__(self,data) 

315 

316class ESENT_LEAF_HEADER(Structure): 

317 structure = ( 

318 ('CommonPageKey',':'), 

319 ) 

320 

321class ESENT_LEAF_ENTRY(Structure): 

322 common = ( 

323 ('CommonPageKeySize','<H=0'), 

324 ) 

325 structure = ( 

326 ('LocalPageKeySize','<H=0'), 

327 ('_LocalPageKey','_-LocalPageKey','self["LocalPageKeySize"]'), 

328 ('LocalPageKey',':'), 

329 ('EntryData',':'), 

330 ) 

331 def __init__(self, flags, data=None): 

332 if flags & TAG_COMMON > 0: 

333 # Include the common header 

334 self.structure = self.common + self.structure 

335 Structure.__init__(self,data) 

336 

337class ESENT_SPACE_TREE_HEADER(Structure): 

338 structure = ( 

339 ('Unknown','<Q=0'), 

340 ) 

341 

342class ESENT_SPACE_TREE_ENTRY(Structure): 

343 structure = ( 

344 ('PageKeySize','<H=0'), 

345 ('LastPageNumber','<L=0'), 

346 ('NumberOfPages','<L=0'), 

347 ) 

348 

349class ESENT_INDEX_ENTRY(Structure): 

350 structure = ( 

351 ('RecordPageKey',':'), 

352 ) 

353 

354class ESENT_DATA_DEFINITION_HEADER(Structure): 

355 structure = ( 

356 ('LastFixedSize','<B=0'), 

357 ('LastVariableDataType','<B=0'), 

358 ('VariableSizeOffset','<H=0'), 

359 ) 

360 

361class ESENT_CATALOG_DATA_DEFINITION_ENTRY(Structure): 

362 fixed = ( 

363 ('FatherDataPageID','<L=0'), 

364 ('Type','<H=0'), 

365 ('Identifier','<L=0'), 

366 ) 

367 

368 column_stuff = ( 

369 ('ColumnType','<L=0'), 

370 ('SpaceUsage','<L=0'), 

371 ('ColumnFlags','<L=0'), 

372 ('CodePage','<L=0'), 

373 ) 

374 

375 other = ( 

376 ('FatherDataPageNumber','<L=0'), 

377 ) 

378 

379 table_stuff = ( 

380 ('SpaceUsage','<L=0'), 

381# ('TableFlags','<L=0'), 

382# ('InitialNumberOfPages','<L=0'), 

383 ) 

384 

385 index_stuff = ( 

386 ('SpaceUsage','<L=0'), 

387 ('IndexFlags','<L=0'), 

388 ('Locale','<L=0'), 

389 ) 

390 

391 lv_stuff = ( 

392 ('SpaceUsage','<L=0'), 

393# ('LVFlags','<L=0'), 

394# ('InitialNumberOfPages','<L=0'), 

395 ) 

396 common = ( 

397# ('RootFlag','<B=0'), 

398# ('RecordOffset','<H=0'), 

399# ('LCMapFlags','<L=0'), 

400# ('KeyMost','<H=0'), 

401 ('Trailing',':'), 

402 ) 

403 

404 def __init__(self,data): 

405 # Depending on the type of data we'll end up building a different struct 

406 dataType = unpack('<H', data[4:][:2])[0] 

407 self.structure = self.fixed 

408 

409 if dataType == CATALOG_TYPE_TABLE: 

410 self.structure += self.other + self.table_stuff 

411 elif dataType == CATALOG_TYPE_COLUMN: 

412 self.structure += self.column_stuff 

413 elif dataType == CATALOG_TYPE_INDEX: 

414 self.structure += self.other + self.index_stuff 

415 elif dataType == CATALOG_TYPE_LONG_VALUE: 415 ↛ 417line 415 didn't jump to line 417, because the condition on line 415 was never false

416 self.structure += self.other + self.lv_stuff 

417 elif dataType == CATALOG_TYPE_CALLBACK: 

418 raise Exception('CallBack types not supported!') 

419 else: 

420 LOG.error('Unknown catalog type 0x%x' % dataType) 

421 self.structure = () 

422 Structure.__init__(self,data) 

423 

424 self.structure += self.common 

425 

426 Structure.__init__(self,data) 

427 

428 

429#def pretty_print(x): 

430# if x in '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~ ': 

431# return x 

432# else: 

433# return '.' 

434# 

435#def hexdump(data): 

436# x=str(data) 

437# strLen = len(x) 

438# i = 0 

439# while i < strLen: 

440# print "%04x " % i, 

441# for j in range(16): 

442# if i+j < strLen: 

443# print "%02X" % ord(x[i+j]), 

444# 

445# else: 

446# print " ", 

447# if j%16 == 7: 

448# print "", 

449# print " ", 

450# print ''.join(pretty_print(x) for x in x[i:i+16] ) 

451# i += 16 

452 

453def getUnixTime(t): 

454 t -= 116444736000000000 

455 t //= 10000000 

456 return t 

457 

458class ESENT_PAGE: 

459 def __init__(self, db, data=None): 

460 self.__DBHeader = db 

461 self.data = data 

462 self.record = None 

463 if data is not None: 463 ↛ exitline 463 didn't return from function '__init__', because the condition on line 463 was never false

464 self.record = ESENT_PAGE_HEADER(self.__DBHeader['Version'], self.__DBHeader['FileFormatRevision'], self.__DBHeader['PageSize'], data) 

465 

466 def printFlags(self): 

467 flags = self.record['PageFlags'] 

468 if flags & FLAGS_EMPTY: 

469 print("\tEmpty") 

470 if flags & FLAGS_INDEX: 

471 print("\tIndex") 

472 if flags & FLAGS_LEAF: 

473 print("\tLeaf") 

474 else: 

475 print("\tBranch") 

476 if flags & FLAGS_LONG_VALUE: 

477 print("\tLong Value") 

478 if flags & FLAGS_NEW_CHECKSUM: 

479 print("\tNew Checksum") 

480 if flags & FLAGS_NEW_FORMAT: 

481 print("\tNew Format") 

482 if flags & FLAGS_PARENT: 

483 print("\tParent") 

484 if flags & FLAGS_ROOT: 

485 print("\tRoot") 

486 if flags & FLAGS_SPACE_TREE: 

487 print("\tSpace Tree") 

488 

489 def dump(self): 

490 baseOffset = len(self.record) 

491 self.record.dump() 

492 tags = self.data[-4*self.record['FirstAvailablePageTag']:] 

493 

494 print("FLAGS: ") 

495 self.printFlags() 

496 

497 print() 

498 

499 for i in range(self.record['FirstAvailablePageTag']): 

500 tag = tags[-4:] 

501 if self.__DBHeader['Version'] == 0x620 and self.__DBHeader['FileFormatRevision'] > 11 and self.__DBHeader['PageSize'] > 8192: 

502 valueSize = unpack('<H', tag[:2])[0] & 0x7fff 

503 valueOffset = unpack('<H',tag[2:])[0] & 0x7fff 

504 hexdump((self.data[baseOffset+valueOffset:][:6])) 

505 pageFlags = ord(self.data[baseOffset+valueOffset:][1]) >> 5 

506 #print "TAG FLAG: 0x%x " % (unpack('<L', self.data[baseOffset+valueOffset:][:4]) ) >> 5 

507 #print "TAG FLAG: 0x " , ord(self.data[baseOffset+valueOffset:][0]) 

508 else: 

509 valueSize = unpack('<H', tag[:2])[0] & 0x1fff 

510 pageFlags = (unpack('<H', tag[2:])[0] & 0xe000) >> 13 

511 valueOffset = unpack('<H',tag[2:])[0] & 0x1fff 

512 

513 print("TAG %-8d offset:0x%-6x flags:0x%-4x valueSize:0x%x" % (i,valueOffset,pageFlags,valueSize)) 

514 #hexdump(self.getTag(i)[1]) 

515 tags = tags[:-4] 

516 

517 if self.record['PageFlags'] & FLAGS_ROOT > 0: 

518 rootHeader = ESENT_ROOT_HEADER(self.getTag(0)[1]) 

519 rootHeader.dump() 

520 elif self.record['PageFlags'] & FLAGS_LEAF == 0: 

521 # Branch Header 

522 flags, data = self.getTag(0) 

523 branchHeader = ESENT_BRANCH_HEADER(data) 

524 branchHeader.dump() 

525 else: 

526 # Leaf Header 

527 flags, data = self.getTag(0) 

528 if self.record['PageFlags'] & FLAGS_SPACE_TREE > 0: 

529 # Space Tree 

530 spaceTreeHeader = ESENT_SPACE_TREE_HEADER(data) 

531 spaceTreeHeader.dump() 

532 else: 

533 leafHeader = ESENT_LEAF_HEADER(data) 

534 leafHeader.dump() 

535 

536 # Print the leaf/branch tags 

537 for tagNum in range(1,self.record['FirstAvailablePageTag']): 

538 flags, data = self.getTag(tagNum) 

539 if self.record['PageFlags'] & FLAGS_LEAF == 0: 

540 # Branch page 

541 branchEntry = ESENT_BRANCH_ENTRY(flags, data) 

542 branchEntry.dump() 

543 elif self.record['PageFlags'] & FLAGS_LEAF > 0: 

544 # Leaf page 

545 if self.record['PageFlags'] & FLAGS_SPACE_TREE > 0: 

546 # Space Tree 

547 spaceTreeEntry = ESENT_SPACE_TREE_ENTRY(data) 

548 #spaceTreeEntry.dump() 

549 

550 elif self.record['PageFlags'] & FLAGS_INDEX > 0: 

551 # Index Entry 

552 indexEntry = ESENT_INDEX_ENTRY(data) 

553 #indexEntry.dump() 

554 elif self.record['PageFlags'] & FLAGS_LONG_VALUE > 0: 

555 # Long Page Value 

556 raise Exception('Long value still not supported') 

557 else: 

558 # Table Value 

559 leafEntry = ESENT_LEAF_ENTRY(flags, data) 

560 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(leafEntry['EntryData']) 

561 dataDefinitionHeader.dump() 

562 catalogEntry = ESENT_CATALOG_DATA_DEFINITION_ENTRY(leafEntry['EntryData'][len(dataDefinitionHeader):]) 

563 catalogEntry.dump() 

564 hexdump(leafEntry['EntryData']) 

565 

566 def getTag(self, tagNum): 

567 if self.record['FirstAvailablePageTag'] < tagNum: 567 ↛ 568line 567 didn't jump to line 568, because the condition on line 567 was never true

568 raise Exception('Trying to grab an unknown tag 0x%x' % tagNum) 

569 

570 tags = self.data[-4*self.record['FirstAvailablePageTag']:] 

571 baseOffset = len(self.record) 

572 for i in range(tagNum): 

573 tags = tags[:-4] 

574 

575 tag = tags[-4:] 

576 

577 if self.__DBHeader['Version'] == 0x620 and self.__DBHeader['FileFormatRevision'] >= 17 and self.__DBHeader['PageSize'] > 8192: 577 ↛ 578line 577 didn't jump to line 578, because the condition on line 577 was never true

578 valueSize = unpack('<H', tag[:2])[0] & 0x7fff 

579 valueOffset = unpack('<H',tag[2:])[0] & 0x7fff 

580 tmpData = list(self.data[baseOffset+valueOffset:][:valueSize]) 

581 pageFlags = ord(tmpData[1]) >> 5 

582 tmpData[1] = chr(ord(tmpData[1:2]) & 0x1f) 

583 tagData = "".join(tmpData) 

584 else: 

585 valueSize = unpack('<H', tag[:2])[0] & 0x1fff 

586 pageFlags = (unpack('<H', tag[2:])[0] & 0xe000) >> 13 

587 valueOffset = unpack('<H',tag[2:])[0] & 0x1fff 

588 tagData = self.data[baseOffset+valueOffset:][:valueSize] 

589 

590 #return pageFlags, self.data[baseOffset+valueOffset:][:valueSize] 

591 return pageFlags, tagData 

592 

593class ESENT_DB: 

594 def __init__(self, fileName, pageSize = 8192, isRemote = False): 

595 self.__fileName = fileName 

596 self.__pageSize = pageSize 

597 self.__DB = None 

598 self.__DBHeader = None 

599 self.__totalPages = None 

600 self.__tables = OrderedDict() 

601 self.__currentTable = None 

602 self.__isRemote = isRemote 

603 self.mountDB() 

604 

605 def mountDB(self): 

606 LOG.debug("Mounting DB...") 

607 if self.__isRemote is True: 607 ↛ 611line 607 didn't jump to line 611, because the condition on line 607 was never false

608 self.__DB = self.__fileName 

609 self.__DB.open() 

610 else: 

611 self.__DB = open(self.__fileName,"rb") 

612 mainHeader = self.getPage(-1) 

613 self.__DBHeader = ESENT_DB_HEADER(mainHeader) 

614 self.__pageSize = self.__DBHeader['PageSize'] 

615 self.__DB.seek(0,2) 

616 self.__totalPages = (self.__DB.tell() // self.__pageSize) -2 

617 LOG.debug("Database Version:0x%x, Revision:0x%x"% (self.__DBHeader['Version'], self.__DBHeader['FileFormatRevision'])) 

618 LOG.debug("Page Size: %d" % self.__pageSize) 

619 LOG.debug("Total Pages in file: %d" % self.__totalPages) 

620 self.parseCatalog(CATALOG_PAGE_NUMBER) 

621 

622 def printCatalog(self): 

623 indent = ' ' 

624 

625 print("Database version: 0x%x, 0x%x" % (self.__DBHeader['Version'], self.__DBHeader['FileFormatRevision'] )) 

626 print("Page size: %d " % self.__pageSize) 

627 print("Number of pages: %d" % self.__totalPages) 

628 print() 

629 print("Catalog for %s" % self.__fileName) 

630 for table in list(self.__tables.keys()): 

631 print("[%s]" % table.decode('utf8')) 

632 print("%sColumns " % indent) 

633 for column in list(self.__tables[table]['Columns'].keys()): 

634 record = self.__tables[table]['Columns'][column]['Record'] 

635 print("%s%-5d%-30s%s" % (indent*2, record['Identifier'], column.decode('utf-8'),ColumnTypeToName[record['ColumnType']])) 

636 print("%sIndexes"% indent) 

637 for index in list(self.__tables[table]['Indexes'].keys()): 

638 print("%s%s" % (indent*2, index.decode('utf-8'))) 

639 print("") 

640 

641 def __addItem(self, entry): 

642 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(entry['EntryData']) 

643 catalogEntry = ESENT_CATALOG_DATA_DEFINITION_ENTRY(entry['EntryData'][len(dataDefinitionHeader):]) 

644 itemName = self.__parseItemName(entry) 

645 

646 if catalogEntry['Type'] == CATALOG_TYPE_TABLE: 

647 self.__tables[itemName] = OrderedDict() 

648 self.__tables[itemName]['TableEntry'] = entry 

649 self.__tables[itemName]['Columns'] = OrderedDict() 

650 self.__tables[itemName]['Indexes'] = OrderedDict() 

651 self.__tables[itemName]['LongValues'] = OrderedDict() 

652 self.__currentTable = itemName 

653 elif catalogEntry['Type'] == CATALOG_TYPE_COLUMN: 

654 self.__tables[self.__currentTable]['Columns'][itemName] = entry 

655 self.__tables[self.__currentTable]['Columns'][itemName]['Header'] = dataDefinitionHeader 

656 self.__tables[self.__currentTable]['Columns'][itemName]['Record'] = catalogEntry 

657 elif catalogEntry['Type'] == CATALOG_TYPE_INDEX: 

658 self.__tables[self.__currentTable]['Indexes'][itemName] = entry 

659 elif catalogEntry['Type'] == CATALOG_TYPE_LONG_VALUE: 659 ↛ 662line 659 didn't jump to line 662, because the condition on line 659 was never false

660 self.__addLongValue(entry) 

661 else: 

662 raise Exception('Unknown type 0x%x' % catalogEntry['Type']) 

663 

664 def __parseItemName(self,entry): 

665 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(entry['EntryData']) 

666 

667 if dataDefinitionHeader['LastVariableDataType'] > 127: 667 ↛ 670line 667 didn't jump to line 670, because the condition on line 667 was never false

668 numEntries = dataDefinitionHeader['LastVariableDataType'] - 127 

669 else: 

670 numEntries = dataDefinitionHeader['LastVariableDataType'] 

671 

672 itemLen = unpack('<H',entry['EntryData'][dataDefinitionHeader['VariableSizeOffset']:][:2])[0] 

673 itemName = entry['EntryData'][dataDefinitionHeader['VariableSizeOffset']:][2*numEntries:][:itemLen] 

674 return itemName 

675 

676 def __addLongValue(self, entry): 

677 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(entry['EntryData']) 

678 lvLen = unpack('<H',entry['EntryData'][dataDefinitionHeader['VariableSizeOffset']:][:2])[0] 

679 lvName = entry['EntryData'][dataDefinitionHeader['VariableSizeOffset']:][7:][:lvLen] 

680 self.__tables[self.__currentTable]['LongValues'][lvName] = entry 

681 

682 def parsePage(self, page): 

683 # Print the leaf/branch tags 

684 for tagNum in range(1,page.record['FirstAvailablePageTag']): 

685 flags, data = page.getTag(tagNum) 

686 if page.record['PageFlags'] & FLAGS_LEAF > 0: 

687 # Leaf page 

688 if page.record['PageFlags'] & FLAGS_SPACE_TREE > 0: 688 ↛ 689line 688 didn't jump to line 689, because the condition on line 688 was never true

689 pass 

690 elif page.record['PageFlags'] & FLAGS_INDEX > 0: 690 ↛ 691line 690 didn't jump to line 691, because the condition on line 690 was never true

691 pass 

692 elif page.record['PageFlags'] & FLAGS_LONG_VALUE > 0: 692 ↛ 693line 692 didn't jump to line 693, because the condition on line 692 was never true

693 pass 

694 else: 

695 # Table Value 

696 leafEntry = ESENT_LEAF_ENTRY(flags, data) 

697 self.__addItem(leafEntry) 

698 

699 def parseCatalog(self, pageNum): 

700 # Parse all the pages starting at pageNum and commit table data 

701 page = self.getPage(pageNum) 

702 self.parsePage(page) 

703 

704 for i in range(1, page.record['FirstAvailablePageTag']): 

705 flags, data = page.getTag(i) 

706 if page.record['PageFlags'] & FLAGS_LEAF == 0: 

707 # Branch page 

708 branchEntry = ESENT_BRANCH_ENTRY(flags, data) 

709 self.parseCatalog(branchEntry['ChildPageNumber']) 

710 

711 

712 def readHeader(self): 

713 LOG.debug("Reading Boot Sector for %s" % self.__volumeName) 

714 

715 def getPage(self, pageNum): 

716 LOG.debug("Trying to fetch page %d (0x%x)" % (pageNum, (pageNum+1)*self.__pageSize)) 

717 self.__DB.seek((pageNum+1)*self.__pageSize, 0) 

718 data = self.__DB.read(self.__pageSize) 

719 while len(data) < self.__pageSize: 719 ↛ 720line 719 didn't jump to line 720, because the condition on line 719 was never true

720 remaining = self.__pageSize - len(data) 

721 data += self.__DB.read(remaining) 

722 # Special case for the first page 

723 if pageNum <= 0: 

724 return data 

725 else: 

726 return ESENT_PAGE(self.__DBHeader, data) 

727 

728 def close(self): 

729 self.__DB.close() 

730 

731 def openTable(self, tableName): 

732 # Returns a cursos for later use 

733 

734 if isinstance(tableName, bytes) is not True: 734 ↛ 737line 734 didn't jump to line 737, because the condition on line 734 was never false

735 tableName = b(tableName) 

736 

737 if tableName in self.__tables: 737 ↛ 768line 737 didn't jump to line 768, because the condition on line 737 was never false

738 entry = self.__tables[tableName]['TableEntry'] 

739 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(entry['EntryData']) 

740 catalogEntry = ESENT_CATALOG_DATA_DEFINITION_ENTRY(entry['EntryData'][len(dataDefinitionHeader):]) 

741 

742 # Let's position the cursor at the leaf levels for fast reading 

743 pageNum = catalogEntry['FatherDataPageNumber'] 

744 done = False 

745 while done is False: 

746 page = self.getPage(pageNum) 

747 if page.record['FirstAvailablePageTag'] <= 1: 747 ↛ 749line 747 didn't jump to line 749, because the condition on line 747 was never true

748 # There are no records 

749 done = True 

750 for i in range(1, page.record['FirstAvailablePageTag']): 750 ↛ 745line 750 didn't jump to line 745, because the loop on line 750 didn't complete

751 flags, data = page.getTag(i) 

752 if page.record['PageFlags'] & FLAGS_LEAF == 0: 

753 # Branch page, move on to the next page 

754 branchEntry = ESENT_BRANCH_ENTRY(flags, data) 

755 pageNum = branchEntry['ChildPageNumber'] 

756 break 

757 else: 

758 done = True 

759 break 

760 

761 cursor = TABLE_CURSOR 

762 cursor['TableData'] = self.__tables[tableName] 

763 cursor['FatherDataPageNumber'] = catalogEntry['FatherDataPageNumber'] 

764 cursor['CurrentPageData'] = page 

765 cursor['CurrentTag'] = 0 

766 return cursor 

767 else: 

768 return None 

769 

770 def __getNextTag(self, cursor): 

771 page = cursor['CurrentPageData'] 

772 

773 if cursor['CurrentTag'] >= page.record['FirstAvailablePageTag']: 

774 # No more data in this page, chau 

775 return None 

776 

777 flags, data = page.getTag(cursor['CurrentTag']) 

778 if page.record['PageFlags'] & FLAGS_LEAF > 0: 778 ↛ 791line 778 didn't jump to line 791, because the condition on line 778 was never false

779 # Leaf page 

780 if page.record['PageFlags'] & FLAGS_SPACE_TREE > 0: 780 ↛ 781line 780 didn't jump to line 781, because the condition on line 780 was never true

781 raise Exception('FLAGS_SPACE_TREE > 0') 

782 elif page.record['PageFlags'] & FLAGS_INDEX > 0: 782 ↛ 783line 782 didn't jump to line 783, because the condition on line 782 was never true

783 raise Exception('FLAGS_INDEX > 0') 

784 elif page.record['PageFlags'] & FLAGS_LONG_VALUE > 0: 784 ↛ 785line 784 didn't jump to line 785, because the condition on line 784 was never true

785 raise Exception('FLAGS_LONG_VALUE > 0') 

786 else: 

787 # Table Value 

788 leafEntry = ESENT_LEAF_ENTRY(flags, data) 

789 return leafEntry 

790 

791 return None 

792 

793 def getNextRow(self, cursor): 

794 cursor['CurrentTag'] += 1 

795 

796 tag = self.__getNextTag(cursor) 

797 #hexdump(tag) 

798 

799 if tag is None: 

800 # No more tags in this page, search for the next one on the right 

801 page = cursor['CurrentPageData'] 

802 if page.record['NextPageNumber'] == 0: 

803 # No more pages, chau 

804 return None 

805 else: 

806 cursor['CurrentPageData'] = self.getPage(page.record['NextPageNumber']) 

807 cursor['CurrentTag'] = 0 

808 return self.getNextRow(cursor) 

809 else: 

810 return self.__tagToRecord(cursor, tag['EntryData']) 

811 

812 def __tagToRecord(self, cursor, tag): 

813 # So my brain doesn't forget, the data record is composed of: 

814 # Header 

815 # Fixed Size Data (ID < 127) 

816 # The easiest to parse. Their size is fixed in the record. You can get its size 

817 # from the Column Record, field SpaceUsage 

818 # Variable Size Data (127 < ID < 255) 

819 # At VariableSizeOffset you get an array of two bytes per variable entry, pointing 

820 # to the length of the value. Values start at: 

821 # numEntries = LastVariableDataType - 127 

822 # VariableSizeOffset + numEntries * 2 (bytes) 

823 # Tagged Data ( > 255 ) 

824 # After the Variable Size Value, there's more data for the tagged values. 

825 # Right at the beginning there's another array (taggedItems), pointing to the 

826 # values, size. 

827 # 

828 # The interesting thing about this DB records is there's no need for all the columns to be there, hence 

829 # saving space. That's why I got over all the columns, and if I find data (of any type), i assign it. If  

830 # not, the column's empty. 

831 # 

832 # There are a lot of caveats in the code, so take your time to explore it.  

833 # 

834 # ToDo: Better complete this description 

835 # 

836 

837 record = OrderedDict() 

838 taggedItems = OrderedDict() 

839 taggedItemsParsed = False 

840 

841 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(tag) 

842 #dataDefinitionHeader.dump() 

843 variableDataBytesProcessed = (dataDefinitionHeader['LastVariableDataType'] - 127) * 2 

844 prevItemLen = 0 

845 tagLen = len(tag) 

846 fixedSizeOffset = len(dataDefinitionHeader) 

847 variableSizeOffset = dataDefinitionHeader['VariableSizeOffset'] 

848 

849 columns = cursor['TableData']['Columns'] 

850 

851 for column in list(columns.keys()): 

852 columnRecord = columns[column]['Record'] 

853 #columnRecord.dump() 

854 if columnRecord['Identifier'] <= dataDefinitionHeader['LastFixedSize']: 

855 # Fixed Size column data type, still available data 

856 record[column] = tag[fixedSizeOffset:][:columnRecord['SpaceUsage']] 

857 fixedSizeOffset += columnRecord['SpaceUsage'] 

858 

859 elif 127 < columnRecord['Identifier'] <= dataDefinitionHeader['LastVariableDataType']: 859 ↛ 861line 859 didn't jump to line 861, because the condition on line 859 was never true

860 # Variable data type 

861 index = columnRecord['Identifier'] - 127 - 1 

862 itemLen = unpack('<H',tag[variableSizeOffset+index*2:][:2])[0] 

863 

864 if itemLen & 0x8000: 

865 # Empty item 

866 itemLen = prevItemLen 

867 record[column] = None 

868 else: 

869 itemValue = tag[variableSizeOffset+variableDataBytesProcessed:][:itemLen-prevItemLen] 

870 record[column] = itemValue 

871 

872 #if columnRecord['Identifier'] <= dataDefinitionHeader['LastVariableDataType']: 

873 variableDataBytesProcessed +=itemLen-prevItemLen 

874 

875 prevItemLen = itemLen 

876 

877 elif columnRecord['Identifier'] > 255: 

878 # Have we parsed the tagged items already? 

879 if taggedItemsParsed is False and (variableDataBytesProcessed+variableSizeOffset) < tagLen: 

880 index = variableDataBytesProcessed+variableSizeOffset 

881 #hexdump(tag[index:]) 

882 endOfVS = self.__pageSize 

883 firstOffsetTag = (unpack('<H', tag[index+2:][:2])[0] & 0x3fff) + variableDataBytesProcessed+variableSizeOffset 

884 while True: 

885 taggedIdentifier = unpack('<H', tag[index:][:2])[0] 

886 index += 2 

887 taggedOffset = (unpack('<H', tag[index:][:2])[0] & 0x3fff) 

888 # As of Windows 7 and later ( version 0x620 revision 0x11) the  

889 # tagged data type flags are always present 

890 if self.__DBHeader['Version'] == 0x620 and self.__DBHeader['FileFormatRevision'] >= 17 and self.__DBHeader['PageSize'] > 8192: 890 ↛ 891line 890 didn't jump to line 891, because the condition on line 890 was never true

891 flagsPresent = 1 

892 else: 

893 flagsPresent = (unpack('<H', tag[index:][:2])[0] & 0x4000) 

894 index += 2 

895 if taggedOffset < endOfVS: 

896 endOfVS = taggedOffset 

897 taggedItems[taggedIdentifier] = (taggedOffset, tagLen, flagsPresent) 

898 #print "ID: %d, Offset:%d, firstOffset:%d, index:%d, flag: 0x%x" % (taggedIdentifier, taggedOffset,firstOffsetTag,index, flagsPresent) 

899 if index >= firstOffsetTag: 

900 # We reached the end of the variable size array 

901 break 

902 

903 # Calculate length of variable items 

904 # Ugly.. should be redone 

905 prevKey = list(taggedItems.keys())[0] 

906 for i in range(1,len(taggedItems)): 

907 offset0, length, flags = taggedItems[prevKey] 

908 offset, _, _ = list(taggedItems.items())[i][1] 

909 taggedItems[prevKey] = (offset0, offset-offset0, flags) 

910 #print "ID: %d, Offset: %d, Len: %d, flags: %d" % (prevKey, offset0, offset-offset0, flags) 

911 prevKey = list(taggedItems.keys())[i] 

912 taggedItemsParsed = True 

913 

914 # Tagged data type 

915 if columnRecord['Identifier'] in taggedItems: 

916 offsetItem = variableDataBytesProcessed + variableSizeOffset + taggedItems[columnRecord['Identifier']][0] 

917 itemSize = taggedItems[columnRecord['Identifier']][1] 

918 # If item have flags, we should skip them 

919 if taggedItems[columnRecord['Identifier']][2] > 0: 

920 itemFlag = ord(tag[offsetItem:offsetItem+1]) 

921 offsetItem += 1 

922 itemSize -= 1 

923 else: 

924 itemFlag = 0 

925 

926 #print "ID: %d, itemFlag: 0x%x" %( columnRecord['Identifier'], itemFlag) 

927 if itemFlag & (TAGGED_DATA_TYPE_COMPRESSED ): 927 ↛ 928line 927 didn't jump to line 928, because the condition on line 927 was never true

928 LOG.error('Unsupported tag column: %s, flag:0x%x' % (column, itemFlag)) 

929 record[column] = None 

930 elif itemFlag & TAGGED_DATA_TYPE_MULTI_VALUE: 

931 # ToDo: Parse multi-values properly 

932 LOG.debug('Multivalue detected in column %s, returning raw results' % (column)) 

933 record[column] = (hexlify(tag[offsetItem:][:itemSize]),) 

934 else: 

935 record[column] = tag[offsetItem:][:itemSize] 

936 

937 else: 

938 record[column] = None 

939 else: 

940 record[column] = None 

941 

942 # If we understand the data type, we unpack it and cast it accordingly 

943 # otherwise, we just encode it in hex 

944 if type(record[column]) is tuple: 

945 # A multi value data, we won't decode it, just leave it this way 

946 record[column] = record[column][0] 

947 elif columnRecord['ColumnType'] == JET_coltypText or columnRecord['ColumnType'] == JET_coltypLongText: 

948 # Let's handle strings 

949 if record[column] is not None: 

950 if columnRecord['CodePage'] not in StringCodePages: 950 ↛ 951line 950 didn't jump to line 951, because the condition on line 950 was never true

951 raise Exception('Unknown codepage 0x%x'% columnRecord['CodePage']) 

952 stringDecoder = StringCodePages[columnRecord['CodePage']] 

953 

954 try: 

955 record[column] = record[column].decode(stringDecoder) 

956 except Exception: 

957 LOG.debug("Exception:", exc_info=True) 

958 LOG.debug('Fixing Record[%r][%d]: %r' % (column, columnRecord['ColumnType'], record[column])) 

959 record[column] = record[column].decode(stringDecoder, "replace") 

960 pass 

961 else: 

962 unpackData = ColumnTypeSize[columnRecord['ColumnType']] 

963 if record[column] is not None: 

964 if unpackData is None: 

965 record[column] = hexlify(record[column]) 

966 else: 

967 unpackStr = unpackData[1] 

968 record[column] = unpack(unpackStr, record[column])[0] 

969 

970 return record