Coverage for /root/GitHubProjects/impacket/impacket/ese.py : 68%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# SECUREAUTH LABS. Copyright 2018 SecureAuth Corporation. All rights reserved.
2#
3# This software is provided under under a slightly modified version
4# of the Apache Software License. See the accompanying LICENSE file
5# for more information.
6#
7# Description:
8# Microsoft Extensive Storage Engine parser, just focused on trying
9# to parse NTDS.dit files (not meant as a full parser, although it might work)
10#
11# Author:
12# Alberto Solino (@agsolino)
13#
14# Reference for:
15# Structure.
16#
17# Excellent reference done by Joachim Metz
18# http://forensic-proof.com/wp-content/uploads/2011/07/Extensible-Storage-Engine-ESE-Database-File-EDB-format.pdf
19#
20# ToDo:
21# [ ] Parse multi-values properly
22# [ ] Support long values properly
23from __future__ import division
24from __future__ import print_function
25from impacket import LOG
26try:
27 from collections import OrderedDict
28except:
29 try:
30 from ordereddict.ordereddict import OrderedDict
31 except:
32 from ordereddict import OrderedDict
33from impacket.structure import Structure, hexdump
34from struct import unpack
35from binascii import hexlify
36from six import b
38# Constants
40FILE_TYPE_DATABASE = 0
41FILE_TYPE_STREAMING_FILE = 1
43# Database state
44JET_dbstateJustCreated = 1
45JET_dbstateDirtyShutdown = 2
46JET_dbstateCleanShutdown = 3
47JET_dbstateBeingConverted = 4
48JET_dbstateForceDetach = 5
50# Page Flags
51FLAGS_ROOT = 1
52FLAGS_LEAF = 2
53FLAGS_PARENT = 4
54FLAGS_EMPTY = 8
55FLAGS_SPACE_TREE = 0x20
56FLAGS_INDEX = 0x40
57FLAGS_LONG_VALUE = 0x80
58FLAGS_NEW_FORMAT = 0x2000
59FLAGS_NEW_CHECKSUM = 0x2000
61# Tag Flags
62TAG_UNKNOWN = 0x1
63TAG_DEFUNCT = 0x2
64TAG_COMMON = 0x4
66# Fixed Page Numbers
67DATABASE_PAGE_NUMBER = 1
68CATALOG_PAGE_NUMBER = 4
69CATALOG_BACKUP_PAGE_NUMBER = 24
71# Fixed FatherDataPages
72DATABASE_FDP = 1
73CATALOG_FDP = 2
74CATALOG_BACKUP_FDP = 3
76# Catalog Types
77CATALOG_TYPE_TABLE = 1
78CATALOG_TYPE_COLUMN = 2
79CATALOG_TYPE_INDEX = 3
80CATALOG_TYPE_LONG_VALUE = 4
81CATALOG_TYPE_CALLBACK = 5
83# Column Types
84JET_coltypNil = 0
85JET_coltypBit = 1
86JET_coltypUnsignedByte = 2
87JET_coltypShort = 3
88JET_coltypLong = 4
89JET_coltypCurrency = 5
90JET_coltypIEEESingle = 6
91JET_coltypIEEEDouble = 7
92JET_coltypDateTime = 8
93JET_coltypBinary = 9
94JET_coltypText = 10
95JET_coltypLongBinary = 11
96JET_coltypLongText = 12
97JET_coltypSLV = 13
98JET_coltypUnsignedLong = 14
99JET_coltypLongLong = 15
100JET_coltypGUID = 16
101JET_coltypUnsignedShort= 17
102JET_coltypMax = 18
104ColumnTypeToName = {
105 JET_coltypNil : 'NULL',
106 JET_coltypBit : 'Boolean',
107 JET_coltypUnsignedByte : 'Signed byte',
108 JET_coltypShort : 'Signed short',
109 JET_coltypLong : 'Signed long',
110 JET_coltypCurrency : 'Currency',
111 JET_coltypIEEESingle : 'Single precision FP',
112 JET_coltypIEEEDouble : 'Double precision FP',
113 JET_coltypDateTime : 'DateTime',
114 JET_coltypBinary : 'Binary',
115 JET_coltypText : 'Text',
116 JET_coltypLongBinary : 'Long Binary',
117 JET_coltypLongText : 'Long Text',
118 JET_coltypSLV : 'Obsolete',
119 JET_coltypUnsignedLong : 'Unsigned long',
120 JET_coltypLongLong : 'Long long',
121 JET_coltypGUID : 'GUID',
122 JET_coltypUnsignedShort: 'Unsigned short',
123 JET_coltypMax : 'Max',
124}
126ColumnTypeSize = {
127 JET_coltypNil : None,
128 JET_coltypBit : (1,'B'),
129 JET_coltypUnsignedByte : (1,'B'),
130 JET_coltypShort : (2,'<h'),
131 JET_coltypLong : (4,'<l'),
132 JET_coltypCurrency : (8,'<Q'),
133 JET_coltypIEEESingle : (4,'<f'),
134 JET_coltypIEEEDouble : (8,'<d'),
135 JET_coltypDateTime : (8,'<Q'),
136 JET_coltypBinary : None,
137 JET_coltypText : None,
138 JET_coltypLongBinary : None,
139 JET_coltypLongText : None,
140 JET_coltypSLV : None,
141 JET_coltypUnsignedLong : (4,'<L'),
142 JET_coltypLongLong : (8,'<Q'),
143 JET_coltypGUID : (16,'16s'),
144 JET_coltypUnsignedShort: (2,'<H'),
145 JET_coltypMax : None,
146}
148# Tagged Data Type Flags
149TAGGED_DATA_TYPE_VARIABLE_SIZE = 1
150TAGGED_DATA_TYPE_COMPRESSED = 2
151TAGGED_DATA_TYPE_STORED = 4
152TAGGED_DATA_TYPE_MULTI_VALUE = 8
153TAGGED_DATA_TYPE_WHO_KNOWS = 10
155# Code pages
156CODEPAGE_UNICODE = 1200
157CODEPAGE_ASCII = 20127
158CODEPAGE_WESTERN = 1252
160StringCodePages = {
161 CODEPAGE_UNICODE : 'utf-16le',
162 CODEPAGE_ASCII : 'ascii',
163 CODEPAGE_WESTERN : 'cp1252',
164}
166# Structures
168TABLE_CURSOR = {
169 'TableData' : b'',
170 'FatherDataPageNumber': 0,
171 'CurrentPageData' : b'',
172 'CurrentTag' : 0,
173}
175class ESENT_JET_SIGNATURE(Structure):
176 structure = (
177 ('Random','<L=0'),
178 ('CreationTime','<Q=0'),
179 ('NetBiosName','16s=b""'),
180 )
182class ESENT_DB_HEADER(Structure):
183 structure = (
184 ('CheckSum','<L=0'),
185 ('Signature','"\xef\xcd\xab\x89'),
186 ('Version','<L=0'),
187 ('FileType','<L=0'),
188 ('DBTime','<Q=0'),
189 ('DBSignature',':',ESENT_JET_SIGNATURE),
190 ('DBState','<L=0'),
191 ('ConsistentPosition','<Q=0'),
192 ('ConsistentTime','<Q=0'),
193 ('AttachTime','<Q=0'),
194 ('AttachPosition','<Q=0'),
195 ('DetachTime','<Q=0'),
196 ('DetachPosition','<Q=0'),
197 ('LogSignature',':',ESENT_JET_SIGNATURE),
198 ('Unknown','<L=0'),
199 ('PreviousBackup','24s=b""'),
200 ('PreviousIncBackup','24s=b""'),
201 ('CurrentFullBackup','24s=b""'),
202 ('ShadowingDisables','<L=0'),
203 ('LastObjectID','<L=0'),
204 ('WindowsMajorVersion','<L=0'),
205 ('WindowsMinorVersion','<L=0'),
206 ('WindowsBuildNumber','<L=0'),
207 ('WindowsServicePackNumber','<L=0'),
208 ('FileFormatRevision','<L=0'),
209 ('PageSize','<L=0'),
210 ('RepairCount','<L=0'),
211 ('RepairTime','<Q=0'),
212 ('Unknown2','28s=b""'),
213 ('ScrubTime','<Q=0'),
214 ('RequiredLog','<Q=0'),
215 ('UpgradeExchangeFormat','<L=0'),
216 ('UpgradeFreePages','<L=0'),
217 ('UpgradeSpaceMapPages','<L=0'),
218 ('CurrentShadowBackup','24s=b""'),
219 ('CreationFileFormatVersion','<L=0'),
220 ('CreationFileFormatRevision','<L=0'),
221 ('Unknown3','16s=b""'),
222 ('OldRepairCount','<L=0'),
223 ('ECCCount','<L=0'),
224 ('LastECCTime','<Q=0'),
225 ('OldECCFixSuccessCount','<L=0'),
226 ('ECCFixErrorCount','<L=0'),
227 ('LastECCFixErrorTime','<Q=0'),
228 ('OldECCFixErrorCount','<L=0'),
229 ('BadCheckSumErrorCount','<L=0'),
230 ('LastBadCheckSumTime','<Q=0'),
231 ('OldCheckSumErrorCount','<L=0'),
232 ('CommittedLog','<L=0'),
233 ('PreviousShadowCopy','24s=b""'),
234 ('PreviousDifferentialBackup','24s=b""'),
235 ('Unknown4','40s=b""'),
236 ('NLSMajorVersion','<L=0'),
237 ('NLSMinorVersion','<L=0'),
238 ('Unknown5','148s=b""'),
239 ('UnknownFlags','<L=0'),
240 )
242class ESENT_PAGE_HEADER(Structure):
243 structure_2003_SP0 = (
244 ('CheckSum','<L=0'),
245 ('PageNumber','<L=0'),
246 )
247 structure_0x620_0x0b = (
248 ('CheckSum','<L=0'),
249 ('ECCCheckSum','<L=0'),
250 )
251 structure_win7 = (
252 ('CheckSum','<Q=0'),
253 )
254 common = (
255 ('LastModificationTime','<Q=0'),
256 ('PreviousPageNumber','<L=0'),
257 ('NextPageNumber','<L=0'),
258 ('FatherDataPage','<L=0'),
259 ('AvailableDataSize','<H=0'),
260 ('AvailableUncommittedDataSize','<H=0'),
261 ('FirstAvailableDataOffset','<H=0'),
262 ('FirstAvailablePageTag','<H=0'),
263 ('PageFlags','<L=0'),
264 )
265 extended_win7 = (
266 ('ExtendedCheckSum1','<Q=0'),
267 ('ExtendedCheckSum2','<Q=0'),
268 ('ExtendedCheckSum3','<Q=0'),
269 ('PageNumber','<Q=0'),
270 ('Unknown','<Q=0'),
271 )
272 def __init__(self, version, revision, pageSize=8192, data=None):
273 if (version < 0x620) or (version == 0x620 and revision < 0x0b): 273 ↛ 275line 273 didn't jump to line 275, because the condition on line 273 was never true
274 # For sure the old format
275 self.structure = self.structure_2003_SP0 + self.common
276 elif version == 0x620 and revision < 0x11: 276 ↛ 278line 276 didn't jump to line 278, because the condition on line 276 was never true
277 # Exchange 2003 SP1 and Windows Vista and later
278 self.structure = self.structure_0x620_0x0b + self.common
279 else:
280 # Windows 7 and later
281 self.structure = self.structure_win7 + self.common
282 if pageSize > 8192: 282 ↛ 283line 282 didn't jump to line 283, because the condition on line 282 was never true
283 self.structure += self.extended_win7
285 Structure.__init__(self,data)
287class ESENT_ROOT_HEADER(Structure):
288 structure = (
289 ('InitialNumberOfPages','<L=0'),
290 ('ParentFatherDataPage','<L=0'),
291 ('ExtentSpace','<L=0'),
292 ('SpaceTreePageNumber','<L=0'),
293 )
295class ESENT_BRANCH_HEADER(Structure):
296 structure = (
297 ('CommonPageKey',':'),
298 )
300class ESENT_BRANCH_ENTRY(Structure):
301 common = (
302 ('CommonPageKeySize','<H=0'),
303 )
304 structure = (
305 ('LocalPageKeySize','<H=0'),
306 ('_LocalPageKey','_-LocalPageKey','self["LocalPageKeySize"]'),
307 ('LocalPageKey',':'),
308 ('ChildPageNumber','<L=0'),
309 )
310 def __init__(self, flags, data=None):
311 if flags & TAG_COMMON > 0:
312 # Include the common header
313 self.structure = self.common + self.structure
314 Structure.__init__(self,data)
316class ESENT_LEAF_HEADER(Structure):
317 structure = (
318 ('CommonPageKey',':'),
319 )
321class ESENT_LEAF_ENTRY(Structure):
322 common = (
323 ('CommonPageKeySize','<H=0'),
324 )
325 structure = (
326 ('LocalPageKeySize','<H=0'),
327 ('_LocalPageKey','_-LocalPageKey','self["LocalPageKeySize"]'),
328 ('LocalPageKey',':'),
329 ('EntryData',':'),
330 )
331 def __init__(self, flags, data=None):
332 if flags & TAG_COMMON > 0:
333 # Include the common header
334 self.structure = self.common + self.structure
335 Structure.__init__(self,data)
337class ESENT_SPACE_TREE_HEADER(Structure):
338 structure = (
339 ('Unknown','<Q=0'),
340 )
342class ESENT_SPACE_TREE_ENTRY(Structure):
343 structure = (
344 ('PageKeySize','<H=0'),
345 ('LastPageNumber','<L=0'),
346 ('NumberOfPages','<L=0'),
347 )
349class ESENT_INDEX_ENTRY(Structure):
350 structure = (
351 ('RecordPageKey',':'),
352 )
354class ESENT_DATA_DEFINITION_HEADER(Structure):
355 structure = (
356 ('LastFixedSize','<B=0'),
357 ('LastVariableDataType','<B=0'),
358 ('VariableSizeOffset','<H=0'),
359 )
361class ESENT_CATALOG_DATA_DEFINITION_ENTRY(Structure):
362 fixed = (
363 ('FatherDataPageID','<L=0'),
364 ('Type','<H=0'),
365 ('Identifier','<L=0'),
366 )
368 column_stuff = (
369 ('ColumnType','<L=0'),
370 ('SpaceUsage','<L=0'),
371 ('ColumnFlags','<L=0'),
372 ('CodePage','<L=0'),
373 )
375 other = (
376 ('FatherDataPageNumber','<L=0'),
377 )
379 table_stuff = (
380 ('SpaceUsage','<L=0'),
381# ('TableFlags','<L=0'),
382# ('InitialNumberOfPages','<L=0'),
383 )
385 index_stuff = (
386 ('SpaceUsage','<L=0'),
387 ('IndexFlags','<L=0'),
388 ('Locale','<L=0'),
389 )
391 lv_stuff = (
392 ('SpaceUsage','<L=0'),
393# ('LVFlags','<L=0'),
394# ('InitialNumberOfPages','<L=0'),
395 )
396 common = (
397# ('RootFlag','<B=0'),
398# ('RecordOffset','<H=0'),
399# ('LCMapFlags','<L=0'),
400# ('KeyMost','<H=0'),
401 ('Trailing',':'),
402 )
404 def __init__(self,data):
405 # Depending on the type of data we'll end up building a different struct
406 dataType = unpack('<H', data[4:][:2])[0]
407 self.structure = self.fixed
409 if dataType == CATALOG_TYPE_TABLE:
410 self.structure += self.other + self.table_stuff
411 elif dataType == CATALOG_TYPE_COLUMN:
412 self.structure += self.column_stuff
413 elif dataType == CATALOG_TYPE_INDEX:
414 self.structure += self.other + self.index_stuff
415 elif dataType == CATALOG_TYPE_LONG_VALUE: 415 ↛ 417line 415 didn't jump to line 417, because the condition on line 415 was never false
416 self.structure += self.other + self.lv_stuff
417 elif dataType == CATALOG_TYPE_CALLBACK:
418 raise Exception('CallBack types not supported!')
419 else:
420 LOG.error('Unknown catalog type 0x%x' % dataType)
421 self.structure = ()
422 Structure.__init__(self,data)
424 self.structure += self.common
426 Structure.__init__(self,data)
429#def pretty_print(x):
430# if x in '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~ ':
431# return x
432# else:
433# return '.'
434#
435#def hexdump(data):
436# x=str(data)
437# strLen = len(x)
438# i = 0
439# while i < strLen:
440# print "%04x " % i,
441# for j in range(16):
442# if i+j < strLen:
443# print "%02X" % ord(x[i+j]),
444#
445# else:
446# print " ",
447# if j%16 == 7:
448# print "",
449# print " ",
450# print ''.join(pretty_print(x) for x in x[i:i+16] )
451# i += 16
453def getUnixTime(t):
454 t -= 116444736000000000
455 t //= 10000000
456 return t
458class ESENT_PAGE:
459 def __init__(self, db, data=None):
460 self.__DBHeader = db
461 self.data = data
462 self.record = None
463 if data is not None: 463 ↛ exitline 463 didn't return from function '__init__', because the condition on line 463 was never false
464 self.record = ESENT_PAGE_HEADER(self.__DBHeader['Version'], self.__DBHeader['FileFormatRevision'], self.__DBHeader['PageSize'], data)
466 def printFlags(self):
467 flags = self.record['PageFlags']
468 if flags & FLAGS_EMPTY:
469 print("\tEmpty")
470 if flags & FLAGS_INDEX:
471 print("\tIndex")
472 if flags & FLAGS_LEAF:
473 print("\tLeaf")
474 else:
475 print("\tBranch")
476 if flags & FLAGS_LONG_VALUE:
477 print("\tLong Value")
478 if flags & FLAGS_NEW_CHECKSUM:
479 print("\tNew Checksum")
480 if flags & FLAGS_NEW_FORMAT:
481 print("\tNew Format")
482 if flags & FLAGS_PARENT:
483 print("\tParent")
484 if flags & FLAGS_ROOT:
485 print("\tRoot")
486 if flags & FLAGS_SPACE_TREE:
487 print("\tSpace Tree")
489 def dump(self):
490 baseOffset = len(self.record)
491 self.record.dump()
492 tags = self.data[-4*self.record['FirstAvailablePageTag']:]
494 print("FLAGS: ")
495 self.printFlags()
497 print()
499 for i in range(self.record['FirstAvailablePageTag']):
500 tag = tags[-4:]
501 if self.__DBHeader['Version'] == 0x620 and self.__DBHeader['FileFormatRevision'] > 11 and self.__DBHeader['PageSize'] > 8192:
502 valueSize = unpack('<H', tag[:2])[0] & 0x7fff
503 valueOffset = unpack('<H',tag[2:])[0] & 0x7fff
504 hexdump((self.data[baseOffset+valueOffset:][:6]))
505 pageFlags = ord(self.data[baseOffset+valueOffset:][1]) >> 5
506 #print "TAG FLAG: 0x%x " % (unpack('<L', self.data[baseOffset+valueOffset:][:4]) ) >> 5
507 #print "TAG FLAG: 0x " , ord(self.data[baseOffset+valueOffset:][0])
508 else:
509 valueSize = unpack('<H', tag[:2])[0] & 0x1fff
510 pageFlags = (unpack('<H', tag[2:])[0] & 0xe000) >> 13
511 valueOffset = unpack('<H',tag[2:])[0] & 0x1fff
513 print("TAG %-8d offset:0x%-6x flags:0x%-4x valueSize:0x%x" % (i,valueOffset,pageFlags,valueSize))
514 #hexdump(self.getTag(i)[1])
515 tags = tags[:-4]
517 if self.record['PageFlags'] & FLAGS_ROOT > 0:
518 rootHeader = ESENT_ROOT_HEADER(self.getTag(0)[1])
519 rootHeader.dump()
520 elif self.record['PageFlags'] & FLAGS_LEAF == 0:
521 # Branch Header
522 flags, data = self.getTag(0)
523 branchHeader = ESENT_BRANCH_HEADER(data)
524 branchHeader.dump()
525 else:
526 # Leaf Header
527 flags, data = self.getTag(0)
528 if self.record['PageFlags'] & FLAGS_SPACE_TREE > 0:
529 # Space Tree
530 spaceTreeHeader = ESENT_SPACE_TREE_HEADER(data)
531 spaceTreeHeader.dump()
532 else:
533 leafHeader = ESENT_LEAF_HEADER(data)
534 leafHeader.dump()
536 # Print the leaf/branch tags
537 for tagNum in range(1,self.record['FirstAvailablePageTag']):
538 flags, data = self.getTag(tagNum)
539 if self.record['PageFlags'] & FLAGS_LEAF == 0:
540 # Branch page
541 branchEntry = ESENT_BRANCH_ENTRY(flags, data)
542 branchEntry.dump()
543 elif self.record['PageFlags'] & FLAGS_LEAF > 0:
544 # Leaf page
545 if self.record['PageFlags'] & FLAGS_SPACE_TREE > 0:
546 # Space Tree
547 spaceTreeEntry = ESENT_SPACE_TREE_ENTRY(data)
548 #spaceTreeEntry.dump()
550 elif self.record['PageFlags'] & FLAGS_INDEX > 0:
551 # Index Entry
552 indexEntry = ESENT_INDEX_ENTRY(data)
553 #indexEntry.dump()
554 elif self.record['PageFlags'] & FLAGS_LONG_VALUE > 0:
555 # Long Page Value
556 raise Exception('Long value still not supported')
557 else:
558 # Table Value
559 leafEntry = ESENT_LEAF_ENTRY(flags, data)
560 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(leafEntry['EntryData'])
561 dataDefinitionHeader.dump()
562 catalogEntry = ESENT_CATALOG_DATA_DEFINITION_ENTRY(leafEntry['EntryData'][len(dataDefinitionHeader):])
563 catalogEntry.dump()
564 hexdump(leafEntry['EntryData'])
566 def getTag(self, tagNum):
567 if self.record['FirstAvailablePageTag'] < tagNum: 567 ↛ 568line 567 didn't jump to line 568, because the condition on line 567 was never true
568 raise Exception('Trying to grab an unknown tag 0x%x' % tagNum)
570 tags = self.data[-4*self.record['FirstAvailablePageTag']:]
571 baseOffset = len(self.record)
572 for i in range(tagNum):
573 tags = tags[:-4]
575 tag = tags[-4:]
577 if self.__DBHeader['Version'] == 0x620 and self.__DBHeader['FileFormatRevision'] >= 17 and self.__DBHeader['PageSize'] > 8192: 577 ↛ 578line 577 didn't jump to line 578, because the condition on line 577 was never true
578 valueSize = unpack('<H', tag[:2])[0] & 0x7fff
579 valueOffset = unpack('<H',tag[2:])[0] & 0x7fff
580 tmpData = list(self.data[baseOffset+valueOffset:][:valueSize])
581 pageFlags = ord(tmpData[1]) >> 5
582 tmpData[1] = chr(ord(tmpData[1:2]) & 0x1f)
583 tagData = "".join(tmpData)
584 else:
585 valueSize = unpack('<H', tag[:2])[0] & 0x1fff
586 pageFlags = (unpack('<H', tag[2:])[0] & 0xe000) >> 13
587 valueOffset = unpack('<H',tag[2:])[0] & 0x1fff
588 tagData = self.data[baseOffset+valueOffset:][:valueSize]
590 #return pageFlags, self.data[baseOffset+valueOffset:][:valueSize]
591 return pageFlags, tagData
593class ESENT_DB:
594 def __init__(self, fileName, pageSize = 8192, isRemote = False):
595 self.__fileName = fileName
596 self.__pageSize = pageSize
597 self.__DB = None
598 self.__DBHeader = None
599 self.__totalPages = None
600 self.__tables = OrderedDict()
601 self.__currentTable = None
602 self.__isRemote = isRemote
603 self.mountDB()
605 def mountDB(self):
606 LOG.debug("Mounting DB...")
607 if self.__isRemote is True: 607 ↛ 611line 607 didn't jump to line 611, because the condition on line 607 was never false
608 self.__DB = self.__fileName
609 self.__DB.open()
610 else:
611 self.__DB = open(self.__fileName,"rb")
612 mainHeader = self.getPage(-1)
613 self.__DBHeader = ESENT_DB_HEADER(mainHeader)
614 self.__pageSize = self.__DBHeader['PageSize']
615 self.__DB.seek(0,2)
616 self.__totalPages = (self.__DB.tell() // self.__pageSize) -2
617 LOG.debug("Database Version:0x%x, Revision:0x%x"% (self.__DBHeader['Version'], self.__DBHeader['FileFormatRevision']))
618 LOG.debug("Page Size: %d" % self.__pageSize)
619 LOG.debug("Total Pages in file: %d" % self.__totalPages)
620 self.parseCatalog(CATALOG_PAGE_NUMBER)
622 def printCatalog(self):
623 indent = ' '
625 print("Database version: 0x%x, 0x%x" % (self.__DBHeader['Version'], self.__DBHeader['FileFormatRevision'] ))
626 print("Page size: %d " % self.__pageSize)
627 print("Number of pages: %d" % self.__totalPages)
628 print()
629 print("Catalog for %s" % self.__fileName)
630 for table in list(self.__tables.keys()):
631 print("[%s]" % table.decode('utf8'))
632 print("%sColumns " % indent)
633 for column in list(self.__tables[table]['Columns'].keys()):
634 record = self.__tables[table]['Columns'][column]['Record']
635 print("%s%-5d%-30s%s" % (indent*2, record['Identifier'], column.decode('utf-8'),ColumnTypeToName[record['ColumnType']]))
636 print("%sIndexes"% indent)
637 for index in list(self.__tables[table]['Indexes'].keys()):
638 print("%s%s" % (indent*2, index.decode('utf-8')))
639 print("")
641 def __addItem(self, entry):
642 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(entry['EntryData'])
643 catalogEntry = ESENT_CATALOG_DATA_DEFINITION_ENTRY(entry['EntryData'][len(dataDefinitionHeader):])
644 itemName = self.__parseItemName(entry)
646 if catalogEntry['Type'] == CATALOG_TYPE_TABLE:
647 self.__tables[itemName] = OrderedDict()
648 self.__tables[itemName]['TableEntry'] = entry
649 self.__tables[itemName]['Columns'] = OrderedDict()
650 self.__tables[itemName]['Indexes'] = OrderedDict()
651 self.__tables[itemName]['LongValues'] = OrderedDict()
652 self.__currentTable = itemName
653 elif catalogEntry['Type'] == CATALOG_TYPE_COLUMN:
654 self.__tables[self.__currentTable]['Columns'][itemName] = entry
655 self.__tables[self.__currentTable]['Columns'][itemName]['Header'] = dataDefinitionHeader
656 self.__tables[self.__currentTable]['Columns'][itemName]['Record'] = catalogEntry
657 elif catalogEntry['Type'] == CATALOG_TYPE_INDEX:
658 self.__tables[self.__currentTable]['Indexes'][itemName] = entry
659 elif catalogEntry['Type'] == CATALOG_TYPE_LONG_VALUE: 659 ↛ 662line 659 didn't jump to line 662, because the condition on line 659 was never false
660 self.__addLongValue(entry)
661 else:
662 raise Exception('Unknown type 0x%x' % catalogEntry['Type'])
664 def __parseItemName(self,entry):
665 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(entry['EntryData'])
667 if dataDefinitionHeader['LastVariableDataType'] > 127: 667 ↛ 670line 667 didn't jump to line 670, because the condition on line 667 was never false
668 numEntries = dataDefinitionHeader['LastVariableDataType'] - 127
669 else:
670 numEntries = dataDefinitionHeader['LastVariableDataType']
672 itemLen = unpack('<H',entry['EntryData'][dataDefinitionHeader['VariableSizeOffset']:][:2])[0]
673 itemName = entry['EntryData'][dataDefinitionHeader['VariableSizeOffset']:][2*numEntries:][:itemLen]
674 return itemName
676 def __addLongValue(self, entry):
677 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(entry['EntryData'])
678 lvLen = unpack('<H',entry['EntryData'][dataDefinitionHeader['VariableSizeOffset']:][:2])[0]
679 lvName = entry['EntryData'][dataDefinitionHeader['VariableSizeOffset']:][7:][:lvLen]
680 self.__tables[self.__currentTable]['LongValues'][lvName] = entry
682 def parsePage(self, page):
683 # Print the leaf/branch tags
684 for tagNum in range(1,page.record['FirstAvailablePageTag']):
685 flags, data = page.getTag(tagNum)
686 if page.record['PageFlags'] & FLAGS_LEAF > 0:
687 # Leaf page
688 if page.record['PageFlags'] & FLAGS_SPACE_TREE > 0: 688 ↛ 689line 688 didn't jump to line 689, because the condition on line 688 was never true
689 pass
690 elif page.record['PageFlags'] & FLAGS_INDEX > 0: 690 ↛ 691line 690 didn't jump to line 691, because the condition on line 690 was never true
691 pass
692 elif page.record['PageFlags'] & FLAGS_LONG_VALUE > 0: 692 ↛ 693line 692 didn't jump to line 693, because the condition on line 692 was never true
693 pass
694 else:
695 # Table Value
696 leafEntry = ESENT_LEAF_ENTRY(flags, data)
697 self.__addItem(leafEntry)
699 def parseCatalog(self, pageNum):
700 # Parse all the pages starting at pageNum and commit table data
701 page = self.getPage(pageNum)
702 self.parsePage(page)
704 for i in range(1, page.record['FirstAvailablePageTag']):
705 flags, data = page.getTag(i)
706 if page.record['PageFlags'] & FLAGS_LEAF == 0:
707 # Branch page
708 branchEntry = ESENT_BRANCH_ENTRY(flags, data)
709 self.parseCatalog(branchEntry['ChildPageNumber'])
712 def readHeader(self):
713 LOG.debug("Reading Boot Sector for %s" % self.__volumeName)
715 def getPage(self, pageNum):
716 LOG.debug("Trying to fetch page %d (0x%x)" % (pageNum, (pageNum+1)*self.__pageSize))
717 self.__DB.seek((pageNum+1)*self.__pageSize, 0)
718 data = self.__DB.read(self.__pageSize)
719 while len(data) < self.__pageSize: 719 ↛ 720line 719 didn't jump to line 720, because the condition on line 719 was never true
720 remaining = self.__pageSize - len(data)
721 data += self.__DB.read(remaining)
722 # Special case for the first page
723 if pageNum <= 0:
724 return data
725 else:
726 return ESENT_PAGE(self.__DBHeader, data)
728 def close(self):
729 self.__DB.close()
731 def openTable(self, tableName):
732 # Returns a cursos for later use
734 if isinstance(tableName, bytes) is not True: 734 ↛ 737line 734 didn't jump to line 737, because the condition on line 734 was never false
735 tableName = b(tableName)
737 if tableName in self.__tables: 737 ↛ 768line 737 didn't jump to line 768, because the condition on line 737 was never false
738 entry = self.__tables[tableName]['TableEntry']
739 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(entry['EntryData'])
740 catalogEntry = ESENT_CATALOG_DATA_DEFINITION_ENTRY(entry['EntryData'][len(dataDefinitionHeader):])
742 # Let's position the cursor at the leaf levels for fast reading
743 pageNum = catalogEntry['FatherDataPageNumber']
744 done = False
745 while done is False:
746 page = self.getPage(pageNum)
747 if page.record['FirstAvailablePageTag'] <= 1: 747 ↛ 749line 747 didn't jump to line 749, because the condition on line 747 was never true
748 # There are no records
749 done = True
750 for i in range(1, page.record['FirstAvailablePageTag']): 750 ↛ 745line 750 didn't jump to line 745, because the loop on line 750 didn't complete
751 flags, data = page.getTag(i)
752 if page.record['PageFlags'] & FLAGS_LEAF == 0:
753 # Branch page, move on to the next page
754 branchEntry = ESENT_BRANCH_ENTRY(flags, data)
755 pageNum = branchEntry['ChildPageNumber']
756 break
757 else:
758 done = True
759 break
761 cursor = TABLE_CURSOR
762 cursor['TableData'] = self.__tables[tableName]
763 cursor['FatherDataPageNumber'] = catalogEntry['FatherDataPageNumber']
764 cursor['CurrentPageData'] = page
765 cursor['CurrentTag'] = 0
766 return cursor
767 else:
768 return None
770 def __getNextTag(self, cursor):
771 page = cursor['CurrentPageData']
773 if cursor['CurrentTag'] >= page.record['FirstAvailablePageTag']:
774 # No more data in this page, chau
775 return None
777 flags, data = page.getTag(cursor['CurrentTag'])
778 if page.record['PageFlags'] & FLAGS_LEAF > 0: 778 ↛ 791line 778 didn't jump to line 791, because the condition on line 778 was never false
779 # Leaf page
780 if page.record['PageFlags'] & FLAGS_SPACE_TREE > 0: 780 ↛ 781line 780 didn't jump to line 781, because the condition on line 780 was never true
781 raise Exception('FLAGS_SPACE_TREE > 0')
782 elif page.record['PageFlags'] & FLAGS_INDEX > 0: 782 ↛ 783line 782 didn't jump to line 783, because the condition on line 782 was never true
783 raise Exception('FLAGS_INDEX > 0')
784 elif page.record['PageFlags'] & FLAGS_LONG_VALUE > 0: 784 ↛ 785line 784 didn't jump to line 785, because the condition on line 784 was never true
785 raise Exception('FLAGS_LONG_VALUE > 0')
786 else:
787 # Table Value
788 leafEntry = ESENT_LEAF_ENTRY(flags, data)
789 return leafEntry
791 return None
793 def getNextRow(self, cursor):
794 cursor['CurrentTag'] += 1
796 tag = self.__getNextTag(cursor)
797 #hexdump(tag)
799 if tag is None:
800 # No more tags in this page, search for the next one on the right
801 page = cursor['CurrentPageData']
802 if page.record['NextPageNumber'] == 0:
803 # No more pages, chau
804 return None
805 else:
806 cursor['CurrentPageData'] = self.getPage(page.record['NextPageNumber'])
807 cursor['CurrentTag'] = 0
808 return self.getNextRow(cursor)
809 else:
810 return self.__tagToRecord(cursor, tag['EntryData'])
812 def __tagToRecord(self, cursor, tag):
813 # So my brain doesn't forget, the data record is composed of:
814 # Header
815 # Fixed Size Data (ID < 127)
816 # The easiest to parse. Their size is fixed in the record. You can get its size
817 # from the Column Record, field SpaceUsage
818 # Variable Size Data (127 < ID < 255)
819 # At VariableSizeOffset you get an array of two bytes per variable entry, pointing
820 # to the length of the value. Values start at:
821 # numEntries = LastVariableDataType - 127
822 # VariableSizeOffset + numEntries * 2 (bytes)
823 # Tagged Data ( > 255 )
824 # After the Variable Size Value, there's more data for the tagged values.
825 # Right at the beginning there's another array (taggedItems), pointing to the
826 # values, size.
827 #
828 # The interesting thing about this DB records is there's no need for all the columns to be there, hence
829 # saving space. That's why I got over all the columns, and if I find data (of any type), i assign it. If
830 # not, the column's empty.
831 #
832 # There are a lot of caveats in the code, so take your time to explore it.
833 #
834 # ToDo: Better complete this description
835 #
837 record = OrderedDict()
838 taggedItems = OrderedDict()
839 taggedItemsParsed = False
841 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(tag)
842 #dataDefinitionHeader.dump()
843 variableDataBytesProcessed = (dataDefinitionHeader['LastVariableDataType'] - 127) * 2
844 prevItemLen = 0
845 tagLen = len(tag)
846 fixedSizeOffset = len(dataDefinitionHeader)
847 variableSizeOffset = dataDefinitionHeader['VariableSizeOffset']
849 columns = cursor['TableData']['Columns']
851 for column in list(columns.keys()):
852 columnRecord = columns[column]['Record']
853 #columnRecord.dump()
854 if columnRecord['Identifier'] <= dataDefinitionHeader['LastFixedSize']:
855 # Fixed Size column data type, still available data
856 record[column] = tag[fixedSizeOffset:][:columnRecord['SpaceUsage']]
857 fixedSizeOffset += columnRecord['SpaceUsage']
859 elif 127 < columnRecord['Identifier'] <= dataDefinitionHeader['LastVariableDataType']: 859 ↛ 861line 859 didn't jump to line 861, because the condition on line 859 was never true
860 # Variable data type
861 index = columnRecord['Identifier'] - 127 - 1
862 itemLen = unpack('<H',tag[variableSizeOffset+index*2:][:2])[0]
864 if itemLen & 0x8000:
865 # Empty item
866 itemLen = prevItemLen
867 record[column] = None
868 else:
869 itemValue = tag[variableSizeOffset+variableDataBytesProcessed:][:itemLen-prevItemLen]
870 record[column] = itemValue
872 #if columnRecord['Identifier'] <= dataDefinitionHeader['LastVariableDataType']:
873 variableDataBytesProcessed +=itemLen-prevItemLen
875 prevItemLen = itemLen
877 elif columnRecord['Identifier'] > 255:
878 # Have we parsed the tagged items already?
879 if taggedItemsParsed is False and (variableDataBytesProcessed+variableSizeOffset) < tagLen:
880 index = variableDataBytesProcessed+variableSizeOffset
881 #hexdump(tag[index:])
882 endOfVS = self.__pageSize
883 firstOffsetTag = (unpack('<H', tag[index+2:][:2])[0] & 0x3fff) + variableDataBytesProcessed+variableSizeOffset
884 while True:
885 taggedIdentifier = unpack('<H', tag[index:][:2])[0]
886 index += 2
887 taggedOffset = (unpack('<H', tag[index:][:2])[0] & 0x3fff)
888 # As of Windows 7 and later ( version 0x620 revision 0x11) the
889 # tagged data type flags are always present
890 if self.__DBHeader['Version'] == 0x620 and self.__DBHeader['FileFormatRevision'] >= 17 and self.__DBHeader['PageSize'] > 8192: 890 ↛ 891line 890 didn't jump to line 891, because the condition on line 890 was never true
891 flagsPresent = 1
892 else:
893 flagsPresent = (unpack('<H', tag[index:][:2])[0] & 0x4000)
894 index += 2
895 if taggedOffset < endOfVS:
896 endOfVS = taggedOffset
897 taggedItems[taggedIdentifier] = (taggedOffset, tagLen, flagsPresent)
898 #print "ID: %d, Offset:%d, firstOffset:%d, index:%d, flag: 0x%x" % (taggedIdentifier, taggedOffset,firstOffsetTag,index, flagsPresent)
899 if index >= firstOffsetTag:
900 # We reached the end of the variable size array
901 break
903 # Calculate length of variable items
904 # Ugly.. should be redone
905 prevKey = list(taggedItems.keys())[0]
906 for i in range(1,len(taggedItems)):
907 offset0, length, flags = taggedItems[prevKey]
908 offset, _, _ = list(taggedItems.items())[i][1]
909 taggedItems[prevKey] = (offset0, offset-offset0, flags)
910 #print "ID: %d, Offset: %d, Len: %d, flags: %d" % (prevKey, offset0, offset-offset0, flags)
911 prevKey = list(taggedItems.keys())[i]
912 taggedItemsParsed = True
914 # Tagged data type
915 if columnRecord['Identifier'] in taggedItems:
916 offsetItem = variableDataBytesProcessed + variableSizeOffset + taggedItems[columnRecord['Identifier']][0]
917 itemSize = taggedItems[columnRecord['Identifier']][1]
918 # If item have flags, we should skip them
919 if taggedItems[columnRecord['Identifier']][2] > 0:
920 itemFlag = ord(tag[offsetItem:offsetItem+1])
921 offsetItem += 1
922 itemSize -= 1
923 else:
924 itemFlag = 0
926 #print "ID: %d, itemFlag: 0x%x" %( columnRecord['Identifier'], itemFlag)
927 if itemFlag & (TAGGED_DATA_TYPE_COMPRESSED ): 927 ↛ 928line 927 didn't jump to line 928, because the condition on line 927 was never true
928 LOG.error('Unsupported tag column: %s, flag:0x%x' % (column, itemFlag))
929 record[column] = None
930 elif itemFlag & TAGGED_DATA_TYPE_MULTI_VALUE:
931 # ToDo: Parse multi-values properly
932 LOG.debug('Multivalue detected in column %s, returning raw results' % (column))
933 record[column] = (hexlify(tag[offsetItem:][:itemSize]),)
934 else:
935 record[column] = tag[offsetItem:][:itemSize]
937 else:
938 record[column] = None
939 else:
940 record[column] = None
942 # If we understand the data type, we unpack it and cast it accordingly
943 # otherwise, we just encode it in hex
944 if type(record[column]) is tuple:
945 # A multi value data, we won't decode it, just leave it this way
946 record[column] = record[column][0]
947 elif columnRecord['ColumnType'] == JET_coltypText or columnRecord['ColumnType'] == JET_coltypLongText:
948 # Let's handle strings
949 if record[column] is not None:
950 if columnRecord['CodePage'] not in StringCodePages: 950 ↛ 951line 950 didn't jump to line 951, because the condition on line 950 was never true
951 raise Exception('Unknown codepage 0x%x'% columnRecord['CodePage'])
952 stringDecoder = StringCodePages[columnRecord['CodePage']]
954 try:
955 record[column] = record[column].decode(stringDecoder)
956 except Exception:
957 LOG.debug("Exception:", exc_info=True)
958 LOG.debug('Fixing Record[%r][%d]: %r' % (column, columnRecord['ColumnType'], record[column]))
959 record[column] = record[column].decode(stringDecoder, "replace")
960 pass
961 else:
962 unpackData = ColumnTypeSize[columnRecord['ColumnType']]
963 if record[column] is not None:
964 if unpackData is None:
965 record[column] = hexlify(record[column])
966 else:
967 unpackStr = unpackData[1]
968 record[column] = unpack(unpackStr, record[column])[0]
970 return record