From 1690e746db40c2acec22888e6c2407989aa97e53 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Tue, 14 Jun 2022 01:20:50 -0700 Subject: [PATCH 1/2] v0.34.1 --- CHANGELOG.md | 5 ++ README.rst | 4 +- extract_msg/__init__.py | 6 +- extract_msg/exceptions.py | 7 ++ extract_msg/message_signed_base.py | 23 +++--- extract_msg/utils.py | 121 ++++++++++++++++++++++++++++- 6 files changed, 148 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a93cb17..a94e1c10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +**0.34.1** +* Added convenience function `utils.openMsgBulk` (imported to `exract_msg` namespace) for opening message paths with wildcards. Allows you to open several messages with one function, returning a list of the opened messages. +* Added convenience function `utils.unwrapMsg` which recurses through an `MSGFile` and it's attachments, creating a series of linear structures stored in a dictionary. Useful for analyzing, saving, etc. all messages and attachments down through the structure without having to recurse yourself. +* Fixed an issue that would cause signed attachments to not properly generate. + **v0.34.0** * [[TeamMsgExtractor #102](https://github.com/TeamMsgExtractor/msg-extractor/issues/102)] Added the option to directly save body to pdf. This requires that you either have wkhtmltopdf on your path or that you provide a path directly to it in order to work. Simply pass `pdf = True` to save to turn it on. More details in the doc for the save function. You can also do this from the command line using the `--pdf` option, incompatible with other body types. * Added `--glob` option for allowing you to provide an msg path that will evaluate wildcards. diff --git a/README.rst b/README.rst index cade4379..3b693970 100644 --- a/README.rst +++ b/README.rst @@ -219,8 +219,8 @@ your access to the newest major version of extract-msg. .. |License: GPL v3| image:: https://img.shields.io/badge/License-GPLv3-blue.svg :target: LICENSE.txt -.. |PyPI3| image:: https://img.shields.io/badge/pypi-0.34.0-blue.svg - :target: https://pypi.org/project/extract-msg/0.34.0/ +.. |PyPI3| image:: https://img.shields.io/badge/pypi-0.34.1-blue.svg + :target: https://pypi.org/project/extract-msg/0.34.1/ .. |PyPI2| image:: https://img.shields.io/badge/python-3.6+-brightgreen.svg :target: https://www.python.org/downloads/release/python-367/ diff --git a/extract_msg/__init__.py b/extract_msg/__init__.py index 5d226677..14fe875b 100644 --- a/extract_msg/__init__.py +++ b/extract_msg/__init__.py @@ -27,8 +27,8 @@ # along with this program. If not, see . __author__ = 'Destiny Peterson & Matthew Walker' -__date__ = '2022-06-11' -__version__ = '0.34.0' +__date__ = '2022-06-14' +__version__ = '0.34.1' import logging @@ -46,4 +46,4 @@ from .properties import Properties from .recipient import Recipient from .task import Task -from .utils import openMsg, properHex +from .utils import openMsg, openMsgBulk, properHex diff --git a/extract_msg/exceptions.py b/extract_msg/exceptions.py index 0b82be5a..95237563 100644 --- a/extract_msg/exceptions.py +++ b/extract_msg/exceptions.py @@ -60,6 +60,13 @@ class InvalidVersionError(Exception): """ pass +class StandardViolationError(Exception): + """ + A critical violation of the MSG standards was detected and could not be + recovered from. Recoverable violations will result in log messages instead. + """ + pass + class UnknownCodepageError(Exception): """ The codepage provided was not one we know of. diff --git a/extract_msg/message_signed_base.py b/extract_msg/message_signed_base.py index 11950890..d9346512 100644 --- a/extract_msg/message_signed_base.py +++ b/extract_msg/message_signed_base.py @@ -5,6 +5,7 @@ import mailbits +from .exceptions import StandardViolationError from .message_base import MessageBase from .signed_attachment import SignedAttachment from .utils import inputToString @@ -68,25 +69,23 @@ def headerInit(self) -> bool: def attachments(self) -> list: """ Returns a list of all attachments. + + :raises StandardViolationError: The standard for signed messages was + blatantly violated. """ try: return self._sAttachments except AttributeError: atts = super().attachments + + if len(atts) != 1: + raise StandardViolationError('Signed messages without exactly 1 (regular) attachment constitue a violation of the standard.') + self._sAttachments = [] self._signedBody = None self._signedHtmlBody = None - if len(atts) > 1: - logger.warning('Found more than one regular attachment when parsing signed message.') - elif len(atts) == 0: - logger.warning('Failed to access any attachments from the signed message.') - return self._sAttachments - - try: - mainAttachment = next(att for att in atts if hasattr(att, 'getFilename') and att.getFilename() == 'smime.p7m') - except StopIteration: - logger.warning('Failed to find signed attachment.') - return self._sAttachments + + mainAttachment = atts[0] # If we are here, we should have the attachment. So now we need to # try to parse and unwrap the data. @@ -103,7 +102,7 @@ def attachments(self) -> list: else: output.append(part) - # At this point, `output` has out parts. + # At this point, `output` has our parts. for part in output: # Get the mime type. mime = part['headers']['content-type']['content_type'] diff --git a/extract_msg/utils.py b/extract_msg/utils.py index 6a5ce598..9de341a4 100644 --- a/extract_msg/utils.py +++ b/extract_msg/utils.py @@ -22,8 +22,10 @@ import tzlocal from html import escape as htmlEscape +from typing import Dict, List, Tuple, Union from . import constants +from .enums import AttachmentType from .exceptions import BadHtmlError, ConversionError, IncompatibleOptionsError, InvaildPropertyIdError, UnknownCodepageError, UnknownTypeError, UnrecognizedMSGTypeError, UnsupportedMSGTypeError @@ -45,6 +47,7 @@ def addNumToDir(dirName : pathlib.Path) -> pathlib.Path: pass return None + def addNumToZipDir(dirName : pathlib.Path, _zip): """ Attempt to create the directory with a '(n)' appended. @@ -56,6 +59,7 @@ def addNumToZipDir(dirName : pathlib.Path, _zip): return newDirName return None + def bitwiseAdjust(inp : int, mask : int) -> int: """ Uses a given mask to adjust the location of bits after an operation like @@ -74,6 +78,7 @@ def bitwiseAdjust(inp : int, mask : int) -> int: raise ValueError('Mask MUST be greater than 0') return inp >> bin(mask)[::-1].index('1') + def bitwiseAdjustedAnd(inp : int, mask : int) -> int: """ Preforms the bitwise AND operation between :param inp: and :param mask: and @@ -85,6 +90,7 @@ def bitwiseAdjustedAnd(inp : int, mask : int) -> int: raise ValueError('Mask MUST be greater than 0') return (inp & mask) >> bin(mask)[::-1].index('1') + def bytesToGuid(bytesInput : bytes) -> str: """ Converts a bytes instance to a GUID. @@ -92,6 +98,7 @@ def bytesToGuid(bytesInput : bytes) -> str: guidVals = constants.ST_GUID.unpack(bytesInput) return f'{{{guidVals[0]:08X}-{guidVals[1]:04X}-{guidVals[2]:04X}-{guidVals[3][:2].hex().upper()}-{guidVals[3][2:].hex().upper()}}}' + def ceilDiv(n : int, d : int) -> int: """ Returns the int from the ceil division of n / d. @@ -102,6 +109,7 @@ def ceilDiv(n : int, d : int) -> int: """ return -(n // -d) + def createZipOpen(func): """ Creates a wrapper for the open function of a ZipFile that will automatically @@ -115,6 +123,7 @@ def _open(name, mode, *args, **kwargs): return _open + def divide(string, length : int) -> list: """ Divides a string into multiple substrings of equal length. If there is not @@ -135,6 +144,7 @@ def divide(string, length : int) -> list: """ return [string[length * x:length * (x + 1)] for x in range(int(ceilDiv(len(string), length)))] + def findWk(path = None): """ Attempt to find the path of the wkhtmltopdf executable. If :param path: is @@ -158,12 +168,14 @@ def findWk(path = None): raise ExecutableNotFound('Could not find wkhtmltopdf.') + def fromTimeStamp(stamp) -> datetime.datetime: """ Returns a datetime from the UTC timestamp given the current timezone. """ return datetime.datetime.fromtimestamp(stamp, tzlocal.get_localzone()) + def getCommandArgs(args): """ Parse command-line arguments. @@ -308,6 +320,7 @@ def getCommandArgs(args): return options + def getEncodingName(codepage : int) -> str: """ Returns the name of the encoding with the specified codepage. @@ -323,15 +336,18 @@ def getEncodingName(codepage : int) -> str: except LookupError: raise UnsupportedEncodingError(f'The codepage {codepage} ({constants.CODE_PAGES[codepage]}) is not currently supported by your version of Python.') + def getFullClassName(inp): return inp.__class__.__module__ + '.' + inp.__class__.__name__ + def hasLen(obj) -> bool: """ Checks if :param obj: has a __len__ attribute. """ return hasattr(obj, '__len__') + def injectHtmlHeader(msgFile, prepared : bool = False) -> bytes: """ Returns the HTML body from the MSG file (will check that it has one) with @@ -357,7 +373,6 @@ def injectHtmlHeader(msgFile, prepared : bool = False) -> bytes: # If the body is not valid or not found, raise an AttributeError. if not body: raise AttributeError('Cannot find a prepared HTML body to inject into.') - else: body = msgFile.htmlBody @@ -453,6 +468,7 @@ def replace(bodyMarker): # Use the previously defined function to inject the HTML header. return constants.RE_HTML_BODY_START.sub(replace, body, 1) + def injectRtfHeader(msgFile) -> bytes: """ Returns the RTF body from the MSG file (will check that it has one) with the @@ -533,6 +549,7 @@ def replace(bodyMarker): raise RuntimeError('All injection attempts failed. Please report this to the developer.') + def inputToBytes(stringInputVar, encoding) -> bytes: """ Converts the input into bytes. @@ -548,6 +565,7 @@ def inputToBytes(stringInputVar, encoding) -> bytes: else: raise ConversionError('Cannot convert to bytes.') + def inputToMsgpath(inp) -> list: """ Converts the input into an msg path. @@ -557,6 +575,7 @@ def inputToMsgpath(inp) -> list: ret = inputToString(inp, 'utf-8').replace('\\', '/').split('/') return ret if ret[0] != '' else [] + def inputToString(bytesInputVar, encoding) -> str: """ Converts the input into a string. @@ -572,6 +591,7 @@ def inputToString(bytesInputVar, encoding) -> str: else: raise ConversionError('Cannot convert to str type.') + def isEncapsulatedRtf(inp : bytes) -> bool: """ Currently the destection is made to be *extremly* basic, but this will work @@ -580,12 +600,14 @@ def isEncapsulatedRtf(inp : bytes) -> bool: """ return b'\\fromhtml' in inp + def isEmptyString(inp : str) -> bool: """ Returns true if the input is None or is an Empty string. """ return (inp == '' or inp is None) + def knownMsgClass(classType : str) -> bool: """ Checks if the specified class type is recognized by the module. Usually used @@ -601,12 +623,14 @@ def knownMsgClass(classType : str) -> bool: return False + def filetimeToUtc(inp : int) -> float: """ Converts a FILETIME into a unix timestamp. """ return (inp - 116444736000000000) / 10000000.0 + def msgpathToString(inp) -> str: """ Converts an msgpath (one of the internal paths inside an msg file) into a @@ -619,6 +643,7 @@ def msgpathToString(inp) -> str: inp.replace('\\', '/') return inp + def openMsg(path, **kwargs): """ Function to automatically open an MSG file and detect what type it is. @@ -691,6 +716,31 @@ def openMsg(path, **kwargs): logger.error(f'Could not recognize msg class type "{msg.classType}". This most likely means it hasn\'t been implemented yet, and you should ask the developers to add support for it.') return msg + +def openMsgBulk(path, **kwargs) -> Union[List, Tuple]: + """ + Takes the same arguments as openMsg, but opens a collection of msg files + based on a wild card. Returns a list if successful, otherwise returns a + tuple. + + :param ignoreFailures: If this is True, will return a list of all successful + files, ignoring any failures. Otherwise, will close all that + successfully opened, and return a tuple of the exception and the path of + the file that failed. + """ + files = [] + for x in glob.glob(path): + try: + files.append(openMsg(x, **kwargs)) + except Exception as e: + if not kwargs.get('ignoreFailures', False): + for msg in files: + msg.close() + return (e, x) + + return files + + def parseType(_type : int, stream, encoding, extras): """ Converts the data in :param stream: to a much more accurate type, specified @@ -825,6 +875,7 @@ def parseType(_type : int, stream, encoding, extras): raise NotImplementedError(f'Parsing for type {_type} has not yet been implmented. If you need this type, please create a new issue labeled "NotImplementedError: parseType {_type}".') return value + def prepareFilename(filename) -> str: """ Adjusts :param filename: so that it can succesfully be used as an actual @@ -833,6 +884,7 @@ def prepareFilename(filename) -> str: # I would use re here, but it tested to be slightly slower than this. return ''.join(i for i in filename if i not in r'\/:*?"<>|' + '\x00') + def properHex(inp, length : int = 0) -> str: """ Takes in various input types and converts them into a hex string whose @@ -849,12 +901,14 @@ def properHex(inp, length : int = 0) -> str: a = '0' + a return a.rjust(length, '0').upper() + def roundUp(inp : int, mult : int) -> int: """ Rounds :param inp: up to the nearest multiple of :param mult:. """ return inp + (mult - inp) % mult + def rtfSanitizeHtml(inp : str) -> str: """ Sanitizes input to an RTF stream that has encapsulated HTML. @@ -885,6 +939,7 @@ def rtfSanitizeHtml(inp : str) -> str: return output + def rtfSanitizePlain(inp : str) -> str: """ Sanitizes input to a plain RTF stream. @@ -907,6 +962,7 @@ def rtfSanitizePlain(inp : str) -> str: return output + def setupLogging(defaultPath = None, defaultLevel = logging.WARN, logfile = None, enableFileLogging : bool = False, env_key = 'EXTRACT_MSG_LOG_CFG') -> bool: """ @@ -985,6 +1041,66 @@ def setupLogging(defaultPath = None, defaultLevel = logging.WARN, logfile = None logging.getLogger().setLevel(defaultLevel) return True + +def unwrapMsg(msg : "MSGFile") -> Dict: + """ + Takes a recursive message-attachment structure and unwraps it into a linear + dictionary for easy iteration. Dictionary contains 4 keys: "attachments" for + main message attachments, not including embedded MSG files, "embedded" for + attachments representing embedded MSG files, "msg" for all MSG files + (including the original in the first index), and "raw_attachments" for raw + attachments from signed messages. + """ + from .message_signed_base import MessageSignedBase + + # Here is where we store main attachments. + attachments = [] + # Here is where we are going to store embedded msg files. + msgFiles = [msg] + # Here is where we store embedded attachments. + embedded = [] + # Here is where we store raw attachments from signed messages. + raw = [] + + # Normally we would need a recursive function to unwrap a recursive + # structure like the message-attachment structure. Essentially, a function + # that calls itself. Here, I have designed code capable of circumventing + # this to do it in a single function, which is a lot more efficient and + # safer. That is why we store the `toProcess` and use a while loop + # surrounding a for loop. The for loop would be the main body of the + # function, while the append to toProcess would be the recursive call. + toProcess = [msg] + + while len(toProcess) > 0: + # Remove the last item from the list of things to process, and store it + # in `currentItem`. We will be processing it in the for loop. + currentItem = toProcess.pop() + # iterate through the attachments and + for att in currentItem.attachments: + # If it is a regular attachment, add it to the list. Otherwise, add + # it to be processed + if att.type in (AttachmentType.DATA, AttachmentType.SIGNED): + attachments.append(att) + elif att.type == AttachmentType.MSG: + # Here we do two things. The first is we store it to the output + # so we can return it. The second is we add it to the processing + # list. The reason this is two steps is because we need to be + # able to remove items from the processing list, but can't + # do that from the output. + embedded.append(att) + msgFiles.append(att.data) + toProcess.append(att.data) + if isinstance(currentItem, MessageSignedBase): + raw += currentItem._rawAttachments + + return { + 'attachments': attachments, + 'embedded': embedded, + 'msg': msgFiles, + 'raw_attachments': raw, + } + + def validateHtml(html : bytes) -> bool: """ Checks whether the HTML is considered valid. To be valid, the HTML must, at @@ -995,6 +1111,7 @@ def validateHtml(html : bytes) -> bool: return False return True + def verifyPropertyId(id : str) -> None: """ Determines whether a property ID is valid for vertain functions. Property @@ -1014,6 +1131,7 @@ def verifyPropertyId(id : str) -> None: except ValueError: raise InvaildPropertyIdError('ID was not a 4 digit hexadecimal string') + def verifyType(_type): """ Verifies that the type is valid. Raises an exception if it is not. @@ -1024,5 +1142,6 @@ def verifyType(_type): if (_type not in constants.VARIABLE_LENGTH_PROPS_STRING) and (_type not in constants.FIXED_LENGTH_PROPS_STRING): raise UnknownTypeError(f'Unknown type {_type}.') + def windowsUnicode(string): return str(string, 'utf-16-le') if string is not None else None From c0680481b70875f3628d6131dd00fe497b2d27ef Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Tue, 14 Jun 2022 01:29:33 -0700 Subject: [PATCH 2/2] Fix changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a94e1c10..a3c585d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ **0.34.1** -* Added convenience function `utils.openMsgBulk` (imported to `exract_msg` namespace) for opening message paths with wildcards. Allows you to open several messages with one function, returning a list of the opened messages. +* Added convenience function `utils.openMsgBulk` (imported to `extract_msg` namespace) for opening message paths with wildcards. Allows you to open several messages with one function, returning a list of the opened messages. * Added convenience function `utils.unwrapMsg` which recurses through an `MSGFile` and it's attachments, creating a series of linear structures stored in a dictionary. Useful for analyzing, saving, etc. all messages and attachments down through the structure without having to recurse yourself. * Fixed an issue that would cause signed attachments to not properly generate.