From 7dddbc893da2711119eabd5ea806284b1ebf2957 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Fri, 10 Jun 2022 01:17:36 -0700 Subject: [PATCH 1/6] Moved the code for getting parent Named to the Named property. --- extract_msg/msg.py | 31 ++++++++++++++++--------------- extract_msg/utils.py | 1 - 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/extract_msg/msg.py b/extract_msg/msg.py index 142027d0..5aac335f 100644 --- a/extract_msg/msg.py +++ b/extract_msg/msg.py @@ -63,22 +63,13 @@ def __init__(self, path, **kwargs): """ # Retrieve all the kwargs that we need. prefix = kwargs.get('prefix', '') - parentMsg = kwargs.get('parentMsg') + self.__parentMsg = kwargs.get('parentMsg') + # Verify it is a valid class. + if self.__parentMsg is not None and not isinstance(self.__parentMsg, MSGFile): + raise TypeError(':param parentMsg: must be an instance of MSGFile or a subclass.') filename = kwargs.get('filename', None) overrideEncoding = kwargs.get('overrideEncoding', None) - # Handle the parent msg file existing. - if parentMsg: - # Verify it is a valid class. - if not isinstance(parentMsg, MSGFile): - raise TypeError(':param parentMsg: must be an instance of MSGFile or a subclass.') - # Try to get the named properties and use that for our main - # instance. - try: - self.__named = parentMsg.named - except Exception: - pass - # WARNING DO NOT MANUALLY MODIFY PREFIX. Let the program set it. self.__path = path self.__attachmentClass = kwargs.get('attachmentClass', Attachment) @@ -131,7 +122,7 @@ def __init__(self, path, **kwargs): self.__prefixList = prefixl self.__prefixLen = len(prefixl) if prefix: - filename = self._getStringStream(prefixl[:-1] + ['__substg1.0_3001'], prefix=False) + filename = self._getStringStream(prefixl[:-1] + ['__substg1.0_3001'], prefix = False) if filename: self.filename = filename elif hasLen(path): @@ -692,7 +683,17 @@ def named(self) -> Named: try: return self.__named except AttributeError: - self.__named = Named(self) + self.__named = None + # Handle the parent msg file existing. + if self.__parentMsg: + # Try to get the named properties and use that for our main + # instance. + try: + self.__named = self.__parentMsg.named + except Exception: + pass + if not self.__named: + self.__named = Named(self) return self.__named @property diff --git a/extract_msg/utils.py b/extract_msg/utils.py index 7ec6d2d8..2b8196d9 100644 --- a/extract_msg/utils.py +++ b/extract_msg/utils.py @@ -593,7 +593,6 @@ def openMsg(path, **kwargs): :raises UnrecognizedMSGTypeError: if the type is not recognized. """ from .appointment import Appointment - from .attachment import Attachment from .contact import Contact from .message import Message from .msg import MSGFile From 8faf78dec3cb16d305febbe18f6c7c1f9f21a577 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Fri, 10 Jun 2022 17:30:10 -0700 Subject: [PATCH 2/6] Stated implementation for pdf --- extract_msg/message.py | 5 +++-- extract_msg/utils.py | 11 +++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/extract_msg/message.py b/extract_msg/message.py index 57d2b5f0..bda801b1 100644 --- a/extract_msg/message.py +++ b/extract_msg/message.py @@ -115,6 +115,7 @@ def save(self, **kwargs): html = kwargs.get('html', False) rtf = kwargs.get('rtf', False) raw = kwargs.get('raw', False) + pdf = kwargs.get('pdf', False) allowFallback = kwargs.get('allowFallback', False) _zip = kwargs.get('zip') maxNameLength = kwargs.get('maxNameLength', 256) @@ -157,8 +158,8 @@ def save(self, **kwargs): kwargs['customFilename'] = None # Check if incompatible options have been provided in any way. - if _json + html + rtf + raw + attachOnly > 1: - raise IncompatibleOptionsError('Only one of the following options may be used at a time: json, raw, html, rtf, attachmentsOnly.') + if _json + html + rtf + raw + attachOnly + pdf > 1: + raise IncompatibleOptionsError('Only one of the following options may be used at a time: json, raw, html, rtf, attachmentsOnly, pdf.') # TODO: insert code here that will handle checking all of the msg files to see if the path with overflow. diff --git a/extract_msg/utils.py b/extract_msg/utils.py index 7ec6d2d8..efe1925c 100644 --- a/extract_msg/utils.py +++ b/extract_msg/utils.py @@ -183,6 +183,13 @@ def getCommandArgs(args): # --html parser.add_argument('--html', dest='html', action='store_true', help='Sets whether the output should be HTML. If this is not possible, will error.') + # --pdf + parser.add_argument('--pdf', dest = 'pdf', action='store_true', + help='Saves the body as a PDF. If this is not possible, will error.') + + # --wk-path + parser.add_argument('--wk-path', desk = 'wkPath' + help='Overrides the path for finding wkhtmltopdf.') # --prepared-html parser.add_argument('--prepared-html', dest='preparedHtml', action='store_true', help='When used in conjunction with --html, sets whether the HTML output should be prepared for embedded attachments.') @@ -214,8 +221,8 @@ def getCommandArgs(args): options = parser.parse_args(args) # Check if more than one of the following arguments has been specified - if options.html + options.rtf + options.json + options.raw + options.attachmentsOnly > 1: - raise IncompatibleOptionsError('Only one of these options may be selected at a time: --html, --json, --raw, --rtf, --attachments-only') + if options.html + options.rtf + options.json + options.raw + options.pdf + options.attachmentsOnly > 1: + raise IncompatibleOptionsError('Only one of these options may be selected at a time: --html, --json, --raw, --rtf, --attachments-only, --pdf') if options.dev or options.file_logging: options.verbose = True From e6b64c14ac6c2dad621850b5b89d8f63d2b3abe6 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sat, 11 Jun 2022 00:43:27 -0700 Subject: [PATCH 3/6] v0.34.0 --- CHANGELOG.md | 12 ++- README.rst | 4 +- extract_msg/__init__.py | 4 +- extract_msg/__main__.py | 31 +++++--- extract_msg/exceptions.py | 13 +++ extract_msg/message.py | 80 +++++++++++++++---- extract_msg/message_base.py | 9 ++- extract_msg/prop.py | 20 +++-- extract_msg/recipient.py | 2 +- extract_msg/utils.py | 152 ++++++++++++++++++++++++------------ requirements.txt | 1 + 11 files changed, 233 insertions(+), 95 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5dc17a4..c4a6ae84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,19 @@ +**v0.34.0** +* [[TeamMsgExtractor #102](https://github.com/TeamMsgExtractor/msg-extractor/issues/102)] Added the option to directly save body to pdf. This requires that you either have wkhtmltopdf on your path or that you provide a path directly to it in order to work. Simply pass `pdf = True` to save to turn it on. More details in the doc for the save function. You can also do this from the command line using the `--pdf` option, incompatible with other body types. +* Added `--glob` option for allowing you to provide an msg path that will evaluate wildcards. +* Removed per-file output names as they weren't actually functional and currently add too much complexity. If anyone knows a way to handle it directly with `argparse` let me know. +* Added `chardet` as a requirement to help work around an error in `RTFDE`. + **v0.33.0** * [[TeamMsgExtractor #212](https://github.com/TeamMsgExtractor/msg-extractor/issues/212)] Added support for Task objects. -* Added additional parameter `overrideClass` to all `_ensureSetX` type functions. This parameter is a class to initialize using the data, if provided. The data will be provided as the first argument to the `__init__` function of the class *if* the data is not `None`. If the data is done, the value set and returned from `_ensureSetX` will be `None`. This can be overriden using the second added parameter, which defaults to this behavior. Simply set `preserveNone` to `False` to force the data to be passed to the class. Keep in mind that this means the class will receive `None` as it's first parameter. +* Added additional parameter `overrideClass` to all `_ensureSetX` type functions. This parameter is a class to initialize using the data, if provided. The data will be provided as the first argument to the `__init__` function of the class *if* the data is not `None`. If the data is done, the value set and returned from `_ensureSetX` will be `None`. This can be overridden using the second added parameter, which defaults to this behavior. Simply set `preserveNone` to `False` to force the data to be passed to the class. Keep in mind that this means the class will receive `None` as it's first parameter. * Updated `Named` class to make it more like the dictionary it is build on top of. Specifically, added `keys`, `values`, and `items` as functions. * Fixed issue in `Named` where old code wasn't deleted, causing some named properties to be completely omitted. * Improved efficiency of `Named` by making it so `get` no longer copied the entire dictionary repeatedly while looking for the property. * Fixed typo in `Attachment` that caused embedded msg files to still regenerate the `Named` instance even though they should have been using the parent's. * Updated fields in `MSGFile` to use enums. -* Added additional properties to `MSGFile`. -* Significant cleanup. +* Added additional properties to `MSGFile`. +* Significant cleanup. **v0.32.0** * [[TeamMsgExtractor #217](https://github.com/TeamMsgExtractor/msg-extractor/issues/217)] Redesigned named properties to be much more efficient. All in all, load times for MSG files are significantly improved all around. diff --git a/README.rst b/README.rst index d53c2b0a..c11bd591 100644 --- a/README.rst +++ b/README.rst @@ -221,8 +221,8 @@ your access to the newest major version of extract-msg. .. |License: GPL v3| image:: https://img.shields.io/badge/License-GPLv3-blue.svg :target: LICENSE.txt -.. |PyPI3| image:: https://img.shields.io/badge/pypi-0.33.0-blue.svg - :target: https://pypi.org/project/extract-msg/0.33.0/ +.. |PyPI3| image:: https://img.shields.io/badge/pypi-0.34.0-blue.svg + :target: https://pypi.org/project/extract-msg/0.34.0/ .. |PyPI2| image:: https://img.shields.io/badge/python-3.6+-brightgreen.svg :target: https://www.python.org/downloads/release/python-367/ diff --git a/extract_msg/__init__.py b/extract_msg/__init__.py index 1df63cc8..5d226677 100644 --- a/extract_msg/__init__.py +++ b/extract_msg/__init__.py @@ -27,8 +27,8 @@ # along with this program. If not, see . __author__ = 'Destiny Peterson & Matthew Walker' -__date__ = '2022-06-08' -__version__ = '0.33.0' +__date__ = '2022-06-11' +__version__ = '0.34.0' import logging diff --git a/extract_msg/__main__.py b/extract_msg/__main__.py index d9632638..e341a938 100644 --- a/extract_msg/__main__.py +++ b/extract_msg/__main__.py @@ -15,14 +15,14 @@ def main() -> None: # Determine where to save the files to. currentDir = os.getcwd() # Store this in case the path changes. if not args.zip: - if args.out_path: - if not os.path.exists(args.out_path): - os.makedirs(args.out_path) - out = args.out_path + if args.outPath: + if not os.path.exists(args.outPath): + os.makedirs(args.outPath) + out = args.outPath else: out = currentDir else: - out = args.out_path if args.out_path else '' + out = args.outPath if args.outPath else '' if args.dev: import extract_msg.dev @@ -34,7 +34,7 @@ def main() -> None: from extract_msg import validation - valResults = {x[0]: validation.validate(x[0]) for x in args.msgs} + valResults = {x: validation.validate(x) for x in args.msgs} filename = f'validation {int(time.time())}.json' print('Validation Results:') pprint.pprint(valResults) @@ -43,8 +43,8 @@ def main() -> None: json.dump(valResults, fil) input('Press enter to exit...') else: - if not args.dump_stdout: - utils.setupLogging(args.config_path, level, args.log, args.file_logging) + if not args.dumpStdout: + utils.setupLogging(args.configPath, level, args.log, args.fileLogging) # Quickly make a dictionary for the keyword arguments. kwargs = { @@ -52,25 +52,30 @@ def main() -> None: 'attachmentsOnly': args.attachmentsOnly, 'charset': args.charset, 'contentId': args.cid, - 'customFilename': args.out_name, + 'customFilename': args.outName, 'customPath': out, 'html': args.html, 'json': args.json, + 'pdf': args.pdf, 'preparedHtml': args.preparedHtml, 'rtf': args.rtf, - 'useMsgFilename': args.use_filename, + 'useMsgFilename': args.useFilename, + 'wkOptions': args.wkOptions, + 'wkPath': args.wkPath, 'zip': args.zip, } for x in args.msgs: try: - with utils.openMsg(x[0]) as msg: - if args.dump_stdout: + if args.progress: + print(f'Saving file "{x}"...') + with utils.openMsg(x) as msg: + if args.dumpStdout: print(msg.body) else: msg.save(**kwargs) except Exception as e: - print(f'Error with file "{x[0]}": {traceback.format_exc()}') + print(f'Error with file "{x}": {traceback.format_exc()}') if __name__ == '__main__': diff --git a/extract_msg/exceptions.py b/extract_msg/exceptions.py index 01341658..0b82be5a 100644 --- a/extract_msg/exceptions.py +++ b/extract_msg/exceptions.py @@ -12,6 +12,7 @@ logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) + class BadHtmlError(ValueError): """ HTML failed to pass validation. @@ -30,6 +31,12 @@ class DataNotFoundError(Exception): """ pass +class ExecutableNotFound(Exception): + """ + Could not find the specified executable. + """ + pass + class IncompatibleOptionsError(Exception): """ Provided options are incompatible with each other. @@ -77,3 +84,9 @@ class UnrecognizedMSGTypeError(TypeError): open a specific class of msg file. """ pass + +class WKError(RuntimeError): + """ + An error occured while running wkhtmltopdf. + """ + pass diff --git a/extract_msg/message.py b/extract_msg/message.py index bda801b1..f5b9483c 100644 --- a/extract_msg/message.py +++ b/extract_msg/message.py @@ -2,6 +2,7 @@ import logging import os import pathlib +import subprocess import zipfile import bs4 @@ -9,9 +10,9 @@ from imapclient.imapclient import decode_utf7 from . import constants -from .exceptions import DataNotFoundError, IncompatibleOptionsError +from .exceptions import DataNotFoundError, IncompatibleOptionsError, WKError from .message_base import MessageBase -from .utils import addNumToDir, addNumToZipDir, createZipOpen, injectHtmlHeader, injectRtfHeader, inputToBytes, inputToString, prepareFilename +from .utils import addNumToDir, addNumToZipDir, createZipOpen, findWk, injectHtmlHeader, injectRtfHeader, inputToBytes, inputToString, prepareFilename logger = logging.getLogger(__name__) @@ -109,6 +110,14 @@ def save(self, **kwargs): usage, doing things like adding tags, injecting attachments, etc. This is useful for things like trying to convert the HTML body directly to PDF. + :param pdf: Used to enable saving the body as a PDF file. + :param wkPath: Used to manually specify the path of the wkhtmltopdf + executable. If not specified, the function will try to find it. + Useful if wkhtmltopdf is not on the path. If :param pdf: is False, + this argument is ignored. + :param wkOptions: Used to specify additional options to wkhtmltopdf. + this must be a list or list-like object composed of strings and + bytes. """ # Move keyword arguments into variables. _json = kwargs.get('json', False) @@ -130,6 +139,11 @@ def save(self, **kwargs): # Track if we are skipping attachments. skipAttachments = kwargs.get('skipAttachments', False) + # If we are doing PDF, we immediately need to try to find wkhtmltopdf. + if pdf: + wkPath = findWk(kwargs.get('wkPath')) + kwargs['preparedHtml'] = True + # ZipFile handling. if _zip: # `raw` and `zip` are incompatible. @@ -252,21 +266,55 @@ def save(self, **kwargs): if not attachOnly: # Determine the extension to use for the body. - fext = 'json' if _json else fext - - with _open(str(path / ('message.' + fext)), mode) as f: - if _json: - emailObj = json.loads(self.getJson()) - if not skipAttachments: - emailObj['attachments'] = attachmentNames - - f.write(inputToBytes(json.dumps(emailObj), 'utf-8')) - elif useHtml: - f.write(self.getSaveHtmlBody(**kwargs)) - elif useRtf: - f.write(self.getSaveRtfBody(**kwargs)) + if pdf: + # First thing is first, we need to parse our wkOptions if + # they exist. + wkOptions = kwargs.get('wkOptions') + if wkOptions: + try: + # Try to convert to a list, whatever it is, and + # fail if it is not possible. + parsedWkOptions = [*wkOptions] + except TypeError: + raise TypeError(':param wkOptions: must be an iterable, not {type(wkOptions)}.') else: - f.write(self.getSaveBody(**kwargs)) + parsedWkOptions = [] + + # Confirm that all of our options we now have are either + # strings or bytes. + if not all(isinstance(option, (str, bytes)) for option in parsedWkOptions): + raise TypeError(':param wkOptions: must be an iterable of strings and bytes.') + + # We call the program to convert the html, but give tell it + # the data will go in and come out through stdin and stdout, + # respectively. This way we don't have to write temporary + # files to the disk. We also ask that it be quiet about it. + process = subprocess.Popen([wkPath, *parsedWkOptions, '-', '-'], shell = True, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE) + # Give the program the data and wait for the program to + # finish. + output = process.communicate(self.getSaveHtmlBody(**kwargs)) + if process.returncode != 0: + raise WKError(output[1].decode('utf-8')) + + with _open(str(path / 'message.pdf'), mode) as f: + f.write(output[0]) + + else: + fext = 'json' if _json else fext + + with _open(str(path / ('message.' + fext)), mode) as f: + if _json: + emailObj = json.loads(self.getJson()) + if not skipAttachments: + emailObj['attachments'] = attachmentNames + + f.write(inputToBytes(json.dumps(emailObj), 'utf-8')) + elif useHtml: + f.write(self.getSaveHtmlBody(**kwargs)) + elif useRtf: + f.write(self.getSaveRtfBody(**kwargs)) + else: + f.write(self.getSaveBody(**kwargs)) except Exception: if not _zip: diff --git a/extract_msg/message_base.py b/extract_msg/message_base.py index 59509909..f2d6cbe9 100644 --- a/extract_msg/message_base.py +++ b/extract_msg/message_base.py @@ -5,6 +5,7 @@ import re import bs4 +import chardet import compressed_rtf import RTFDE @@ -203,7 +204,13 @@ def deencapsulatedRtf(self) -> RTFDE.DeEncapsulator: if self.rtfBody: # If there is an RTF body, we try to deencapsulate it. try: - self._deencapsultor = RTFDE.DeEncapsulator(self.rtfBody) + try: + self._deencapsultor = RTFDE.DeEncapsulator(self.rtfBody) + except UnicodeDecodeError: + # There is a known issue that bytes are not well decoded + # by RTFDE right now, so let's see if we can't manually + # decode it and see if that will work. + self._deencapsultor = RTFDE.DeEncapsulator(self.rtfBody.decode(chardet.detect(self.rtfBody)['encoding'])) self._deencapsultor.deencapsulate() except RTFDE.exceptions.NotEncapsulatedRtf as e: logger.debug("RTF body is not encapsulated.") diff --git a/extract_msg/prop.py b/extract_msg/prop.py index 44462046..62e5868e 100644 --- a/extract_msg/prop.py +++ b/extract_msg/prop.py @@ -1,9 +1,12 @@ import datetime import logging +import olefile + from . import constants from .utils import fromTimeStamp, filetimeToUtc, properHex + logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) @@ -90,7 +93,7 @@ class FixedLengthProp(PropBase): """ def __init__(self, string): - super(FixedLengthProp, self).__init__(string) + super().__init__(string) self.__value = self.parseType(self.type, constants.STFIX.unpack(string)[0]) def parseType(self, _type, stream): @@ -134,11 +137,16 @@ def parseType(self, _type, stream): elif _type == 0x0040: # PtypTime try: rawtime = constants.ST3.unpack(value)[0] - if rawtime != 915151392000000000: - value = fromTimeStamp(filetimeToUtc(rawtime)) + if rawtime < 116444736000000000: + # We can't properly parse this with our current setup, so + # we will rely on olefile to handle this one. + value = olefile.olefile.filetime2datetime(rawtime) else: - # Temporarily just set to max time to signify a null date. - value = datetime.datetime.max + if rawtime != 915151392000000000: + value = fromTimeStamp(filetimeToUtc(rawtime)) + else: + # Temporarily just set to max time to signify a null date. + value = datetime.datetime.max except Exception as e: logger.exception(e) logger.error(f'Timestamp value of {filetimeToUtc(constants.ST3.unpack(value)[0])} caused an exception. This was probably caused by the time stamp being too far in the future.') @@ -162,7 +170,7 @@ class VariableLengthProp(PropBase): """ def __init__(self, string): - super(VariableLengthProp, self).__init__(string) + super().__init__(string) self.__length, self.__reserved = constants.STVAR.unpack(string) if self.type == 0x001E: self.__realLength = self.__length - 1 diff --git a/extract_msg/recipient.py b/extract_msg/recipient.py index 98b5b5f1..8f07adf0 100644 --- a/extract_msg/recipient.py +++ b/extract_msg/recipient.py @@ -15,7 +15,7 @@ class Recipient: """ Contains the data of one of the recipients in an msg file. """ - + def __init__(self, _dir, msg): self.__msg = msg # Allows calls to original msg file. self.__dir = _dir diff --git a/extract_msg/utils.py b/extract_msg/utils.py index cec51aac..b4837a2b 100644 --- a/extract_msg/utils.py +++ b/extract_msg/utils.py @@ -6,11 +6,13 @@ import codecs import copy import datetime +import glob import json import logging import logging.config import os import pathlib +import shutil import struct # Not actually sure if this needs to be here for the logging, so just in case. import sys @@ -133,6 +135,29 @@ def divide(string, length : int) -> list: """ return [string[length * x:length * (x + 1)] for x in range(int(ceilDiv(len(string), length)))] +def findWk(path = None): + """ + Attempt to find the path of the wkhtmltopdf executable. If :param path: is + provided, verifies that it is executable and returns the path if it is. + + :raises ExecutableNotFound: A valid executable could not be found. + """ + if path: + if os.path.isfile(path): + # Check if executable. + if os.access(path, os.X_OK): + return path + else: + raise ExecutableNotFound('Path provided was not a valid executable (execution bit not set).') + else: + raise ExecutableNotFound('Path provided was not a valid executable (not a file).') + + candidate = shutil.which('wkhtmltopdf') + if candidate: + return candidate + + raise ExecutableNotFound('Could not find wkhtmltopdf.') + def fromTimeStamp(stamp) -> datetime.datetime: """ Returns a datetime from the UTC timestamp given the current timezone. @@ -147,6 +172,8 @@ def getCommandArgs(args): incompatible. """ parser = argparse.ArgumentParser(description = constants.MAINDOC, prog = 'extract_msg') + outFormat = parser.add_mutually_exclusive_group() + inputFormat = parser.add_mutually_exclusive_group() # --use-content-id, --cid parser.add_argument('--use-content-id', '--cid', dest='cid', action='store_true', help='Save attachments by their Content ID, if they have one. Useful when working with the HTML body.') @@ -157,10 +184,10 @@ def getCommandArgs(args): parser.add_argument('--validate', dest='validate', action='store_true', help='Turns on file validation mode. Turns off regular file output.') # --json - parser.add_argument('--json', dest='json', action='store_true', + outFormat.add_argument('--json', dest='json', action='store_true', help='Changes to write output files as json.') # --file-logging - parser.add_argument('--file-logging', dest='file_logging', action='store_true', + parser.add_argument('--file-logging', dest='fileLogging', action='store_true', help='Enables file logging. Implies --verbose.') # --verbose parser.add_argument('--verbose', dest='verbose', action='store_true', @@ -169,27 +196,29 @@ def getCommandArgs(args): parser.add_argument('--log', dest='log', help='Set the path to write the file log to.') # --config PATH - parser.add_argument('--config', dest='config_path', + parser.add_argument('--config', dest='configPath', help='Set the path to load the logging config from.') # --out PATH - parser.add_argument('--out', dest='out_path', + parser.add_argument('--out', dest='outPath', help='Set the folder to use for the program output. (Default: Current directory)') # --use-filename - parser.add_argument('--use-filename', dest='use_filename', action='store_true', + parser.add_argument('--use-filename', dest='useFilename', action='store_true', help='Sets whether the name of each output is based on the msg filename.') # --dump-stdout - parser.add_argument('--dump-stdout', dest='dump_stdout', action='store_true', + parser.add_argument('--dump-stdout', dest='dumpStdout', action='store_true', help='Tells the program to dump the message body (plain text) to stdout. Overrides saving arguments.') # --html - parser.add_argument('--html', dest='html', action='store_true', + outFormat.add_argument('--html', dest='html', action='store_true', help='Sets whether the output should be HTML. If this is not possible, will error.') # --pdf - parser.add_argument('--pdf', dest = 'pdf', action='store_true', + outFormat.add_argument('--pdf', dest='pdf', action='store_true', help='Saves the body as a PDF. If this is not possible, will error.') - - # --wk-path - parser.add_argument('--wk-path', desk = 'wkPath' + # --wk-path PATH + parser.add_argument('--wk-path', dest='wkPath', help='Overrides the path for finding wkhtmltopdf.') + # --wk-options OPTIONS + parser.add_argument('--wk-options', dest='wkOptions', nargs='*', + help='Sets additional options to be used in wkhtmltopdf. Should be a series of options and values, replacing the - or -- in the beginning with + or ++, respectively. For example: --wk-options "+O Landscape"') # --prepared-html parser.add_argument('--prepared-html', dest='preparedHtml', action='store_true', help='When used in conjunction with --html, sets whether the HTML output should be prepared for embedded attachments.') @@ -197,10 +226,10 @@ def getCommandArgs(args): parser.add_argument('--charset', dest='charset', default='utf-8', help='Character set to use for the prepared HTML in the added tag. (Default: utf-8)') # --raw - parser.add_argument('--raw', dest='raw', action='store_true', + outFormat.add_argument('--raw', dest='raw', action='store_true', help='Sets whether the output should be raw. If this is not possible, will error.') # --rtf - parser.add_argument('--rtf', dest='rtf', action='store_true', + outFormat.add_argument('--rtf', dest='rtf', action='store_true', help='Sets whether the output should be RTF. If this is not possible, will error.') # --allow-fallback parser.add_argument('--allow-fallback', dest='allowFallback', action='store_true', @@ -209,55 +238,71 @@ def getCommandArgs(args): parser.add_argument('--zip', dest='zip', help='Path to use for saving to a zip file.') # --attachments-only - parser.add_argument('--attachments-only', dest='attachmentsOnly', action='store_true', + outFormat.add_argument('--attachments-only', dest='attachmentsOnly', action='store_true', help='Specify to only save attachments from an msg file.') # --out-name NAME - parser.add_argument('--out-name', dest='out_name', - help='Name to be used with saving the file output. Should come immediately after the file name.') + inputFormat.add_argument('--out-name', dest='outName', + help='Name to be used with saving the file output. Cannot be used if you are saving more than one file.') + # --glob + inputFormat.add_argument('--glob', '--wildcard', dest='glob', action='store_true', + help='Interpret all paths as having wildcards. Incompatible with --out-name.') + # --progress + parser.add_argument('--progress', dest='progress', action='store_true', + help='Shows what file the program is currently working on during it\'s progress.') # [msg files] parser.add_argument('msgs', metavar='msg', nargs='+', - help='An msg file to be parsed.') + help='An MSG file to be parsed.') options = parser.parse_args(args) - # Check if more than one of the following arguments has been specified - if options.html + options.rtf + options.json + options.raw + options.pdf + options.attachmentsOnly > 1: - raise IncompatibleOptionsError('Only one of these options may be selected at a time: --html, --json, --raw, --rtf, --attachments-only, --pdf') - - if options.dev or options.file_logging: + if options.dev or options.fileLogging: options.verbose = True + # Handle the wkOptions if they exist. + if options.wkOptions: + wkOptions = [] + for option in options.wkOptions: + if option.startswith('++'): + option = '--' + option[2:] + elif option.startswith('+'): + option = '-' + option[1:] + + # Now that we have corrected to the correct start, split the argument if + # necessary. + split = option.split(' ') + if len(split) == 1: + # No spaces means we just pass that directly. + wkOptions.append(option) + else: + wkOptions.append(split[0]) + wkOptions.append(' '.join(split[1:])) + + options.wkOptions = wkOptions + # If dump_stdout is True, we need to unset all arguments used in files. # Technically we actually only *need* to unset `out_path`, but that may # change in the future, so let's be thorough. - if options.dump_stdout: - options.out_path = None + if options.dumpStdout: + options.outPath = None options.json = False options.rtf = False options.html = False - options.use_filename = False + options.useFilename = False options.cid = False - file_args = options.msgs - file_tables = [] # This is where we will store the separated files and their arguments - temp_table = [] # temp_table will store each table while it is still being built. - need_arg = True # This tells us if the last argument was something like - # --out-name which requires a string name after it. - # We start on true to make it so that we use don't have to have something checking if we are on the first table. - for x in file_args: # Iterate through each - if need_arg: - temp_table.append(x) - need_arg = False - elif x in constants.KNOWN_FILE_FLAGS: - temp_table.append(x) - if x in constants.NEEDS_ARG: - need_arg = True - else: - file_tables.append(temp_table) - temp_table = [x] + if options.glob: + fileLists = [] + for path in options.msgs: + fileLists += glob.glob(path) + + if len(fileLists) == 0: + raise ValueError('Could not find any msg files using the specified wildcards.') + options.msgs = fileLists + + # Make it so outName can only be used on single files. + if options.outName and options.fileArgs and len(options.fileArgs) > 0: + raise ValueError('--out-name is not supported when saving multiple MSG files.') - file_tables.append(temp_table) - options.msgs = file_tables return options def getEncodingName(codepage : int) -> str: @@ -327,7 +372,7 @@ def injectHtmlHeader(msgFile, prepared : bool = False) -> bytes: # tag are missing, we determine where to put the body tag (around # everything if there is no tag, otherwise at the end) and then # wrap it all in the tag. - parser = bs4.BeautifulSoup(body) + parser = bs4.BeautifulSoup(body, features = 'html.parser') if not parser.find('html') and not parser.find('body'): if parser.find('head') or parser.find('footer'): # Create the parser we will be using for the corrections. @@ -368,7 +413,7 @@ def injectHtmlHeader(msgFile, prepared : bool = False) -> bytes: # pointing to the wrong place after that. To compensate we first # create a tuple and iterate over that. for tag in tuple(parser.find('html').children): - if tag.name.lower() not in ('head', 'footer'): + if tag.name and tag.name.lower() not in ('head', 'footer'): bodyTag.append(tag) # All the tags should now be properly in the body, so let's @@ -395,7 +440,7 @@ def replace(bodyMarker): 'cc': inputToString(htmlEscape(msgFile.cc) if msgFile.cc else '', 'utf-8'), 'bcc': inputToString(htmlEscape(msgFile.bcc) if msgFile.bcc else '', 'utf-8'), 'date': inputToString(msgFile.date, 'utf-8'), - 'subject': inputToString(htmlEscape(msgFile.subject), 'utf-8'), + 'subject': inputToString(htmlEscape(msgFile.subject) if msgFile.subject else '', 'utf-8'), }).encode('utf-8') # Use the previously defined function to inject the HTML header. @@ -696,11 +741,16 @@ def parseType(_type : int, stream, encoding, extras): return value.decode('utf-16-le') elif _type == 0x0040: # PtypTime rawtime = constants.ST3.unpack(value)[0] - if rawtime != 915151392000000000: - value = fromTimeStamp(filetimeToUtc(rawtime)) + if rawtime < 116444736000000000: + # We can't properly parse this with our current setup, so + # we will rely on olefile to handle this one. + value = olefile.olefile.filetime2datetime(rawtime) else: - # Temporarily just set to max time to signify a null date. - value = datetime.datetime.max + if rawtime != 915151392000000000: + value = fromTimeStamp(filetimeToUtc(rawtime)) + else: + # Temporarily just set to max time to signify a null date. + value = datetime.datetime.max return value elif _type == 0x0048: # PtypGuid return bytesToGuid(value) diff --git a/requirements.txt b/requirements.txt index 723873ff..774a7188 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ ebcdic>=1.1.1 beautifulsoup4>=4.10.0 RTFDE>=0.0.2 mailbits>=0.2.1 +chardet>=4.0.0 From f42d016ed93b9e51f619ef23bf68106b9f58aff4 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sat, 11 Jun 2022 22:00:47 -0700 Subject: [PATCH 4/6] Making further progress on next major version. --- CHANGELOG.md | 1 + extract_msg/constants.py | 123 ++++++++++++++++--------------- extract_msg/message.py | 135 ++++++++++++++++++++-------------- extract_msg/message_base.py | 29 +++++++- extract_msg/message_signed.py | 105 ++++++++++++++++++++++---- extract_msg/utils.py | 6 +- 6 files changed, 264 insertions(+), 135 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4a6ae84..17e240e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ * Added `--glob` option for allowing you to provide an msg path that will evaluate wildcards. * Removed per-file output names as they weren't actually functional and currently add too much complexity. If anyone knows a way to handle it directly with `argparse` let me know. * Added `chardet` as a requirement to help work around an error in `RTFDE`. +* Looking into some of the unsupported encodings revealed at least one to be supported, but was missed due to the name not being registered as an alias. **v0.33.0** * [[TeamMsgExtractor #212](https://github.com/TeamMsgExtractor/msg-extractor/issues/212)] Added support for Task objects. diff --git a/extract_msg/constants.py b/extract_msg/constants.py index 42fc8033..ad815424 100644 --- a/extract_msg/constants.py +++ b/extract_msg/constants.py @@ -38,7 +38,9 @@ RE_RTF_BODY_FALLBACK_FS = re.compile(br'\\fs[0-9]*[^a-zA-Z]') RE_RTF_BODY_FALLBACK_F = re.compile(br'\\f[0-9]*[^a-zA-Z]') RE_RTF_FALLBACK_PLAIN = re.compile(br'\\plain[^a-zA-Z0-9]') - +# This is used in the workaround for decoding issues in RTFDE. We find `\bin` +# sections and try to remove all of them to help with the decoding. +RE_BIN = re.compile(br'\\bin([0-9]+) ?') # EntryID UID Types. EUID_PUBLIC_MESSAGE_STORE = b'\x1A\x44\x73\x90\xAA\x66\x11\xCD\x9B\xC8\x00\xAA\x00\x2F\xC4\x5A' @@ -318,11 +320,11 @@ 437: 'IBM437', # OEM United States 500: 'IBM500', # IBM EBCDIC International 708: 'ASMO-708', # Arabic (ASMO 708) - # UNSUPPORTED + # UNSUPPORTED. 709: '', # Arabic (ASMO-449+, BCON V4) - # UNSUPPORTED + # UNSUPPORTED. 710: '', # Arabic - Transparent Arabic - # UNSUPPORTED + # UNSUPPORTED. 720: 'DOS-720', # Arabic (Transparent ASMO); Arabic (DOS) 737: 'cp737', # OEM Greek (formerly 437G); Greek (DOS) 775: 'ibm775', # OEM Baltic; Baltic (DOS) @@ -330,8 +332,7 @@ 852: 'ibm852', # OEM Latin 2; Central European (DOS) 855: 'IBM855', # OEM Cyrillic (primarily Russian) 857: 'ibm857', # OEM Turkish; Turkish (DOS) - # UNSUPPORTED - 858: 'IBM00858', # OEM Multilingual Latin 1 + Euro symbol + 858: 'cp858', # OEM Multilingual Latin 1 + Euro symbol 860: 'IBM860', # OEM Portuguese; Portuguese (DOS) 861: 'ibm861', # OEM Icelandic; Icelandic (DOS) 862: 'cp862', # OEM Hebrew; Hebrew (DOS) @@ -341,7 +342,7 @@ 866: 'cp866', # OEM Russian; Cyrillic (DOS) 869: 'ibm869', # OEM Modern Greek; Greek, Modern (DOS) 870: 'cp870', # IBM870 # IBM EBCDIC Multilingual/ROECE (Latin 2); IBM EBCDIC Multilingual Latin 2 - # UNSUPPORTED + # UNSUPPORTED. 874: 'windows-874', # ANSI/OEM Thai (ISO 8859-11); Thai (Windows) 875: 'cp875', # IBM EBCDIC Greek Modern 932: 'shift_jis', # ANSI/OEM Japanese; Japanese (Shift-JIS) @@ -374,59 +375,59 @@ 1361: 'Johab', # Korean (Johab) 10000: 'macintosh', # MAC Roman; Western European (Mac) 10001: 'x-mac-japanese', # Japanese (Mac) - # UNSUPPORTED + # UNSUPPORTED. 10002: 'x-mac-chinesetrad', # MAC Traditional Chinese (Big5); Chinese Traditional (Mac) 10003: 'x-mac-korean', # Korean (Mac) - # UNSUPPORTED + # UNSUPPORTED. 10004: 'x-mac-arabic', # Arabic (Mac) - # UNSUPPORTED + # UNSUPPORTED. 10005: 'x-mac-hebrew', # Hebrew (Mac) - # UNSUPPORTED + # UNSUPPORTED. 10006: 'x-mac-greek', # Greek (Mac) - # UNSUPPORTED + # UNSUPPORTED. 10007: 'x-mac-cyrillic', # Cyrillic (Mac) - # UNSUPPORTED + # UNSUPPORTED. 10008: 'x-mac-chinesesimp', # MAC Simplified Chinese (GB 2312); Chinese Simplified (Mac) - # UNSUPPORTED + # UNSUPPORTED. 10010: 'x-mac-romanian', # Romanian (Mac) - # UNSUPPORTED + # UNSUPPORTED. 10017: 'x-mac-ukrainian', # Ukrainian (Mac) - # UNSUPPORTED + # UNSUPPORTED. 10021: 'x-mac-thai', # Thai (Mac) - # UNSUPPORTED + # UNSUPPORTED. 10029: 'x-mac-ce', # MAC Latin 2; Central European (Mac) - # UNSUPPORTED + # UNSUPPORTED. 10079: 'x-mac-icelandic', # Icelandic (Mac) - # UNSUPPORTED + # UNSUPPORTED. 10081: 'x-mac-turkish', # Turkish (Mac) - # UNSUPPORTED + # UNSUPPORTED. 10082: 'x-mac-croatian', # Croatian (Mac) 12000: 'utf-32', # Unicode UTF-32, little endian byte order; available only to managed applications 12001: 'utf-32BE', # Unicode UTF-32, big endian byte order; available only to managed applications - # UNSUPPORTED + # UNSUPPORTED. 20000: 'x-Chinese_CNS', # CNS Taiwan; Chinese Traditional (CNS) - # UNSUPPORTED + # UNSUPPORTED. 20001: 'x-cp20001', # TCA Taiwan - # UNSUPPORTED + # UNSUPPORTED. 20002: 'x_Chinese-Eten', # Eten Taiwan; Chinese Traditional (Eten) - # UNSUPPORTED + # UNSUPPORTED. 20003: 'x-cp20003', # IBM5550 Taiwan - # UNSUPPORTED + # UNSUPPORTED. 20004: 'x-cp20004', # TeleText Taiwan - # UNSUPPORTED + # UNSUPPORTED. 20005: 'x-cp20005', # Wang Taiwan - # UNSUPPORTED + # UNSUPPORTED. 20105: 'x-IA5', # IA5 (IRV International Alphabet No. 5, 7-bit); Western European (IA5) - # UNSUPPORTED + # UNSUPPORTED. 20106: 'x-IA5-German', # IA5 German (7-bit) - # UNSUPPORTED + # UNSUPPORTED. 20107: 'x-IA5-Swedish', # IA5 Swedish (7-bit) - # UNSUPPORTED + # UNSUPPORTED. 20108: 'x-IA5-Norwegian', # IA5 Norwegian (7-bit) 20127: 'us-ascii', # US-ASCII (7-bit) - # UNSUPPORTED + # UNSUPPORTED. 20261: 'x-cp20261', # T.61 - # UNSUPPORTED + # UNSUPPORTED. 20269: 'x-cp20269', # ISO 6937 Non-Spacing Accent 20273: 'IBM273', # IBM EBCDIC Germany 20277: 'cp277', # IBM EBCDIC Denmark-Norway @@ -437,26 +438,26 @@ 20290: 'cp290', # IBM EBCDIC Japanese Katakana Extended 20297: 'cp297', # IBM EBCDIC France 20420: 'cp420', # IBM EBCDIC Arabic - # UNSUPPORTED + # UNSUPPORTED. 20423: 'IBM423', # IBM EBCDIC Greek 20424: 'IBM424', # IBM EBCDIC Hebrew 20833: 'cp833', # IBM EBCDIC Korean Extended 20838: 'cp838', # IBM EBCDIC Thai 20866: 'koi8-r', # Russian (KOI8-R); Cyrillic (KOI8-R) 20871: 'cp871', # IBM EBCDIC Icelandic - # UNSUPPORTED + # UNSUPPORTED. 20880: 'IBM880', # IBM EBCDIC Cyrillic Russian - # UNSUPPORTED + # UNSUPPORTED. 20905: 'IBM905', # IBM EBCDIC Turkish - # UNSUPPORTED + # UNSUPPORTED. 20924: 'IBM00924', # IBM EBCDIC Latin 1/Open System (1047 + Euro symbol) 20932: 'EUC-JP', # Japanese (JIS 0208-1990 and 0212-1990) - # UNSUPPORTED + # UNSUPPORTED. 20936: 'x-cp20936', # Simplified Chinese (GB2312); Chinese Simplified (GB2312-80) - # UNSUPPORTED + # UNSUPPORTED. 20949: 'x-cp20949', # Korean Wansung 21025: 'cp1025', # IBM EBCDIC Cyrillic Serbian-Bulgarian - # UNSUPPORTED + # UNSUPPORTED. 21027: '', # (deprecated) 21866: 'koi8-u', # Ukrainian (KOI8-U); Cyrillic (KOI8-U) 28591: 'iso-8859-1', # ISO 8859-1 Latin 1; Western European (ISO) @@ -470,58 +471,58 @@ 28599: 'iso-8859-9', # ISO 8859-9 Turkish 28603: 'iso-8859-13', # ISO 8859-13 Estonian 28605: 'iso-8859-15', # ISO 8859-15 Latin 9 - # UNSUPPORTED + # UNSUPPORTED. 29001: 'x-Europa', # Europa 3 - # UNSUPPORTED + # UNSUPPORTED. 38598: 'iso-8859-8-i', # ISO 8859-8 Hebrew; Hebrew (ISO-Logical) 50220: 'iso-2022-jp', # ISO 2022 Japanese with no halfwidth Katakana; Japanese (JIS) 50221: 'csISO2022JP', # ISO 2022 Japanese with halfwidth Katakana; Japanese (JIS-Allow 1 byte Kana) 50222: 'iso-2022-jp', # ISO 2022 Japanese JIS X 0201-1989; Japanese (JIS-Allow 1 byte Kana - SO/SI) 50225: 'iso-2022-kr', # ISO 2022 Korean - # UNSUPPORTED + # UNSUPPORTED. 50227: 'x-cp50227', # ISO 2022 Simplified Chinese; Chinese Simplified (ISO 2022) - # UNSUPPORTED + # UNSUPPORTED. 50229: '', # ISO 2022 Traditional Chinese - # UNSUPPORTED + # UNSUPPORTED. 50930: '', # EBCDIC Japanese (Katakana) Extended - # UNSUPPORTED + # UNSUPPORTED. 50931: '', # EBCDIC US-Canada and Japanese - # UNSUPPORTED + # UNSUPPORTED. 50933: '', # EBCDIC Korean Extended and Korean - # UNSUPPORTED + # UNSUPPORTED. 50935: '', # EBCDIC Simplified Chinese Extended and Simplified Chinese - # UNSUPPORTED + # UNSUPPORTED. 50936: '', # EBCDIC Simplified Chinese - # UNSUPPORTED + # UNSUPPORTED. 50937: '', # EBCDIC US-Canada and Traditional Chinese - # UNSUPPORTED + # UNSUPPORTED. 50939: '', # EBCDIC Japanese (Latin) Extended and Japanese 51932: 'euc-jp', # EUC Japanese 51936: 'EUC-CN', # EUC Simplified Chinese; Chinese Simplified (EUC) 51949: 'euc-kr', # EUC Korean - # UNSUPPORTED + # UNSUPPORTED. 51950: '', # EUC Traditional Chinese 52936: 'hz-gb-2312', # HZ-GB2312 Simplified Chinese; Chinese Simplified (HZ) 54936: 'GB18030', # Windows XP and later: GB18030 Simplified Chinese (4 byte); Chinese Simplified (GB18030) - # UNSUPPORTED + # UNSUPPORTED. 57002: 'x-iscii-de', # ISCII Devanagari - # UNSUPPORTED + # UNSUPPORTED. 57003: 'x-iscii-be', # ISCII Bangla - # UNSUPPORTED + # UNSUPPORTED. 57004: 'x-iscii-ta', # ISCII Tamil - # UNSUPPORTED + # UNSUPPORTED. 57005: 'x-iscii-te', # ISCII Telugu - # UNSUPPORTED + # UNSUPPORTED. 57006: 'x-iscii-as', # ISCII Assamese - # UNSUPPORTED + # UNSUPPORTED. 57007: 'x-iscii-or', # ISCII Odia - # UNSUPPORTED + # UNSUPPORTED. 57008: 'x-iscii-ka', # ISCII Kannada - # UNSUPPORTED + # UNSUPPORTED. 57009: 'x-iscii-ma', # ISCII Malayalam - # UNSUPPORTED + # UNSUPPORTED. 57010: 'x-iscii-gu', # ISCII Gujarati - # UNSUPPORTED + # UNSUPPORTED. 57011: 'x-iscii-pa', # ISCII Punjabi 65000: 'utf-7', # Unicode (UTF-7) 65001: 'utf-8', # Unicode (UTF-8) diff --git a/extract_msg/message.py b/extract_msg/message.py index f5b9483c..5818dd48 100644 --- a/extract_msg/message.py +++ b/extract_msg/message.py @@ -139,9 +139,7 @@ def save(self, **kwargs): # Track if we are skipping attachments. skipAttachments = kwargs.get('skipAttachments', False) - # If we are doing PDF, we immediately need to try to find wkhtmltopdf. if pdf: - wkPath = findWk(kwargs.get('wkPath')) kwargs['preparedHtml'] = True # ZipFile handling. @@ -241,80 +239,53 @@ def save(self, **kwargs): try: if not attachOnly: - # Check whether we should be using HTML or RTF. - fext = 'txt' + # Check what to save the body with. + fext = 'json' if _json else 'txt' useHtml = False + usePdf = False useRtf = False if html: if self.htmlBody: useHtml = True fext = 'html' elif not allowFallback: - raise DataNotFoundError('Could not find the htmlBody') + raise DataNotFoundError('Could not find the htmlBody.') - if rtf or (html and not useHtml): + if pdf: + if self.htmlBody: + usePdf = True + fext = 'pdf' + elif not allowFallback: + raise DataNotFoundError('Count not find the htmlBody to convert to pdf.') + + if rtf or (html and not useHtml) or (pdf and not usePdf): if self.rtfBody: useRtf = True fext = 'rtf' elif not allowFallback: - raise DataNotFoundError('Could not find the rtfBody') + raise DataNotFoundError('Could not find the rtfBody.') if not skipAttachments: # Save the attachments. attachmentNames = [attachment.save(**kwargs) for attachment in self.attachments] if not attachOnly: - # Determine the extension to use for the body. - if pdf: - # First thing is first, we need to parse our wkOptions if - # they exist. - wkOptions = kwargs.get('wkOptions') - if wkOptions: - try: - # Try to convert to a list, whatever it is, and - # fail if it is not possible. - parsedWkOptions = [*wkOptions] - except TypeError: - raise TypeError(':param wkOptions: must be an iterable, not {type(wkOptions)}.') + with _open(str(path / ('message.' + fext)), mode) as f: + if _json: + emailObj = json.loads(self.getJson()) + if not skipAttachments: + emailObj['attachments'] = attachmentNames + + f.write(inputToBytes(json.dumps(emailObj), 'utf-8')) + elif useHtml: + f.write(self.getSaveHtmlBody(**kwargs)) + elif usePdf: + f.write(self.getSavePdfBody(**kwargs)) + elif useRtf: + f.write(self.getSaveRtfBody(**kwargs)) else: - parsedWkOptions = [] - - # Confirm that all of our options we now have are either - # strings or bytes. - if not all(isinstance(option, (str, bytes)) for option in parsedWkOptions): - raise TypeError(':param wkOptions: must be an iterable of strings and bytes.') - - # We call the program to convert the html, but give tell it - # the data will go in and come out through stdin and stdout, - # respectively. This way we don't have to write temporary - # files to the disk. We also ask that it be quiet about it. - process = subprocess.Popen([wkPath, *parsedWkOptions, '-', '-'], shell = True, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE) - # Give the program the data and wait for the program to - # finish. - output = process.communicate(self.getSaveHtmlBody(**kwargs)) - if process.returncode != 0: - raise WKError(output[1].decode('utf-8')) - - with _open(str(path / 'message.pdf'), mode) as f: - f.write(output[0]) - - else: - fext = 'json' if _json else fext - - with _open(str(path / ('message.' + fext)), mode) as f: - if _json: - emailObj = json.loads(self.getJson()) - if not skipAttachments: - emailObj['attachments'] = attachmentNames - - f.write(inputToBytes(json.dumps(emailObj), 'utf-8')) - elif useHtml: - f.write(self.getSaveHtmlBody(**kwargs)) - elif useRtf: - f.write(self.getSaveRtfBody(**kwargs)) - else: - f.write(self.getSaveBody(**kwargs)) + f.write(self.getSaveBody(**kwargs)) except Exception: if not _zip: @@ -401,6 +372,58 @@ def getSaveHtmlBody(self, preparedHtml : bool = False, charset : str = 'utf-8', else: return self.htmlBody + def getSavePdfBody(self, **kwargs) -> bytes: + """ + Returns the PDF body that will be used in saving based on the arguments. + + :param wkPath: Used to manually specify the path of the wkhtmltopdf + executable. If not specified, the function will try to find it. + Useful if wkhtmltopdf is not on the path. If :param pdf: is False, + this argument is ignored. + :param wkOptions: Used to specify additional options to wkhtmltopdf. + this must be a list or list-like object composed of strings and + bytes. + :param kwargs: Used to allow kwargs expansion in the save function. + Arguments absorbed by this are simply ignored. + + :raises ExecutableNotFound: The wkhtmltopdf executable could not be + found. + :raises WKError: Something went wrong in creating the PDF body. + """ + # Immediately try to find the executable. + wkPath = findWk(kwargs.get('wkPath')) + + # First thing is first, we need to parse our wkOptions if + # they exist. + wkOptions = kwargs.get('wkOptions') + if wkOptions: + try: + # Try to convert to a list, whatever it is, and + # fail if it is not possible. + parsedWkOptions = [*wkOptions] + except TypeError: + raise TypeError(':param wkOptions: must be an iterable, not {type(wkOptions)}.') + else: + parsedWkOptions = [] + + # Confirm that all of our options we now have are either + # strings or bytes. + if not all(isinstance(option, (str, bytes)) for option in parsedWkOptions): + raise TypeError(':param wkOptions: must be an iterable of strings and bytes.') + + # We call the program to convert the html, but give tell it + # the data will go in and come out through stdin and stdout, + # respectively. This way we don't have to write temporary + # files to the disk. We also ask that it be quiet about it. + process = subprocess.Popen([wkPath, *parsedWkOptions, '-', '-'], shell = True, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE) + # Give the program the data and wait for the program to + # finish. + output = process.communicate(self.getSaveHtmlBody(**kwargs)) + if process.returncode != 0: + raise WKError(output[1].decode('utf-8')) + + return output[0] + def getSaveRtfBody(self, **kwargs) -> bytes: """ Returns the RTF body that will be used in saving based on the arguments. diff --git a/extract_msg/message_base.py b/extract_msg/message_base.py index f2d6cbe9..cab72646 100644 --- a/extract_msg/message_base.py +++ b/extract_msg/message_base.py @@ -9,6 +9,7 @@ import compressed_rtf import RTFDE +from . import constants from .enums import RecipientType from .msg import MSGFile from .recipient import Recipient @@ -203,14 +204,38 @@ def deencapsulatedRtf(self) -> RTFDE.DeEncapsulator: except AttributeError: if self.rtfBody: # If there is an RTF body, we try to deencapsulate it. + body = self.rtfBody + # Sometimes you get MSG files whose RTF body has stuff + # *after* the body, and RTFDE can't handle that. Here is + # how we compensate. + while body and body[-1] != 125: + body = body[:-1] + try: try: - self._deencapsultor = RTFDE.DeEncapsulator(self.rtfBody) + self._deencapsultor = RTFDE.DeEncapsulator(body) except UnicodeDecodeError: # There is a known issue that bytes are not well decoded # by RTFDE right now, so let's see if we can't manually # decode it and see if that will work. - self._deencapsultor = RTFDE.DeEncapsulator(self.rtfBody.decode(chardet.detect(self.rtfBody)['encoding'])) + # + # There is also the fact that it is decoded *at all* + # before binary data is stripped out. This data should + # almost certainly be stripped out, so let's log it and + # then log if we removed any of them before trying this. + logger.warn(f'RTFDE failed to decode rtfBody for message with subject "{self.subject}". Attempting to cut out unnecessary data and override decoding.') + + match = constants.RE_BIN.search(body) + # Because we are going to be actively removing things, + # we want to search the entire thing over again. + while match: + logger.info(f'Found match to badData starting at location {match.start()}. Replacing with nothing.') + length = int(match.group(1)) + # Extract the entire binary section and replace it. + body = body.replace(body[match.start():match.end() + length], b'', 1) + match = constants.RE_BIN.search(body) + + self._deencapsultor = RTFDE.DeEncapsulator(body.decode(chardet.detect(body)['encoding'])) self._deencapsultor.deencapsulate() except RTFDE.exceptions.NotEncapsulatedRtf as e: logger.debug("RTF body is not encapsulated.") diff --git a/extract_msg/message_signed.py b/extract_msg/message_signed.py index 141d0879..222fd1d2 100644 --- a/extract_msg/message_signed.py +++ b/extract_msg/message_signed.py @@ -98,23 +98,32 @@ def save(self, **kwargs): :param attachmentsOnly: Turns off saving the body and only saves the attachments when set. + :param skipAttachments: Turns off saving attachments. :param charset: If the html is being prepared, the charset to use for the Content-Type meta tag to insert. This exists to ensure that something parsing the html can properly determine the encoding (as not having this tag can cause errors in some programs). Set this to `None` or an empty string to not insert the tag (Default: 'utf-8'). - :param kwargs: Used to allow kwags expansion in the save function. + :param kwargs: Used to allow kwargs expansion in the save function. :param preparedHtml: When set, prepares the HTML body for standalone usage, doing things like adding tags, injecting attachments, etc. This is useful for things like trying to convert the HTML body directly to PDF. + :param pdf: Used to enable saving the body as a PDF file. + :param wkPath: Used to manually specify the path of the wkhtmltopdf + executable. If not specified, the function will try to find it. + Useful if wkhtmltopdf is not on the path. If :param pdf: is False, + this argument is ignored. + :param wkOptions: Used to specify additional options to wkhtmltopdf. + this must be a list or list-like object composed of strings and + bytes. """ - # Move keyword arguments into variables. _json = kwargs.get('json', False) html = kwargs.get('html', False) rtf = kwargs.get('rtf', False) raw = kwargs.get('raw', False) + pdf = kwargs.get('pdf', False) allowFallback = kwargs.get('allowFallback', False) _zip = kwargs.get('zip') maxNameLength = kwargs.get('maxNameLength', 256) @@ -126,6 +135,11 @@ def save(self, **kwargs): # Track if we are only saving the attachments. attachOnly = kwargs.get('attachmentsOnly', False) + # Track if we are skipping attachments. + skipAttachments = kwargs.get('skipAttachments', False) + + if pdf: + kwargs['preparedHtml'] = True # ZipFile handling. if _zip: @@ -155,8 +169,8 @@ def save(self, **kwargs): kwargs['customFilename'] = None # Check if incompatible options have been provided in any way. - if _json + html + rtf + raw + attachOnly > 1: - raise IncompatibleOptionsError('Only one of the following options may be used at a time: json, raw, html, rtf, attachmentsOnly.') + if _json + html + rtf + raw + attachOnly + pdf > 1: + raise IncompatibleOptionsError('Only one of the following options may be used at a time: json, raw, html, rtf, attachmentsOnly, pdf.') # TODO: insert code here that will handle checking all of the msg files to see if the path with overflow. @@ -224,40 +238,49 @@ def save(self, **kwargs): try: if not attachOnly: - # Check whether we should be using HTML or RTF. - fext = 'txt' + # Check what to save the body with. + fext = 'json' if _json else 'txt' useHtml = False + usePdf = False useRtf = False if html: if self.htmlBody: useHtml = True fext = 'html' elif not allowFallback: - raise DataNotFoundError('Could not find the htmlBody') + raise DataNotFoundError('Could not find the htmlBody.') + + if pdf: + if self.htmlBody: + usePdf = True + fext = 'pdf' + elif not allowFallback: + raise DataNotFoundError('Count not find the htmlBody to convert to pdf.') - if rtf or (html and not useHtml): + if rtf or (html and not useHtml) or (pdf and not usePdf): if self.rtfBody: useRtf = True fext = 'rtf' elif not allowFallback: - raise DataNotFoundError('Could not find the rtfBody') + raise DataNotFoundError('Could not find the rtfBody.') - # Save the attachments. - attachmentNames = [attachment.save(**kwargs) for attachment in self.attachments] + if not skipAttachments: + # Save the attachments. + attachmentNames = [attachment.save(**kwargs) for attachment in self.attachments] if not attachOnly: - # Determine the extension to use for the body. - fext = 'json' if _json else fext - with _open(str(path / ('message.' + fext)), mode) as f: if _json: emailObj = json.loads(self.getJson()) - emailObj['attachments'] = attachmentNames + if not skipAttachments: + emailObj['attachments'] = attachmentNames f.write(inputToBytes(json.dumps(emailObj), 'utf-8')) elif useHtml: f.write(self.getSaveHtmlBody(**kwargs)) + elif usePdf: + f.write(self.getSavePdfBody(**kwargs)) elif useRtf: f.write(self.getSaveRtfBody(**kwargs)) else: @@ -348,6 +371,58 @@ def getSaveHtmlBody(self, preparedHtml : bool = False, charset : str = 'utf-8', else: return self.htmlBody + def getSavePdfBody(self, **kwargs) -> bytes: + """ + Returns the PDF body that will be used in saving based on the arguments. + + :param wkPath: Used to manually specify the path of the wkhtmltopdf + executable. If not specified, the function will try to find it. + Useful if wkhtmltopdf is not on the path. If :param pdf: is False, + this argument is ignored. + :param wkOptions: Used to specify additional options to wkhtmltopdf. + this must be a list or list-like object composed of strings and + bytes. + :param kwargs: Used to allow kwargs expansion in the save function. + Arguments absorbed by this are simply ignored. + + :raises ExecutableNotFound: The wkhtmltopdf executable could not be + found. + :raises WKError: Something went wrong in creating the PDF body. + """ + # Immediately try to find the executable. + wkPath = findWk(kwargs.get('wkPath')) + + # First thing is first, we need to parse our wkOptions if + # they exist. + wkOptions = kwargs.get('wkOptions') + if wkOptions: + try: + # Try to convert to a list, whatever it is, and + # fail if it is not possible. + parsedWkOptions = [*wkOptions] + except TypeError: + raise TypeError(':param wkOptions: must be an iterable, not {type(wkOptions)}.') + else: + parsedWkOptions = [] + + # Confirm that all of our options we now have are either + # strings or bytes. + if not all(isinstance(option, (str, bytes)) for option in parsedWkOptions): + raise TypeError(':param wkOptions: must be an iterable of strings and bytes.') + + # We call the program to convert the html, but give tell it + # the data will go in and come out through stdin and stdout, + # respectively. This way we don't have to write temporary + # files to the disk. We also ask that it be quiet about it. + process = subprocess.Popen([wkPath, *parsedWkOptions, '-', '-'], shell = True, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE) + # Give the program the data and wait for the program to + # finish. + output = process.communicate(self.getSaveHtmlBody(**kwargs)) + if process.returncode != 0: + raise WKError(output[1].decode('utf-8')) + + return output[0] + def getSaveRtfBody(self, **kwargs) -> bytes: """ Returns the RTF body that will be used in saving based on the arguments. diff --git a/extract_msg/utils.py b/extract_msg/utils.py index b4837a2b..3d62050d 100644 --- a/extract_msg/utils.py +++ b/extract_msg/utils.py @@ -396,8 +396,12 @@ def injectHtmlHeader(msgFile, prepared : bool = False) -> bytes: # insert it. if correctedHtml.find('head'): correctedHtml.find('head').insert_after(bodyTag) - else: + elif correctedHtml.find('footer'): correctedHtml.find('footer').insert_before(bodyTag) + else: + # Neither a head or a body are present, so just append it to + # the main tag. + htmlTag.append(bodyTag) else: # If there is no , ,