fix(build): use custom .spec file to properly import modules

Author: StefanoStone

.gitignore

@@ -4,5 +4,4 @@ __pycache__/
 requirements.txt
 .DS_Store
 build/
-dist/
-main.spec
+dist/

Justfile

@@ -31,8 +31,8 @@ set-up-project: install-python create-virtual-environment sync-dependencies
 sync-dependencies:
     uv sync --all-packages --all-groups --all-extras
 
-build-package:
-    uv run pyinstaller src/lwPython/main.py
+build-package: clean-generated-files
+    uv run pyinstaller main.spec
 
 run-tests:
     uv run pytest -vv

main.spec

@@ -0,0 +1,104 @@
+# -*- mode: python ; coding: utf-8 -*-
+
+from pathlib import Path
+
+from PyInstaller import compat
+from PyInstaller.utils.hooks import collect_data_files, copy_metadata
+
+
+Path('build/main').mkdir(parents=True, exist_ok=True)
+Path('dist').mkdir(parents=True, exist_ok=True)
+
+
+hiddenimports = [
+    'docling.backend.docling_parse_v2_backend',
+    'docling.backend.docling_parse_v4_backend',
+    'docling.backend.pypdfium2_backend',
+    'docling.models.plugins',
+    'docling.models.plugins.defaults',
+    'markitdown.__main__',
+    'textractor.cli',
+    'textractor.cli.cli',
+]
+
+datas = (
+    collect_data_files('docling')
+    + collect_data_files('docling_parse')
+    + collect_data_files('magika')
+    + copy_metadata('docling')
+    + copy_metadata('docling-core')
+    + copy_metadata('docling-ibm-models')
+    + copy_metadata('docling-parse')
+    + copy_metadata('markitdown')
+)
+
+excludes = [
+    'pytest',
+    '_pytest',
+    'tensorflow',
+    'tensorboard',
+    'torch.utils.tensorboard',
+    'torchaudio',
+    'speech_recognition',
+]
+
+if compat.is_linux:
+    excludes += [
+        'nvidia',
+        'nvidia.cublas',
+        'nvidia.cuda_cupti',
+        'nvidia.cuda_nvrtc',
+        'nvidia.cuda_runtime',
+        'nvidia.cudnn',
+        'nvidia.cufft',
+        'nvidia.curand',
+        'nvidia.cusolver',
+        'nvidia.cusparse',
+        'nvidia.cusparselt',
+        'nvidia.nccl',
+        'nvidia.nvjitlink',
+        'nvidia.nvtx',
+    ]
+
+
+a = Analysis(
+    ['src/lwPython/main.py'],
+    pathex=['src'],
+    binaries=[],
+    datas=datas,
+    hiddenimports=hiddenimports,
+    hookspath=[],
+    hooksconfig={},
+    runtime_hooks=[],
+    excludes=excludes,
+    noarchive=False,
+    optimize=0,
+)
+pyz = PYZ(a.pure)
+
+exe = EXE(
+    pyz,
+    a.scripts,
+    [],
+    exclude_binaries=True,
+    name='main',
+    debug=False,
+    bootloader_ignore_signals=False,
+    strip=True,
+    upx=True,
+    console=True,
+    disable_windowed_traceback=False,
+    argv_emulation=False,
+    target_arch=None,
+    codesign_identity=None,
+    entitlements_file=None,
+)
+coll = COLLECT(
+    exe,
+    a.binaries,
+    a.datas,
+    strip=True,
+    upx=True,
+    upx_exclude=[],
+    name='main',
+)

scripts/clean.py

@@ -9,9 +9,6 @@
 from shutil import rmtree
 from from_root import from_root
 
-file = from_root("main.spec")
-file.unlink(missing_ok=True)
-
 for name in ("dist", "build"):
     directory = from_root(name)
     rmtree(directory, ignore_errors=True)

src/lwPython/main.py

@@ -1,5 +1,6 @@
 import click
 import json
+import multiprocessing
 from lwPython._utils import *
 
 @click.group()
@@ -45,4 +46,5 @@ def markitdown_file_to_markdown(input, output):
 cli.add_command(textract_to_markdown)
 
 if __name__ == '__main__':
+    multiprocessing.freeze_support()
     cli()