Skip to content

fixes pickle mod #3416

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Nic-Ma merged 3 commits into from
Nov 30, 2021
Merged

fixes pickle mod #3416

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
35afe1a
fixes pickle mod
wyli Nov 29, 2021
b830c03
update pickle
wyli Nov 29, 2021
ed3f6b3
docstring
wyli Nov 29, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 11 additions & 9 deletions monai/data/dataset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@
from torch.utils.data import Dataset as _TorchDataset
from torch.utils.data import Subset

from monai.data.utils import convert_tables_to_dicts, pickle_hashing
from monai.data.utils import SUPPORTED_PICKLE_MOD, convert_tables_to_dicts, pickle_hashing
from monai.transforms import Compose, Randomizable, ThreadUnsafe, Transform, apply_transform
from monai.utils import MAX_SEED, ensure_tuple, get_seed, min_version, optional_import
from monai.utils import MAX_SEED, ensure_tuple, get_seed, look_up_option, min_version, optional_import
from monai.utils.misc import first

if TYPE_CHECKING:
Expand Down Expand Up @@ -152,7 +152,7 @@ def __init__(
transform: Union[Sequence[Callable], Callable],
cache_dir: Optional[Union[Path, str]],
hash_func: Callable[..., bytes] = pickle_hashing,
pickle_module=pickle,
pickle_module: str = "pickle",
pickle_protocol=pickle.DEFAULT_PROTOCOL,
) -> None:
"""
Expand All @@ -169,9 +169,10 @@ def __init__(
If `cache_dir` is `None`, there is effectively no caching.
hash_func: a callable to compute hash from data items to be cached.
defaults to `monai.data.utils.pickle_hashing`.
pickle_module: module used for pickling metadata and objects, default to `pickle`.
pickle_module: string representing the module used for pickling metadata and objects, default to `"pickle"`.
this arg is used by `torch.save`, for more details, please check:
https://pytorch.org/docs/stable/generated/torch.save.html#torch.save.
https://pytorch.org/docs/stable/generated/torch.save.html#torch.save,
and ``monai.data.utils.SUPPORTED_PICKLE_MOD``.
pickle_protocol: can be specified to override the default protocol, default to `2`.
this arg is used by `torch.save`, for more details, please check:
https://pytorch.org/docs/stable/generated/torch.save.html#torch.save.
Expand Down Expand Up @@ -287,7 +288,7 @@ def _cachecheck(self, item_transformed):
torch.save(
obj=_item_transformed,
f=temp_hash_file,
pickle_module=self.pickle_module,
pickle_module=look_up_option(self.pickle_module, SUPPORTED_PICKLE_MOD),
pickle_protocol=self.pickle_protocol,
)
if temp_hash_file.is_file() and not hashfile.is_file():
Expand Down Expand Up @@ -317,7 +318,7 @@ def __init__(
cache_n_trans: int,
cache_dir: Optional[Union[Path, str]],
hash_func: Callable[..., bytes] = pickle_hashing,
pickle_module=pickle,
pickle_module: str = "pickle",
pickle_protocol=pickle.DEFAULT_PROTOCOL,
) -> None:
"""
Expand All @@ -335,9 +336,10 @@ def __init__(
If `cache_dir` is `None`, there is effectively no caching.
hash_func: a callable to compute hash from data items to be cached.
defaults to `monai.data.utils.pickle_hashing`.
pickle_module: module used for pickling metadata and objects, default to `pickle`.
pickle_module: string representing the module used for pickling metadata and objects, default to `"pickle"`.
this arg is used by `torch.save`, for more details, please check:
https://pytorch.org/docs/stable/generated/torch.save.html#torch.save.
https://pytorch.org/docs/stable/generated/torch.save.html#torch.save,
and ``monai.data.utils.SUPPORTED_PICKLE_MOD``.
pickle_protocol: can be specified to override the default protocol, default to `2`.
this arg is used by `torch.save`, for more details, please check:
https://pytorch.org/docs/stable/generated/torch.save.html#torch.save.
Expand Down
4 changes: 4 additions & 0 deletions monai/data/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,12 @@
"pad_list_data_collate",
"no_collation",
"convert_tables_to_dicts",
"SUPPORTED_PICKLE_MOD",
]

# module to be used by `torch.save`
SUPPORTED_PICKLE_MOD = {"pickle": pickle}
Comment thread
Nic-Ma marked this conversation as resolved.


def get_random_patch(
dims: Sequence[int], patch_size: Sequence[int], rand_state: Optional[np.random.RandomState] = None
Expand Down
2 changes: 1 addition & 1 deletion tests/test_persistentdataset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ def test_cache(self):
data=items,
transform=_InplaceXform(),
cache_dir=tempdir,
pickle_module=pickle,
pickle_module="pickle",
pickle_protocol=pickle.HIGHEST_PROTOCOL,
)
self.assertEqual(items, [[[]], [[0]], [[0, 1]], [[0, 1, 2]], [[0, 1, 2, 3]]])
Expand Down