From 5cef17fca0b438dff20e4a98786a8cb4f64c5afa Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 12 Jan 2022 22:54:40 +0000 Subject: [PATCH 1/4] update folder layout Signed-off-by: Wenqi Li --- docs/source/data.rst | 6 +++ monai/data/__init__.py | 1 + monai/data/folder_layout.py | 99 +++++++++++++++++++++++++++++++++++++ tests/test_folder_layout.py | 75 ++++++++++++++++++++++++++++ 4 files changed, 181 insertions(+) create mode 100644 monai/data/folder_layout.py create mode 100644 tests/test_folder_layout.py diff --git a/docs/source/data.rst b/docs/source/data.rst index 7f95354e8c..1e6b535b12 100644 --- a/docs/source/data.rst +++ b/docs/source/data.rst @@ -178,6 +178,12 @@ Synthetic :members: +Ouput folder layout +------------------- +.. automodule:: monai.data.folder_layout + :members: + + Utilities --------- .. automodule:: monai.data.utils diff --git a/monai/data/__init__.py b/monai/data/__init__.py index ae0bb86c35..bd49f40273 100644 --- a/monai/data/__init__.py +++ b/monai/data/__init__.py @@ -31,6 +31,7 @@ load_decathlon_datalist, load_decathlon_properties, ) +from .folder_layout import FolderLayout from .grid_dataset import GridPatchDataset, PatchDataset, PatchIter from .image_dataset import ImageDataset from .image_reader import ImageReader, ITKReader, NibabelReader, NumpyReader, PILReader, WSIReader diff --git a/monai/data/folder_layout.py b/monai/data/folder_layout.py new file mode 100644 index 0000000000..730c80813a --- /dev/null +++ b/monai/data/folder_layout.py @@ -0,0 +1,99 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from monai.config import PathLike +from monai.data.utils import create_file_basename + +__all__ = ["FolderLayout"] + + +class FolderLayout: + """ + A utility class to create organized filenames within ``output_dir``. The + ``filename`` method could be used to create a filename following the folder structure. + + Example: + + .. code-block:: python + + from monai.data import FolderLayout + + layout = FolderLayout( + output_dir="/test_run_1/", + postfix="seg", + extension=".nii", + makedirs=False) + layout.filename(subject="Sub-A", idx="00", modality="T1") + # return value: "/test_run_1/Sub-A_seg_00_modality-T1.nii" + + The output filename is a string starting with a ``subject`` ID, and + includes additional information about a customized index and image + modality. This utility class doesn't alter the underlying image data, but + provides a convenient way to create filenames. + """ + + def __init__( + self, + output_dir: PathLike, + postfix: str = "", + extension: str = "", + parent: bool = False, + makedirs: bool = False, + data_root_dir: PathLike = "", + ): + """ + Args: + output_dir: output directory. + postfix: a postfix string for output file name appended to ``subject``. + extension: output file extension to be appended to the end of an output filename. + parent: whether to add a level of parent folder to contain each image to the output filename. + makedirs: whether to create the output parent directories if they do not exist. + data_root_dir: an optional `PathLike` object to preserve the folder structure of the input `subject`. + Please see :py:func:`monai.data.utils.create_file_basename` for more details. + """ + self.output_dir = output_dir + self.postfix = postfix + self.ext = extension + self.parent = parent + self.makedirs = makedirs + self.data_root_dir = data_root_dir + + def filename(self, subject="subject", idx=None, **kwargs): + """ + Create a filename based on the input ``subject`` and ``idx``. + + The output filename is formed as: + + ``output_dir/[subject/]subject[_postfix][_idx][_key-value][ext]`` + + Args: + subject: subject name, used as the primary id of the output filename. + When a `PathLike` object is provided, the base filename will be used as the subject name, + the extension name of `subject` will be ignored, in favor of ``extension`` + from this class's constructor. + idx: additional index name of the image. + kwargs: additional keyword arguments to be used to form the output filename. + The key-value pairs will be appended to the output filename. + """ + full_name = create_file_basename( + postfix=self.postfix, + input_file_name=subject, + folder_path=self.output_dir, + data_root_dir=self.data_root_dir, + separate_folder=self.parent, + patch_index=idx, + makedirs=self.makedirs, + ) + for k, v in kwargs.items(): + full_name += f"_{k}-{v}" + if self.ext is not None: + full_name += f"{self.ext}" + return full_name diff --git a/tests/test_folder_layout.py b/tests/test_folder_layout.py new file mode 100644 index 0000000000..f7291933a3 --- /dev/null +++ b/tests/test_folder_layout.py @@ -0,0 +1,75 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import tempfile +import unittest +from pathlib import Path + +from parameterized import parameterized + +from monai.data.folder_layout import FolderLayout + +TEST_CASES = [ + ({"output_dir": ""}, {}, "subject"), + ({"output_dir": Path(".")}, {}, "subject"), + ({"output_dir": Path(".")}, {"idx": 1}, "subject_1"), + (dict(output_dir=Path("/test_run_1"), extension=".seg", makedirs=False), {}, "/test_run_1/subject.seg"), + (dict(output_dir=Path("/test_run_1"), extension=None, makedirs=False), {}, "/test_run_1/subject"), + ( + dict(output_dir=Path("/test_run_1"), postfix="seg", extension=".test", makedirs=False), + {}, # using the default subject name + "/test_run_1/subject_seg.test", + ), + ( + dict(output_dir=Path("/test_run_1"), postfix="seg", extension=".test", makedirs=False), + {"subject": "test.abc"}, + "/test_run_1/test_seg.test", # subject's extension is ignored + ), + ( + dict(output_dir=Path("/test_run_1/dest/test1/"), data_root_dir="/test_run", makedirs=False), + {"subject": "/test_run/source/test.abc"}, + "/test_run_1/dest/test1/source/test", # preserves the structure from `subject` + ), + ( + dict(output_dir=Path("/test_run_1/dest/test1/"), makedirs=False), + {"subject": "/test_run/source/test.abc"}, + "/test_run_1/dest/test1/test", # data_root_dir used + ), + ( + dict(output_dir=Path("/test_run_1/dest/test1/"), makedirs=False), + {"subject": "/test_run/source/test.abc", "key": "value"}, + "/test_run_1/dest/test1/test_key-value", # data_root_dir used + ), + ( + dict(output_dir=Path("/test_run_1/"), postfix="seg", extension=".nii", makedirs=False), + dict(subject=Path("Sub-A"), idx="00", modality="T1"), + "/test_run_1/Sub-A_seg_00_modality-T1.nii", # test the code example + ), +] + + +class TestFolderLayout(unittest.TestCase): + @parameterized.expand(TEST_CASES) + def test_value(self, con_params, f_params, expected): + fname = FolderLayout(**con_params).filename(**f_params) + self.assertEqual(Path(fname), Path(expected)) + + def test_mkdir(self): + """mkdir=True should create the directory if it does not exist.""" + with tempfile.TemporaryDirectory() as tempdir: + output_tmp = os.path.join(tempdir, "output") + FolderLayout(output_tmp, makedirs=True).filename("subject_test", "001") + self.assertTrue(os.path.exists(os.path.join(output_tmp))) + + +if __name__ == "__main__": + unittest.main() From c2eaebe1910d34b1ab16e13f4abe2af36daa61a5 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Mon, 17 Jan 2022 12:51:46 +0000 Subject: [PATCH 2/4] revise create_file_basename util Signed-off-by: Wenqi Li --- monai/data/utils.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/monai/data/utils.py b/monai/data/utils.py index 0cd3c1594a..d983a67ecd 100644 --- a/monai/data/utils.py +++ b/monai/data/utils.py @@ -682,17 +682,24 @@ def create_file_basename( """ Utility function to create the path to the output file based on the input filename (file name extension is not added by this function). - When `data_root_dir` is not specified, the output file name is: + When ``data_root_dir`` is not specified, the output file name is: `folder_path/input_file_name (no ext.) /input_file_name (no ext.)[_postfix][_patch_index]` - otherwise the relative path with respect to `data_root_dir` will be inserted, for example: - input_file_name: /foo/bar/test1/image.png, - postfix: seg - folder_path: /output, - data_root_dir: /foo/bar, - output will be: /output/test1/image/image_seg + otherwise the relative path with respect to ``data_root_dir`` will be inserted, for example: + .. code-block:: python + + from monai.data import create_file_basename + create_file_basename( + postfix="seg", + input_file_name="/foo/bar/test1/image.png", + folder_path="/output", + data_root_dir="/foo/bar", + separate_folder=True, + makedirs=False) + # output: /output/test1/image/image_seg + Args: postfix: output name's postfix input_file_name: path to the input image file. @@ -730,7 +737,7 @@ def create_file_basename( os.makedirs(output, exist_ok=True) # add the sub-folder plus the postfix name to become the file basename in the output path - output = os.path.join(output, (filename + "_" + postfix) if len(postfix) > 0 else filename) + output = os.path.join(output, filename + "_" + postfix if postfix != "" else filename) if patch_index is not None: output += f"_{patch_index}" From 561c09424bd2b99db810705a6bbdc6788c582af2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 17 Jan 2022 12:56:06 +0000 Subject: [PATCH 3/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- monai/data/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/data/utils.py b/monai/data/utils.py index d983a67ecd..85088a07dd 100644 --- a/monai/data/utils.py +++ b/monai/data/utils.py @@ -699,7 +699,7 @@ def create_file_basename( separate_folder=True, makedirs=False) # output: /output/test1/image/image_seg - + Args: postfix: output name's postfix input_file_name: path to the input image file. From feed9311ddba1c10deaec243374847e33dab0112 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Tue, 18 Jan 2022 19:18:59 +0000 Subject: [PATCH 4/4] update based on comments Signed-off-by: Wenqi Li --- monai/data/folder_layout.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monai/data/folder_layout.py b/monai/data/folder_layout.py index 730c80813a..d8ce162c27 100644 --- a/monai/data/folder_layout.py +++ b/monai/data/folder_layout.py @@ -66,7 +66,7 @@ def __init__( self.makedirs = makedirs self.data_root_dir = data_root_dir - def filename(self, subject="subject", idx=None, **kwargs): + def filename(self, subject: PathLike = "subject", idx=None, **kwargs): """ Create a filename based on the input ``subject`` and ``idx``. @@ -81,7 +81,7 @@ def filename(self, subject="subject", idx=None, **kwargs): from this class's constructor. idx: additional index name of the image. kwargs: additional keyword arguments to be used to form the output filename. - The key-value pairs will be appended to the output filename. + The key-value pairs will be appended to the output filename as ``f"_{k}-{v}"``. """ full_name = create_file_basename( postfix=self.postfix,