From 31d19b96358f082919ad2042d7d05dfe5b9899cd Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Fri, 30 May 2025 16:41:01 +0200 Subject: [PATCH 01/40] use method in apply instead of duplicating code --- rsa/model_rdm.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rsa/model_rdm.py b/rsa/model_rdm.py index 4050dbe..456131e 100755 --- a/rsa/model_rdm.py +++ b/rsa/model_rdm.py @@ -41,8 +41,7 @@ def dissimilarity(self, fp_row, fp_col, idx): def apply(self, processes=1, chunksize=10, do_disable_tqdm=False): triu_rows, triu_cols = self.get_triu_rows_cols() - if self.model_rdm_triu is None: - self.model_rdm_triu = np.zeros((triu_rows.size,)) + ENTRY_EMPTY + self._init_model_rdm_triu() with mp.get_context("spawn").Pool(processes=processes) as pool: result = pool.starmap(self.dissimilarity, From ae3bdc957b853b51d492a7f6617349c9e3d40676 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Tue, 3 Jun 2025 15:28:00 +0200 Subject: [PATCH 02/40] classes to handle file pairs --- rsa/fpairs/__init__.py | 0 rsa/fpairs/file_pairs.py | 68 ++++++++++++ rsa/fpairs/file_pairs_cross_list.py | 26 +++++ rsa/fpairs/file_pairs_from_list.py | 27 +++++ rsa/fpairs/test_file_pairs.py | 57 ++++++++++ rsa/fpairs/test_file_pairs_cross_list.py | 133 +++++++++++++++++++++++ rsa/fpairs/test_file_pairs_from_list.py | 25 +++++ 7 files changed, 336 insertions(+) create mode 100644 rsa/fpairs/__init__.py create mode 100644 rsa/fpairs/file_pairs.py create mode 100644 rsa/fpairs/file_pairs_cross_list.py create mode 100644 rsa/fpairs/file_pairs_from_list.py create mode 100644 rsa/fpairs/test_file_pairs.py create mode 100644 rsa/fpairs/test_file_pairs_cross_list.py create mode 100644 rsa/fpairs/test_file_pairs_from_list.py diff --git a/rsa/fpairs/__init__.py b/rsa/fpairs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rsa/fpairs/file_pairs.py b/rsa/fpairs/file_pairs.py new file mode 100644 index 0000000..78c699f --- /dev/null +++ b/rsa/fpairs/file_pairs.py @@ -0,0 +1,68 @@ +import os +from itertools import product + + +class FilePairs: + + def __init__(self, fpairs): + self.pairs = None + self.flist = None + self.tuple_indices = None + self._validate_pairs(fpairs) + + if fpairs is not None and len(fpairs) > 0: + self.pairs = fpairs + self.generate_unique_strings_and_indices() + + @staticmethod + def _validate_pairs(pairs): + """ + Validates the pairs attribute. + + Args: + pairs (list): A list of tuples containing strings. + + Raises: + TypeError: If pairs is not a list. + ValueError: If pairs contains invalid elements. + """ + if not isinstance(pairs, list): + raise TypeError("pairs must be a list.") + for item in pairs: + if not isinstance(item, tuple) or not all(isinstance(sub_item, str) for sub_item in item): + raise ValueError("pairs must contain only tuples of strings.") + + def generate_unique_strings_and_indices(self): + if not self.pairs: + raise ValueError("fpath_list must be a list of tuples of strings to generate indices.") + self.flist = sorted(list(set(s for tup in self.pairs for s in tup))) + + self.tuple_indices = [tuple(self.flist.index(s) for s in tup) for tup in self.pairs] + + def save_to_files(self, save_dir, flist_path='flist.txt', indices_path='indices.txt'): + flist_full_path = os.path.join(save_dir, flist_path) + indices_full_path = os.path.join(save_dir, indices_path) + + with open(flist_full_path, 'w') as f: + f.write('\n'.join(self.flist)) + + with open(indices_full_path, 'w') as f: + for indices in self.tuple_indices: + f.write(','.join(map(str, indices)) + '\n') + + def load(self, load_dir, flist_path='flist.txt', indices_path='indices.txt'): + + flist_full_path = os.path.join(load_dir, flist_path) + with open(flist_full_path, 'r') as f: + self.flist = f.read().splitlines() + + indices_full_path = os.path.join(load_dir, indices_path) + if os.path.isfile(indices_full_path): + with open(indices_full_path, 'r') as f: + self.tuple_indices = [tuple(map(int, line.split(','))) for line in f] + + else: + numels = len(self.flist) + self.tuple_indices = sorted(list(product(range(numels), repeat=2))) + + self.pairs = [tuple(self.flist[i] for i in indices) for indices in self.tuple_indices] diff --git a/rsa/fpairs/file_pairs_cross_list.py b/rsa/fpairs/file_pairs_cross_list.py new file mode 100644 index 0000000..56d12d8 --- /dev/null +++ b/rsa/fpairs/file_pairs_cross_list.py @@ -0,0 +1,26 @@ +# Python +from itertools import product +from rsa.fpairs.file_pairs import FilePairs + + +class FilePairsCrossList(FilePairs): + def __init__(self, flist1, flist2): + """ + Initializes the CrossFilePairs class with pairs constructed + between two lists of files, excluding pairs from the same list. + + Args: + list1 (list): First list of file names. + list2 (list): Second list of file names. + """ + if not isinstance(flist1, list) or not isinstance(flist2, list): + raise TypeError("Both inputs must be lists.") + + if not all(isinstance(item, str) for item in flist1 + flist2): + raise ValueError("Both lists must contain only strings.") + + # Generate pairs between the two lists + fpairs = list(product(flist1, flist2)) + fpairs = sorted([(a, b) for a, b in fpairs if a != b]) # remove duplicates + print(fpairs) + super().__init__(fpairs) diff --git a/rsa/fpairs/file_pairs_from_list.py b/rsa/fpairs/file_pairs_from_list.py new file mode 100644 index 0000000..a0e608e --- /dev/null +++ b/rsa/fpairs/file_pairs_from_list.py @@ -0,0 +1,27 @@ +from itertools import product +import numpy as np +from rsa.fpairs.file_pairs import FilePairs + + +class FilePairsFromList(FilePairs): + def __init__(self, flist): + """ + Initializes the CrossFilePairs class with pairs constructed + between two lists of files, excluding pairs from the same list. + + Args: + list1 (list): First list of file names. + list2 (list): Second list of file names. + """ + if not isinstance(flist, list): + raise TypeError("Input must be a list.") + + if not all(isinstance(item, str) for item in flist): + raise ValueError("List must contain only strings.") + + # Generate pairs between the two lists + num_rows = len(flist) + triu_rows, triu_cols = np.triu_indices(num_rows, k=1) + fpairs = sorted([(flist[r], flist[c]) for r, c in zip(triu_rows, triu_cols)]) + # print('fpairs', fpairs) + super().__init__(fpairs) diff --git a/rsa/fpairs/test_file_pairs.py b/rsa/fpairs/test_file_pairs.py new file mode 100644 index 0000000..f0e6bcf --- /dev/null +++ b/rsa/fpairs/test_file_pairs.py @@ -0,0 +1,57 @@ +from nose.tools import assert_equal, assert_raises, assert_true +import os +import tempfile +import shutil +from rsa.fpairs.file_pairs import FilePairs + + +class TestFilePairs: + + @classmethod + def setup_class(cls): + cls.test_dir = tempfile.mkdtemp() + + @classmethod + def teardown_class(cls): + shutil.rmtree(cls.test_dir) + + def setup(self): + self.fpairs = [("file1", "file2"), ("file3", "file4"), ("file1", "file3")] + self.file_pairs = FilePairs(self.fpairs) + + def test_initialization(self): + assert_equal(self.file_pairs.pairs, self.fpairs) + assert_equal(self.file_pairs.flist, ["file1", "file2", "file3", "file4"]) + assert_equal(self.file_pairs.tuple_indices, [(0, 1), (2, 3), (0, 2)]) + + def test_validate_pairs(self): + with assert_raises(TypeError): + FilePairs._validate_pairs("invalid") + with assert_raises(ValueError): + FilePairs._validate_pairs([("file1", 123)]) + + def test_save_to_files(self): + self.file_pairs.save_to_files(self.test_dir) + flist_path = os.path.join(self.test_dir, "flist.txt") + indices_path = os.path.join(self.test_dir, "indices.txt") + + assert_true(os.path.exists(flist_path)) + assert_true(os.path.exists(indices_path)) + + with open(flist_path, "r") as f: + assert_equal(f.read().splitlines(), self.file_pairs.flist) + + with open(indices_path, "r") as f: + assert_equal( + [tuple(map(int, line.split(","))) for line in f], + self.file_pairs.tuple_indices, + ) + + def test_load(self): + self.file_pairs.save_to_files(self.test_dir) + new_file_pairs = FilePairs([]) + new_file_pairs.load(self.test_dir) + + assert_equal(new_file_pairs.flist, self.file_pairs.flist) + assert_equal(new_file_pairs.tuple_indices, self.file_pairs.tuple_indices) + diff --git a/rsa/fpairs/test_file_pairs_cross_list.py b/rsa/fpairs/test_file_pairs_cross_list.py new file mode 100644 index 0000000..91aa8bc --- /dev/null +++ b/rsa/fpairs/test_file_pairs_cross_list.py @@ -0,0 +1,133 @@ +from nose.tools import assert_equal, assert_raises, assert_true +import os +import tempfile +import shutil +from rsa.fpairs.file_pairs_cross_list import FilePairsCrossList +from rsa.fpairs.test_file_pairs import TestFilePairs + + +class TestFilePairsCrossList(TestFilePairs): + + def setup(self): + self.flist1 = ["file1", "file2", "file3"] + self.flist2 = ["file4", "file5", "file6"] + self.fpairs = [("file1", "file4"), + ("file1", "file5"), + ("file1", "file6"), + ("file2", "file4"), + ("file2", "file5"), + ("file2", "file6"), + ("file3", "file4"), + ("file3", "file5"), + ("file3", "file6")] + self.file_pairs = FilePairsCrossList(self.flist1, self.flist2) + + def test_initialization(self): + assert_equal(self.file_pairs.pairs, self.fpairs) + + flist = sorted(self.flist1 + self.flist2) + + assert_equal(self.file_pairs.flist, flist) + assert_equal(self.file_pairs.tuple_indices, [(0, 3), + (0, 4), + (0, 5), + (1, 3), + (1, 4), + (1, 5), + (2, 3), + (2, 4), + (2, 5)]) + + def test_validate_pairs(self): + with assert_raises(TypeError): + FilePairsCrossList._validate_pairs("invalid") + with assert_raises(ValueError): + FilePairsCrossList._validate_pairs([("file1", 123)]) + + def test_save_to_files(self): + self.file_pairs.save_to_files(self.test_dir) + flist_path = os.path.join(self.test_dir, "flist.txt") + indices_path = os.path.join(self.test_dir, "indices.txt") + + assert_true(os.path.exists(flist_path)) + assert_true(os.path.exists(indices_path)) + + with open(flist_path, "r") as f: + assert_equal(f.read().splitlines(), self.file_pairs.flist) + + with open(indices_path, "r") as f: + assert_equal( + [tuple(map(int, line.split(","))) for line in f], + self.file_pairs.tuple_indices, + ) + + def test_load(self): + self.file_pairs.save_to_files(self.test_dir) + new_file_pairs = FilePairsCrossList([], []) + new_file_pairs.load(self.test_dir) + + assert_equal(new_file_pairs.flist, self.file_pairs.flist) + assert_equal(new_file_pairs.tuple_indices, self.file_pairs.tuple_indices) + + +class TestFilePairsCrossListWithDuplicates(TestFilePairs): + + def setup(self): + self.flist1 = ["file1", "file2", "file3"] + self.flist2 = ["file1", "file4", "file5"] + self.fpairs = [("file1", "file4"), + ("file1", "file5"), + ("file2", "file1"), + ("file2", "file4"), + ("file2", "file5"), + ("file3", "file1"), + ("file3", "file4"), + ("file3", "file5")] + self.file_pairs = FilePairsCrossList(self.flist1, self.flist2) + + def test_initialization(self): + assert_equal(self.file_pairs.pairs, self.fpairs) + + flist = self.flist1 + self.flist2 + flist = sorted(set(flist)) # Remove duplicates + + assert_equal(self.file_pairs.flist, flist) + assert_equal(self.file_pairs.tuple_indices, [(0, 3), + (0, 4), + (1, 0), + (1, 3), + (1, 4), + (2, 0), + (2, 3), + (2, 4)]) + + def test_validate_pairs(self): + with assert_raises(TypeError): + FilePairsCrossList._validate_pairs("invalid") + with assert_raises(ValueError): + FilePairsCrossList._validate_pairs([("file1", 123)]) + + def test_save_to_files(self): + self.file_pairs.save_to_files(self.test_dir) + flist_path = os.path.join(self.test_dir, "flist.txt") + indices_path = os.path.join(self.test_dir, "indices.txt") + + assert_true(os.path.exists(flist_path)) + assert_true(os.path.exists(indices_path)) + + with open(flist_path, "r") as f: + assert_equal(f.read().splitlines(), self.file_pairs.flist) + + with open(indices_path, "r") as f: + assert_equal( + [tuple(map(int, line.split(","))) for line in f], + self.file_pairs.tuple_indices, + ) + + def test_load(self): + self.file_pairs.save_to_files(self.test_dir) + new_file_pairs = FilePairsCrossList([], []) + new_file_pairs.load(self.test_dir) + + assert_equal(new_file_pairs.flist, self.file_pairs.flist) + assert_equal(new_file_pairs.tuple_indices, self.file_pairs.tuple_indices) diff --git a/rsa/fpairs/test_file_pairs_from_list.py b/rsa/fpairs/test_file_pairs_from_list.py new file mode 100644 index 0000000..b5a375e --- /dev/null +++ b/rsa/fpairs/test_file_pairs_from_list.py @@ -0,0 +1,25 @@ +from nose.tools import assert_equal, assert_raises, assert_true +import os +import tempfile +import shutil +from rsa.fpairs.file_pairs_from_list import FilePairsFromList +from rsa.fpairs.test_file_pairs import TestFilePairs + +class TestFilePairsFromList(TestFilePairs): + + def setup(self): + self.flist = ["file1", "file2", "file3"] + self.fpairs = [("file1", "file2"), + ("file1", "file3"), + ("file2", "file3")] + self.file_pairs = FilePairsFromList(self.flist) + + def test_initialization(self): + assert_equal(self.file_pairs.pairs, self.fpairs) + + flist = sorted(self.flist) + + assert_equal(self.file_pairs.flist, flist) + assert_equal(self.file_pairs.tuple_indices, [(0, 1), + (0, 2), + (1, 2)]) \ No newline at end of file From a31c9223630df012a8f31cc2c8db99c2c6e03127 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Tue, 3 Jun 2025 15:40:14 +0200 Subject: [PATCH 03/40] install coverage to ensure latest version --- .github/workflows/nosetests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nosetests.yml b/.github/workflows/nosetests.yml index 2b0e84a..83194eb 100755 --- a/.github/workflows/nosetests.yml +++ b/.github/workflows/nosetests.yml @@ -50,7 +50,7 @@ jobs: run: | source activate test-env python --version - conda install -q nose coveralls + conda install -q nose coverage coveralls nosetests --with-coverage - name: Coveralls uses: AndreMiras/coveralls-python-action@develop From ea3d585cdf43a812022ebcd1e9c8894fb06ff18b Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Tue, 3 Jun 2025 15:44:25 +0200 Subject: [PATCH 04/40] use forge channel --- .github/workflows/nosetests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nosetests.yml b/.github/workflows/nosetests.yml index 83194eb..8242ef9 100755 --- a/.github/workflows/nosetests.yml +++ b/.github/workflows/nosetests.yml @@ -50,7 +50,7 @@ jobs: run: | source activate test-env python --version - conda install -q nose coverage coveralls + conda install -q nose coveralls conda-forge::coverage nosetests --with-coverage - name: Coveralls uses: AndreMiras/coveralls-python-action@develop From 93397f630c94a9f8946a87f5af4a528d9cd37d8d Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Tue, 3 Jun 2025 15:46:50 +0200 Subject: [PATCH 05/40] earlier version --- .github/workflows/nosetests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nosetests.yml b/.github/workflows/nosetests.yml index 8242ef9..de90727 100755 --- a/.github/workflows/nosetests.yml +++ b/.github/workflows/nosetests.yml @@ -50,7 +50,7 @@ jobs: run: | source activate test-env python --version - conda install -q nose coveralls conda-forge::coverage + conda install -q nose coveralls conda-forge::coverage=6.5.2 nosetests --with-coverage - name: Coveralls uses: AndreMiras/coveralls-python-action@develop From 29825030b931084d672db0553ac65e99c641495a Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Tue, 3 Jun 2025 15:51:34 +0200 Subject: [PATCH 06/40] no coverage for python 3.11 --- .github/workflows/nosetests.yml | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/.github/workflows/nosetests.yml b/.github/workflows/nosetests.yml index de90727..8bb7ce9 100755 --- a/.github/workflows/nosetests.yml +++ b/.github/workflows/nosetests.yml @@ -46,17 +46,35 @@ jobs: - name: show python path run: | python -c "import sys; print('\n'.join(sys.path))" - - name: Unit Testing + + - name: Unit Testing with coverage + if: matrix.python-version != '3.11' run: | - source activate test-env - python --version - conda install -q nose coveralls conda-forge::coverage=6.5.2 - nosetests --with-coverage + source activate test-env + python --version + conda install -q nose coveralls + nosetests --with-coverage + + # Step to run nosetests without coverage for Python 3.11 + - name: Unit Testing without coverage + if: matrix.python-version == '3.11' + run: | + source activate test-env + python --version + conda install -q nose coveralls + nosetests + + #~ - name: Unit Testing + #~ run: | + #~ source activate test-env + #~ python --version + #~ conda install -q nose coveralls conda-forge::coverage=6.5.2 + #~ nosetests --with-coverage - name: Coveralls uses: AndreMiras/coveralls-python-action@develop with: parallel: true - flag-name: Unit Testing + flag-name: Unit Testing with coverage finish: needs: build From 959c431d9a84c82a1770adcb4389e806795088e0 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Tue, 3 Jun 2025 15:57:19 +0200 Subject: [PATCH 07/40] run nosetests verbose --- .github/workflows/nosetests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nosetests.yml b/.github/workflows/nosetests.yml index 8bb7ce9..d516eb2 100755 --- a/.github/workflows/nosetests.yml +++ b/.github/workflows/nosetests.yml @@ -62,7 +62,7 @@ jobs: source activate test-env python --version conda install -q nose coveralls - nosetests + nosetests -v #~ - name: Unit Testing #~ run: | From 46b84aed6892250fdb3086729c45629a5904fae7 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Tue, 3 Jun 2025 15:57:57 +0200 Subject: [PATCH 08/40] check dir --- .github/workflows/nosetests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/nosetests.yml b/.github/workflows/nosetests.yml index d516eb2..9b70086 100755 --- a/.github/workflows/nosetests.yml +++ b/.github/workflows/nosetests.yml @@ -62,6 +62,8 @@ jobs: source activate test-env python --version conda install -q nose coveralls + pwd + ls nosetests -v #~ - name: Unit Testing From 7e7d6c2f2561a7727a0f9a53c6f069acdf64abbd Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Tue, 3 Jun 2025 15:59:49 +0200 Subject: [PATCH 09/40] go up one level --- .github/workflows/nosetests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nosetests.yml b/.github/workflows/nosetests.yml index 9b70086..dfaf370 100755 --- a/.github/workflows/nosetests.yml +++ b/.github/workflows/nosetests.yml @@ -64,6 +64,7 @@ jobs: conda install -q nose coveralls pwd ls + cd .. nosetests -v #~ - name: Unit Testing From c19da1e56b892292203de23067047cb66b40c5cf Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Tue, 3 Jun 2025 16:03:18 +0200 Subject: [PATCH 10/40] add init to get detected by test --- rsa/corr/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100755 rsa/corr/__init__.py diff --git a/rsa/corr/__init__.py b/rsa/corr/__init__.py new file mode 100755 index 0000000..e69de29 From 85b7e220d5c6c283f183c5b7c6cb349dd50068d9 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Tue, 3 Jun 2025 16:05:52 +0200 Subject: [PATCH 11/40] add documentation --- rsa/cache/rdm_cache.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/rsa/cache/rdm_cache.py b/rsa/cache/rdm_cache.py index b6962a0..00166b3 100644 --- a/rsa/cache/rdm_cache.py +++ b/rsa/cache/rdm_cache.py @@ -30,5 +30,16 @@ def add(self, x, y, value): self.cache_dict[key] = value def get(self, x, y, default_value): + """ + Retrieves a cached value for a pair of elements. + + Args: + x (str): First element. + y (str): Second element. + default_value: Value to return if the pair is not in the cache. + + Returns: + Cached value or the default value. + """ key = _get_key(x, y) return self.cache_dict.get(key, default_value) From 5e42a2c8a16c841e6589b88c3f579218f0f0f180 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Tue, 3 Jun 2025 16:10:12 +0200 Subject: [PATCH 12/40] intentionally break test --- rsa/test_input_rdm_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsa/test_input_rdm_utils.py b/rsa/test_input_rdm_utils.py index 3a1633b..cc85416 100755 --- a/rsa/test_input_rdm_utils.py +++ b/rsa/test_input_rdm_utils.py @@ -175,7 +175,7 @@ def test_calc_and_save_input_rdm_do_triu_false_values_offdiag(self): fpath_dst = os.path.join(self.dir_tmp, 'inrdm.npy') input_rdm_utils.calc_and_save_input_rdm(self.fpath_acts, fpath_dst, key=self.key, do_triu=False) in_rdm = np.load(fpath_dst) - assert_equal(in_rdm[0, 1], 0) + assert_equal(in_rdm[0, 1], 1) assert_equal(in_rdm[0, 2], 2) assert_equal(in_rdm[1, 2], 2) From dedeae910665cbd37fe6ac4c77869058843b714d Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Tue, 3 Jun 2025 16:14:43 +0200 Subject: [PATCH 13/40] add argument to nose --- .github/workflows/nosetests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nosetests.yml b/.github/workflows/nosetests.yml index dfaf370..da23909 100755 --- a/.github/workflows/nosetests.yml +++ b/.github/workflows/nosetests.yml @@ -65,7 +65,7 @@ jobs: pwd ls cd .. - nosetests -v + nosetests --exe -v #~ - name: Unit Testing #~ run: | From 47323518b4ef2d688c48d4dfe6a79c4ff3e13897 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Tue, 3 Jun 2025 16:16:31 +0200 Subject: [PATCH 14/40] add exe arg for test discovery --- .github/workflows/nosetests.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/nosetests.yml b/.github/workflows/nosetests.yml index da23909..62b6a78 100755 --- a/.github/workflows/nosetests.yml +++ b/.github/workflows/nosetests.yml @@ -53,7 +53,7 @@ jobs: source activate test-env python --version conda install -q nose coveralls - nosetests --with-coverage + nosetests --exe --with-coverage # Step to run nosetests without coverage for Python 3.11 - name: Unit Testing without coverage @@ -66,13 +66,7 @@ jobs: ls cd .. nosetests --exe -v - - #~ - name: Unit Testing - #~ run: | - #~ source activate test-env - #~ python --version - #~ conda install -q nose coveralls conda-forge::coverage=6.5.2 - #~ nosetests --with-coverage + - name: Coveralls uses: AndreMiras/coveralls-python-action@develop with: From 6225da7275e845bee17d5ab0f10f3bf193c1786a Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Tue, 3 Jun 2025 16:16:40 +0200 Subject: [PATCH 15/40] restore test value --- rsa/test_input_rdm_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsa/test_input_rdm_utils.py b/rsa/test_input_rdm_utils.py index cc85416..3a1633b 100755 --- a/rsa/test_input_rdm_utils.py +++ b/rsa/test_input_rdm_utils.py @@ -175,7 +175,7 @@ def test_calc_and_save_input_rdm_do_triu_false_values_offdiag(self): fpath_dst = os.path.join(self.dir_tmp, 'inrdm.npy') input_rdm_utils.calc_and_save_input_rdm(self.fpath_acts, fpath_dst, key=self.key, do_triu=False) in_rdm = np.load(fpath_dst) - assert_equal(in_rdm[0, 1], 1) + assert_equal(in_rdm[0, 1], 0) assert_equal(in_rdm[0, 2], 2) assert_equal(in_rdm[1, 2], 2) From 08ac3fe4ac2b02da3038f2fbec3f8512ecdcdfe2 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Tue, 3 Jun 2025 16:17:14 +0200 Subject: [PATCH 16/40] remove verbose output --- .github/workflows/nosetests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nosetests.yml b/.github/workflows/nosetests.yml index 62b6a78..4e20e42 100755 --- a/.github/workflows/nosetests.yml +++ b/.github/workflows/nosetests.yml @@ -65,7 +65,7 @@ jobs: pwd ls cd .. - nosetests --exe -v + nosetests --exe - name: Coveralls uses: AndreMiras/coveralls-python-action@develop From 88be977ab049df33ad96c7400144e02e38fc4da4 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Tue, 3 Jun 2025 16:17:58 +0200 Subject: [PATCH 17/40] keep directory --- .github/workflows/nosetests.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/nosetests.yml b/.github/workflows/nosetests.yml index 4e20e42..e0246a2 100755 --- a/.github/workflows/nosetests.yml +++ b/.github/workflows/nosetests.yml @@ -62,9 +62,6 @@ jobs: source activate test-env python --version conda install -q nose coveralls - pwd - ls - cd .. nosetests --exe - name: Coveralls From b8638fca8c6dc154c248e338991aed2992d1b9c7 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Thu, 5 Jun 2025 11:14:45 +0200 Subject: [PATCH 18/40] test cache --- .github/workflows/nosetests.yml | 10 +++- rsa/cache/cache_key.py | 15 ++++++ rsa/cache/rdm_cache.py | 61 +++++++++++++++++++---- rsa/cache/test_cache_key.py | 77 +++++++++++++++++++++++++++++ rsa/cache/test_rdm_cache.py | 56 +++++++++++++++++++++ rsa/fpairs/file_pairs.py | 9 ++++ rsa/fpairs/file_pairs_cross_list.py | 1 - rsa/fpairs/file_pairs_from_list.py | 2 +- rsa/fpairs/file_pairs_utils.py | 30 +++++++++++ 9 files changed, 247 insertions(+), 14 deletions(-) create mode 100644 rsa/cache/cache_key.py create mode 100644 rsa/cache/test_cache_key.py create mode 100644 rsa/fpairs/file_pairs_utils.py diff --git a/.github/workflows/nosetests.yml b/.github/workflows/nosetests.yml index e0246a2..b3402d4 100755 --- a/.github/workflows/nosetests.yml +++ b/.github/workflows/nosetests.yml @@ -47,16 +47,22 @@ jobs: run: | python -c "import sys; print('\n'.join(sys.path))" - - name: Unit Testing with coverage + - name: Unit Testing with coverage using nosetests if: matrix.python-version != '3.11' run: | source activate test-env python --version conda install -q nose coveralls nosetests --exe --with-coverage + + - name: Run unittest tests + if: matrix.python-version != '3.11' + run: | + source activate test-env + python -m unittest discover # Step to run nosetests without coverage for Python 3.11 - - name: Unit Testing without coverage + - name: Unit Testing without coverage using nosetest if: matrix.python-version == '3.11' run: | source activate test-env diff --git a/rsa/cache/cache_key.py b/rsa/cache/cache_key.py new file mode 100644 index 0000000..12a4de3 --- /dev/null +++ b/rsa/cache/cache_key.py @@ -0,0 +1,15 @@ + +class CacheKey: + + def __init__(self, separator='__-__'): + + self.separator = separator + + def join(self, x, y): + pair = sorted([x, y]) + key = self.separator.join(pair) + return key + + def split(self, key): + x, y = key.split(self.separator) + return x, y \ No newline at end of file diff --git a/rsa/cache/rdm_cache.py b/rsa/cache/rdm_cache.py index 00166b3..d193055 100644 --- a/rsa/cache/rdm_cache.py +++ b/rsa/cache/rdm_cache.py @@ -1,24 +1,65 @@ import yaml - - -def _get_key(x, y): - pair = sorted([x, y]) - key = ''.join(pair) - return key +from rsa.cache.cache_key import CacheKey class RDMCache: def __init__(self): self.cache_dict = {} + self.key_handler = CacheKey() def load_from_file(self, fp_cache): with open(fp_cache, 'r') as h: - self.cache_dict = yaml.safe_load(h) + data = yaml.safe_load(h) + + version = data.get('version', None) + + if version is None: + self.cache_dict = data + + elif version == 250605: + + if 'cache' in self.cache_dict and 'flist' in self.cache_dict: + flist = self.cache_dict['flist'] + cache_dict_mapped = self.cache_dict['cache'] + separator = self.cache_dict.get('separator', self.key_handler.separator) + self.key_handler.separator = separator + for k, v in self.cache_dict.items(): + xi, yi = self.key_handler.split(k) + x = flist[xi] + y = flist[yi] + k_new = self.key_handler.join(x, y) + cache_dict_mapped[k_new] = v + self.cache_dict = cache_dict_mapped + else: + raise KeyError("Cache file does not contain 'cache' or 'flist' keys.") def save_to_file(self, fp_dst): with open(fp_dst, 'w') as h: - h.write(yaml.dump(self.cache_dict)) + + keys = list(self.cache_dict.keys()) + flist = set() + for k in keys: + x, y = self.key_handler.split(k) + flist.add(x) + flist.add(y) + + self.cache_dict['flist'] = list(flist) + + cache_dict_mapped = {} + for k, v in self.cache_dict.items(): + x, y = self.key_handler.split(k) + xi = flist.index(x) + yi = flist.index(y) + k_new = self.key_handler.join(xi, yi) + cache_dict_mapped[k_new] = v + + h.write(yaml.dump({ + 'version': 250605, # Add version information + 'cache': cache_dict_mapped, + 'flist': list(flist), + 'separator': self.key_handler.separator + })) # def is_in(self, x, y): # @@ -26,7 +67,7 @@ def save_to_file(self, fp_dst): # return key in self.cache_dict def add(self, x, y, value): - key = _get_key(x, y) + key = self.key_handler.join(x, y) self.cache_dict[key] = value def get(self, x, y, default_value): @@ -41,5 +82,5 @@ def get(self, x, y, default_value): Returns: Cached value or the default value. """ - key = _get_key(x, y) + key = self.key_handler.join(x, y) return self.cache_dict.get(key, default_value) diff --git a/rsa/cache/test_cache_key.py b/rsa/cache/test_cache_key.py new file mode 100644 index 0000000..61c33d9 --- /dev/null +++ b/rsa/cache/test_cache_key.py @@ -0,0 +1,77 @@ +import unittest +import tempfile +import os +import yaml +from rsa.cache.rdm_cache import RDMCache + + +class TestRDMCache(unittest.TestCase): + + def setUp(self): + self.cache = RDMCache() + + def test_add_and_get(self): + # Test adding and retrieving a value with file path-like strings + self.cache.add("/path/to/file1", "/path/to/file2", 42) + self.assertEqual(self.cache.get("/path/to/file1", "/path/to/file2", None), 42) + self.assertEqual(self.cache.get("/path/to/file2", "/path/to/file1", None), 42) # Ensure symmetry + self.assertEqual(self.cache.get("/path/to/unknown", "/path/to/file2", "default"), + "default") # Test default value + + def test_save_to_file(self): + # Test saving cache to a file + self.cache.add("/path/to/file1", "/path/to/file2", 42) + self.cache.add("/path/to/file3", "/path/to/file4", 84) + + with tempfile.NamedTemporaryFile(delete=False) as temp_file: + fp = temp_file.name + + try: + self.cache.save_to_file(fp) + with open(fp, 'r') as f: + data = yaml.safe_load(f) + self.assertIn("cache", data) + self.assertIn("flist", data) + self.assertIn("separator", data) + finally: + os.remove(fp) + + def test_load_from_file(self): + # Test loading cache from a file + cache_data = { + "cache": {"0__-__1": 42, "2__-__3": 84}, + "flist": ["/path/to/file1", "/path/to/file2", "/path/to/file3", "/path/to/file4"], + "separator": "__-__" + } + + with tempfile.NamedTemporaryFile(delete=False, mode='w') as temp_file: + fp = temp_file.name + yaml.dump(cache_data, temp_file) + + try: + self.cache.load_from_file(fp) + self.assertEqual(self.cache.get("/path/to/file1", "/path/to/file2", None), 42) + self.assertEqual(self.cache.get("/path/to/file3", "/path/to/file4", None), 84) + finally: + os.remove(fp) + + def test_save_and_load(self): + # Test saving and loading cache + self.cache.add("/path/to/file1", "/path/to/file2", 42) + self.cache.add("/path/to/file3", "/path/to/file4", 84) + + with tempfile.NamedTemporaryFile(delete=False) as temp_file: + fp = temp_file.name + + try: + self.cache.save_to_file(fp) + new_cache = RDMCache() + new_cache.load_from_file(fp) + self.assertEqual(new_cache.get("/path/to/file1", "/path/to/file2", None), 42) + self.assertEqual(new_cache.get("/path/to/file3", "/path/to/file4", None), 84) + finally: + os.remove(fp) + + +if __name__ == "__main__": + unittest.main() diff --git a/rsa/cache/test_rdm_cache.py b/rsa/cache/test_rdm_cache.py index e69de29..634f4fb 100644 --- a/rsa/cache/test_rdm_cache.py +++ b/rsa/cache/test_rdm_cache.py @@ -0,0 +1,56 @@ +# Python +import unittest +import tempfile +import os +import yaml +from rsa.cache.rdm_cache import RDMCache + +class TestRDMCacheVersion250605(unittest.TestCase): + + def setUp(self): + self.cache = RDMCache() + + def test_load_version_250605(self): + # Prepare cache data for version 250605 + cache_data = { + "version": 250605, + "cache": {"0__-__1": 42, "2__-__3": 84}, + "flist": ["/path/to/file1", "/path/to/file2", "/path/to/file3", "/path/to/file4"], + "separator": "__-__" + } + + with tempfile.NamedTemporaryFile(delete=False, mode='w') as temp_file: + fp = temp_file.name + yaml.dump(cache_data, temp_file) + + try: + self.cache.load_from_file(fp) + self.assertEqual(self.cache.get("/path/to/file1", "/path/to/file2", None), 42) + self.assertEqual(self.cache.get("/path/to/file3", "/path/to/file4", None), 84) + finally: + os.remove(fp) + +class TestRDMCacheNoVersion(unittest.TestCase): + + def setUp(self): + self.cache = RDMCache() + + def test_load_no_version(self): + # Prepare cache data without version + cache_data = { + "cache": {"key1": 42, "key2": 84} + } + + with tempfile.NamedTemporaryFile(delete=False, mode='w') as temp_file: + fp = temp_file.name + yaml.dump(cache_data, temp_file) + + try: + self.cache.load_from_file(fp) + self.assertEqual(self.cache.get("key1", None, None), 42) + self.assertEqual(self.cache.get("key2", None, None), 84) + finally: + os.remove(fp) + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/rsa/fpairs/file_pairs.py b/rsa/fpairs/file_pairs.py index 78c699f..ec31a9c 100644 --- a/rsa/fpairs/file_pairs.py +++ b/rsa/fpairs/file_pairs.py @@ -66,3 +66,12 @@ def load(self, load_dir, flist_path='flist.txt', indices_path='indices.txt'): self.tuple_indices = sorted(list(product(range(numels), repeat=2))) self.pairs = [tuple(self.flist[i] for i in indices) for indices in self.tuple_indices] + + def __len__(self): + """ + Returns the number of file pairs. + + Returns: + int: Number of file pairs. + """ + return len(self.pairs) diff --git a/rsa/fpairs/file_pairs_cross_list.py b/rsa/fpairs/file_pairs_cross_list.py index 56d12d8..15059b3 100644 --- a/rsa/fpairs/file_pairs_cross_list.py +++ b/rsa/fpairs/file_pairs_cross_list.py @@ -22,5 +22,4 @@ def __init__(self, flist1, flist2): # Generate pairs between the two lists fpairs = list(product(flist1, flist2)) fpairs = sorted([(a, b) for a, b in fpairs if a != b]) # remove duplicates - print(fpairs) super().__init__(fpairs) diff --git a/rsa/fpairs/file_pairs_from_list.py b/rsa/fpairs/file_pairs_from_list.py index a0e608e..dfbde28 100644 --- a/rsa/fpairs/file_pairs_from_list.py +++ b/rsa/fpairs/file_pairs_from_list.py @@ -24,4 +24,4 @@ def __init__(self, flist): triu_rows, triu_cols = np.triu_indices(num_rows, k=1) fpairs = sorted([(flist[r], flist[c]) for r, c in zip(triu_rows, triu_cols)]) # print('fpairs', fpairs) - super().__init__(fpairs) + super().__init__(fpairs) \ No newline at end of file diff --git a/rsa/fpairs/file_pairs_utils.py b/rsa/fpairs/file_pairs_utils.py new file mode 100644 index 0000000..53d5399 --- /dev/null +++ b/rsa/fpairs/file_pairs_utils.py @@ -0,0 +1,30 @@ +from rsa.fpairs import (FilePairsCrossList, # pairs from two lists + FilePairsFromList, # all pairs from one list + FilePairs) # pairs already provided + + +def create_file_pairs(flist1, flist2=None): + """ + Create file pairs based on the provided lists. + + Args: + flist1 (list): First list of file names or a list of tuples of strings. + flist2 (list, optional): Second list of file names. + + Returns: + FilePairs: An instance of FilePairsCrossList, FilePairsFromList, or FilePairs. + """ + if flist1 is not None and flist2 is not None: + # both lists provided + return FilePairsCrossList(flist1, flist2) + elif flist1 is not None: + # only one list provided + if isinstance(flist1, list): + # Check if flist1 is a list of tuples of strings or a list of strings + if all(isinstance(item, tuple) and all(isinstance(sub_item, str) for sub_item in item) for item in flist1): + return FilePairs(flist1) + return FilePairsFromList(flist1) + else: + raise TypeError("flist1 must be a list.") + else: + raise ValueError("At least one file list must be provided.") From b55b56b0c231575da391c70ce34f074cee0ab426 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Thu, 5 Jun 2025 12:35:31 +0200 Subject: [PATCH 19/40] fix cache indexing --- rsa/cache/rdm_cache.py | 22 +++++++-------- rsa/cache/test_rdm_cache.py | 54 ++++++++++++++++++++++++++++++------- 2 files changed, 55 insertions(+), 21 deletions(-) diff --git a/rsa/cache/rdm_cache.py b/rsa/cache/rdm_cache.py index d193055..4efb042 100644 --- a/rsa/cache/rdm_cache.py +++ b/rsa/cache/rdm_cache.py @@ -19,18 +19,18 @@ def load_from_file(self, fp_cache): elif version == 250605: - if 'cache' in self.cache_dict and 'flist' in self.cache_dict: - flist = self.cache_dict['flist'] - cache_dict_mapped = self.cache_dict['cache'] - separator = self.cache_dict.get('separator', self.key_handler.separator) + if 'cache' in data and 'flist' in data: + flist = data['flist'] + cache_dict_mapped = data['cache'] + separator = data.get('separator', self.key_handler.separator) + # update separator self.key_handler.separator = separator - for k, v in self.cache_dict.items(): + for k, v in data['cache'].items(): xi, yi = self.key_handler.split(k) - x = flist[xi] - y = flist[yi] + x = flist[int(xi)] + y = flist[int(yi)] k_new = self.key_handler.join(x, y) - cache_dict_mapped[k_new] = v - self.cache_dict = cache_dict_mapped + self.cache_dict[k_new] = v else: raise KeyError("Cache file does not contain 'cache' or 'flist' keys.") @@ -43,8 +43,8 @@ def save_to_file(self, fp_dst): x, y = self.key_handler.split(k) flist.add(x) flist.add(y) - - self.cache_dict['flist'] = list(flist) + flist = sorted(list(flist)) + self.cache_dict['flist'] = flist cache_dict_mapped = {} for k, v in self.cache_dict.items(): diff --git a/rsa/cache/test_rdm_cache.py b/rsa/cache/test_rdm_cache.py index 634f4fb..141c297 100644 --- a/rsa/cache/test_rdm_cache.py +++ b/rsa/cache/test_rdm_cache.py @@ -5,14 +5,47 @@ import yaml from rsa.cache.rdm_cache import RDMCache -class TestRDMCacheVersion250605(unittest.TestCase): + +class TestRDMCacheAddAndGet(unittest.TestCase): + + def setUp(self): + self.cache = RDMCache() + + def test_add_and_get(self): + # Add a pair to the cache + self.cache.add("file1.txt", "file2.txt", 42) + + # Retrieve the value for the added pair + value = self.cache.get("file1.txt", "file2.txt", None) + self.assertEqual(value, 42, "The value retrieved from the cache should match the added value.") + + def test_get_default_value(self): + # Attempt to retrieve a value for a pair not in the cache + default_value = self.cache.get("file3.txt", "file4.txt", -1) + self.assertEqual(default_value, -1, "The default value should be returned for a missing pair.") + default_value = self.cache.get("file3.txt", "file4.txt", -2) + self.assertEqual(default_value, -2, "The default value should be returned for a missing pair.") + + def test_overwrite_existing_value(self): + # Add a pair to the cache + self.cache.add("file1.txt", "file2.txt", 42) + + # Overwrite the value for the same pair + self.cache.add("file1.txt", "file2.txt", 84) + + # Retrieve the updated value + updated_value = self.cache.get("file1.txt", "file2.txt", None) + self.assertEqual(updated_value, 84, "The value should be updated when the pair is added again.") + + +class TestRDMCacheSerializationVersion250605(unittest.TestCase): def setUp(self): self.cache = RDMCache() def test_load_version_250605(self): # Prepare cache data for version 250605 - cache_data = { + data = { "version": 250605, "cache": {"0__-__1": 42, "2__-__3": 84}, "flist": ["/path/to/file1", "/path/to/file2", "/path/to/file3", "/path/to/file4"], @@ -21,7 +54,7 @@ def test_load_version_250605(self): with tempfile.NamedTemporaryFile(delete=False, mode='w') as temp_file: fp = temp_file.name - yaml.dump(cache_data, temp_file) + yaml.dump(data, temp_file) try: self.cache.load_from_file(fp) @@ -30,27 +63,28 @@ def test_load_version_250605(self): finally: os.remove(fp) -class TestRDMCacheNoVersion(unittest.TestCase): + +class TestRDMCacheSerializationNoVersion(unittest.TestCase): def setUp(self): self.cache = RDMCache() def test_load_no_version(self): # Prepare cache data without version - cache_data = { - "cache": {"key1": 42, "key2": 84} - } + cache_data = {"0__-__1": 42, "2__-__3": 84} + # avoid using save_to_file and write to yaml directly with tempfile.NamedTemporaryFile(delete=False, mode='w') as temp_file: fp = temp_file.name yaml.dump(cache_data, temp_file) try: self.cache.load_from_file(fp) - self.assertEqual(self.cache.get("key1", None, None), 42) - self.assertEqual(self.cache.get("key2", None, None), 84) + self.assertEqual(self.cache.get("0", "1", None), 42) + self.assertEqual(self.cache.get("2", "3", None), 84) finally: os.remove(fp) + if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() From 60fedd1c8440bf7bc31da67d65025da2ca0da4e1 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Thu, 5 Jun 2025 14:07:15 +0200 Subject: [PATCH 20/40] allow setting chunksize --- rsa/input_rdm.py | 4 ++-- rsa/input_rdm_utils.py | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/rsa/input_rdm.py b/rsa/input_rdm.py index d36da47..66014f3 100755 --- a/rsa/input_rdm.py +++ b/rsa/input_rdm.py @@ -25,7 +25,7 @@ def dissimilarity(self, act_row, act_col, dst_idx): rho, _ = pearsonr(act_row, act_col) return dst_idx, 1 - rho - def apply(self, processes=1, do_disable_tqdm=False): + def apply(self, processes=1, chunksize=10, do_disable_tqdm=False): triu_rows, triu_cols = np.triu_indices(self.num_rows, k=1) self.input_rdm_triu = np.zeros((triu_rows.size,)) + ENTRY_EMPTY_INPUT_RDM @@ -38,7 +38,7 @@ def apply(self, processes=1, do_disable_tqdm=False): for idx, (row, col) in enumerate(zip(triu_rows, triu_cols))], total=len(triu_rows), disable=do_disable_tqdm), - chunksize=10, + chunksize=chunksize, ) for idx, dissimilarity in result: diff --git a/rsa/input_rdm_utils.py b/rsa/input_rdm_utils.py index b12496c..483d45b 100755 --- a/rsa/input_rdm_utils.py +++ b/rsa/input_rdm_utils.py @@ -9,7 +9,8 @@ def calc_input_rdm(fpath_src_activations, key="", do_keep_mem_low=False, - num_processes=None): + num_processes=1, + chunksize=10): """ Calculate Input RDM @@ -31,6 +32,7 @@ def calc_input_rdm(fpath_src_activations, key="", if do_keep_mem_low: ir = InputRDM(acts.reshape(num_samples, -1)) in_rdm = ir.apply(processes=1 if num_processes is None else num_processes, + chunksize=chunksize, do_disable_tqdm=True) from rsa.rdm_utils import triu_off_diag_vec_to_rdm in_rdm = triu_off_diag_vec_to_rdm(in_rdm) @@ -45,10 +47,12 @@ def calc_input_rdm(fpath_src_activations, key="", def calc_and_save_input_rdm(fpath_src_activations, fpath_dst, key="", do_triu=True, do_keep_mem_low=False, - num_processes=None): + num_processes=None, + chunksize=10): in_rdm = calc_input_rdm(fpath_src_activations, key=key, do_keep_mem_low=do_keep_mem_low, - num_processes=num_processes) + num_processes=num_processes, + chunksize=chunksize) if do_triu: in_rdm = get_triu_off_diag_flat(in_rdm) # print(in_rdm.shape) From 549e349acfb333450092609543174054a855a38a Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Thu, 5 Jun 2025 14:07:39 +0200 Subject: [PATCH 21/40] keep flist out of dict, cast keys to string --- rsa/cache/rdm_cache.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rsa/cache/rdm_cache.py b/rsa/cache/rdm_cache.py index 4efb042..0814f96 100644 --- a/rsa/cache/rdm_cache.py +++ b/rsa/cache/rdm_cache.py @@ -44,14 +44,13 @@ def save_to_file(self, fp_dst): flist.add(x) flist.add(y) flist = sorted(list(flist)) - self.cache_dict['flist'] = flist cache_dict_mapped = {} for k, v in self.cache_dict.items(): x, y = self.key_handler.split(k) xi = flist.index(x) yi = flist.index(y) - k_new = self.key_handler.join(xi, yi) + k_new = self.key_handler.join(str(xi), str(yi)) cache_dict_mapped[k_new] = v h.write(yaml.dump({ From e5d2c79807f9a6cee50999ece89c3e6a27dd6fa2 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Thu, 5 Jun 2025 14:08:14 +0200 Subject: [PATCH 22/40] mock method to speed up testing --- rsa/test_input_rdm_utils_parallel.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/rsa/test_input_rdm_utils_parallel.py b/rsa/test_input_rdm_utils_parallel.py index 71225e0..f016f70 100755 --- a/rsa/test_input_rdm_utils_parallel.py +++ b/rsa/test_input_rdm_utils_parallel.py @@ -3,6 +3,7 @@ assert_almost_equals, \ assert_true, assert_false, \ assert_raises, assert_list_equal +from unittest.mock import patch import os import tempfile import shutil @@ -48,11 +49,21 @@ def test_calc_input_rdm_values_diag(self): assert_equal(el, 0.0) def test_calc_and_save_input_rdm_return_path(self): - for idx in range(5): - fpath_dst_arg = os.path.join(self.dir_tmp, 'inrdm_%d.npy' % idx) - fpath_dst_ret = input_rdm_utils.calc_and_save_input_rdm(self.fpath_acts, fpath_dst_arg, key=self.key, - do_keep_mem_low=True) - assert_equal(fpath_dst_ret, fpath_dst_arg) + + with patch('rsa.input_rdm_utils.InputRDM') as MockInputRDM: + # Mock the PearsonCorrcoef instance and its calculate method + mock_instance = MockInputRDM.return_value + mock_instance.apply.return_value = np.zeros((3,)) # Mocked return value + + for idx in range(5): + fpath_dst_arg = os.path.join(self.dir_tmp, 'inrdm_%d.npy' % idx) + fpath_dst_ret = input_rdm_utils.calc_and_save_input_rdm(self.fpath_acts, fpath_dst_arg, key=self.key, + do_keep_mem_low=True) + assert_equal(fpath_dst_ret, fpath_dst_arg) + + # Assertions to verify behavior + # MockPearsonCorrcoef.assert_called_once_with((4, 3)) # Example shape + assert_equal(mock_instance.apply.call_count, 5) def test_calc_and_save_input_rdm_default_dims(self): @@ -177,7 +188,7 @@ def setup(self): def test_calc_input_rdm_values_offdiag_2D_input(self): in_rdm = input_rdm_utils.calc_input_rdm(self.fpath_acts, key=self.key, do_keep_mem_low=True) - assert_almost_equals(in_rdm[0, 1], 0,15) + assert_almost_equals(in_rdm[0, 1], 0, 15) assert_equal(in_rdm[0, 2], 2) assert_equal(in_rdm[1, 2], 2) @@ -197,7 +208,7 @@ def test_calc_and_save_input_rdm_do_triu_true_values_offdiag(self): in_rdm = np.load(fpath_dst) assert_equal(in_rdm.size, self.num_samples * (self.num_samples - 1) / 2) - assert_almost_equals(in_rdm[0], 0,15) + assert_almost_equals(in_rdm[0], 0, 15) assert_equal(in_rdm[1], 2) assert_equal(in_rdm[2], 2) From 1c77388ebe08a34bf6e73fd8d0af5569818cf99a Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Thu, 5 Jun 2025 14:08:32 +0200 Subject: [PATCH 23/40] reduce iterations to speed up testing --- rsa/test_model_rdm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsa/test_model_rdm.py b/rsa/test_model_rdm.py index 6814a7c..dfa4fc4 100755 --- a/rsa/test_model_rdm.py +++ b/rsa/test_model_rdm.py @@ -65,7 +65,7 @@ def helper_calc_model_rdm(self, flist): return mrdm def test_calc_model_rdm_size(self): - for sz_in_rdm in range(3, 7): + for sz_in_rdm in range(3, 4): for num_rdms in range(2, 5): fp_in_rdms = [] for rdm_idx in range(num_rdms): From d9aabc9009bf8e2aa3a8f0108d24cb17de1d1011 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Thu, 5 Jun 2025 16:16:02 +0200 Subject: [PATCH 24/40] restrict coverage --- .github/workflows/nosetests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nosetests.yml b/.github/workflows/nosetests.yml index b3402d4..6582f7a 100755 --- a/.github/workflows/nosetests.yml +++ b/.github/workflows/nosetests.yml @@ -53,7 +53,7 @@ jobs: source activate test-env python --version conda install -q nose coveralls - nosetests --exe --with-coverage + nosetests --exe --with-coverage --cover-package rsa - name: Run unittest tests if: matrix.python-version != '3.11' From 0558ad09455560a7853efd775c994b93d75d9e89 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Thu, 5 Jun 2025 16:22:53 +0200 Subject: [PATCH 25/40] omit unused imports --- rsa/model_rdm_cached.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/rsa/model_rdm_cached.py b/rsa/model_rdm_cached.py index 57d07bb..d41b599 100755 --- a/rsa/model_rdm_cached.py +++ b/rsa/model_rdm_cached.py @@ -1,12 +1,4 @@ -import multiprocessing as mp -import errno -from pathlib import Path -import os -from tqdm import tqdm import numpy as np -from rsa.model_rdm_utils import calc_spearman_rank_corr_from_files, ENTRY_EMPTY -from rsa.rdm_loader import RDMLoaderNPY -import rsa.mat_utils as mutils from rsa.cache.rdm_cache import RDMCache from rsa.model_rdm import ModelRDM From ac282b1f722f713a731d8da67c59b97a36a2e498 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Thu, 5 Jun 2025 16:24:12 +0200 Subject: [PATCH 26/40] omit obsolete comment --- rsa/test_input_rdm_utils_parallel.py | 1 - 1 file changed, 1 deletion(-) diff --git a/rsa/test_input_rdm_utils_parallel.py b/rsa/test_input_rdm_utils_parallel.py index f016f70..315fa56 100755 --- a/rsa/test_input_rdm_utils_parallel.py +++ b/rsa/test_input_rdm_utils_parallel.py @@ -51,7 +51,6 @@ def test_calc_input_rdm_values_diag(self): def test_calc_and_save_input_rdm_return_path(self): with patch('rsa.input_rdm_utils.InputRDM') as MockInputRDM: - # Mock the PearsonCorrcoef instance and its calculate method mock_instance = MockInputRDM.return_value mock_instance.apply.return_value = np.zeros((3,)) # Mocked return value From cb3bb9317b1f5bc7ad789086bae414ddf23b4d5a Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Thu, 5 Jun 2025 16:26:36 +0200 Subject: [PATCH 27/40] remove unused imports --- rsa/model_rdm_utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/rsa/model_rdm_utils.py b/rsa/model_rdm_utils.py index 41b6fd0..b00f38f 100755 --- a/rsa/model_rdm_utils.py +++ b/rsa/model_rdm_utils.py @@ -3,9 +3,7 @@ import numpy as np import pandas as pd from scipy.stats import spearmanr -import rsa.mat_utils as mutils from rsa.input_rdm_utils import get_input_rdm_flat_from_file -from rsa.rdm_loader import RDMLoaderNPY ENTRY_EMPTY = -999 From 46307fa73e293964d095819bf3238b6281b27ea9 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Thu, 5 Jun 2025 16:35:19 +0200 Subject: [PATCH 28/40] restore import of constant --- rsa/model_rdm_cached.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsa/model_rdm_cached.py b/rsa/model_rdm_cached.py index d41b599..703447b 100755 --- a/rsa/model_rdm_cached.py +++ b/rsa/model_rdm_cached.py @@ -1,6 +1,7 @@ import numpy as np from rsa.cache.rdm_cache import RDMCache from rsa.model_rdm import ModelRDM +from rsa.model_rdm_utils import ENTRY_EMPTY class ModelRDMCached(ModelRDM): @@ -45,4 +46,3 @@ def apply(self, processes=1, chunksize=10, do_disable_tqdm=False): self.cache.save_to_file(self.fp_cache) return self.model_rdm_triu - From 99c69489d76ba9ac379d2cb73583893f13773703 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Thu, 5 Jun 2025 17:46:35 +0200 Subject: [PATCH 29/40] speed up test with less iterations --- rsa/test_model_rdm.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/rsa/test_model_rdm.py b/rsa/test_model_rdm.py index dfa4fc4..d0146dc 100755 --- a/rsa/test_model_rdm.py +++ b/rsa/test_model_rdm.py @@ -2,6 +2,7 @@ from nose.tools import assert_equal, \ assert_true, assert_false, \ assert_raises, assert_list_equal +from unittest.mock import patch import os import tempfile import shutil @@ -65,8 +66,9 @@ def helper_calc_model_rdm(self, flist): return mrdm def test_calc_model_rdm_size(self): + for sz_in_rdm in range(3, 4): - for num_rdms in range(2, 5): + for num_rdms in [2, 5]: fp_in_rdms = [] for rdm_idx in range(num_rdms): rdm = rand_rdm(sz_in_rdm) @@ -121,7 +123,6 @@ def test_calc_model_rdm_values(self): class TestModelRDMInput2DMatNPZ(TestModelRDMInput2DMat): def helper_calc_model_rdm(self, flist): - # switch from npy to npz flist_npz = [] for fp in flist: @@ -137,10 +138,10 @@ def helper_calc_model_rdm(self, flist): mrdm = m.apply(do_disable_tqdm=True) return mrdm + class TestModelRDMInput2DMaInMemory(TestModelRDMInput2DMat): def helper_calc_model_rdm(self, flist): - # switch from npy to in-memory list_irdm = [np.load(fp) for fp in flist] From 86fd5a7779c3e60d831e7f4c26f9826ae3534ad9 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Fri, 6 Jun 2025 10:00:34 +0200 Subject: [PATCH 30/40] cache key tests --- rsa/cache/test_cache_key.py | 88 +++++++++---------------------------- 1 file changed, 21 insertions(+), 67 deletions(-) diff --git a/rsa/cache/test_cache_key.py b/rsa/cache/test_cache_key.py index 61c33d9..17b97fd 100644 --- a/rsa/cache/test_cache_key.py +++ b/rsa/cache/test_cache_key.py @@ -1,77 +1,31 @@ +# Python import unittest -import tempfile -import os -import yaml -from rsa.cache.rdm_cache import RDMCache +from rsa.cache.cache_key import CacheKey - -class TestRDMCache(unittest.TestCase): +class TestCacheKey(unittest.TestCase): def setUp(self): - self.cache = RDMCache() - - def test_add_and_get(self): - # Test adding and retrieving a value with file path-like strings - self.cache.add("/path/to/file1", "/path/to/file2", 42) - self.assertEqual(self.cache.get("/path/to/file1", "/path/to/file2", None), 42) - self.assertEqual(self.cache.get("/path/to/file2", "/path/to/file1", None), 42) # Ensure symmetry - self.assertEqual(self.cache.get("/path/to/unknown", "/path/to/file2", "default"), - "default") # Test default value - - def test_save_to_file(self): - # Test saving cache to a file - self.cache.add("/path/to/file1", "/path/to/file2", 42) - self.cache.add("/path/to/file3", "/path/to/file4", 84) - - with tempfile.NamedTemporaryFile(delete=False) as temp_file: - fp = temp_file.name - - try: - self.cache.save_to_file(fp) - with open(fp, 'r') as f: - data = yaml.safe_load(f) - self.assertIn("cache", data) - self.assertIn("flist", data) - self.assertIn("separator", data) - finally: - os.remove(fp) - - def test_load_from_file(self): - # Test loading cache from a file - cache_data = { - "cache": {"0__-__1": 42, "2__-__3": 84}, - "flist": ["/path/to/file1", "/path/to/file2", "/path/to/file3", "/path/to/file4"], - "separator": "__-__" - } - - with tempfile.NamedTemporaryFile(delete=False, mode='w') as temp_file: - fp = temp_file.name - yaml.dump(cache_data, temp_file) - - try: - self.cache.load_from_file(fp) - self.assertEqual(self.cache.get("/path/to/file1", "/path/to/file2", None), 42) - self.assertEqual(self.cache.get("/path/to/file3", "/path/to/file4", None), 84) - finally: - os.remove(fp) + self.cache_key = CacheKey(separator="__-__") - def test_save_and_load(self): - # Test saving and loading cache - self.cache.add("/path/to/file1", "/path/to/file2", 42) - self.cache.add("/path/to/file3", "/path/to/file4", 84) + def test_join_creates_key(self): + key = self.cache_key.join("file1", "file2") + self.assertEqual(key, "file1__-__file2", "The key should be correctly joined with the separator.") - with tempfile.NamedTemporaryFile(delete=False) as temp_file: - fp = temp_file.name + def test_join_is_order_independent(self): + key1 = self.cache_key.join("file1", "file2") + key2 = self.cache_key.join("file2", "file1") + self.assertEqual(key1, key2, "The key should be the same regardless of the order of x and y.") - try: - self.cache.save_to_file(fp) - new_cache = RDMCache() - new_cache.load_from_file(fp) - self.assertEqual(new_cache.get("/path/to/file1", "/path/to/file2", None), 42) - self.assertEqual(new_cache.get("/path/to/file3", "/path/to/file4", None), 84) - finally: - os.remove(fp) + def test_split_key(self): + key = "file1__-__file2" + x, y = self.cache_key.split(key) + self.assertEqual(x, "file1", "The first element should be correctly split from the key.") + self.assertEqual(y, "file2", "The second element should be correctly split from the key.") + def test_split_and_join_consistency(self): + key = self.cache_key.join("file1", "file2") + x, y = self.cache_key.split(key) + self.assertEqual(key, self.cache_key.join(x, y), "Splitting and rejoining the key should result in the same key.") if __name__ == "__main__": - unittest.main() + unittest.main() \ No newline at end of file From ae27c0c30f65cedf11438f15c18ba7af24bc4ae4 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Fri, 6 Jun 2025 10:07:42 +0200 Subject: [PATCH 31/40] specify directory --- .github/workflows/nosetests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nosetests.yml b/.github/workflows/nosetests.yml index 6582f7a..480afdd 100755 --- a/.github/workflows/nosetests.yml +++ b/.github/workflows/nosetests.yml @@ -59,7 +59,7 @@ jobs: if: matrix.python-version != '3.11' run: | source activate test-env - python -m unittest discover + python -m unittest /home/runner/work/rsatools/rsatools # Step to run nosetests without coverage for Python 3.11 - name: Unit Testing without coverage using nosetest From 2c2042c32f4c475356a1d5985e3cd715a771149b Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Fri, 6 Jun 2025 10:59:27 +0200 Subject: [PATCH 32/40] add init --- rsa/cache/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 rsa/cache/__init__.py diff --git a/rsa/cache/__init__.py b/rsa/cache/__init__.py new file mode 100644 index 0000000..e69de29 From cc48c819943d86cf2a4ef2ab715184a270865abb Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Fri, 6 Jun 2025 10:59:44 +0200 Subject: [PATCH 33/40] remove dir --- .github/workflows/nosetests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nosetests.yml b/.github/workflows/nosetests.yml index 480afdd..f6557a4 100755 --- a/.github/workflows/nosetests.yml +++ b/.github/workflows/nosetests.yml @@ -59,7 +59,7 @@ jobs: if: matrix.python-version != '3.11' run: | source activate test-env - python -m unittest /home/runner/work/rsatools/rsatools + python -m unittest # Step to run nosetests without coverage for Python 3.11 - name: Unit Testing without coverage using nosetest From 4aa527a682ca1f5a3002a1ee7f640dd2b1c8f97e Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Fri, 6 Jun 2025 12:44:53 +0200 Subject: [PATCH 34/40] no need for extra unit test run --- .github/workflows/nosetests.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/nosetests.yml b/.github/workflows/nosetests.yml index f6557a4..0e5949a 100755 --- a/.github/workflows/nosetests.yml +++ b/.github/workflows/nosetests.yml @@ -54,12 +54,7 @@ jobs: python --version conda install -q nose coveralls nosetests --exe --with-coverage --cover-package rsa - - - name: Run unittest tests - if: matrix.python-version != '3.11' - run: | - source activate test-env - python -m unittest + #~ python -m unittest # Step to run nosetests without coverage for Python 3.11 - name: Unit Testing without coverage using nosetest From 9eac18e1c40b6ae9d610654d5ba3bc0d20575763 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Mon, 23 Jun 2025 16:08:25 +0200 Subject: [PATCH 35/40] model rdm with file pairs --- rsa/fpairs/file_pairs.py | 4 + rsa/fpairs/file_pairs_utils.py | 7 +- rsa/model_rdm.py | 2 +- rsa/model_rdm_paired.py | 52 +++++++++ rsa/test_model_rdm_paired.py | 205 +++++++++++++++++++++++++++++++++ 5 files changed, 265 insertions(+), 5 deletions(-) create mode 100755 rsa/model_rdm_paired.py create mode 100755 rsa/test_model_rdm_paired.py diff --git a/rsa/fpairs/file_pairs.py b/rsa/fpairs/file_pairs.py index ec31a9c..1f00963 100644 --- a/rsa/fpairs/file_pairs.py +++ b/rsa/fpairs/file_pairs.py @@ -75,3 +75,7 @@ def __len__(self): int: Number of file pairs. """ return len(self.pairs) + + def get(self, idx): + + return self.pairs[idx] \ No newline at end of file diff --git a/rsa/fpairs/file_pairs_utils.py b/rsa/fpairs/file_pairs_utils.py index 53d5399..bb7af7b 100644 --- a/rsa/fpairs/file_pairs_utils.py +++ b/rsa/fpairs/file_pairs_utils.py @@ -1,7 +1,6 @@ -from rsa.fpairs import (FilePairsCrossList, # pairs from two lists - FilePairsFromList, # all pairs from one list - FilePairs) # pairs already provided - +from rsa.fpairs.file_pairs import FilePairs # pairs already provided +from rsa.fpairs.file_pairs_cross_list import FilePairsCrossList # pairs from two lists +from rsa.fpairs.file_pairs_from_list import FilePairsFromList # all pairs from one list def create_file_pairs(flist1, flist2=None): """ diff --git a/rsa/model_rdm.py b/rsa/model_rdm.py index 456131e..f78fa7a 100755 --- a/rsa/model_rdm.py +++ b/rsa/model_rdm.py @@ -17,7 +17,7 @@ def __init__(self, fpath_list): # if not Path(fp).is_file(): # raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), fp) self.fp_list = fpath_list - self.num_rows = len(fpath_list) + self.num_rows = len(fpath_list) if self.fp_list else 0 self.loader = RDMLoaderNPY() self.model_rdm_triu = None diff --git a/rsa/model_rdm_paired.py b/rsa/model_rdm_paired.py new file mode 100755 index 0000000..e4156bb --- /dev/null +++ b/rsa/model_rdm_paired.py @@ -0,0 +1,52 @@ +import multiprocessing as mp +import errno +from pathlib import Path +import os +from tqdm import tqdm +import numpy as np +from rsa.model_rdm_utils import calc_spearman_rank_corr_from_files, ENTRY_EMPTY +from rsa.rdm_loader import RDMLoaderNPY +from rsa.model_rdm import ModelRDM +import rsa.mat_utils as mutils +from rsa.fpairs.file_pairs_utils import create_file_pairs + + +class ModelRDMPaired(ModelRDM): + + def __init__(self, fpath_list): + self.pairs = create_file_pairs(fpath_list) if fpath_list else None + super().__init__(self.pairs.flist if self.pairs else None) + + def set_pairs(self, pairs): + self.pairs = pairs + self.fp_list = self.pairs.flist + + def _init_model_rdm_triu(self): + self.numels = len(self.pairs) if self.pairs else 0 + self.model_rdm_triu = np.zeros((self.numels,)) + ENTRY_EMPTY + + def dissimilarity(self, fp_row, fp_col, idx): + + if self.model_rdm_triu[idx] == ENTRY_EMPTY: + idx, _, _, spearman = calc_spearman_rank_corr_from_files(fp_row, fp_col, -1, -1, idx, loader=self.loader) + return idx, 1 - spearman.correlation + + def apply(self, processes=1, chunksize=10, do_disable_tqdm=False): + + self._init_model_rdm_triu() + + with mp.get_context("spawn").Pool(processes=processes) as pool: + result = pool.starmap(self.dissimilarity, + tqdm( + [(self.pairs.get(idx)[0], + self.pairs.get(idx)[1], idx) + for idx in range(self.numels)], + total=self.numels, + disable=do_disable_tqdm), + chunksize=chunksize, + ) + + for idx, dissimilarity in result: + self.model_rdm_triu[idx] = dissimilarity + return self.model_rdm_triu + diff --git a/rsa/test_model_rdm_paired.py b/rsa/test_model_rdm_paired.py new file mode 100755 index 0000000..3232be0 --- /dev/null +++ b/rsa/test_model_rdm_paired.py @@ -0,0 +1,205 @@ +from nose import tools +from nose.tools import assert_equal, \ + assert_true, assert_false, \ + assert_raises, assert_list_equal +from unittest.mock import patch +import os +import tempfile +import shutil +import numpy as np +from scipy.stats import spearmanr + +from rsa.model_rdm_paired import ModelRDMPaired +from rsa.mat_utils import get_triu_off_diag_flat, triu_off_diag_to_mat +from rsa.rdm_loader import RDMLoaderNPZ, RDMLoaderInMemory + + +def rand_rdm(n): + rng = np.random.default_rng() + rdm = np.zeros((n, n)) + rows, cols = np.triu_indices_from(rdm, k=1) + vals = rng.uniform(0, 2, size=n * (n - 1) // 2) + for idx, val in enumerate(vals): + rdm[rows[idx], cols[idx]] = vals[idx] + # rdm += rdm.T + return get_triu_off_diag_flat(rdm) + + +class TestModelRDMPairedNoInput: + + def test_calc_model_rdm_flist_none(self): + + m = ModelRDMPaired(None) + mrdm = m.apply(do_disable_tqdm=True) + assert_equal(mrdm.size, 0) + + def test_calc_model_rdm_flist_empty(self): + m = ModelRDMPaired([]) + mrdm = m.apply(do_disable_tqdm=True) + assert_equal(mrdm.size, 0) + +class TestModelRDMPairedInput2DMat: + + @classmethod + def setup_class(cls): + cls.dir_tmp = tempfile.mkdtemp() + + @classmethod + def teardown_class(cls): + shutil.rmtree(cls.dir_tmp) + pass + + @staticmethod + def assert_rdm_shape(rdm): + assert_equal(rdm.ndim, 2) + assert_equal(rdm.shape[0], 3) + assert_equal(rdm.shape[1], 3) + + def setup(self): + in_rdm1 = np.array([[0, 1, 2], + [1, 0, 0.5], + [2, 0.5, 0]]) + self.fpath_in1 = os.path.join(self.dir_tmp, 'in1.npy') + np.save(self.fpath_in1, in_rdm1) + in_rdm2 = in_rdm1 + self.fpath_in2 = os.path.join(self.dir_tmp, 'in2.npy') + np.save(self.fpath_in2, in_rdm2) + in_rdm3 = np.array([[0, 2, 1], + [2, 0, 0.5], + [1, 0.5, 0]]) + self.fpath_in3 = os.path.join(self.dir_tmp, 'in3.npy') + np.save(self.fpath_in3, in_rdm3) + in_rdm4 = in_rdm3 + self.fpath_in4 = os.path.join(self.dir_tmp, 'in4.npy') + np.save(self.fpath_in4, in_rdm4) + + def helper_calc_model_rdm(self, flist): + m = ModelRDMPaired(flist) + mrdm = m.apply(do_disable_tqdm=True) + return mrdm + + def test_calc_model_rdm_size(self): + + for sz_in_rdm in range(3, 4): + for num_rdms in [2, 5]: + fp_in_rdms = [] + for rdm_idx in range(num_rdms): + rdm = rand_rdm(sz_in_rdm) + # print(rdm.shape) + fp = os.path.join(self.dir_tmp, 'rand_inrdm_%d-%d-%d.npy' % (sz_in_rdm, num_rdms, rdm_idx)) + np.save(fp, rdm) + fp_in_rdms.append(fp) + mrdm = self.helper_calc_model_rdm(fp_in_rdms) + # print(mrdm.shape) + assert_equal(mrdm.size, num_rdms * (num_rdms - 1) // 2) + + def test_calc_model_rdm_identical(self): + fp_list = [self.fpath_in1, + self.fpath_in2, + self.fpath_in3, + self.fpath_in4] + mrdm = self.helper_calc_model_rdm(fp_list) + mrdm = triu_off_diag_to_mat(mrdm) + for r, fp_r in enumerate(fp_list): + for c, fp_c in enumerate(fp_list): + if fp_r == fp_c: + assert_equal(mrdm[r, c], 0) + assert_equal(mrdm[c, r], 0) + elif np.all(np.load(fp_r) == np.load(fp_c)): + # print(r,c) + assert_equal(mrdm[r, c], 0) + assert_equal(mrdm[c, r], 0) + else: + # print(mrdm[r, c], r, c) + assert_true(0 <= mrdm[r, c] <= 2) + + def test_calc_model_rdm_values(self): + fp_list = [self.fpath_in1, + self.fpath_in2, + self.fpath_in3, + self.fpath_in4] + mrdm = self.helper_calc_model_rdm(fp_list) + mrdm = triu_off_diag_to_mat(mrdm) + mrdm += mrdm.T + for r, fp_r in enumerate(fp_list): + rdm_r = np.load(fp_r) + self.assert_rdm_shape(rdm_r) + rdm_r = get_triu_off_diag_flat(rdm_r) if rdm_r.ndim > 1 else rdm_r + for c, fp_c in enumerate(fp_list): + rdm_c = np.load(fp_c) + self.assert_rdm_shape(rdm_c) + rdm_c = get_triu_off_diag_flat(rdm_c) if rdm_c.ndim > 1 else rdm_c + corr = spearmanr(rdm_r, rdm_c).correlation + assert_equal(mrdm[r, c], 1 - corr) + + +class TestModelRDMPairedInput2DMatNPZ(TestModelRDMPairedInput2DMat): + + def helper_calc_model_rdm(self, flist): + # switch from npy to npz + flist_npz = [] + for fp in flist: + my_in_rdm = np.load(fp) + fp_new = os.path.splitext(fp)[0] + '.npz' + np.savez(fp_new, in_rdm=my_in_rdm) + flist_npz.append(fp_new) + + loader = RDMLoaderNPZ() + loader.set_key('in_rdm') + m = ModelRDMPaired(flist_npz) + m.set_loader(loader) + mrdm = m.apply(do_disable_tqdm=True) + return mrdm + + +class TestModelRDMPairedInputTriuVec(TestModelRDMPairedInput2DMat): + + @staticmethod + def assert_rdm_shape(rdm): + assert_equal(rdm.ndim, 1) + assert_equal(rdm.shape[0], 3 * (3 - 1) // 2) + + def setup(self): + TestModelRDMPairedInput2DMat.setup(self) + in_rdm1 = np.array([[0, 1, 2], + [1, 0, 0.5], + [2, 0.5, 0]]) + for idx in range(1, 5): + fp = os.path.join(self.dir_tmp, 'in%d.npy' % idx) + in_rdm = np.load(fp) + np.save(fp, get_triu_off_diag_flat(in_rdm)) + + +class ModelRDMPairedScaled(ModelRDMPaired): + def dissimilarity(self, fp_row, fp_col, idx): + idx, dissimilarity = ModelRDMPaired.dissimilarity(self, fp_row, fp_col, idx) + return idx, 100 * dissimilarity + + +class TestModelRDMCorrelation(TestModelRDMPairedInputTriuVec): + + @staticmethod + def assert_rdm_shape(rdm): + assert_equal(rdm.ndim, 1) + assert_equal(rdm.shape[0], 3 * (3 - 1) // 2) + + def test_calc_model_rdm_values(self): + fp_list = [self.fpath_in1, + self.fpath_in2, + self.fpath_in3, + self.fpath_in4] + mrdm = self.helper_calc_model_rdm(fp_list) + + m = ModelRDMPairedScaled(fp_list) + mrdm = m.apply(do_disable_tqdm=True) + mrdm = triu_off_diag_to_mat(mrdm) + mrdm += mrdm.T + for r, fp_r in enumerate(fp_list): + rdm_r = np.load(fp_r) + self.assert_rdm_shape(rdm_r) + rdm_r = get_triu_off_diag_flat(rdm_r) if rdm_r.ndim > 1 else rdm_r + for c, fp_c in enumerate(fp_list): + rdm_c = np.load(fp_c) + self.assert_rdm_shape(rdm_c) + rdm_c = get_triu_off_diag_flat(rdm_c) if rdm_c.ndim > 1 else rdm_c + assert_equal(mrdm[r, c], 100 * (1 - spearmanr(rdm_r, rdm_c).correlation)) From 2246c922d15921229015b847886492453fd8b2f2 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Thu, 26 Jun 2025 12:02:49 +0200 Subject: [PATCH 36/40] metho to load from dictionary --- rsa/cache/rdm_cache.py | 45 ++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/rsa/cache/rdm_cache.py b/rsa/cache/rdm_cache.py index 0814f96..c001a56 100644 --- a/rsa/cache/rdm_cache.py +++ b/rsa/cache/rdm_cache.py @@ -8,31 +8,34 @@ def __init__(self): self.cache_dict = {} self.key_handler = CacheKey() + def load_from_dict(self, data): + + version = data.get('version', None) + if version is None: + self.cache_dict = data + + elif version == 250605: + + if 'cache' in data and 'flist' in data: + flist = data['flist'] + cache_dict_mapped = data['cache'] + separator = data.get('separator', self.key_handler.separator) + # update separator + self.key_handler.separator = separator + for k, v in data['cache'].items(): + xi, yi = self.key_handler.split(k) + x = flist[int(xi)] + y = flist[int(yi)] + k_new = self.key_handler.join(x, y) + self.cache_dict[k_new] = v + else: + raise KeyError("Cache file does not contain 'cache' or 'flist' keys.") + def load_from_file(self, fp_cache): with open(fp_cache, 'r') as h: data = yaml.safe_load(h) - version = data.get('version', None) - - if version is None: - self.cache_dict = data - - elif version == 250605: - - if 'cache' in data and 'flist' in data: - flist = data['flist'] - cache_dict_mapped = data['cache'] - separator = data.get('separator', self.key_handler.separator) - # update separator - self.key_handler.separator = separator - for k, v in data['cache'].items(): - xi, yi = self.key_handler.split(k) - x = flist[int(xi)] - y = flist[int(yi)] - k_new = self.key_handler.join(x, y) - self.cache_dict[k_new] = v - else: - raise KeyError("Cache file does not contain 'cache' or 'flist' keys.") + self.load_from_dict(data) def save_to_file(self, fp_dst): with open(fp_dst, 'w') as h: From 35b33a9be811db624b7cb2b7e5fdcc909c8b2eab Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Thu, 7 Aug 2025 18:00:56 +0200 Subject: [PATCH 37/40] handle empty results --- rsa/model_rdm.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rsa/model_rdm.py b/rsa/model_rdm.py index f78fa7a..1d6b529 100755 --- a/rsa/model_rdm.py +++ b/rsa/model_rdm.py @@ -54,7 +54,9 @@ def apply(self, processes=1, chunksize=10, do_disable_tqdm=False): chunksize=chunksize, ) - for idx, dissimilarity in result: - self.model_rdm_triu[idx] = dissimilarity + for item in result: + if item is not None: + idx, dissimilarity = item + self.model_rdm_triu[idx] = dissimilarity return self.model_rdm_triu From a217375de4f18e0fa39a3ac43e2ee121cf254837 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Thu, 7 Aug 2025 18:01:16 +0200 Subject: [PATCH 38/40] keep track of cache hits --- rsa/model_rdm_cached.py | 7 +++++++ rsa/test_model_rdm_cached.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/rsa/model_rdm_cached.py b/rsa/model_rdm_cached.py index 703447b..2be28f2 100755 --- a/rsa/model_rdm_cached.py +++ b/rsa/model_rdm_cached.py @@ -8,6 +8,7 @@ class ModelRDMCached(ModelRDM): def __init__(self, fpath_list, fp_cache): super().__init__(fpath_list) + self.cache_hits = -1 self.cache = None self.fp_cache = fp_cache self.load_cache(self.fp_cache) @@ -30,6 +31,8 @@ def _init_model_rdm_triu(self): self.model_rdm_triu[idx] = self.cache.get(fp_row, fp_col, ENTRY_EMPTY) + self.cache_hits = np.count_nonzero(self.model_rdm_triu != ENTRY_EMPTY) + def apply(self, processes=1, chunksize=10, do_disable_tqdm=False): self.model_rdm_triu = super().apply(processes=processes, @@ -46,3 +49,7 @@ def apply(self, processes=1, chunksize=10, do_disable_tqdm=False): self.cache.save_to_file(self.fp_cache) return self.model_rdm_triu + + + def get_cache_hits(self): + return self.cache_hits diff --git a/rsa/test_model_rdm_cached.py b/rsa/test_model_rdm_cached.py index f4bde57..d45ffdb 100755 --- a/rsa/test_model_rdm_cached.py +++ b/rsa/test_model_rdm_cached.py @@ -2,9 +2,12 @@ from nose.tools import assert_equal, \ assert_true, assert_false, \ assert_raises, assert_list_equal +import shutil +import tempfile import os import yaml +from rsa.cache.rdm_cache import RDMCache from rsa.model_rdm_cached import ModelRDMCached from rsa.test_model_rdm import TestModelRDMInput2DMat, \ @@ -64,3 +67,34 @@ def helper_calc_model_rdm(self, flist): mrdm = helper_calc_model_rdm_with_cache(flist, fp_cache) return mrdm + + +class TestModelRDMCachedCacheHits: + + @classmethod + def setup_class(cls): + cls.dir_tmp = tempfile.mkdtemp() + + @classmethod + def teardown_class(cls): + shutil.rmtree(cls.dir_tmp) + pass + + def test_cache_hits_cache_hits_all(self): + flist = [os.path.join(self.dir_tmp, fp) for fp in ['a.npy', 'b.npy', 'c.npy']] + + data = { + "version": 250605, + "cache": {"0__-__1": 0.1, "0__-__2": 0.2, "1__-__2": 0.12}, + "flist": flist, + "separator": "__-__" + } + cache = RDMCache() + cache.load_from_dict(data) + + fp_cache = os.path.join(self.dir_tmp, 'my_cache.yml') + cache.save_to_file(fp_cache) + + m = ModelRDMCached(flist, fp_cache) + m.apply(do_disable_tqdm=True) + assert_equal(m.get_cache_hits(), 3) \ No newline at end of file From 43d7ff0a3873e4a3201b99b5a6b87a48ca9905ce Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Thu, 28 Aug 2025 16:45:00 +0200 Subject: [PATCH 39/40] format --- rsa/test_model_rdm_paired.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rsa/test_model_rdm_paired.py b/rsa/test_model_rdm_paired.py index 3232be0..a4b428b 100755 --- a/rsa/test_model_rdm_paired.py +++ b/rsa/test_model_rdm_paired.py @@ -28,15 +28,15 @@ def rand_rdm(n): class TestModelRDMPairedNoInput: def test_calc_model_rdm_flist_none(self): - m = ModelRDMPaired(None) mrdm = m.apply(do_disable_tqdm=True) assert_equal(mrdm.size, 0) def test_calc_model_rdm_flist_empty(self): - m = ModelRDMPaired([]) - mrdm = m.apply(do_disable_tqdm=True) - assert_equal(mrdm.size, 0) + m = ModelRDMPaired([]) + mrdm = m.apply(do_disable_tqdm=True) + assert_equal(mrdm.size, 0) + class TestModelRDMPairedInput2DMat: From 93dac0934908c5a6e7f07fa65986b0e775affe39 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Mon, 29 Sep 2025 16:22:55 +0200 Subject: [PATCH 40/40] add logging to mrdm cache --- rsa/model_rdm.py | 11 +++++++++-- rsa/model_rdm_cached.py | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/rsa/model_rdm.py b/rsa/model_rdm.py index 1d6b529..fe3421a 100755 --- a/rsa/model_rdm.py +++ b/rsa/model_rdm.py @@ -1,7 +1,6 @@ import multiprocessing as mp import errno -from pathlib import Path -import os +import logging from tqdm import tqdm import numpy as np from rsa.model_rdm_utils import calc_spearman_rank_corr_from_files, ENTRY_EMPTY @@ -12,6 +11,14 @@ class ModelRDM: def __init__(self, fpath_list): + self.logger = logging.getLogger(self.__class__.__name__) + if not self.logger.hasHandlers(): + handler = logging.StreamHandler() + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formatter) + self.logger.addHandler(handler) + self.logger.setLevel(logging.INFO) + self.logger.debug("Logger initialized.") # Oflloaded to loader, but should still check before apply if applicable with loader TODO # for fp in fpath_list: # if not Path(fp).is_file(): diff --git a/rsa/model_rdm_cached.py b/rsa/model_rdm_cached.py index 2be28f2..9f92d6e 100755 --- a/rsa/model_rdm_cached.py +++ b/rsa/model_rdm_cached.py @@ -32,6 +32,7 @@ def _init_model_rdm_triu(self): self.model_rdm_triu[idx] = self.cache.get(fp_row, fp_col, ENTRY_EMPTY) self.cache_hits = np.count_nonzero(self.model_rdm_triu != ENTRY_EMPTY) + self.logger.debug(f"Cache hits: {self.cache_hits} / {self.model_rdm_triu.size} ({(self.cache_hits/self.model_rdm_triu.size)*100:.2f}%)") def apply(self, processes=1, chunksize=10, do_disable_tqdm=False):