Merge remote-tracking branch 'origin/main'

Author: tachella

.github/workflows/benchopt_test.yml

@@ -0,0 +1,40 @@
+name: Tests
+
+on:
+  push:
+    branches:
+      - main
+  create:
+    tags:
+      - '**'
+  pull_request:
+  schedule:
+    # Run every 1st of the month at 7:42am UTC.
+    - cron:  '42 7 1 * *'
+
+jobs:
+  find-benchmarks:
+    runs-on: ubuntu-latest
+    outputs:
+      benchmark-dirs: ${{ steps.find-dirs.outputs.dirs }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Find benchmark directories
+        id: find-dirs
+        run: |
+          # Find all directories containing an objective.py file
+          dirs=$(find . -maxdepth 2 -name "objective.py" -type f | xargs dirname | sed 's|./||' | jq -R -s -c 'split("\n")[:-1]')
+          echo "dirs=$dirs" >> $GITHUB_OUTPUT
+
+  benchopt_dev:
+    needs: find-benchmarks
+    strategy:
+      matrix:
+        benchmark_dir: ${{ fromJson(needs.find-benchmarks.outputs.benchmark-dirs) }}
+    uses: benchopt/template_benchmark/.github/workflows/test_benchmarks.yml@main
+    with:
+      benchopt_branch: benchopt@main
+      benchmark_dir: ${{ matrix.benchmark_dir }}
+
+  lint:
+    uses: benchopt/template_benchmark/.github/workflows/lint_benchmarks.yml@main

.gitignore

@@ -2,6 +2,7 @@
 **/__pycache__/
 **/__cache__/
 **/data/
+**/outputs/
 
 # IDE config folders
 **/.vscode/

benchmark_template/objective.py

@@ -1,6 +1,5 @@
 from benchopt import BaseObjective
 
-import time
 import deepinv as dinv
 from torch.utils.data import DataLoader
 
@@ -16,7 +15,9 @@ class Objective(BaseObjective):
 
     # Minimal version of benchopt required to run this benchmark.
     # Bump it up if the benchmark depends on a new feature of benchopt.
-    min_benchopt_version = "1.7"
+    min_benchopt_version = "1.8"
+
+    sampling_strategy = 'run_once'
 
     def set_data(self, dataset, physics):
         self.dataset = dataset

benchmark_template/solvers/solver1.py

@@ -10,10 +10,6 @@ class Solver(BaseSolver):
     # add any hyper-parameters here
     parameters = {}
 
-    sampling_strategy = 'run_once'
-
-    requirements = []
-
     def set_objective(self, train_dataset=None, physics=None):
         device = (
             dinv.utils.get_freer_gpu() if torch.cuda.is_available() else "cpu"

cbsd500_gaussian_denoising/datasets/cbsd68.py

@@ -13,15 +13,18 @@
 class Dataset(BaseDataset):
 
     name = "CBSD68"
-
     parameters = {
         'physics' : ['Denoising'],
         'noise' : ['GaussianNoise'],
-        'img_size': [256],
         'sigma': [0.1],
+        'img_size': [256],
         'debug': [False],
     }
 
+    test_parameters = {
+        "debug": [True]
+    }
+
     requirements = ["datasets"]
 
     def get_data(self):

cbsd500_gaussian_denoising/objective.py

@@ -1,6 +1,5 @@
 from benchopt import BaseObjective
 
-import time
 import deepinv as dinv
 from torch.utils.data import DataLoader
 
@@ -16,7 +15,10 @@ class Objective(BaseObjective):
 
     # Minimal version of benchopt required to run this benchmark.
     # Bump it up if the benchmark depends on a new feature of benchopt.
-    min_benchopt_version = "1.7"
+    min_benchopt_version = "1.8"
+
+    # Deactivate multiple runs for each solver
+    sampling_strategy = "run_once"
 
     def set_data(self, dataset, physics):
         self.dataset = dataset
@@ -44,7 +46,14 @@ def evaluate_result(self, model):
         return results
 
     def get_one_result(self):
-        return dict(model=lambda x: x)
+
+        class DummyModel:
+            def eval(self): pass
+
+            def __call__(self, x, physics=None):
+                return physics.A_adjoint(x)
+
+        return dict(model=DummyModel())
 
     def get_objective(self):
         return dict(

cbsd500_gaussian_denoising/solvers/drunet.py

@@ -8,10 +8,6 @@ class Solver(BaseSolver):
 
     parameters = {}
 
-    sampling_strategy = 'run_once'
-
-    requirements = []
-
     def set_objective(self, train_dataset=None, physics=None):
         self.model = dinv.models.ArtifactRemoval(
             dinv.models.DRUNet()

cbsd500_gaussian_denoising/solvers/restformer.py

@@ -8,10 +8,6 @@ class Solver(BaseSolver):
 
     parameters = {}
 
-    sampling_strategy = 'run_once'
-
-    requirements = []
-
     def set_objective(self, train_dataset=None, physics=None):
         self.model = dinv.models.ArtifactRemoval(
             dinv.models.Restormer()

div2k_gaussian_deblurring/datasets/div2k.py

@@ -21,6 +21,10 @@ class Dataset(BaseDataset):
         'debug': [False],
     }
 
+    test_parameters = {
+        "debug": [True]
+    }
+
     requirements = ["datasets"]
 
     def get_data(self):

div2k_gaussian_deblurring/objective.py

@@ -15,7 +15,10 @@ class Objective(BaseObjective):
 
     # Minimal version of benchopt required to run this benchmark.
     # Bump it up if the benchmark depends on a new feature of benchopt.
-    min_benchopt_version = "1.7"
+    min_benchopt_version = "1.8"
+
+    # Deactivate multiple runs for each solver
+    sampling_strategy = "run_once"
 
     def set_data(self, dataset, physics):
         self.dataset = dataset
@@ -44,7 +47,14 @@ def evaluate_result(self, model):
         return results
 
     def get_one_result(self):
-        return dict(model=lambda x: x)
+
+        class DummyModel:
+            def eval(self): pass
+
+            def __call__(self, x, physics=None):
+                return physics.A_adjoint(x)
+
+        return dict(model=DummyModel())
 
     def get_objective(self):
         return dict(