Updated modules to support module combinator builder pattern

Author: ncguilbeault

src/Bonsai.ML.Torch/NeuralNets/ActivationFunctionBuilder.cs

@@ -0,0 +1,72 @@
+using System.ComponentModel;
+using System.Xml.Serialization;
+
+namespace Bonsai.ML.Torch.NeuralNets;
+
+/// <summary>
+/// Represents an operator that creates an activation function.
+/// </summary>
+[XmlInclude(typeof(NonLinearActivations.ContinuouslyDifferentiableExponential))]
+[XmlInclude(typeof(NonLinearActivations.Exponential))]
+[XmlInclude(typeof(NonLinearActivations.Gated))]
+[XmlInclude(typeof(NonLinearActivations.GaussianError))]
+[XmlInclude(typeof(NonLinearActivations.HardShrinkage))]
+[XmlInclude(typeof(NonLinearActivations.HardSigmoid))]
+[XmlInclude(typeof(NonLinearActivations.Hardswish))]
+[XmlInclude(typeof(NonLinearActivations.HardTanh))]
+[XmlInclude(typeof(NonLinearActivations.LeakyRectified))]
+[XmlInclude(typeof(NonLinearActivations.LogSigmoid))]
+[XmlInclude(typeof(NonLinearActivations.LogSoftmax))]
+[XmlInclude(typeof(NonLinearActivations.Mish))]
+[XmlInclude(typeof(NonLinearActivations.MultiheadAttention))]
+[XmlInclude(typeof(NonLinearActivations.ParametricRectified))]
+[XmlInclude(typeof(NonLinearActivations.RandomizedLeakyRectified))]
+[XmlInclude(typeof(NonLinearActivations.Rectified))]
+[XmlInclude(typeof(NonLinearActivations.RectifiedBounded))]
+[XmlInclude(typeof(NonLinearActivations.ScaledExponential))]
+[XmlInclude(typeof(NonLinearActivations.Sigmoid))]
+[XmlInclude(typeof(NonLinearActivations.SigmoidWeighted))]
+[XmlInclude(typeof(NonLinearActivations.Softmax))]
+[XmlInclude(typeof(NonLinearActivations.Softmax2d))]
+[XmlInclude(typeof(NonLinearActivations.Softmin))]
+[XmlInclude(typeof(NonLinearActivations.Softplus))]
+[XmlInclude(typeof(NonLinearActivations.SoftShrinkage))]
+[XmlInclude(typeof(NonLinearActivations.Softsign))]
+[XmlInclude(typeof(NonLinearActivations.Tanh))]
+[XmlInclude(typeof(NonLinearActivations.TanhShrinkage))]
+[XmlInclude(typeof(NonLinearActivations.Threshold))]
+[DefaultProperty(nameof(ActivationFunction))]
+[Combinator]
+[Description("Creates an activation function.")]
+[WorkflowElementCategory(ElementCategory.Source)]
+public class ActivationFunctionBuilder : ModuleCombinatorBuilder, INamedElement
+{
+    /// <inheritdoc/>
+    public override Range<int> ArgumentRange => Range.Create(0, 1);
+
+    string INamedElement.Name => $"ActivationFunction.{GetElementDisplayName(ActivationFunction)}";
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="ActivationFunctionBuilder"/> class.
+    /// </summary>
+    public ActivationFunctionBuilder()
+    {
+        Module = new NonLinearActivations.Rectified();
+    }
+    
+    /// <summary>
+    /// Gets or sets the specific activation function to create.
+    /// </summary>
+    [DesignOnly(true)]
+    [DisplayName("Module")]
+    [Externalizable(false)]
+    [RefreshProperties(RefreshProperties.All)]
+    [Category(nameof(CategoryAttribute.Design))]
+    [Description("The specific activation function to create.")]
+    [TypeConverter(typeof(ModuleTypeConverter))]
+    public object ActivationFunction
+    {
+        get => Module;
+        set => Module = value;
+    }
+}

src/Bonsai.ML.Torch/NeuralNets/Backward.cs

@@ -1,23 +1,14 @@
 using System;
-using System.Linq;
 using System.ComponentModel;
-using System.Reactive;
-using System.Reactive.Disposables;
 using System.Reactive.Linq;
-using System.Xml.Serialization;
 using static TorchSharp.torch;
-using static TorchSharp.torch.nn;
-using static TorchSharp.torch.optim;
-using static TorchSharp.torch.optim.lr_scheduler;
-using System.Threading.Tasks;
 
 namespace Bonsai.ML.Torch.NeuralNets;
 
 /// <summary>
 /// Represents an operator that computes backward on the input tensor.
 /// </summary>
 [Combinator]
-[ResetCombinator]
 [Description("Computes backward on the input tensor.")]
 [WorkflowElementCategory(ElementCategory.Sink)]
 public class Backward

src/Bonsai.ML.Torch/NeuralNets/CollectParameters.cs

@@ -1,14 +1,10 @@
-using Bonsai.Expressions;
 using System;
 using System.ComponentModel;
 using System.Reactive.Linq;
 using System.Linq;
-using System.Linq.Expressions;
 using System.Collections.Generic;
-using System.Reflection;
-using TorchSharp;
 using TorchSharp.Modules;
-using static TorchSharp.torch;
+using static TorchSharp.torch.nn;
 
 namespace Bonsai.ML.Torch.NeuralNets;
 
@@ -23,9 +19,13 @@ public class CollectParameters
     /// <summary>
     /// Collects the parameters from torch modules into a collection.
     /// </summary>
-    public IObservable<IEnumerable<Parameter>> Process(params IObservable<nn.Module>[] sources)
+    /// <param name="sources"></param>
+    /// <returns></returns>
+    public IObservable<IEnumerable<Parameter>> Process(params IObservable<Module>[] sources)
     {
-        return Observable.Concat(sources)
+        return Observable
+            .Concat(sources.Select(source =>
+                source.Take(1)))
             .SelectMany(module =>
             {
                 return module.parameters(recurse: true);

src/Bonsai.ML.Torch/NeuralNets/Container/Sequential.cs

@@ -1,44 +1,27 @@
 using System;
-using System.Collections.Generic;
 using System.ComponentModel;
 using System.Reactive.Linq;
-using TorchSharp;
 using static TorchSharp.torch;
 using static TorchSharp.torch.nn;
-using System.Xml.Serialization;
 using System.Linq;
 
 namespace Bonsai.ML.Torch.NeuralNets.Container;
 
 /// <summary>
-/// Creates a sequential model from the specified modules.
+/// Represents an operator that creates a sequential container.
 /// </summary>
-[Combinator]
-[ResetCombinator]
-[Description("Creates a sequential model from the specified modules.")]
-[WorkflowElementCategory(ElementCategory.Source)]
+/// <remarks>
+/// See <see href="https://pytorch.org/docs/stable/generated/torch.nn.Sequential.html"/> for more information.
+/// </remarks>
+[Description("Creates a sequential container.")]
 public class Sequential
 {
     /// <summary>
-    /// The device on which to create the sequential model.
-    /// </summary>
-    [XmlIgnore]
-    public Device? Device { get; set; } = null;
-    
-    /// <summary>
-    /// Generates an observable sequence that creates a sequential model from the input modules.
+    /// Creates a sequential container from the input modules.
     /// </summary>
     /// <returns></returns>
     public IObservable<Module<Tensor, Tensor>> Process(IObservable<Module<Tensor, Tensor>[]> source)
     {
-        return source.SelectMany(modules =>
-        {
-            var sequential = Sequential(modules);
-            if (Device is not null && Device != CPU)
-            {
-                sequential.to(Device);
-            }
-            return Observable.Return(sequential);
-        });
+        return source.Select(modules => Sequential(modules));
     }
 }

src/Bonsai.ML.Torch/NeuralNets/Convolution/Conv1d.cs

@@ -1,10 +1,8 @@
 using System;
 using System.ComponentModel;
-using System.Collections.Generic;
 using System.Reactive.Linq;
 using System.Xml.Serialization;
 using TorchSharp;
-using TorchSharp.Modules;
 using static TorchSharp.torch;
 using static TorchSharp.torch.nn;
 
@@ -13,61 +11,58 @@ namespace Bonsai.ML.Torch.NeuralNets.Convolution;
 /// <summary>
 /// Represents an operator that creates a 1D convolution module.
 /// </summary>
+/// <remarks>
+/// See <see href="https://pytorch.org/docs/stable/generated/torch.nn.Conv1d.html"/> for more information.
+/// </remarks>
 [Description("Creates a 1D convolution module.")]
 public class Conv1d
 {
     /// <summary>
-    /// The in_channels parameter for the Conv1d module.
+    /// The number of input channels in the input tensor.
     /// </summary>
-    [Description("The in_channels parameter for the Conv1d module")]
+    [Description("The number of input channels in the input tensor.")]
     public long InChannels { get; set; }
 
     /// <summary>
-    /// The out_channels parameter for the Conv1d module.
+    /// The number of output channels produced by the convolution.
     /// </summary>
-    [Description("The out_channels parameter for the Conv1d module")]
+    [Description("The number of output channels produced by the convolution.")]
     public long OutChannels { get; set; }
 
     /// <summary>
-    /// The kernelsize parameter for the Conv1d module.
+    /// The size of the convolution kernel.
     /// </summary>
-    [Description("The kernelsize parameter for the Conv1d module")]
+    [Description("The size of the convolution kernel.")]
     public long KernelSize { get; set; }
 
     /// <summary>
-    /// The stride parameter for the Conv1d module.
+    /// The stride of the convolution.
     /// </summary>
-    [Description("The stride parameter for the Conv1d module")]
+    [Description("The stride of the convolution.")]
     public long Stride { get; set; } = 1;
 
     /// <summary>
-    /// The padding parameter for the Conv1d module.
+    /// The padding added to both sides of the input.
     /// </summary>
-    [Description("The padding parameter for the Conv1d module")]
+    [Description("The padding added to both sides of the input.")]
     public long Padding { get; set; } = 0;
 
     /// <summary>
-    /// The output_padding parameter for the Conv1d module.
+    /// The spacing between kernel elements.
     /// </summary>
-    [Description("The output_padding parameter for the ConvTransposed1d module")]
-    public long OutputPadding { get; set; } = 0;
-
-    /// <summary>
-    /// The dilation parameter for the Conv1d module.
-    /// </summary>
-    [Description("The dilation parameter for the Conv1d module")]
+    [Description("The spacing between kernel elements.")]
     public long Dilation { get; set; } = 1;
 
     /// <summary>
-    /// The padding_mode parameter for the Conv1d module.
+    /// The mode of padding.
     /// </summary>
-    [Description("The padding_mode parameter for the Conv1d module")]
+    [Description("The mode of padding.")]
     public PaddingModes PaddingMode { get; set; } = PaddingModes.Zeros;
 
     /// <summary>
-    /// The groups parameter for the Conv1d module.
+    /// The number of blocked connections from input channels to output channels.
     /// </summary>
-    [Description("The groups parameter for the Conv1d module")]
+    [Description("The number of blocked connections from input channels to output channels.")]
     public long Groups { get; set; } = 1;
 
     /// <summary>
@@ -77,21 +72,22 @@ public class Conv1d
     public bool Bias { get; set; } = true;
 
     /// <summary>
-    /// The desired device of returned tensor.
+    /// The desired device of the returned tensor.
     /// </summary>
     [XmlIgnore]
-    [Description("The desired device of returned tensor")]
+    [Description("The desired device of the returned tensor")]
     public Device Device { get; set; } = null;
 
     /// <summary>
-    /// The desired data type of returned tensor.
+    /// The desired data type of the returned tensor.
     /// </summary>
-    [Description("The desired data type of returned tensor")]
+    [Description("The desired data type of the returned tensor")]
     public ScalarType? Type { get; set; } = null;
 
     /// <summary>
     /// Creates a Conv1d module.
     /// </summary>
+    /// <returns></returns>
     public IObservable<Module<Tensor, Tensor>> Process()
     {
         return Observable.Return(Conv1d(InChannels, OutChannels, KernelSize, Stride, Padding, Dilation, PaddingMode, Groups, Bias, Device, Type));
@@ -103,7 +99,6 @@ public IObservable<Module<Tensor, Tensor>> Process()
     /// <typeparam name="T"></typeparam>
     /// <param name="source"></param>
     /// <returns></returns>
-    /// <exception cref="InvalidOperationException"></exception>
     public IObservable<Module<Tensor, Tensor>> Process<T>(IObservable<T> source)
     {
         return source.Select(_ => Conv1d(InChannels, OutChannels, KernelSize, Stride, Padding, Dilation, PaddingMode, Groups, Bias, Device, Type));