Merge pull request #48 from bonsai-rx/dev/torchsharp

Integrating torchsharp functionality into Bonsai

Author: glopesdev

Bonsai.ML.sln

@@ -30,6 +30,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bonsai.ML.LinearDynamicalSy
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bonsai.ML.HiddenMarkovModels.Design", "src\Bonsai.ML.HiddenMarkovModels.Design\Bonsai.ML.HiddenMarkovModels.Design.csproj", "{FC395DDC-62A4-4E14-A198-272AB05B33C7}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bonsai.ML.Torch", "src\Bonsai.ML.Torch\Bonsai.ML.Torch.csproj", "{06FCC9AF-CE38-44BB-92B3-0D451BE88537}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -72,6 +74,10 @@ Global
         {FC395DDC-62A4-4E14-A198-272AB05B33C7}.Debug|Any CPU.Build.0 = Debug|Any CPU
         {FC395DDC-62A4-4E14-A198-272AB05B33C7}.Release|Any CPU.ActiveCfg = Release|Any CPU
         {FC395DDC-62A4-4E14-A198-272AB05B33C7}.Release|Any CPU.Build.0 = Release|Any CPU
+        {06FCC9AF-CE38-44BB-92B3-0D451BE88537}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+        {06FCC9AF-CE38-44BB-92B3-0D451BE88537}.Debug|Any CPU.Build.0 = Debug|Any CPU
+        {06FCC9AF-CE38-44BB-92B3-0D451BE88537}.Release|Any CPU.ActiveCfg = Release|Any CPU
+        {06FCC9AF-CE38-44BB-92B3-0D451BE88537}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -86,6 +92,7 @@ Global
 		{39A4414F-52B1-42D7-82FA-E65DAD885264} = {12312384-8828-4786-AE19-EFCEDF968290}
         {A135C7DB-EA50-4FC6-A6CB-6A5A5CC5FA13} = {12312384-8828-4786-AE19-EFCEDF968290}
         {17DF50BE-F481-4904-A4C8-5DF9725B2CA1} = {12312384-8828-4786-AE19-EFCEDF968290}
+        {06FCC9AF-CE38-44BB-92B3-0D451BE88537} = {12312384-8828-4786-AE19-EFCEDF968290}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {B6468F13-97CD-45E0-9E1E-C122D7F1E09F}

README.md

@@ -42,6 +42,9 @@ Facilitates inference using Hidden Markov Models (HMMs). It interfaces with the
 ### Bonsai.ML.HiddenMarkovModels.Design
 Visualizers and editor features for the HiddenMarkovModels package.
 
+### Bonsai.ML.Torch
+Interfaces with the [TorchSharp](https://github.com/dotnet/TorchSharp) package, a C# wrapper around the torch library. Provides tooling for manipulating tensors, performing linear algebra, training and inference with deep neural networks, and more. 
+
 > [!NOTE]
 > Bonsai.ML packages can be installed through Bonsai's integrated package manager and are generally ready for immediate use. However, some packages may require additional installation steps. Refer to the specific package section for detailed installation guides and documentation.
 

docs/articles/Torch/torch-overview.md

@@ -0,0 +1,44 @@
+# Bonsai.ML.Torch - Overview
+
+The Torch package provides a Bonsai interface to interact with [TorchSharp](https://github.com/dotnet/TorchSharp). This package adds powerful functionality into Bonsai, namely the ability to perform tensor manipulations, type conversions, complex linear algebra, deep neural networks, and support for GPU processing.
+
+## Installation Guide
+
+The Bonsai.ML.Torch package can be installed through the Bonsai Package Manager and depends on the TorchSharp library. Additionally, running the package requires installing the specific torch DLLs needed for your desired application. The steps for installing are outlined below.
+
+### Running on the CPU 
+For running the package using the CPU, the `TorchSharp-cpu` library should be installed through the NuGet package manager.
+
+### Running on the GPU
+To run torch on the GPU, you first need to ensure that you have a CUDA compatible device installed on your system. 
+
+Next, you must follow the [CUDA installation guide for Windows](https://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/index.html) or the [guide for Linux](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html). Please make sure to install the correct CUDA version ([v12.1](https://developer.nvidia.com/cuda-12-1-0-download-archive)), as `TorchSharp` currently only supports this version.
+
+Next, you need to install the `cuDNN v9` library following the [guide for Windows](https://docs.nvidia.com/deeplearning/cudnn/latest/installation/windows.html) or the [guide for Linux](https://docs.nvidia.com/deeplearning/cudnn/latest/installation/linux.html). Again, you need to ensure you have the correct version installed (v9). You should consult [nvidia's support matrix](https://docs.nvidia.com/deeplearning/cudnn/latest/reference/support-matrix.html) to ensure the versions of CUDA and cuDNN you installed are compatible with your specific OS, graphics driver, and hardware.
+
+Once complete, you need to install the cuda-compatible torch libraries and place them into the correct location. You can download the libraries from [the PyTorch website](https://pytorch.org/get-started/locally/) with the following options selected:
+
+- PyTorch Build: Stable (2.5.1)
+- OS: [Your OS]
+- Package: LibTorch
+- Language: C++/Java
+- Compute Platform: CUDA 12.1
+
+Finally, extract the zip folder and copy the contents of the `lib` folder into the `Extensions` folder of your bonsai installation directory.
+
+> [!WARNING]
+> You can only install one of the above CPU or GPU packages, not both. The GPU package contains all of the support of the CPU package. 
+
+## Getting Started
+
+The `Bonsai.ML.Torch` package primarily provides tooling and functionality for users to interact with and manipulate `Tensor` objects, the core data type of torch which underlies most advanced operations. Additionally, the package provides some capabilities for defining neural network architectures, running forward inference, and learning via back propagation.
+
+## Tensor Operations
+The package provides several ways to work with tensors. Users can initialize tensors, (`Ones`, `Zeros`, etc.), create tensors from .NET data types, (`ToTensor`), and define custom tensors using Python-like syntax (`CreateTensor`). Tensors can be converted back to .NET types using the `ToArray` node (for flattening tensors into a unidimensional array) or the `ToNDArray` node (for preserving multidimensional array shapes). Furthermore, the `Tensor` object contains many extension methods which can be used via scripting with `ExpressionTransform` (for example, `it.sum()` to sum a tensor, or `it.T` to transpose), and works with overloaded operators (for example, `Zip` -> `Multiply`). It is also possible to use the `ExpressionTransform` node to access individual elements of a tensor, using the syntax `it.ReadCpuT(0)` where `T` is a primitive .NET data type (i.e. `Single`, `Double`, `Int64`, etc.).
+
+
+## Running on the GPU
+Users must be explicit about running computations on the GPU. First, the `InitializeDeviceType` node must run with a CUDA-compatible GPU. If successful, the node will return a `Device` object representing the GPU. Afterwards, the tensors can either be created directly on the GPU by setting the `Device` property to the GPU device, or moved to the GPU using the `ToDevice` node. For most tensor operations to work, all of the tensors involved must be present on the same device.
+
+## Neural Networks
+The package provides initial support for working with torch `Module` objects, the core type representing deep neural networks. The `LoadModuleFromArchitecture` node allows users to select from a list of common architectures, and can optionally load pretrained weights from disk. Additionally, the package supports loading `TorchScript` modules with the `LoadScriptModule` node, which enables users to use torch modules saved in the `.pt` file format. Users can then use the `Forward` node to run inference and the `Backward` node to run back propagation.   

docs/articles/toc.yml

@@ -13,4 +13,7 @@
 - name: Overview
   href: HiddenMarkovModels/hmm-overview.md
 - name: Getting Started
-  href: HiddenMarkovModels/hmm-getting-started.md
+  href: HiddenMarkovModels/hmm-getting-started.md
+- name: Torch
+- name: Overview
+  href: Torch/torch-overview.md

src/Bonsai.ML.Data/ArrayHelper.cs src/Bonsai.ML.Data/JsonDataHelper.cssrc/Bonsai.ML.Data/ArrayHelper.cs renamed to src/Bonsai.ML.Data/JsonDataHelper.cs

@@ -3,32 +3,32 @@
 using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
+using System.Reflection;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 
 namespace Bonsai.ML.Data
 {
     /// <summary>
-    /// Provides a set of static methods for working with arrays.
+    /// Provides a set of static methods for working with JSON data containing numeric and boolean values.
     /// </summary>
-    public static class ArrayHelper
+    public static class JsonDataHelper
     {
         /// <summary>
-        /// Parses the input string into an object of the specified type. 
-        /// If the input is a JSON array, the method will attempt to parse it into a list or array of the specified type. 
+        /// Parses the input string into an object of the specified type.
         /// </summary>
         /// <param name="input">The string to parse.</param>
         /// <param name="dtype">The data type of the object.</param>
         /// <returns>An object of the specified type containing the parsed data.</returns>
-        public static object ParseString(string input, Type dtype)
+        public static object Parse(string input, Type dtype)
         {
             if (!IsValidJson(input))
             {
                 throw new ArgumentException($"Parameter: {nameof(input)} is not valid JSON.");
             }
 
             var token = JsonConvert.DeserializeObject<JToken>(input);
-            
+
             if (token is JValue value)
             {
                 return Convert.ChangeType(value, dtype);
@@ -67,7 +67,7 @@ public static object ParseToken(JToken token, Type dtype)
             }
             else if (token is JArray)
             {
-                if (token[0] is JValue)
+                if (token.Count() == 0 || token[0] is JValue)
                 {
                     if (token.All(item => item is JValue))
                     {
@@ -78,7 +78,8 @@ public static object ParseToken(JToken token, Type dtype)
                 }
                 else
                 {
-                    var subArrayDimensions = token.Cast<JArray>().Select(value => {
+                    var subArrayDimensions = token.Cast<JArray>().Select(value =>
+                    {
                         var depth = ParseDepth(value);
                         return ParseDimensions(value, depth);
                     }).ToList();
@@ -147,7 +148,7 @@ private static int[] ParseDimensions(JToken token, int depth, int currentLevel =
             if (currentLevel > 0 && token is JArray arr && arr.All(x => x is JArray))
             {
                 var subArrayDimensions = new HashSet<string>();
-                foreach (JArray subArr in arr)
+                foreach (JArray subArr in arr.Cast<JArray>())
                 {
                     int[] subDims = ParseDimensions(subArr, depth - currentLevel, currentLevel + 1);
                     subArrayDimensions.Add(string.Join(",", subDims));
@@ -159,7 +160,7 @@ private static int[] ParseDimensions(JToken token, int depth, int currentLevel =
                 }
             }
 
-            return dimensions.ToArray();
+            return [.. dimensions];
         }
 
         private static void PopulateArray(JToken token, Array array, int[] indices, Type dtype)
@@ -185,13 +186,13 @@ private static object CreateList(JToken token, Type dtype)
         {
             var listType = typeof(List<>).MakeGenericType(DetermineListType(token, dtype));
             var list = (IList)Activator.CreateInstance(listType);
-            
+
             foreach (var item in token)
             {
                 var result = ParseToken(item, dtype);
                 list.Add(result);
             }
-            
+
             return list;
         }
 
@@ -224,5 +225,186 @@ private static Type DetermineListType(JToken token, Type type)
                 return typeof(object);
             }
         }
+
+        /// <summary>
+        /// Formats the input object into a string representation that is consistent with JSON syntax.
+        /// </summary>
+        /// <param name="obj">The object to format.</param>
+        /// <returns>A string representation that is consistent with JSON syntax.</returns>
+        public static string Format(object obj)
+        {
+            var sb = new StringBuilder();
+            int depth = 0;
+            Format(obj, sb, depth);
+            return sb.ToString();
+        }
+
+        private static void Format(object obj, StringBuilder sb, int depth)
+        {
+            switch (obj)
+            {
+                case null:
+                    sb.Append("null");
+                    break;
+                case string:
+                case char:
+                    sb.Append('"').Append(obj).Append('"');
+                    break;
+                case bool:
+                    sb.Append(obj.ToString().ToLower());
+                    break;
+                case int:
+                case double:
+                case float:
+                case long:
+                case short:
+                case byte:
+                case ushort:
+                case uint:
+                case ulong:
+                case sbyte:
+                case decimal:
+                    sb.Append(obj);
+                    break;
+                case Array:
+                    FormatArray(obj, sb, depth);
+                    break;
+                case IList:
+                    FormatList(obj, sb, depth);
+                    break;
+                case IDictionary:
+                    FormatDictionary(obj, sb, depth);
+                    break;
+                case object tuple when obj.GetType().IsGenericType && 
+                        obj.GetType().GetGenericTypeDefinition() == typeof(Tuple<,>) || 
+                        obj.GetType().GetGenericTypeDefinition() == typeof(Tuple<,,>) ||
+                        obj.GetType().GetGenericTypeDefinition() == typeof(Tuple<,,,>) ||
+                        obj.GetType().GetGenericTypeDefinition() == typeof(Tuple<,,,,>) ||
+                        obj.GetType().GetGenericTypeDefinition() == typeof(Tuple<,,,,,>) ||
+                        obj.GetType().GetGenericTypeDefinition() == typeof(Tuple<,,,,,,>) ||
+                        obj.GetType().GetGenericTypeDefinition() == typeof(Tuple<,,,,,,,>):
+                    FormatTuple(obj, sb, depth);
+                    break;
+                default:
+                    FormatDefault(obj, sb, depth);
+                    break;
+            }
+        }
+
+        private static void FormatArray(object obj, StringBuilder sb, int depth)
+        {
+            var array = (Array)obj;
+            if (array.Rank == 1)
+            {
+                sb.Append('[');
+                for (int i = 0; i < array.Length; i++)
+                {
+                    if (i > 0)
+                    {
+                        sb.Append(", ");
+                    }
+                    Format(array.GetValue(i), sb, depth + 1);
+                }
+                sb.Append(']');
+            }
+            else
+            {
+                FormatNDArray(array, new int[array.Rank], 0, sb, depth);
+            }
+        }
+
+        private static void FormatNDArray(Array array, int[] indices, int dimension, StringBuilder sb, int depth)
+        {
+            if (dimension == array.Rank)
+            {
+                Format(array.GetValue(indices), sb, depth + 1);
+                return;
+            }
+
+            sb.Append('[');
+            for (int i = 0; i < array.GetLength(dimension); i++)
+            {
+                if (i > 0)
+                {
+                    sb.Append(", ");
+                }
+                indices[dimension] = i;
+                FormatNDArray(array, indices, dimension + 1, sb, depth + 1);
+            }
+            sb.Append(']');
+        }
+
+        private static void FormatList(object obj, StringBuilder sb, int depth)
+        {
+            var list = (IList)obj;
+            sb.Append('[');
+            for (int i = 0; i < list.Count; i++)
+            {
+                if (i > 0)
+                {
+                    sb.Append(", ");
+                }
+                Format(list[i], sb, depth + 1);
+            }
+            sb.Append(']');
+        }
+
+        private static void FormatDictionary(object obj, StringBuilder sb, int depth)
+        {
+            var dict = (IDictionary)obj;
+            sb.Append('{');
+            bool first = true;
+            foreach (DictionaryEntry entry in dict)
+            {
+                if (!first)
+                {
+                    sb.Append(", ");
+                }
+                else
+                {
+                    first = false;
+                }
+                Format(entry.Key, sb, depth + 1);
+                sb.Append(": ");
+                Format(entry.Value, sb, depth + 1);
+            }
+            sb.Append('}');
+        }
+
+        private static void FormatTuple(object obj, StringBuilder sb, int depth)
+        {
+            var itemProperties = obj.GetType()
+                .GetProperties()
+                .Where(p => p.Name.StartsWith("Item"))
+                .OrderBy(p => p.Name)
+                .ToArray();
+
+            sb.Append('(');
+            for (int i = 0; i < itemProperties.Length; i++)
+            {
+                if (i > 0)
+                {
+                    sb.Append(", ");
+                }
+                Format(itemProperties[i].GetValue(obj), sb, depth + 1);
+            }
+            sb.Append(')');
+        }
+
+        private static void FormatDefault(object obj, StringBuilder sb, int depth)
+        {
+            var properties = obj.GetType().GetProperties(BindingFlags.Public | BindingFlags.Instance);
+            sb.Append('{');
+            for (int i = 0; i < properties.Length; i++)
+            {
+                if (i > 0)
+                {
+                    sb.Append(", ");
+                }
+                sb.Append('"').Append(properties[i].Name).Append("\": ");
+                Format(properties[i].GetValue(obj), sb, depth + 1);
+            }
+            sb.Append('}');
+        }
     }
 }