Layers JSON serialization improved, code refactoring

Author: Sergio0694

NeuralNetwork.NET/Helpers/Sha256.cs

@@ -0,0 +1,69 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Security.Cryptography;
+using System.Threading.Tasks;
+using JetBrains.Annotations;
+using NeuralNetworkNET.Extensions;
+
+namespace NeuralNetworkNET.Helpers
+{
+    /// <summary>
+    /// A static class that can be used to quickly calculate hashes from array of an arbitrary <see langword="struct"/> type
+    /// </summary>
+    public static class Sha256
+    {
+        // The SHA256 hash bytes length
+        private const int HashLength = 32;
+
+        /// <summary>
+        /// Calculates an hash for the input <typeparamref name="T"/> array
+        /// </summary>
+        /// <typeparam name="T">The type of items in the input array</typeparam>
+        /// <param name="array">The input array to process</param>
+        [PublicAPI]
+        [Pure, NotNull]
+        public static unsafe byte[] Hash<T>([NotNull] T[] array) where T : struct
+        {
+            int size = Unsafe.SizeOf<T>() * array.Length;
+            fixed (byte* p = &Unsafe.As<T, byte>(ref array[0]))
+            using (UnmanagedMemoryStream stream = new UnmanagedMemoryStream(p, size, size, FileAccess.Read))
+            using (SHA256 provider = SHA256.Create())
+            {
+                return provider.ComputeHash(stream);
+            }
+        }
+
+        /// <summary>
+        /// Calculates an aggregate hash for the input <typeparamref name="T"/> arrays
+        /// </summary>
+        /// <typeparam name="T">The type of items in the input arrays</typeparam>
+        /// <param name="arrays">The arrays to process</param>
+        [PublicAPI]
+        [Pure, NotNull]
+        public static unsafe byte[] Hash<T>([NotNull, ItemNotNull] params T[][] arrays) where T : struct
+        {
+            // Compute the hashes in parallel
+            if (arrays.Length == 0) return new byte[0];
+            if (arrays.Any(v => v.Length == 0)) throw new ArgumentException("The input array can't contain empty vectors");
+            byte[][] hashes = new byte[arrays.Length][];
+            Parallel.For(0, arrays.Length, i => hashes[i] = Hash(arrays[i])).AssertCompleted();
+
+            // Merge the computed hashes into a single bytes array
+            unchecked
+            {
+                byte[] result = new byte[HashLength];
+                fixed (byte* p = result)
+                    for (int i = 0; i < HashLength; i++)
+                    {
+                        uint hash = 17;
+                        for (int j = 0; j < hashes.Length; j++)
+                            hash = hash * 31 + hashes[j][i];
+                        p[i] = (byte)(hash % byte.MaxValue);
+                    }
+                return result;
+            }
+        }
+    }
+}

NeuralNetwork.NET/Networks/Layers/Abstract/BatchNormalizationLayerBase.cs

@@ -4,6 +4,7 @@
 using NeuralNetworkNET.APIs.Enums;
 using NeuralNetworkNET.APIs.Structs;
 using NeuralNetworkNET.Extensions;
+using NeuralNetworkNET.Helpers;
 using NeuralNetworkNET.Networks.Layers.Initialization;
 using NeuralNetworkNET.SupervisedLearning.Optimization;
 using Newtonsoft.Json;
@@ -32,6 +33,15 @@ internal abstract class BatchNormalizationLayerBase : WeightedLayerBase
         // The current iteration number (for the Cumulative Moving Average)
         private int _Iteration;
 
+        /// <summary>
+        /// Gets the current CMA factor used to update the <see cref="Mu"/> and <see cref="Sigma2"/> tensors
+        /// </summary>
+        [JsonProperty(nameof(CumulativeMovingAverageFactor), Order = 6)]
+        public float CumulativeMovingAverageFactor => 1f / (1 + _Iteration);
+
+        /// <inheritdoc/>
+        public override String Hash => Convert.ToBase64String(Sha256.Hash(Weights, Biases, Mu, Sigma2));
+
         /// <inheritdoc/>
         public override LayerType LayerType { get; } = LayerType.BatchNormalization;
 

NeuralNetwork.NET/Networks/Layers/Abstract/WeightedLayerBase.cs

@@ -1,11 +1,11 @@
 using System;
 using System.IO;
-using System.Security.Cryptography;
 using JetBrains.Annotations;
 using NeuralNetworkNET.APIs.Enums;
 using NeuralNetworkNET.APIs.Interfaces;
 using NeuralNetworkNET.APIs.Structs;
 using NeuralNetworkNET.Extensions;
+using NeuralNetworkNET.Helpers;
 using Newtonsoft.Json;
 
 namespace NeuralNetworkNET.Networks.Layers.Abstract
@@ -23,39 +23,7 @@ internal abstract class WeightedLayerBase : NetworkLayerBase
         /// </summary>
         [NotNull]
         [JsonProperty(nameof(Hash), Order = 5)]
-        public unsafe String Hash
-        {
-            [Pure]
-            get
-            {
-                fixed (float* pw = Weights, pb = Biases)
-                {
-                    // Use unmanaged streams to avoid copying the weights and biases
-                    int
-                        weightsSize = sizeof(float) * Weights.Length,
-                        biasesSize = sizeof(float) * Biases.Length;
-                    using (UnmanagedMemoryStream
-                        weightsStream = new UnmanagedMemoryStream((byte*)pw, weightsSize, weightsSize, FileAccess.Read),
-                        biasesStream = new UnmanagedMemoryStream((byte*)pb, biasesSize, biasesSize, FileAccess.Read))
-                    using (SHA256 provider = SHA256.Create())
-                    {
-                        // Compute the two SHA256 hashes and combine them (there isn't a way to concatenate two streams with the hash class)
-                        byte[]
-                            weightsHash = provider.ComputeHash(weightsStream),
-                            biasesHash = provider.ComputeHash(biasesStream),
-                            hash = new byte[32];
-                        unchecked
-                        {
-                            for (int i = 0; i < 32; i++)
-                                hash[i] = (byte)(17 * 31 * weightsHash[i] * 31 * biasesHash[i] % byte.MaxValue); // Trust me
-                        }
-
-                        // Convert the final hash to a base64 string
-                        return Convert.ToBase64String(hash);
-                    }
-                }
-            }
-        }
+        public virtual String Hash => Convert.ToBase64String(Sha256.Hash(Weights, Biases));
 
         /// <summary>
         /// Gets the weights for the current network layer

Unit/NeuralNetwork.NET.Unit/GraphNetworkTest.cs

@@ -328,16 +328,17 @@ public void JsonMetadataSerialization1()
                 var conv1 = root.Layer(NetworkLayers.Convolutional((5, 5), 10, ActivationType.ReLU));
                 var pool1 = conv1.Layer(NetworkLayers.Pooling(ActivationType.Sigmoid));
 
-                var _1x1 = pool1.Layer(NetworkLayers.Convolutional((1, 1), 20, ActivationType.ReLU));
+                var _1x1 = pool1.Layer(NetworkLayers.Convolutional((1, 1), 20, ActivationType.Identity));
                 var _3x3reduce1x1 = pool1.Layer(NetworkLayers.Convolutional((1, 1), 20, ActivationType.ReLU));
-                var _3x3 = _3x3reduce1x1.Layer(NetworkLayers.Convolutional((1, 1), 20, ActivationType.ReLU));
+                var _3x3 = _3x3reduce1x1.Layer(NetworkLayers.Convolutional((1, 1), 20, ActivationType.Identity));
 
                 var split = _3x3.TrainingBranch();
                 var fct = split.Layer(NetworkLayers.FullyConnected(100, ActivationType.LeCunTanh));
                 _ = fct.Layer(NetworkLayers.Softmax(10));
 
                 var stack = _1x1.DepthConcatenation(_3x3);
-                var fc1 = stack.Layer(NetworkLayers.FullyConnected(100, ActivationType.Sigmoid));
+                var bn = stack.Layer(NetworkLayers.BatchNormalization(NormalizationMode.Spatial, ActivationType.ReLU));
+                var fc1 = bn.Layer(NetworkLayers.FullyConnected(100, ActivationType.Sigmoid));
                 _ = fc1.Layer(NetworkLayers.Softmax(10));
             });
             String json = network.SerializeMetadataAsJson();

Unit/NeuralNetwork.NET.Unit/SerializationTest.cs

@@ -90,7 +90,8 @@ public void JsonMetadataSerialization()
                 NetworkLayers.Convolutional((10, 10), 20, ActivationType.Identity),
                 NetworkLayers.Pooling(ActivationType.ReLU),
                 NetworkLayers.Convolutional((10, 10), 20, ActivationType.Identity),
-                NetworkLayers.Pooling(ActivationType.ReLU),
+                NetworkLayers.Pooling(ActivationType.Identity),
+                NetworkLayers.BatchNormalization(NormalizationMode.Spatial, ActivationType.ReLU),
                 NetworkLayers.FullyConnected(125, ActivationType.Tanh),
                 NetworkLayers.Softmax(133));
             String metadata1 = network.SerializeMetadataAsJson();