using System; using System.Linq; // Select using System.Collections.Generic; using Unity.Barracuda.Compiler.Passes; using UnityEngine.Assertions; using UnityEditor; namespace Unity.Barracuda { /// /// Barracuda Model Layer /// public class Layer { /// /// Layer Type /// public enum Type { /// /// No operation / identity layer /// Nop = 0, /// /// Dense layer /// Dense = 1, /// /// Matrix multiplication layer /// MatMul = 2, /// /// Rank-3 Dense Layer /// Dense3 = 3, /// /// 2D Convolution layer /// Conv2D = 20, /// /// Depthwise Convolution layer /// DepthwiseConv2D = 21, /// /// Transpose 2D Convolution layer /// Conv2DTrans = 22, /// /// Upsampling layer /// Upsample2D = 23, /// /// Max Pool layer /// MaxPool2D = 25, /// /// Average Pool layer /// AvgPool2D = 26, /// /// Global Max Pool layer /// GlobalMaxPool2D = 27, /// /// Global Average Pool layer /// GlobalAvgPool2D = 28, /// /// Border / Padding layer /// Border2D = 29, /// /// 3D Convolution layer /// Conv3D = 30, /// /// Transpose 3D Convolution layer (not yet implemented) /// Conv3DTrans = 32, // TODO: NOT IMPLEMENTED /// /// 3D Upsampling layer /// Upsample3D = 33, /// /// 3D Max Pool layer (not yet implemented) /// MaxPool3D = 35, // TODO: NOT IMPLEMENTED /// /// 3D Average Pool layer (not yet implemented) /// AvgPool3D = 36, // TODO: NOT IMPLEMENTED /// /// 3D Global Max Pool layer (not yet implemented) /// GlobalMaxPool3D = 37, // TODO: NOT IMPLEMENTED /// /// 3D Global Average Pool layer (not yet implemented) /// GlobalAvgPool3D = 38, // TODO: NOT IMPLEMENTED /// /// 3D Border / Padding layer /// Border3D = 39, /// /// Activation layer, see `Activation` enum for activation types /// Activation = 50, /// /// Scale + Bias layer /// ScaleBias = 51, /// /// Normalization layer /// Normalization = 52, /// /// LRN (Local Response Normalization) layer /// LRN = 53, /// /// Dropout layer (does nothing in inference) /// Dropout = 60, /// /// Random sampling from normal distribution layer /// RandomNormal = 64, /// /// Random sampling from uniform distribution layer /// RandomUniform = 65, /// /// Random sampling from multinomial distribution layer /// Multinomial = 66, /// /// OneHot layer /// OneHot = 67, /// /// TopK indices layer /// TopKIndices = 68, /// /// TopK values layer /// TopKValues = 69, /// /// NonZero layer /// NonZero = 70, /// /// Range layer /// Range = 71, /// /// RoiAlign layer /// RoiAlign = 72, /// /// Addition layer /// Add = 100, /// /// Subtraction layer /// Sub = 101, /// /// Multiplication layer /// Mul = 102, /// /// Division layer /// Div = 103, /// /// Power layer /// Pow = 104, /// /// Min layer /// Min = 110, /// /// Max layer /// Max = 111, /// /// Mean layer /// Mean = 112, /// /// Reduce L1 layer (not yet implemented) /// ReduceL1 = 120, // TODO: NOT IMPLEMENTED /// /// Reduce L2 layer (not yet implemented) /// ReduceL2 = 121, // TODO: NOT IMPLEMENTED /// /// Reduce LogSum layer (not yet implemented) /// ReduceLogSum = 122, // TODO: NOT IMPLEMENTED /// /// Reduce LogSumExp layer (not yet implemented) /// ReduceLogSumExp = 123, // TODO: NOT IMPLEMENTED /// /// Reduce with Max layer /// ReduceMax = 124, /// /// Reduce with Mean layer /// ReduceMean = 125, /// /// Reduce with Min layer /// ReduceMin = 126, /// /// Reduce with Prod layer /// ReduceProd = 127, /// /// Reduce with Sum layer /// ReduceSum = 128, /// /// Reduce with SumSquare layer (not yet implemented) /// ReduceSumSquare = 129, // TODO: NOT IMPLEMENTED /// /// Logic operation: Greater layer /// Greater = 140, /// /// Logic operation: GreaterEqual layer /// GreaterEqual = 141, /// /// Logic operation: Less layer /// Less = 142, /// /// Logic operation: LessEqual layer /// LessEqual = 143, /// /// Logic operation: Equal layer /// Equal = 144, /// /// Logic operation: LogicalOr layer /// LogicalOr = 145, /// /// Logic operation: LogicalAnd layer /// LogicalAnd = 146, /// /// Logic operation: LogicalNot layer /// LogicalNot = 147, /// /// Logic operation: LogicalXor layer /// LogicalXor = 148, /// /// Logic operation: Where layer /// Where = 149, /// /// Logic operation: Sign layer /// Sign = 150, /// /// Generic Pad layer (not fully supported) /// Pad = 159, // TODO: NOT IMPLEMENTED /// /// Reflection padding layer /// Pad2DReflect = 160, /// /// Symmetric padding layer /// Pad2DSymmetric = 161, /// /// Edge padding layer /// Pad2DEdge = 162, /// /// ArgMax layer /// ArgMax = 163, /// /// ArgMin layer /// ArgMin = 164, /// /// ConstantOfShape layer /// ConstantOfShape = 199, /// /// Flatten layer /// Flatten = 200, /// /// Reshape layer /// Reshape = 201, /// /// Transpose layer /// Transpose = 202, /// /// Squeeze layer (not fully supported) /// Squeeze = 203, // TODO: NOT IMPLEMENTED /// /// Unsqueeze layer (not fully supported) /// Unsqueeze = 204, // TODO: NOT IMPLEMENTED /// /// Gather layer /// Gather = 205, /// /// Depth to space layer /// DepthToSpace = 206, /// /// Space to depth layer /// SpaceToDepth = 207, /// /// Expand layer /// Expand = 208, /// /// 2D Resample layer /// Resample2D = 209, /// /// Concat layer /// Concat = 210, /// /// Strided slice layer /// StridedSlice = 211, /// /// Tile layer /// Tile = 212, /// /// Shape layer /// Shape = 213, /// /// Non max suppression layer /// NonMaxSuppression = 214, /// /// LSTM /// LSTM = 215, /// /// ScatterND /// ScatterND = 216, /// /// Constant load layer (for internal use) /// Load = 255 } //Keep in sync with Tensor.cginc ACTIVATION defines and IsActivationFusable() methods in ModelBuilder.cs and FuseActivationsPass.cs /// /// Fused activations enum /// public enum FusedActivation { /// /// None /// None = Activation.None, /// /// Relu /// Relu = Activation.Relu, /// /// Tanh /// Tanh = Activation.Tanh, /// /// Softplus /// Softplus = Activation.Softplus, /// /// Sigmoid /// Sigmoid = Activation.Sigmoid, /// /// Relu6 /// Relu6 = Activation.Relu6, /// /// Swish /// Swish = Activation.Swish, /// /// Neg /// Neg = Activation.Neg, /// /// Sqrt /// Sqrt = Activation.Sqrt, /// /// Exp /// Exp = Activation.Exp, /// /// Log /// Log = Activation.Log, /// /// Acos /// Acos = Activation.Acos, /// /// Acosh /// Acosh = Activation.Acosh, /// /// Asin /// Asin = Activation.Asin, /// /// Asinh /// Asinh = Activation.Asinh, /// /// Atan /// Atan = Activation.Atan, /// /// Atanh /// Atanh = Activation.Atanh, /// /// Cos /// Cos = Activation.Cos, /// /// Cosh /// Cosh = Activation.Cosh, /// /// Sin /// Sin = Activation.Sin, /// /// Sinh /// Sinh = Activation.Sinh, /// /// Tan /// Tan = Activation.Tan, /// /// Erf /// Erf = Activation.Erf } /// /// Activation enum /// public enum Activation { /// /// None /// None = 0, /// /// Relu /// Relu = 1, /// /// Softmax /// Softmax = 2, /// /// Tanh /// Tanh = 3, /// /// Sigmoid /// Sigmoid = 4, /// /// Elu /// Elu = 5, /// /// Relu6 /// Relu6 = 6, /// /// LeakyRelu /// LeakyRelu = 7, /// /// Selu /// Selu = 8, /// /// Swish /// Swish = 9, /// /// LogSoftmax /// LogSoftmax = 10, /// /// Softplus /// Softplus = 11, /// /// Softsign (not yet implemented) /// Softsign = 12, // TODO: NOT IMPLEMENTED /// /// PRelu /// PRelu = 13, /// /// Hardmax (not yet implemented) /// Hardmax = 20, // TODO: NOT IMPLEMENTED /// /// HardSigmoid /// HardSigmoid = 21, /// /// Abs /// Abs = 100, /// /// Neg /// Neg = 101, /// /// Ceil /// Ceil = 102, /// /// Clip /// Clip = 103, /// /// Floor /// Floor = 104, /// /// Round /// Round = 105, /// /// Reciprocal /// Reciprocal = 110, /// /// Sqrt /// Sqrt = 111, /// /// Pow /// Pow = 112, /// /// Exp /// Exp = 113, /// /// Log /// Log = 114, /// /// Acos /// Acos = 200, /// /// Acosh /// Acosh = 201, /// /// Asin /// Asin = 202, /// /// Asinh /// Asinh = 203, /// /// Atan /// Atan = 204, /// /// Atanh /// Atanh = 205, /// /// Cos /// Cos = 206, /// /// Cosh /// Cosh = 207, /// /// Sin /// Sin = 208, /// /// Sinh /// Sinh = 209, /// /// Tan /// Tan = 210, /// /// Erf /// Erf = 211 } /// /// Auto padding enum /// public enum AutoPad { /// /// NotSet /// NotSet = 1, /// /// Valid /// Valid = 0, /// /// Same upper /// SameUpper = -1, /// /// Same lower /// SameLower = -2, } public enum PadMode { Constant = 0, Reflect = 1, Edge = 2, Symetric = 3, } /// /// Depth to space mode enum /// public enum DepthToSpaceMode { /// /// DCR (Depth Column Row) /// DCR, /// /// CRD (Column Row Depth) /// CRD } /// /// ScatterND reduction mode /// public enum ScatterNDReductionMode { /// /// None /// None = 0, /// /// Add /// Add = 1, /// /// Multiply /// Mul = 2, } /// /// Layer param data structure /// public struct DataSet { /// /// Name /// public string name; /// /// Shape /// public TensorShape shape; /// /// Offset from start /// public Int64 offset; /// /// Item size in bytes /// public Int32 itemSizeInBytes; /// /// Dataset length /// public Int32 length; } /// /// Layer preservation flags /// [Flags] public enum Flags { /// /// No flags defined /// None = 0, /// /// Preserve the layer (e.g. don't remove it in a model pass) /// Preserve = 1 << 1, } /// /// Layer name /// public string name; /// /// Layer type /// public Type type; /// /// Layer flags (not serialized) - used for conversion /// [NonSerialized] public Flags flags; /// /// Layer activation type /// public Activation activation; /// /// Padding shape /// public Int32[] pad; /// /// Stride /// public Int32[] stride; /// /// Pooling /// public Int32[] pool; /// /// Axis /// public Int32 axis; /// /// Alpha /// public float alpha; /// /// Beta /// public float beta; /// /// Input (layer) names /// public string[] inputs; /// /// Output (layer) names (not serialized) - used for conversion /// [NonSerialized] public string[] outputs; /// /// Axes (not serialized) - used for conversion /// [NonSerialized] public Int32[] axes; /// /// Datasets bound to layer /// public DataSet[] datasets; /// /// Flat weights array (for actual shape see `datasets`) /// public BarracudaArray weights; private Layer(string layerName) { name = layerName; type = Type.Nop; activation = Activation.None; pad = new int[0]; stride = new int[0]; pool = new int[0]; axis = -1; alpha = 1.0f; beta = 0.0f; inputs = new string[0]; datasets = new DataSet[0]; weights = new BarracudaArray(0);//TODO fp16? } /// /// Constructs Layer /// /// layer name /// layer type /// layer activation type public Layer(string layerName, Type layerType, Activation activationType = Activation.None) : this(layerName) { type = layerType; activation = activationType; } /// /// Constructs Activation Layer /// /// layer name /// layer activation type public Layer(string layerName, Activation activationType) : this(layerName) { type = Type.Activation; activation = activationType; } /// /// Layer summary string /// /// layer summary string public override string ToString() { return ($"name:{name}, activation:{activation}, inputs:[{string.Join(",", inputs)}], " + $"pad:[{string.Join(",", pad)}], stride:[{string.Join(",", stride)}], pool:[{string.Join(",", pool)}], " + $"alpha:{alpha}, beta:{beta}, axis:{axis}, " + $"weights:[{string.Join(", ", datasets.Select(x => $"{x.name} {x.shape}"))}]".Replace(name+"/","").Replace(name+" ","")). Replace("activation:None, ", "").Replace("inputs:[], ", "").Replace("pad:[], ", ""). Replace("stride:[], ", "").Replace("stride:[1,1], ", "").Replace("pool:[], ", ""). Replace("alpha:1, ", "").Replace("beta:0, ", "").Replace("axis:-1, ", ""). Replace("weights:[]", ""); } /// /// Converts DataSet to Tensor /// /// dataset index /// Tensor public Tensor DataSetToTensor(int index) { Assert.IsTrue(index < datasets.Length); var ds = datasets[index]; return new Tensor(ds.shape, new SharedArrayTensorData(weights, ds.shape, (int)ds.offset), ds.name); } /// /// Converts Tensor to DataSet /// /// input `Tensor` /// dataset index public void ApplyTensorToDataSet(Tensor X, int index) { Assert.IsTrue(index < datasets.Length); var ds = datasets[index]; ds.shape = X.shape; BarracudaArray.Copy(X.ToReadOnlyArray(), 0, weights, ds.offset, ds.shape.length); datasets[index] = ds; } } /// /// Neural Net Model data structure /// public class Model { /// /// Model version, incremented with each data structure change /// public const int Version = 20; internal const int LastVersionWithout8DSupport = 16; public const int LastVersionWithoutWeightsAlignmentSupport = 18; internal const int WeightsAlignment = 16; /// /// Input data structure /// public struct Input { /// /// Name /// public string name; /// /// Shape as `int` array /// public Int32[] shape; // input shape can contain -1 for unspecified dimensions /// /// Input rank /// public int rank; /// /// Creates input structure with specified name /// /// name /// Input structure public Input WithName(string name) { return new Input {name = name, shape = shape}; } } /// /// Memory data structure. Used by recurrent models to store information about recurrent inputs/outputs /// public struct Memory { /// /// Shape /// public TensorShape shape; /// /// Input name /// public string input; /// /// Output name /// public string output; } /// /// Model layout /// public string layout = String.Empty; /// /// All model inputs /// public List inputs = new List(); /// /// All model outputs /// public List outputs = new List(); /// /// All model memories /// public List memories = new List(); /// /// All model layers /// public List layers = new List(); #region Importer info /// /// Model source metadata string /// public string IrSource = "Script"; /// /// Model ONNX version metadata string /// public string IrVersion = "NA"; /// /// Model producer metadata string /// public string ProducerName = "Script"; /// /// Model import warnings /// public List Warnings { get; } = new List(); /// /// Importer warning data structure /// public class ImporterWarning { /// /// Message /// public string Message { get; } /// /// Layer name /// public string LayerName { get; } /// /// Constructs ImporterWarning /// /// layer name /// message public ImporterWarning(string layer, string msg) { Message = msg; LayerName = layer; } } /// /// Metadata properties associated with the model /// public Dictionary Metadata { get; private set; } = new Dictionary(); #endregion /// /// Build shallow copy of the model /// /// shallow copy of the model public Model ShallowCopy() { var model = new Model(); model.inputs.AddRange(inputs); model.outputs.AddRange(outputs); model.memories.AddRange(memories); model.layers.AddRange(layers); model.IrSource = IrSource; model.IrVersion = IrVersion; model.ProducerName = ProducerName; model.Warnings.AddRange(Warnings); model.Metadata = new Dictionary(Metadata); return model; } /// /// Model summary string /// /// Model summary string public override string ToString() { // weights are not loaded for UI, recompute size var totalUniqueWeights = 0; for (var l = 0; l < layers.Count; ++l) for (var d = 0; d < layers[l].datasets.Length; ++d) totalUniqueWeights += layers[l].datasets[d].length; return $"inputs: [{string.Join(", ", inputs.Select(i => $"{i.name} ({string.Join(",", i.shape)})"))}], " + $"memories: [{string.Join(", ", memories.Select(m => $"{m.input} {m.shape} {m.output}"))}], " + $"outputs: [{string.Join(", ", outputs)}] " + $"\n{layers.Count} layers, {totalUniqueWeights:n0} weights: \n{string.Join("\n", layers.Select(i => $"{i.type} ({i})"))}"; } /// /// Convert in place all model weights to given data type /// /// target type for moodel weights internal void ConvertWeights(DataType type) { foreach (var layer in layers) { if (layer.weights != null && layer.weights.Type != type) { var sourceWeights = layer.weights; var targetWeights = new BarracudaArray(layer.weights.Length, type); BarracudaArray.Copy(sourceWeights, targetWeights); layer.weights = targetWeights; } } } } /// /// Model metadata extensions /// public static class ModelMetadataExtensions { /// /// Get model tensor by name /// /// Model /// Tensor name /// Tensor static public Tensor GetTensorByName(this Model model, string name) { foreach (var l in model.layers) foreach (var ds in l.datasets) if (ds.name == name) return new Tensor(ds.shape, new SharedArrayTensorData(l.weights, ds.shape, (int)ds.offset), ds.name); return null; } /// /// Get model tensor shape by name /// /// Model /// Tensor name /// Tensor shape /// static public TensorShape? GetShapeByName(this Model model, string name) { foreach (var i in model.inputs) if (i.name == name) return new TensorShape(i.shape); TensorShape shape; if (ModelAnalyzer.TryGetOutputTensorShape(model, name, out shape)) return shape; foreach (var l in model.layers) foreach (var ds in l.datasets) if (ds.name == name) return ds.shape; foreach (var mem in model.memories) { if (mem.input == name || mem.output == name) return mem.shape; } throw new System.Collections.Generic.KeyNotFoundException("Shape " + name + " not found!"); } /// /// Get count of layers that directly depend on specified input /// /// Model /// input name /// count of layers that directly depend on specified input static public int GetDownStreamLayersCount(this Model model, string name) { return model.layers.Count(x => x.inputs.Contains(name)); } } } // namespace Unity.Barracuda