diff --git a/.github/workflows/runtime-build.yml b/.github/workflows/runtime-build.yml index 3939046ec3..b54f4a5bfe 100644 --- a/.github/workflows/runtime-build.yml +++ b/.github/workflows/runtime-build.yml @@ -95,7 +95,7 @@ jobs: strategy: matrix: config: - - { name: riscv64-linux, shell: bash, arch: riscv64, toolchain: riscv64-unknown-linux, toolchain_env: RISCV_ROOT_PATH, toolchain_file: riscv64-unknown-linux-gnu-12.0.1, qemu: qemu-riscv64, loader_args: '-cpu;rv64,v=true,Zfh=true,vlen=128,elen=64,vext_spec=v1.0;-L', cmakeArgs: '', buildType: Release } + - {name: riscv64-linux, shell: bash, arch: riscv64, toolchain: riscv64-unknown-linux, toolchain_env: RISCV_ROOT_PATH, toolchain_file: riscv64-unknown-linux-gnu-13.2.0, qemu: qemu-riscv64, loader_args: '-cpu;rv64,v=true,Zfh=true,vlen=128,elen=64,vext_spec=v1.0;-L', cmakeArgs: '', buildType: Release} steps: - uses: actions/checkout@v3 @@ -151,4 +151,4 @@ jobs: with: name: nncaseruntime-${{matrix.config.name}} path: ${{github.workspace}}/install - if-no-files-found: error \ No newline at end of file + if-no-files-found: error diff --git a/CMakeLists.txt b/CMakeLists.txt index 636db14888..ce82d0c916 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -57,6 +57,10 @@ if(ENABLE_DUMP_MEM) add_definitions(-DDUMP_MEM) endif() +if(ENABLE_OPENMP) + add_definitions(-DENABLE_OPENMP) +endif() + if(NOT CMAKE_TOOLCHAIN_FILE) include(toolchains/native.toolchain.cmake) endif() @@ -283,5 +287,7 @@ if(BUILD_TESTING) endif() # Modules +add_subdirectory(modules/ncnn) +#add_subdirectory(modules/k210) #add_subdirectory(modules/vulkan) diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index 2827003ab2..ab7bc5b2ed 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -1,3 +1,6 @@ +find_package(gsl-lite REQUIRED) +find_package(ncnn REQUIRED) + if (ENABLE_OPENMP) find_package(OpenMP COMPONENTS CXX REQUIRED) endif () diff --git a/conanfile.py b/conanfile.py index 6ddcfc9ce3..a8cf6991d0 100644 --- a/conanfile.py +++ b/conanfile.py @@ -35,13 +35,14 @@ class nncaseConan(ConanFile): "python": True, "vulkan_runtime": False } - + def imports(self): if self.settings.os == 'Windows': self.copy("nethost.dll", "bin", "bin") self.copy("ortki.dll", "bin", "bin") def requirements(self): + self.requires('ncnn/20240102') if self.options.tests: self.requires('gtest/1.10.0') self.requires('ortki/0.0.3') @@ -64,7 +65,6 @@ def configure(self): if self.settings.os == 'Windows': self.settings.compiler.toolset = 'ClangCL' - if not self.options.runtime: if self.settings.os == 'Windows': self.options["nethost"].shared = True diff --git a/modules/Nncase.Modules.CPU/Targets/CPUTarget.cs b/modules/Nncase.Modules.CPU/Targets/CPUTarget.cs index ad654b9b54..4994fad0b6 100644 --- a/modules/Nncase.Modules.CPU/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.CPU/Targets/CPUTarget.cs @@ -61,6 +61,73 @@ public void RegisterTargetInDependentPass(IPassManager passManager, CompileOptio /// public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions options) { + passManager.AddWithName("LowerNcnnIR").Configure(p => + { + p.Add(); + + // Fold reduce + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + + // single op + p.Add(); + p.Add(); + p.Add(); + p.Add(); + + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + + p.Add(); + p.Add(); + p.Add(); + + // p.Add(); // ncnn dequantize int to float. + }); + + passManager.AddWithName("RemoveGlueOp").Configure(p => + { + p.Add(); + p.Add(); + p.Add(); + }); + + passManager.AddWithName("RemoveSingleSqueezeAndUnsqueeze").Configure(p => + { + p.Add(); + p.Add(); + }); } /// diff --git a/modules/Nncase.Modules.CPU/packages.lock.json b/modules/Nncase.Modules.CPU/packages.lock.json index dded23c243..08721aae0d 100644 --- a/modules/Nncase.Modules.CPU/packages.lock.json +++ b/modules/Nncase.Modules.CPU/packages.lock.json @@ -190,10 +190,18 @@ "nncase.io": { "type": "Project" }, + "nncase.modules.ncnn": { + "type": "Project", + "dependencies": { + "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Passes": "[1.0.0, )" + } + }, "nncase.modules.stackvm": { "type": "Project", "dependencies": { "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Modules.Ncnn": "[1.0.0, )", "Nncase.Passes": "[1.0.0, )" } }, diff --git a/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs b/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs new file mode 100644 index 0000000000..0bbbaf459d --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs @@ -0,0 +1,362 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +namespace Nncase.ArgsStruct; + +/// +/// Ncnn Pooling arguments. +/// +public record PoolingArgs +{ + public PoolingArgs(int poolingType, int kernelW, int kernelH, int strideW, int strideH, int padLeft, int padRight, int padTop, int padBottom, bool globalPooling, int padMode, bool avgPoolCountIncludePad, bool adaptivePooling, int outW, int outH, bool ceilMode) + { + PoolingType = poolingType; + KernelH = kernelH; + KernelW = kernelW; + StrideH = strideH; + StrideW = strideW; + PadLeft = padLeft; + PadRight = padRight; + PadTop = padTop; + PadBottom = padBottom; + GlobalPooling = globalPooling; + PadMode = padMode; + AvgPoolCountIncludePad = avgPoolCountIncludePad; + AdaptivePooling = adaptivePooling; + OutW = outW; + OutH = outH; + CeilMode = ceilMode; + } + + public int PoolingType { get; } + + public int KernelW { get; } + + public int KernelH { get; } + + public int StrideW { get; } + + public int StrideH { get; } + + public int PadLeft { get; } + + public int PadRight { get; } + + public int PadTop { get; } + + public int PadBottom { get; } + + public bool GlobalPooling { get; } + + public int PadMode { get; } + + public bool AvgPoolCountIncludePad { get; } + + public bool AdaptivePooling { get; } + + public int OutW { get; } + + public int OutH { get; } + + public bool CeilMode { get; } +} + +public record ReductionArgs +{ + public ReductionArgs(int opType, int reduceAll, float coeff, long[] axes, int keepdims) + { + OpType = opType; + ReduceAll = reduceAll; + Coeff = coeff; + Axes = axes; + Keepdims = keepdims; + } + + public int OpType { get; } + + public int ReduceAll { get; } + + public float Coeff { get; } + + public long[] Axes { get; } + + public int Keepdims { get; } +} + +public record CropArgs +{ + public CropArgs(int[] starts, int[] ends, int[] axes) + { + Woffset = 0; + Hoffset = 0; + Doffset = 0; + Coffset = 0; + Outw = 0; + Outh = 0; + Outd = 0; + Outc = 0; + Woffset2 = 0; + Hoffset2 = 0; + Doffset2 = 0; + Coffset2 = 0; + Starts = starts; + Ends = ends; + Axes = axes; + } + + public CropArgs(int woffset, int hoffset, int doffset, int coffset, int outw, int outh, int outd, int outc, int woffset2, int hoffset2, int doffset2, int coffset2) + { + Woffset = woffset; + Hoffset = hoffset; + Doffset = doffset; + Coffset = coffset; + Outw = outw; + Outh = outh; + Outd = outd; + Outc = outc; + Woffset2 = woffset2; + Hoffset2 = hoffset2; + Doffset2 = doffset2; + Coffset2 = coffset2; + Starts = null; + Ends = null; + Axes = null; + } + + public int Woffset { get; } + + public int Hoffset { get; } + + public int Doffset { get; } + + public int Coffset { get; } + + public int Outw { get; } + + public int Outh { get; } + + public int Outd { get; } + + public int Outc { get; } + + public int Woffset2 { get; } + + public int Hoffset2 { get; } + + public int Doffset2 { get; } + + public int Coffset2 { get; } + + public int[]? Starts { get; } + + public int[]? Ends { get; } + + public int[]? Axes { get; } +} + +public record ConvTransposeArgs +{ + public ConvTransposeArgs(Tensor? weightData = null, float[]? biasData = null, int numOutput = default, int kernelW = default, int kernelH = default, int dilationW = default, int dilationH = default, int strideW = default, int strideH = default, int padLeft = default, int padRight = default, int padTop = default, int padBottom = default, int biasTerm = default, int weightDataSize = default, int activationType = default, float[]? activationParams = null, int outputPadRight = default, int outputPadBottom = default, int outputW = default, int outputH = default) + { + WeightData = weightData!; + BiasData = biasData!; + NumOutput = numOutput; + KernelW = kernelW; + KernelH = kernelH; + DilationW = dilationW; + DilationH = dilationH; + StrideW = strideW; + StrideH = strideH; + PadLeft = padLeft; + PadRight = padRight; + PadTop = padTop; + PadBottom = padBottom; + BiasTerm = biasTerm; + WeightDataSize = weightDataSize; + ActivationType = activationType; + ActivationParams = activationParams!; + OutputPadRight = outputPadRight; + OutputPadBottom = outputPadBottom; + OutputW = outputW; + OutputH = outputH; + } + + public Tensor WeightData { get; } + + public float[] BiasData { get; } + + public int NumOutput { get; } + + public int KernelW { get; } + + public int KernelH { get; } + + public int DilationW { get; } + + public int DilationH { get; } + + public int StrideW { get; } + + public int StrideH { get; } + + public int PadLeft { get; } + + public int PadRight { get; } + + public int PadTop { get; } + + public int PadBottom { get; } + + public int BiasTerm { get; } + + public int WeightDataSize { get; } + + public int ActivationType { get; } + + public float[] ActivationParams { get; } + + public int OutputPadRight { get; } + + public int OutputPadBottom { get; } + + public int OutputW { get; } + + public int OutputH { get; } + + public override string ToString() => $"{nameof(WeightData)}: {string.Join("_", WeightData.Shape.ToValueArray())}, {nameof(NumOutput)}: {NumOutput}, {nameof(KernelW)}: {KernelW}, {nameof(KernelH)}: {KernelH}, {nameof(DilationW)}: {DilationW}, {nameof(DilationH)}: {DilationH}, {nameof(StrideW)}: {StrideW}, {nameof(StrideH)}: {StrideH}, {nameof(PadLeft)}: {PadLeft}, {nameof(PadRight)}: {PadRight}, {nameof(PadTop)}: {PadTop}, {nameof(PadBottom)}: {PadBottom}, {nameof(BiasTerm)}: {BiasTerm}, {nameof(WeightDataSize)}: {WeightDataSize}, {nameof(OutputPadRight)}: {OutputPadRight}, {nameof(OutputPadBottom)}: {OutputPadBottom}, {nameof(OutputW)}: {OutputW}, {nameof(OutputH)}: {OutputH}"; +} + +public record ConvArgs +{ + public ConvArgs(float[] weightData, float[] biasData, int numOutput, int kernelW, int kernelH, int dilationW, int dilationH, int strideW, int strideH, int padLeft, int padRight, int padTop, int padBottom, float padValue, int biasTerm, int weightDataSize, int int8ScaleTerm, int activationType, float[] activationParams, int dynamicWeight, int groups) + { + WeightData = weightData; + BiasData = biasData; + NumOutput = numOutput; + KernelW = kernelW; + KernelH = kernelH; + DilationW = dilationW; + DilationH = dilationH; + StrideW = strideW; + StrideH = strideH; + PadLeft = padLeft; + PadRight = padRight; + PadTop = padTop; + PadBottom = padBottom; + PadValue = padValue; + BiasTerm = biasTerm; + WeightDataSize = weightDataSize; + Int8ScaleTerm = int8ScaleTerm; + ActivationType = activationType; + ActivationParams = activationParams; + DynamicWeight = dynamicWeight; + Groups = groups; + } + + /// + /// Gets input. + /// + public float[] WeightData { get; } + + /// + /// Gets BiasData. + /// + public float[] BiasData { get; } + + /// + /// Gets NumOutput. + /// + public int NumOutput { get; } + + /// + /// Gets KernelW. + /// + public int KernelW { get; } + + /// + /// Gets KernelH. + /// + public int KernelH { get; } + + /// + /// Gets DilationW. + /// + public int DilationW { get; } + + /// + /// Gets DilationH. + /// + public int DilationH { get; } + + /// + /// Gets StrideW. + /// + public int StrideW { get; } + + /// + /// Gets StrideH. + /// + public int StrideH { get; } + + /// + /// Gets PadLeft. + /// + public int PadLeft { get; } + + /// + /// Gets PadRight. + /// + public int PadRight { get; } + + /// + /// Gets PadTop. + /// + public int PadTop { get; } + + /// + /// Gets PadBottom. + /// + public int PadBottom { get; } + + /// + /// Gets PadValue. + /// + public float PadValue { get; } + + /// + /// Gets BiasTerm. + /// + public int BiasTerm { get; } + + /// + /// Gets WeightDataSize. + /// + public int WeightDataSize { get; } + + /// + /// Gets Int8ScaleTerm. + /// + public int Int8ScaleTerm { get; } + + /// + /// Gets ActivationType. + /// + public int ActivationType { get; } + + /// + /// Gets ActivationParams. + /// + public float[] ActivationParams { get; } + + /// + /// Gets DynamicWeight. + /// + public int DynamicWeight { get; } + + /// + /// Gets Groups. + /// + public int Groups { get; } + + public override string ToString() => $"{nameof(WeightData)}: {WeightData}, {nameof(BiasData)}: {BiasData}, {nameof(NumOutput)}: {NumOutput}, {nameof(KernelW)}: {KernelW}, {nameof(KernelH)}: {KernelH}, {nameof(DilationW)}: {DilationW}, {nameof(DilationH)}: {DilationH}, {nameof(StrideW)}: {StrideW}, {nameof(StrideH)}: {StrideH}, {nameof(PadLeft)}: {PadLeft}, {nameof(PadRight)}: {PadRight}, {nameof(PadTop)}: {PadTop}, {nameof(PadBottom)}: {PadBottom}, {nameof(PadValue)}: {PadValue}, {nameof(BiasTerm)}: {BiasTerm}, {nameof(WeightDataSize)}: {WeightDataSize}, {nameof(Int8ScaleTerm)}: {Int8ScaleTerm}, {nameof(ActivationType)}: {ActivationType}, {nameof(ActivationParams)}: {ActivationParams}, {nameof(DynamicWeight)}: {DynamicWeight}, {nameof(Groups)}: {Groups}"; +} diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs new file mode 100644 index 0000000000..47f24950a5 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -0,0 +1,612 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using DryIoc.ImTools; +using Microsoft.Toolkit.HighPerformance; +using Nncase.ArgsStruct; +using Nncase.IR; +using Nncase.IR.Ncnn; +using Nncase.Runtime.Ncnn; + +namespace Nncase.CodeGen.Ncnn; + +internal class NcnnEmitter +{ + private static long _s_pos; + private readonly NcnnModel _model; + private readonly BinaryWriter _binWriter; + private readonly List? _rData; + + public NcnnEmitter(BinaryWriter binWriter) + { + _model = new NcnnModel(); + _binWriter = binWriter; + _rData = new List(); + } + + public List? GetRData() + { + return _rData; + } + + public void SaveParam(Stream paramStream) + { + using var sw = new StreamWriter(paramStream, Encoding.ASCII, leaveOpen: true); + _model.Serialize(sw); + } + + public void SaveBin(string dumpPath, uint id) + { + if (id == 0) + { + _s_pos = 0; + } + + using (var fileStream = File.Create(Path.Join(dumpPath, $"/ncnn_{id}.bin"))) + { + long now = _binWriter.BaseStream.Position; + if (now != 0 && now != _s_pos) + { + using (var tempStream = new MemoryStream()) + { + _binWriter.BaseStream.Position = _s_pos; // 重置位置到新内容的开始 + _binWriter.BaseStream.CopyTo(tempStream, (int)(now - _s_pos)); + tempStream.Position = 0; + tempStream.CopyTo(fileStream); + } + + _s_pos = _binWriter.BaseStream.Position; + } + } + } + + public void Input(string name) => + AddLayer("Input", name, Array.Empty(), new[] { name }, null); + + public void Softmax(string name, string input, int axis) => + AddLayer("Softmax", name, new[] { input }, new[] { name }, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = axis }, // axis + [1] = new ParamValue { Kind = ParamKind.Int, IntValue = 1 }, // fixbug0 + }); + + public void Unary(string name, string input, UnaryOperationType opTypes) => + AddLayer("UnaryOp", name, new[] { input }, new[] { name }, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = (int)opTypes }, + }); + + public void BatchNorm(string name, string input, int channels, float eps, float[] slopeData, float[] meanData, float[] varData, float[] biasData/*, float[] aData, float[] bData*/) + { + AddLayer("BatchNorm", name, new[] { input }, new[] { name }, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = channels }, + [1] = new ParamValue { Kind = ParamKind.Float, FloatValue = eps }, + }); + + WriteFloatArray(slopeData); + WriteFloatArray(meanData); + WriteFloatArray(varData); + WriteFloatArray(biasData); + } + + public void Binary(string name, string inputA, string inputB, BinaryOperationType opTypes, int lOrR, float[] constInput, int[] constShape) + { + var inputList = new[] { inputA, inputB }; + + if (constInput != null && constShape != null) + { + if (lOrR == 1) + { + inputList[0] = name + "_memorydata"; + } + else + { + inputList[1] = name + "_memorydata"; + } + + var paramDict = new ParamDict(); + for (int i = 0; i < constShape.Length; i++) + { + paramDict[i] = new ParamValue { Kind = ParamKind.Int, IntValue = constShape[constShape.Length - 1 - i] }; + } + + AddLayer("MemoryData", name + "_memorydata", Array.Empty(), new[] { name + "_memorydata" }, paramDict); + + WriteFloatArray(constInput); + } + + AddLayer("BinaryOp", name, inputList, new[] { name }, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = (int)opTypes }, + }); + } + + public void Celu(string name, string input, float alpha) => + AddLayer("CELU", name, new[] { input }, new[] { name }, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Float, FloatValue = alpha }, // alpha + }); + + public void Clip(string name, string input, float min, float max) => + AddLayer("Clip", name, new[] { input }, new[] { name }, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Float, FloatValue = min }, // min + [1] = new ParamValue { Kind = ParamKind.Float, FloatValue = max }, // max + }); + + public void Concat(string name, string[] input, int axis) + { + AddLayer("Concat", name, input, new[] { name }, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = axis }, // axis + }); + } + + public void Conv(string name, string input, ConvArgs args) + { + var actData = new List { args.ActivationParams.Length }; + actData.AddRange(args.ActivationParams); + + var param = new ParamDict(); + param.Add(0, new ParamValue { Kind = ParamKind.Int, IntValue = args.NumOutput }); + param.Add(1, new ParamValue { Kind = ParamKind.Int, IntValue = args.KernelW }); + param.Add(11, new ParamValue { Kind = ParamKind.Int, IntValue = args.KernelH }); + param.Add(2, new ParamValue { Kind = ParamKind.Int, IntValue = args.DilationW }); + param.Add(12, new ParamValue { Kind = ParamKind.Int, IntValue = args.DilationH }); + param.Add(3, new ParamValue { Kind = ParamKind.Int, IntValue = args.StrideW }); + param.Add(13, new ParamValue { Kind = ParamKind.Int, IntValue = args.StrideH }); + param.Add(4, new ParamValue { Kind = ParamKind.Int, IntValue = args.PadLeft }); + param.Add(15, new ParamValue { Kind = ParamKind.Int, IntValue = args.PadRight }); + param.Add(14, new ParamValue { Kind = ParamKind.Int, IntValue = args.PadTop }); + param.Add(16, new ParamValue { Kind = ParamKind.Int, IntValue = args.PadBottom }); + param.Add(18, new ParamValue { Kind = ParamKind.Float, FloatValue = args.PadValue }); + param.Add(5, new ParamValue { Kind = ParamKind.Int, IntValue = args.BiasTerm }); + param.Add(6, new ParamValue { Kind = ParamKind.Int, IntValue = args.WeightDataSize }); + param.Add(7, new ParamValue { Kind = ParamKind.Int, IntValue = args.Groups }); + param.Add(8, new ParamValue { Kind = ParamKind.Int, IntValue = args.Int8ScaleTerm }); + param.Add(9, new ParamValue { Kind = ParamKind.Int, IntValue = args.ActivationType }); + param.Add(-10, new ParamValue { Kind = ParamKind.ArrayOfIntOrFloat, TensorValue = actData.ToArray() }); + param.Add(19, new ParamValue { Kind = ParamKind.Int, IntValue = args.DynamicWeight }); + + if (args.Groups > 1) + { + AddLayer("ConvolutionDepthWise", name, new[] { input }, new[] { name }, param); + } + else + { + AddLayer("Convolution", name, new[] { input }, new[] { name }, param); + } + + WriteFloatArray(new float[] { 0 }); // quantize flag [Not exist in ncnn op.md] + WriteFloatArray(args.WeightData); + WriteFloatArray(args.BiasData); + } + + public void Cumsum(string name, string input, int axis) + { + AddLayer("CumulativeSum", name, new[] { input }, new[] { name }, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = axis }, // axis + }); + } + + public void Elu(string name, string input, float alpha) => + AddLayer("ELU", name, new[] { input }, new[] { name }, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Float, FloatValue = alpha }, // alpha + }); + + public void Erf(string name, string input) => AddLayer("Erf", name, new[] { input }, new[] { name }, new ParamDict { }); + + public void HardSigmoid(string name, string input, float alpha, float beta) => + AddLayer("HardSigmoid", name, new[] { input }, new[] { name }, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Float, FloatValue = alpha }, // alpha + [1] = new ParamValue { Kind = ParamKind.Float, FloatValue = beta }, // beta + }); + + public void HardSwish(string name, string input, float alpha, float beta) => + AddLayer("HardSwish", name, new[] { input }, new[] { name }, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Float, FloatValue = alpha }, // alpha + [1] = new ParamValue { Kind = ParamKind.Float, FloatValue = beta }, // beta + }); + + public void InstanceNorm(string name, string input, int channels, float eps, int affine, float[] gammaData, float[] betaData) + { + AddLayer("InstanceNorm", name, new[] { input }, new[] { name }, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = channels }, // channels + [1] = new ParamValue { Kind = ParamKind.Float, FloatValue = eps }, // eps + [2] = new ParamValue { Kind = ParamKind.Int, IntValue = affine }, // affine + }); + WriteFloatArray(gammaData); + WriteFloatArray(betaData); + } + + public void LayerNorm(string[] name, string input, int affineSize, float eps, int affine, float[] gammaData, float[] betaData) + { + AddLayer("LayerNorm", name[0], new[] { input }, name, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = affineSize }, // affineSize + [1] = new ParamValue { Kind = ParamKind.Float, FloatValue = eps }, // eps + [2] = new ParamValue { Kind = ParamKind.Int, IntValue = affine }, // affine + }); + WriteFloatArray(gammaData); + WriteFloatArray(betaData); + } + + public void LRN(string[] name, string input, float alpha, float beta, float bias, int size) => + AddLayer("LRN", name[0], new[] { input }, name, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = 0 }, // region_type + [1] = new ParamValue { Kind = ParamKind.Int, IntValue = size }, // size + [2] = new ParamValue { Kind = ParamKind.Float, FloatValue = alpha }, // alpha + [3] = new ParamValue { Kind = ParamKind.Float, FloatValue = beta }, // beta + [4] = new ParamValue { Kind = ParamKind.Float, FloatValue = bias }, // bias + }); + + public void LSTM(string[] name, string input, int hiddenSize, int weightDataSize, int direction, float[] w, float[] r, float[] b) + { + AddLayer("LSTM", name[0], new[] { input }, name, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = hiddenSize }, + [1] = new ParamValue { Kind = ParamKind.Int, IntValue = weightDataSize }, + [2] = new ParamValue { Kind = ParamKind.Int, IntValue = direction }, + }); + WriteFloatArray(new float[] { 0 }); + WriteFloatArray(w); + WriteFloatArray(new float[] { 0 }); + WriteFloatArray(b); + WriteFloatArray(new float[] { 0 }); + WriteFloatArray(r); + } + + public void Padding(string[] name, string input, int top, int bottom, int left, int right, int type, float value, int front, int behind) + { + AddLayer("Padding", name[0], new[] { input }, name, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = top }, + [1] = new ParamValue { Kind = ParamKind.Int, IntValue = bottom }, + [2] = new ParamValue { Kind = ParamKind.Int, IntValue = left }, + [3] = new ParamValue { Kind = ParamKind.Int, IntValue = right }, + [4] = new ParamValue { Kind = ParamKind.Int, IntValue = type }, + [5] = new ParamValue { Kind = ParamKind.Float, FloatValue = value }, + + // [6] for perChannelPadDataSize. + [7] = new ParamValue { Kind = ParamKind.Int, IntValue = front }, + [8] = new ParamValue { Kind = ParamKind.Int, IntValue = behind }, + }); + + // TODO: confirm padValue is a tensor. + WriteFloatArray(new float[] { value }); + } + + public void Pooling(string[] name, string input, PoolingArgs poolingArgs) + { + AddLayer("Pooling", name[0], new[] { input }, name, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.PoolingType }, + [1] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.KernelW }, + [11] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.KernelH }, + [2] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.StrideW }, + [12] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.StrideH }, + [3] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.PadLeft }, + [14] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.PadRight }, + [13] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.PadTop }, + [15] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.PadBottom }, + [4] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.GlobalPooling ? 1 : 0 }, + [5] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.PadMode }, + [6] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.AvgPoolCountIncludePad ? 1 : 0 }, + [7] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.AdaptivePooling ? 1 : 0 }, + + // [8] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.OutH }, + // [18] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.OutH }, + }); + } + + public void PReLU(string[] name, string input, float[] slope) + { + AddLayer("PReLU", name[0], new[] { input }, name, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = slope.Length }, + }); + + WriteFloatArray(slope); + } + + public void Reduction(string[] name, string input, ReductionArgs reductionArgs) + { + var args = new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = reductionArgs.OpType }, + [1] = new ParamValue { Kind = ParamKind.Int, IntValue = reductionArgs.ReduceAll }, + }; + + if (reductionArgs.Axes.Length > 0) + { + var axesSizeAndData = new List { reductionArgs.Axes.Length }; + foreach (var item in reductionArgs.Axes) + { + axesSizeAndData.Add(item); + } + + args.Add(-3, new ParamValue { Kind = ParamKind.ArrayOfInt, TensorValue = axesSizeAndData.ToArray() }); + } + + args.Add(4, new ParamValue { Kind = ParamKind.Int, IntValue = reductionArgs.Keepdims }); + args.Add(5, new ParamValue { Kind = ParamKind.Int, IntValue = 1 }); + AddLayer("Reduction", name[0], new[] { input }, name, args); + } + + public void Reshape(string[] name, string input, int[] newshape) + { + var args = new ParamDict(); + List index; + if (newshape.Length < 4) + { + index = Enumerable.Range(0, newshape.Length).ToList(); + } + else + { + index = new List { 0, 1, 11, 2 }; + } + + int i = 0; + foreach (int item in newshape.Reverse()) + { + args.Add(index[i], new ParamValue { Kind = ParamKind.Int, IntValue = item }); + i += 1; + } + + AddLayer("Reshape", name[0], new[] { input }, name, args); + } + + public void SELU(string[] name, string input, float alpha, float gamma) + { + AddLayer("SELU", name[0], new[] { input }, name, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Float, FloatValue = alpha }, + [1] = new ParamValue { Kind = ParamKind.Float, FloatValue = gamma }, + }); + } + + public void Crop(string[] name, string input, CropArgs cropArgs) + { + var args = new ParamDict(); + + // TODO: if need to fit torch crop, add other args into paramDict. + if (cropArgs.Axes!.Length > 0) + { + var startData = new List { cropArgs.Axes.Length }; + var endData = new List { cropArgs.Axes.Length }; + var axisData = new List { cropArgs.Axes.Length }; + for (int i = 0; i < cropArgs.Axes.Length; i++) + { + startData.Add(cropArgs.Starts![i]); + endData.Add(cropArgs.Ends![i]); + axisData.Add(cropArgs.Axes![i]); + } + + args.Add(-9, new ParamValue { Kind = ParamKind.ArrayOfInt, TensorValue = startData.ToArray() }); + args.Add(-10, new ParamValue { Kind = ParamKind.ArrayOfInt, TensorValue = endData.ToArray() }); + args.Add(-11, new ParamValue { Kind = ParamKind.ArrayOfInt, TensorValue = axisData.ToArray() }); + } + + AddLayer("Crop", name[0], new[] { input }, name, args); + } + + public void Sigmoid(string[] name, string input) + { + AddLayer("Sigmoid", name[0], new[] { input }, name); + } + + public void Softplus(string[] name, string input) + { + AddLayer("Softplus", name[0], new[] { input }, name); + } + + public void Slice(string[] name, string input, int[] slices, int axis) + { + var sliceData = new List { slices.Length }; + sliceData.AddRange(slices); + AddLayer("Slice", name[0], new[] { input }, name, new ParamDict + { + [-0] = new ParamValue { Kind = ParamKind.ArrayOfInt, TensorValue = sliceData.ToArray() }, + [1] = new ParamValue { Kind = ParamKind.Int, IntValue = axis }, + }); + } + + public void Tile(string[] name, string input, int[] repeats) + { + var repeatsData = new List { repeats.Length }; + repeatsData.AddRange(repeats); + AddLayer("Tile", name[0], new[] { input }, name, new ParamDict + { + [-2] = new ParamValue { Kind = ParamKind.ArrayOfInt, TensorValue = repeatsData.ToArray() }, + }); + } + + public void Permute(string[] name, string input, int orderType) + { + AddLayer("Permute", name[0], new[] { input }, name, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = orderType }, + }); + } + + public void Matmul(string[] name, string inputA, string inputB, int lOrR, float[] constInput, int[] constShape) + { + var inputList = new[] { inputA, inputB }; + + if (constInput != null && constShape != null) + { + if (lOrR == 1) + { + inputList[0] = name[0] + "_memorydata"; + } + else + { + inputList[1] = name[0] + "_memorydata"; + } + + var paramDict = new ParamDict(); + for (int i = 0; i < constShape.Length; i++) + { + int index = i switch + { + 0 => 0, + 1 => 1, + 2 when constShape.Length == 3 => 2, + 2 when constShape.Length == 4 => 11, + 3 when constShape.Length == 4 => 2, + _ => throw new NotSupportedException("Only support less than 5D"), + }; + paramDict[index] = new ParamValue { Kind = ParamKind.Int, IntValue = constShape[constShape.Length - 1 - i] }; + } + + AddLayer("MemoryData", name[0] + "_memorydata", Array.Empty(), new[] { name[0] + "_memorydata" }, paramDict); + + WriteFloatArray(constInput); + } + + AddLayer("MatMul", name[0], inputList, name, null); + } + + public void ConvTranspose(string[] name, string input, ConvTransposeArgs args) + { + var actData = new List { args.ActivationParams.Length }; + actData.AddRange(args.ActivationParams); + + AddLayer("Deconvolution", name[0], new[] { input }, name, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = args.NumOutput }, + [1] = new ParamValue { Kind = ParamKind.Int, IntValue = args.KernelW }, + [11] = new ParamValue { Kind = ParamKind.Int, IntValue = args.KernelH }, + [2] = new ParamValue { Kind = ParamKind.Int, IntValue = args.DilationW }, + [12] = new ParamValue { Kind = ParamKind.Int, IntValue = args.DilationH }, + [3] = new ParamValue { Kind = ParamKind.Int, IntValue = args.StrideW }, + [13] = new ParamValue { Kind = ParamKind.Int, IntValue = args.StrideH }, + [4] = new ParamValue { Kind = ParamKind.Int, IntValue = args.PadLeft }, + [14] = new ParamValue { Kind = ParamKind.Int, IntValue = args.PadTop }, + [15] = new ParamValue { Kind = ParamKind.Int, IntValue = args.PadRight }, + [16] = new ParamValue { Kind = ParamKind.Int, IntValue = args.PadBottom }, + + [5] = new ParamValue { Kind = ParamKind.Int, IntValue = args.BiasTerm }, + [6] = new ParamValue { Kind = ParamKind.Int, IntValue = args.WeightDataSize }, + + // [9] = new ParamValue { Kind = ParamKind.Int, IntValue = args.ActivationType }, + // [-10] = new ParamValue { Kind = ParamKind.ArrayOfIntOrFloat, TensorValue = actData.ToArray() }, + // + // [18] = new ParamValue { Kind = ParamKind.Int, IntValue = args.OutputPadRight }, + // [19] = new ParamValue { Kind = ParamKind.Int, IntValue = args.OutputPadBottom }, + // [20] = new ParamValue { Kind = ParamKind.Int, IntValue = args.OutputW }, + // [21] = new ParamValue { Kind = ParamKind.Int, IntValue = args.OutputH }, + }); + WriteFloatArray(new float[] { 0 }); // quantize flag [Not exist in ncnn op.md] + WriteFloatArray(args.WeightData.ToArray()); + WriteFloatArray(args.BiasData); + } + + public void Cast(string[] name, string input, int fromType, int toType) + { + AddLayer("Cast", name[0], new[] { input }, name, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = fromType }, + [1] = new ParamValue { Kind = ParamKind.Int, IntValue = toType }, + }); + } + + public void GELU(string[] name, string input) + { + AddLayer("GELU", name[0], new[] { input }, name, null); + } + + public void Dequantize(string[] name, string input, float[] scale, float[] bias) + { + AddLayer("Dequantize", name[0], new[] { input }, name, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = scale.Length }, + [1] = new ParamValue { Kind = ParamKind.Int, IntValue = bias.Length }, + }); + + WriteFloatArray(scale); + WriteFloatArray(bias); + } + + public void Squeeze(string[] name, string input, int[] dims) + { + var repeatsData = new List { dims.Length }; + repeatsData.AddRange(dims); + var args = new ParamDict(); + if (dims.Length == 0) + { + args.Add(0, new ParamValue { Kind = ParamKind.Int, IntValue = 1 }); + args.Add(1, new ParamValue { Kind = ParamKind.Int, IntValue = 1 }); + args.Add(2, new ParamValue { Kind = ParamKind.Int, IntValue = 1 }); + } + else + { + args.Add(-3, new ParamValue { Kind = ParamKind.ArrayOfInt, TensorValue = repeatsData.ToArray() }); + } + + AddLayer("Squeeze", name[0], new[] { input }, name, args); + } + + public void Unsqueeze(string[] name, string input, int[] dims) + { + var repeatsData = new List { dims.Length }; + repeatsData.AddRange(dims); + AddLayer("ExpandDims", name[0], new[] { input }, name, new ParamDict + { + [-3] = new ParamValue { Kind = ParamKind.ArrayOfInt, TensorValue = repeatsData.ToArray() }, + }); + } + + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) + { + var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); + if (paramDict != null) + { + layer.ParamDict = paramDict; + } + + for (int i = 0; i < bottoms.Length; i++) + { + layer.Bottoms[i] = new NcnnTensor { Name = bottoms[i] }; + } + + for (int i = 0; i < tops.Length; i++) + { + layer.Tops[i] = new NcnnTensor { Name = tops[i] }; + } + + switch (type) + { + case "Input": + _model.ModelInputs.Add(layer); + break; + case "MemoryData": + _model.MemoryDatas.Add(layer); + break; + default: + _model.Layers.Add(layer); + break; + } + } + + private void WriteFloatArray(float[] data) + { + _rData!.AddRange(data); + foreach (float value in data) + { + _binWriter.Write(value); + } + } +} diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs new file mode 100644 index 0000000000..a697a077eb --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -0,0 +1,265 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Reactive; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Toolkit.HighPerformance; +using NetFabric.Hyperlinq; +using Nncase.Diagnostics; +using Nncase.IR; +using Nncase.IR.Ncnn; +using Nncase.IR.Tensors; +using Nncase.Utilities; +using static Nncase.CodeGen.CodeGenDumper; + +namespace Nncase.CodeGen.Ncnn; + +/// +/// Ncnn function builder. +/// +internal class NcnnFunctionBuilder : FunctionBuilder +{ + private readonly NcnnEmitter _emitter; + private string[]? _inputs; + private string[]? _outputs; + + public NcnnFunctionBuilder(uint id, SectionManager sectionManager) + : base(id, sectionManager) + { + _emitter = new NcnnEmitter(sectionManager.GetWriter(WellknownSectionNames.Rdata)); + } + + protected override ILinkableFunction CreateLinkableFunction(uint id, BaseFunction callable, IReadOnlyList functionRefs, Stream text) + { + return new NcnnLinkableFunction(id, callable, functionRefs, text, _inputs!, _outputs!, _emitter.GetRData()!.ToArray()!); + } + + protected override void Compile(BaseFunction callable) + { + var visitor = new CodeGenVisitor(_emitter); + _outputs = visitor.Visit(callable).Split(','); + _inputs = visitor.Inputs.ToArray(); + } + + protected override void WriteText() + { + _emitter.SaveParam(TextWriter.BaseStream); + + // dump param and bin. + string dumpPath = Path.Join(DumpScope.Current.Directory, "ncnn_param_dir"); + if (!Directory.Exists(dumpPath)) + { + Directory.CreateDirectory(dumpPath); + } + else if (Id == 0) + { + // clear dir before single case. + foreach (string filePath in Directory.GetFiles(dumpPath, "*", SearchOption.AllDirectories)) + { + File.SetAttributes(filePath, FileAttributes.Normal); // 移除所有特殊属性以便删除 + File.Delete(filePath); + } + } + + if (DumpScope.Current.IsEnabled(DumpFlags.CodeGen)) + { + // dump param. + using (var fileStream = File.Create(Path.Join(dumpPath, $"ncnn_{Id}.param"))) + { + TextWriter.BaseStream.Seek(0, SeekOrigin.Begin); + TextWriter.BaseStream.CopyTo(fileStream); + } + + // dump bin. + _emitter.SaveBin(dumpPath, Id); + } + } + + private class CodeGenVisitor : ExprVisitor + { + private readonly NcnnEmitter _emitter; + + private int _layerId; + + public CodeGenVisitor(NcnnEmitter emitter) + { + _emitter = emitter; + } + + public List Inputs { get; } = new(); + + public List Outputs { get; } = new(); + + protected override string VisitLeafVar(Var expr) + { + var name = GetNextName(); + _emitter.Input(name); + Inputs.Add(name); + return name; + } + + protected override string VisitLeafFunction(Function expr) => ExprMemo[expr.Body]; + + protected override string VisitLeafOp(Op expr) => string.Empty; + + protected override string VisitLeafTuple(IR.Tuple expr) => StringUtility.Join(",", expr.Fields.AsValueEnumerable().Select(x => ExprMemo[x])); + + protected override string VisitLeafCall(Call expr) + { + var names = new List { GetNextName() }; + string[]? inString; + switch (expr.Target) + { + case NcnnSoftmax op: + _emitter.Softmax(names[0], ExprMemo[expr.Arguments[0]], op.Axis); + break; + case NcnnUnary op: + _emitter.Unary(names[0], ExprMemo[expr.Arguments[0]], op.OpType); + break; + case NcnnBatchNorm op: + _emitter.BatchNorm(names[0], ExprMemo[expr.Arguments[0]], op.Channels, op.Eps, op.SlopeData, op.MeanData, op.VarData, op.BiasData); + break; + case NcnnBinary op: + inString = op.LorR switch + { + 0 => new string[] { ExprMemo[expr.Arguments[0]], ExprMemo[expr.Arguments[1]] }, + 1 => new string[] { string.Empty, ExprMemo[expr.Arguments[0]] }, + 2 => new string[] { ExprMemo[expr.Arguments[0]], string.Empty }, + _ => throw new NotImplementedException("Never reach here."), + }; + _emitter.Binary(names[0], inString[0], inString[1], op.OpType, op.LorR, op.ConstInput!, op.ConstShape!); + break; + case NcnnCelu op: + _emitter.Celu(names[0], ExprMemo[expr.Arguments[0]], op.Alpha); + break; + case NcnnClip op: + _emitter.Clip(names[0], ExprMemo[expr.Arguments[0]], op.Min, op.Max); + break; + case NcnnConcat op: + List in_ = new(); + var t = (IR.Tuple)expr.Arguments[0]; + for (int i = 0; i < t.Fields.Length; i++) + { + in_.Add(ExprMemo[t.Fields[i]]); + } + + _emitter.Concat(names[0], in_.ToArray(), op.Axis); + break; + case NcnnConv op: + _emitter.Conv(names[0], ExprMemo[expr.Arguments[0]], op.Args); + break; + case NcnnCumsum op: + _emitter.Cumsum(names[0], ExprMemo[expr.Arguments[0]], op.Axis); + break; + case NcnnElu op: + _emitter.Elu(names[0], ExprMemo[expr.Arguments[0]], op.Alpha); + break; + case NcnnErf: + _emitter.Erf(names[0], ExprMemo[expr.Arguments[0]]); + break; + case NcnnHardSigmoid op: + _emitter.HardSigmoid(names[0], ExprMemo[expr.Arguments[0]], op.Alpha, op.Beta); + break; + case NcnnHardSwish op: + _emitter.HardSwish(names[0], ExprMemo[expr.Arguments[0]], op.Alpha, op.Beta); + break; + case NcnnInstanceNorm op: + _emitter.InstanceNorm(names[0], ExprMemo[expr.Arguments[0]], op.Channels, op.Eps, op.Affine, op.GammaData, op.BetaData); + break; + case NcnnLRN op: + _emitter.LRN(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Alpha, op.Beta, op.Bias, op.Size); + break; + case NcnnLSTM op: + for (int i = 1; i < op.OutputSize; i++) + { + var a = GetNextName(); + names.Add(a); + } + + _emitter.LSTM(names.ToArray(), ExprMemo[expr.Arguments[0]], op.HiddenSize, op.WeightDataSize, op.Direction, op.W, op.R, op.B); + break; + case NcnnPadding op: + _emitter.Padding(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Top, op.Bottom, op.Left, op.Right, op.Type, op.Value, op.Front, op.Behind); + break; + case NcnnPooling op: + _emitter.Pooling(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Args); + break; + case NcnnPReLU op: + _emitter.PReLU(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Slope); + break; + case NcnnReduction op: + _emitter.Reduction(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Args); + break; + case NcnnReshape op: + _emitter.Reshape(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Shape); + break; + case NcnnSELU op: + _emitter.SELU(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Alpha, op.Gamma); + break; + case NcnnSigmoid: + _emitter.Sigmoid(names.ToArray(), ExprMemo[expr.Arguments[0]]); + break; + case NcnnCrop op: + _emitter.Crop(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Args); + break; + case NcnnSoftplus: + _emitter.Softplus(names.ToArray(), ExprMemo[expr.Arguments[0]]); + break; + case NcnnSlice op: + names.AddRange(op.Slices.Select(i => GetNextName())); + _emitter.Slice(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Slices, op.Axis); + break; + case NcnnTile op: + _emitter.Tile(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Repeats); + break; + case NcnnPermute op: + _emitter.Permute(names.ToArray(), ExprMemo[expr.Arguments[0]], op.OrderType); + break; + case NcnnMatMul op: + inString = op.LorR switch + { + 0 => new string[] { ExprMemo[expr.Arguments[0]], ExprMemo[expr.Arguments[1]] }, + 1 => new string[] { string.Empty, ExprMemo[expr.Arguments[0]] }, + 2 => new string[] { ExprMemo[expr.Arguments[0]], string.Empty }, + _ => throw new NotImplementedException("Never reach here."), + }; + _emitter.Matmul(names.ToArray(), inString[0], inString[1], op.LorR, op.ConstInput, op.ConstShape); + break; + case NcnnConvTranspose op: + _emitter.ConvTranspose(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Args); + break; + case NcnnLayerNorm op: + _emitter.LayerNorm(names.ToArray(), ExprMemo[expr.Arguments[0]], op.AffineSize, op.Eps, op.Affine, op.GammaData, op.BetaData); + break; + case NcnnCast op: + _emitter.Cast(names.ToArray(), ExprMemo[expr.Arguments[0]], op.FromType, op.ToType); + break; + case NcnnGELU: + _emitter.GELU(names.ToArray(), ExprMemo[expr.Arguments[0]]); + break; + case NcnnDequantize op: + _emitter.Dequantize(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Scale, op.Bias); + break; + case NcnnSqueeze op: + _emitter.Squeeze(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Dims); + break; + case NcnnUnsqueeze op: + _emitter.Unsqueeze(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Dims); + break; + default: + throw new NotSupportedException($"Not support {nameof(expr.Target)} in Ncnn ops emitter"); + } + + // serialize outputs to string. + string output = string.Join(",", names.Select(x => x.ToString())); + return output; + } + + private string GetNextName() => $"layer{_layerId++}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnLinkableFunction.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnLinkableFunction.cs new file mode 100644 index 0000000000..28fe673c69 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnLinkableFunction.cs @@ -0,0 +1,44 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; + +namespace Nncase.CodeGen.Ncnn; + +internal class NcnnLinkableFunction : ILinkableFunction +{ + public NcnnLinkableFunction(uint id, BaseFunction sourceFunction, IEnumerable functionRefs, Stream text, string[] inputs, string[] outputs, float[] rData) + { + Id = id; + SourceFunction = sourceFunction; + FunctionRefs = functionRefs; + Text = text; + Inputs = inputs; + Outputs = outputs; + Sections = new[] + { + LinkedSection.FromStrings(inputs, ".inputs"), + LinkedSection.FromStrings(outputs, ".outputs"), + LinkedSection.FromData(rData, ".rdata"), + }; + } + + public uint Id { get; } + + public BaseFunction SourceFunction { get; } + + public IEnumerable FunctionRefs { get; } + + public Stream Text { get; } + + public string[] Inputs { get; } + + public string[] Outputs { get; } + + public IReadOnlyList Sections { get; } +} diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnLinkableModule.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnLinkableModule.cs new file mode 100644 index 0000000000..5ea4a751ba --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnLinkableModule.cs @@ -0,0 +1,26 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Nncase.CodeGen.Ncnn; + +internal class NcnnLinkableModule : LinkableModule +{ + public NcnnLinkableModule(IReadOnlyList functions, SectionManager sectionManager) + : base(functions, sectionManager) + { + } + + protected override ILinkedModule CreateLinkedModule(IReadOnlyList linkedFunctions, Stream text) + { + return new NcnnLinkedModule( + linkedFunctions, + text, + SectionManager.GetContent(WellknownSectionNames.Rdata)); + } +} diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnLinkedModule.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnLinkedModule.cs new file mode 100644 index 0000000000..078f9c903c --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnLinkedModule.cs @@ -0,0 +1,31 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Nncase.CodeGen.Ncnn; + +internal class NcnnLinkedModule : ILinkedModule +{ + public NcnnLinkedModule(IReadOnlyList functions, Stream text, Stream? rdata) + { + Functions = functions; + Sections = new[] + { + new LinkedSection(text, ".text", 0, 8, (uint)text.Length), + new LinkedSection(rdata, ".rdata", 0, 8, (uint)(rdata?.Length ?? 0)), + }; + } + + public string ModuleKind => "ncnn"; + + public uint Version => 1; + + public IReadOnlyList Functions { get; } + + public IReadOnlyList Sections { get; } +} diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnModuleBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnModuleBuilder.cs new file mode 100644 index 0000000000..3234727e69 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnModuleBuilder.cs @@ -0,0 +1,32 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; + +namespace Nncase.CodeGen.Ncnn; + +/// +/// Ncnn module builder. +/// +public class NcnnModuleBuilder : ModuleBuilder +{ + /// + public override string ModuleKind => "ncnn"; + + /// + protected override ILinkableModule CreateLinkableModule(IReadOnlyList linkableFunctions) + { + return new NcnnLinkableModule(linkableFunctions, SectionManager); + } + + /// + protected override FunctionBuilder CreateFunctionBuilder(uint id) + { + return new NcnnFunctionBuilder(id, SectionManager); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnBatchNorm.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnBatchNorm.cs new file mode 100644 index 0000000000..e843def456 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnBatchNorm.cs @@ -0,0 +1,73 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnBatchNormEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnBatchNorm batchnorm) + { + var input = context.GetOrtArgumentValue(batchnorm, NcnnBatchNorm.Input); + var eps = batchnorm.Eps; + var slopeData = batchnorm.SlopeData.ToArray(); + var meanData = batchnorm.MeanData.ToArray(); + var varData = batchnorm.VarData.ToArray(); + var biasData = batchnorm.BiasData.ToArray(); + return OrtKI.BatchNormalization(input, slopeData, biasData, meanData, varData, eps, 0.0f).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnBatchNorm target) + { + var input = context.CheckArgumentType(target, NcnnBatchNorm.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnBatchNorm target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnBatchNorm target) + { + var inputType = context.GetArgumentType(target, NcnnBatchNorm.Input); + var returnType = context.GetReturnType(); + var returnF = MetricUtility.GetFLOPs(returnType); + var inputF = MetricUtility.GetFLOPs(inputType); + var inner = inputF / returnF; + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = (inner * 2) + (inputF * (MetricUtility.SubFLOPs + MetricUtility.DivFLOPs + MetricUtility.MulFLOPs + MetricUtility.AddFLOPs)), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnBatchNorm target) => context.GetArgumentShape(target, NcnnBatchNorm.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnBinary.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnBinary.cs new file mode 100644 index 0000000000..a3cc410e3c --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnBinary.cs @@ -0,0 +1,137 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using DryIoc; +using NetFabric.Hyperlinq; +using Nncase.CostModel; +using Nncase.Evaluator; +using Nncase.IR; +using Nncase.IR.Ncnn; +using Nncase.Utilities; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnBinaryEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnBinary binary) + { + var inputA = context.GetOrtArgumentValue(binary, NcnnBinary.InputA); + var inputB = context.GetOrtArgumentValue(binary, NcnnBinary.InputB); + var opType = binary.OpType; + + // return OrtKI. (input, dim).ToValue(); + switch (opType) + { + case BinaryOperationType.ADD: + return OrtKI.Add(inputA, inputB).ToValue(); + case BinaryOperationType.SUB: + return OrtKI.Sub(inputA, inputB).ToValue(); + case BinaryOperationType.MUL: + return OrtKI.Mul(inputA, inputB).ToValue(); + case BinaryOperationType.DIV: + return OrtKI.Div(inputA, inputB).ToValue(); + + // case BinaryOperationType.MAX: + // return System.Math.Min(inputA, inputB).ToValue(); + // case BinaryOperationType.MIN: + // return OrtKI.Min(inputA, inputB).ToValue(); + + // TODO: trunc + default: + throw new NotSupportedException("Ncnn unary ops"); + } + } + + public IRType Visit(NcnnBinary target, TensorType lhs, TensorType rhs) + { + return TypeInference.BroadcastType(lhs, rhs); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnBinary target) + { + return target.LorR switch + { + 1 => Visit(context.CheckArgumentType(target, NcnnBinary.InputA), new TensorType(context.CheckArgumentType(target, NcnnBinary.InputA).DType, target.ConstShape)), + 2 => Visit(new TensorType(context.CheckArgumentType(target, NcnnBinary.InputA).DType, target.ConstShape), context.CheckArgumentType(target, NcnnBinary.InputA)), + 0 => Visit(context.CheckArgumentType(target, NcnnBinary.InputA), context.CheckArgumentType(target, NcnnBinary.InputB)), + _ => throw new NotSupportedException("Never reach here, LorR without fourth situation."), + }; + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnBinary target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnBinary target) + { + var lhsType = context.GetArgumentType(target, NcnnBinary.InputA); + var rhsType = context.GetArgumentType(target, NcnnBinary.InputB); + var outputType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(lhsType) + CostUtility.GetMemoryAccess(rhsType) + CostUtility.GetMemoryAccess(outputType), + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(outputType, (int)MetricUtility.GetBinaryFLOPs(MapBinaryOp(target.OpType))), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnBinary target) + { + var lhs = context.GetArgumentShape(target, NcnnBinary.InputA); + var rhs = context.GetArgumentShape(target, NcnnBinary.InputB); + return ShapeExprUtility.BroadcastShape(lhs, rhs); + } + + private static BinaryOp MapBinaryOp(BinaryOperationType binaryOp) => + binaryOp switch + { + BinaryOperationType.ADD => BinaryOp.Add, + BinaryOperationType.SUB => BinaryOp.Sub, + BinaryOperationType.MUL => BinaryOp.Mul, + BinaryOperationType.DIV => BinaryOp.Div, + BinaryOperationType.MAX => BinaryOp.Max, + BinaryOperationType.MIN => BinaryOp.Min, + BinaryOperationType.POW => BinaryOp.Pow, + _ => throw new ArgumentException("Invalid binary op type"), + + // unsupported Binary ops + // BinaryOp.Mod => + // BitwiseAnd + // BitwiseOr + // BitwiseXor + // LogicalAnd + // LogicalOr + // LogicalXor + // LeftShift + // RightShift + // => BinaryOperationType.RSUB, + // => BinaryOperationType.RDIV, + // => BinaryOperationType.RPOW, + // => BinaryOperationType.ATAN2, + // => BinaryOperationType.RATAN2, + }; + + private IRType Visit(TensorType inputA, TensorType inputB) + { + return TypeInference.BroadcastType(inputA, inputB); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCast.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCast.cs new file mode 100644 index 0000000000..e8e2ebdbb1 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCast.cs @@ -0,0 +1,77 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Data; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnCastEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnCast cast) + { + var input = context.GetOrtArgumentValue(cast, NcnnCast.Input); + return OrtKI.Cast(input, cast.ToType).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnCast target) + { + var input = context.CheckArgumentType(target, NcnnCast.Input); + return Visit(input, target.ToType); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnCast target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnCast target) + { + _ = context.GetArgumentType(target, NcnnCast.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnCast target) => context.GetArgumentShape(target, NcnnCast.Input); + + public DataType RecoverDataType(int num) + { + return num switch + { + 1 => DataTypes.Float32, + 2 => DataTypes.Float16, + 4 => DataTypes.BFloat16, + _ => throw new DataException($"not support DataTypeNum :{num}"), + }; + } + + private IRType Visit(TensorType input, int dT) + { + return new TensorType(RecoverDataType(dT), input.Shape); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCelu.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCelu.cs new file mode 100644 index 0000000000..be3d722b15 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCelu.cs @@ -0,0 +1,66 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnCeluEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnCelu celu) + { + var input = context.GetOrtArgumentValue(celu, NcnnCelu.Input); + var alpha = celu.Alpha; + return OrtKI.Celu(input, alpha).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnCelu target) + { + var input = context.CheckArgumentType(target, NcnnCelu.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnCelu target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnCelu target) + { + var inputType = context.GetArgumentType(target, NcnnCelu.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.DivFLOPs + MetricUtility.ExpFLOPs + MetricUtility.MulFLOPs + MetricUtility.SubFLOPs + MetricUtility.AddFLOPs + (MetricUtility.CmpFLOPs * 2)), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnCelu target) => context.GetArgumentShape(target, NcnnCelu.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnClip.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnClip.cs new file mode 100644 index 0000000000..538d1ed07b --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnClip.cs @@ -0,0 +1,67 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnClipEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnClip clip) + { + var input = context.GetOrtArgumentValue(clip, NcnnClip.Input); + var min = clip.Min; + var max = clip.Max; + return OrtKI.Clip(input, min, max).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnClip target) + { + var input = context.CheckArgumentType(target, NcnnClip.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnClip target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnClip target) + { + var inputType = context.GetArgumentType(target, NcnnClip.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * 2, + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnClip target) => context.GetArgumentShape(target, NcnnClip.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConcat.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConcat.cs new file mode 100644 index 0000000000..f0f758fcea --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConcat.cs @@ -0,0 +1,113 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using Nncase.Utilities; +using OrtKISharp; +using static Nncase.IR.F.Tensors; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnConcatEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnConcat concat) + { + var inputs = context.GetArgumentValueAsTensors(concat, NcnnConcat.Input); + var axis = concat.Axis; + return OrtKI.Concat(inputs.Select(t => t.ToOrtTensor()).ToArray(), axis).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnConcat target) + { + var inputs = context.CheckArgumentType(target, NcnnConcat.Input); + return Visit(inputs.Fields, target.Axis); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnConcat target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnConcat target) => Metric.Zero; + + public Expr Visit(IShapeEvaluateContext context, NcnnConcat target) => context.GetArgumentShape(target, NcnnConcat.Input); + + private IRType? CheckType(TupleType inputs) + { + bool? allScalar = null; + DataType? allDType = null; + foreach (var (i, input) in Enumerable.Range(0, inputs.Count).Select(i => (i, inputs[i]))) + { + TensorType type; + if (input is TensorType a) + { + type = a; + } + else if (input is DistributedType { TensorType: TensorType b }) + { + type = b; + } + else + { + return new InvalidType($"The ConCat Item[{i}] Must Have TensorType But Get {input}"); + } + + if (type.Shape.IsUnranked) + { + return new TensorType(type.DType, Shape.Unranked); + } + + allScalar = (allScalar ?? type.IsScalar) & type.IsScalar; + allDType ??= type.DType; + if (allDType != type.DType) + { + return new InvalidType( + $"The ConCat Item[{i}] Must Be {allDType} But Get {type.DType.GetDisplayName()}"); + } + } + + if (allScalar == true && allDType is not null) + { + return new TensorType(allDType, new[] { inputs.Count }); + } + + return null; + } + + private TensorType GetTensorType(IRType input) => input switch + { + TensorType t => t, + DistributedType d => d.TensorType, + _ => throw new InvalidCastException(), + }; + + private IRType Visit(IRArray inputs, int axisValue) + { + var outputShape = GetTensorType(inputs[0]).Shape.ToArray(); + + foreach (var item in inputs[1..]) + { + outputShape[axisValue] += GetTensorType(item).Shape.ToArray()[axisValue]; + } + + return new TensorType(GetTensorType(inputs[0]).DType, outputShape); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConv.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConv.cs new file mode 100644 index 0000000000..362fdfc601 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConv.cs @@ -0,0 +1,88 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; +using static Nncase.IR.F.Tensors; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnConvEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnConv conv) + { + var inputs = context.GetOrtArgumentValue(conv, NcnnConv.Input); + var weights = new Tensor(conv.Args.WeightData, new[] { conv.Args.NumOutput, conv.Args.WeightDataSize / (conv.Args.KernelH * conv.Args.KernelW), conv.Args.KernelH, conv.Args.KernelW }).ToOrtTensor(); + var bias = new Tensor(conv.Args.BiasData, new[] { conv.Args.NumOutput }).ToOrtTensor(); + var result = OrtKI.Conv(inputs, weights, bias, "NOTSET", new long[] { conv.Args.DilationH, conv.Args.DilationW }, 1, new long[] { conv.Args.KernelH, conv.Args.KernelW }, new long[] { conv.Args.PadLeft, conv.Args.PadTop, conv.Args.PadRight, conv.Args.PadBottom }, new long[] { conv.Args.StrideH, conv.Args.StrideW }); + return OrtKI.Clip(result, conv.Args.ActivationParams[0], conv.Args.ActivationParams[1]).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnConv target) + { + var inputs = context.CheckArgumentType(target, NcnnConv.Input); + return Visit(inputs, target); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnConv target) + { + var inputType = context.GetArgumentType(target, NcnnConv.Input); + var weightsType = new TensorType(DataTypes.Float32, new[] { target.Args.WeightDataSize }); + var biasType = new TensorType(DataTypes.Float32, new[] { target.Args.NumOutput }); + var macPerElement = (2 * target.Args.WeightDataSize / target.Args.NumOutput) - 1; + + var outputType = context.GetReturnType(); + + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(inputType) + CostUtility.GetMemoryAccess(weightsType) + CostUtility.GetMemoryAccess(biasType), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(outputType), + [CostFactorNames.CPUCycles] = CostUtility.GetCPUCycles(outputType, (uint)macPerElement), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnConv target) => Metric.Zero; + + public Expr Visit(IShapeEvaluateContext context, NcnnConv target) => context.GetArgumentShape(target, NcnnConv.Input); + + private TensorType GetTensorType(IRType input) => input switch + { + TensorType t => t, + DistributedType d => d.TensorType, + _ => throw new InvalidCastException(), + }; + + private IRType Visit(TensorType input, NcnnConv conv) + { + var outputShape = input.Shape.ToList(); + var kernelShape = new[] { conv.Args.NumOutput, conv.Args.WeightDataSize / (conv.Args.NumOutput * conv.Args.KernelH * conv.Args.KernelW), conv.Args.KernelH, conv.Args.KernelW }; + outputShape[0] = conv.Args.NumOutput; + + outputShape[1] = IR.TypePatternUtility.GetWindowedOutputSize( + input.Shape[1].FixedValue + conv.Args.PadTop + conv.Args.PadBottom, + kernelShape[2], + conv.Args.StrideH, + conv.Args.DilationH, + false); + outputShape[2] = IR.TypePatternUtility.GetWindowedOutputSize( + input.Shape[2].FixedValue + conv.Args.PadLeft + conv.Args.PadRight, + kernelShape[3], + conv.Args.StrideW, + conv.Args.DilationW, + false); + return new TensorType(GetTensorType(input).DType, outputShape.ToArray()); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConvTranspose.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConvTranspose.cs new file mode 100644 index 0000000000..6182197696 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConvTranspose.cs @@ -0,0 +1,94 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; +using static Nncase.IR.F.Tensors; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnConvTransposeEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnConvTranspose convTranspose) + { + var inputs = context.GetOrtArgumentValue(convTranspose, NcnnConvTranspose.Input); + var weights = Transpose(convTranspose.Args.WeightData, new int[] { 1, 0, 2, 3 }).Evaluate().AsTensor().ToOrtTensor(); + var bias = new Tensor(convTranspose.Args.BiasData, new[] { convTranspose.Args.NumOutput }).ToOrtTensor(); + var result = OrtKI.ConvTranspose(inputs, weights, bias, "NOTSET", new long[] { convTranspose.Args.DilationH, convTranspose.Args.DilationW }, 1, new long[] { convTranspose.Args.KernelH, convTranspose.Args.KernelW }, new long[] { convTranspose.Args.OutputPadBottom, convTranspose.Args.OutputPadRight, }, new long[] { convTranspose.Args.OutputH, convTranspose.Args.OutputW }, new long[] { convTranspose.Args.PadRight, convTranspose.Args.PadBottom, convTranspose.Args.PadRight, convTranspose.Args.PadBottom }, new long[] { convTranspose.Args.StrideH, convTranspose.Args.StrideW }); + return OrtKI.Clip(result, convTranspose.Args.ActivationParams[0], convTranspose.Args.ActivationParams[1]).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnConvTranspose target) + { + var inputs = context.CheckArgumentType(target, NcnnConvTranspose.Input); + return Visit(inputs, target); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnConvTranspose target) + { + var inputType = context.GetArgumentType(target, NcnnConvTranspose.Input); + _ = inputType.Shape.ToValueArray(); + var weightsType = new TensorType(target.Args.WeightData.ElementType, target.Args.WeightData.Shape); + var weightsShape = weightsType.Shape.ToValueArray(); + var biasType = new TensorType(target.Args.WeightData.ElementType, new[] { target.Args.BiasData.Length }); + + var macPerElement = weightsShape[1] * weightsShape[2] * weightsShape[3]; + var outputType = context.GetReturnType(); + + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(inputType) + CostUtility.GetMemoryAccess(weightsType) + CostUtility.GetMemoryAccess(biasType), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(outputType), + [CostFactorNames.CPUCycles] = CostUtility.GetCPUCycles(outputType, (uint)macPerElement * 2), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnConvTranspose target) + { + var returnType = context.GetReturnType(); + _ = returnType.Shape.ToValueArray(); + + var inputType = context.GetArgumentType(target, NcnnConvTranspose.Input); + var inputShape = inputType.Shape.ToValueArray(); + var weightType = new TensorType(target.Args.WeightData.ElementType, target.Args.WeightData.Shape); + var weightShape = weightType.Shape.ToValueArray(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(inputType) + CostUtility.GetMemoryAccess(weightType) + CostUtility.GetMemoryAccess(returnType), + [MetricFactorNames.FLOPs] = (UInt128)(inputShape[0] * weightShape[0] * weightShape[1] * inputShape[2] * inputShape[3] * weightShape[2] * weightShape[3]), + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnConvTranspose target) => context.GetArgumentShape(target, NcnnConvTranspose.Input); + + private TensorType GetTensorType(IRType input) => input switch + { + TensorType t => t, + DistributedType d => d.TensorType, + _ => throw new InvalidCastException(), + }; + + private IRType Visit(TensorType input, NcnnConvTranspose convTranspose) + { + var outputShape = input.Shape.ToList(); + outputShape[0] = convTranspose.Args.NumOutput; + outputShape[1] = convTranspose.Args.OutputH; + outputShape[2] = convTranspose.Args.OutputW; + + return new TensorType(GetTensorType(input).DType, outputShape.ToArray()); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCrop.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCrop.cs new file mode 100644 index 0000000000..02956bc523 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCrop.cs @@ -0,0 +1,75 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.ArgsStruct; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnCropEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnCrop crop) + { + var input = context.GetOrtArgumentValue(crop, NcnnCrop.Input); + var starts = crop.Args.Starts; + var ends = crop.Args.Ends; + var axes = crop.Args.Axes; + var steps = Enumerable.Repeat(1, (starts ?? Array.Empty()).Length).ToArray(); + return OrtKI.Slice(input, starts ?? Array.Empty(), ends ?? Array.Empty(), axes ?? Array.Empty(), steps).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnCrop target) + { + var input = context.CheckArgumentType(target, NcnnCrop.Input); + return Visit(input, target.Args); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnCrop target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnCrop target) + { + var inputType = context.GetArgumentType(target, NcnnCrop.Input); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(inputType) * 2, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnCrop target) => context.GetArgumentShape(target, NcnnCrop.Input); + + private IRType Visit(TensorType input, CropArgs args) + { + var outputShape = input.Shape.ToArray(); + for (int i = 0; i < args.Axes!.Length; i++) + { + int tStart = args.Starts![i] >= 0 ? args.Starts[i] : args.Starts[i] + outputShape[args.Axes[i]].FixedValue; + int tEnd = args.Ends![i] >= 0 ? args.Ends[i] : args.Ends[i] + outputShape[args.Axes[i]].FixedValue; + outputShape[args.Axes[i] < 0 ? args.Axes[i] + outputShape.Length : args.Axes[i]] = System.Math.Abs(tEnd - tStart); + } + + return new TensorType(input.DType, outputShape); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCumsum.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCumsum.cs new file mode 100644 index 0000000000..c4c83ede68 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCumsum.cs @@ -0,0 +1,67 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnCumsumEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnCumsum cumsum) + { + var input = context.GetOrtArgumentValue(cumsum, NcnnCumsum.Input); + var axis = cumsum.Axis; + var axisTensor = new Tensor(new[] { axis }, new Shape(1)); + return OrtKI.CumSum(input, axisTensor.ToOrtTensor(), 0L, 0L).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnCumsum target) + { + var input = context.CheckArgumentType(target, NcnnCumsum.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnCumsum target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnCumsum target) + { + var inputType = context.GetArgumentType(target, NcnnCumsum.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * MetricUtility.AddFLOPs, + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnCumsum target) => context.GetArgumentShape(target, NcnnCumsum.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnDequantize.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnDequantize.cs new file mode 100644 index 0000000000..60a1006b36 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnDequantize.cs @@ -0,0 +1,67 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnDequantizeEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnDequantize dequantize) + { + var input = context.GetOrtArgumentValue(dequantize, NcnnDequantize.Input); + return OrtKI.DequantizeLinear(input, dequantize.Scale, dequantize.Bias, 0).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnDequantize target) + { + var input = context.CheckArgumentType(target, NcnnDequantize.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnDequantize target) + { + var input = context.GetArgumentType(target, NcnnDequantize.Input); + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(input) + + (UInt128)((target.Scale.Length + target.Bias.Length) * sizeof(float)), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnDequantize target) + { + var inputType = context.GetArgumentType(target, NcnnDequantize.Input); + var outputType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(inputType) + CostUtility.GetMemoryAccess(outputType), + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(outputType, 2), + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnDequantize target) => context.GetArgumentShape(target, NcnnDequantize.Input); + + private IRType Visit(TensorType input) + { + // ncnn only support dequantize to float32. + return new TensorType(DataTypes.Float32, input.Shape); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnElu.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnElu.cs new file mode 100644 index 0000000000..e450a140b5 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnElu.cs @@ -0,0 +1,66 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnEluEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnElu elu) + { + var input = context.GetOrtArgumentValue(elu, NcnnElu.Input); + var alpha = elu.Alpha; + return OrtKI.Elu(input, alpha).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnElu target) + { + var input = context.CheckArgumentType(target, NcnnElu.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnElu target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnElu target) + { + var inputType = context.GetArgumentType(target, NcnnElu.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.ExpFLOPs + MetricUtility.MulFLOPs + MetricUtility.SubFLOPs + MetricUtility.CmpFLOPs), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnElu target) => context.GetArgumentShape(target, NcnnElu.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnErf.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnErf.cs new file mode 100644 index 0000000000..bc890a5ace --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnErf.cs @@ -0,0 +1,65 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnErfEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnErf erf) + { + var input = context.GetOrtArgumentValue(erf, NcnnErf.Input); + return OrtKI.Erf(input).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnErf target) + { + var input = context.CheckArgumentType(target, NcnnErf.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnErf target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnErf target) + { + var inputType = context.GetArgumentType(target, NcnnErf.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.ExpFLOPs + MetricUtility.MulFLOPs + MetricUtility.SubFLOPs + MetricUtility.CmpFLOPs), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnErf target) => context.GetArgumentShape(target, NcnnErf.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnGELU.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnGELU.cs new file mode 100644 index 0000000000..5e5b0aacdf --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnGELU.cs @@ -0,0 +1,68 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.IR.NN; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnGELUEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnGELU celu) + { + var input = context.GetOrtArgumentValue(celu, NcnnGELU.Input); + var res = IR.F.Math.Mul(IR.F.NN.Erf(IR.F.Math.Div(input.ToValue().AsTensor(), 1.4142135381698608)) + 1, input.ToValue().AsTensor()) * 0.5; + return res.Evaluate(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnGELU target) + { + var input = context.CheckArgumentType(target, NcnnGELU.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnGELU target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnGELU target) + { + _ = context.GetArgumentType(target, NcnnGELU.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnGELU target) => context.GetArgumentShape(target, NcnnGELU.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSigmoid.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSigmoid.cs new file mode 100644 index 0000000000..2a3f4b9848 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSigmoid.cs @@ -0,0 +1,67 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnHardSigmoidEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnHardSigmoid hardSigmoid) + { + var input = context.GetOrtArgumentValue(hardSigmoid, NcnnHardSigmoid.Input); + var alpha = hardSigmoid.Alpha; + var beta = hardSigmoid.Beta; + return OrtKI.HardSigmoid(input, alpha, beta).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnHardSigmoid target) + { + var input = context.CheckArgumentType(target, NcnnHardSigmoid.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnHardSigmoid target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnHardSigmoid target) + { + var inputType = context.GetArgumentType(target, NcnnHardSigmoid.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.MulFLOPs + MetricUtility.AddFLOPs + (MetricUtility.CmpFLOPs * 2)), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnHardSigmoid target) => context.GetArgumentShape(target, NcnnHardSigmoid.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSwish.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSwish.cs new file mode 100644 index 0000000000..a372b289b9 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSwish.cs @@ -0,0 +1,65 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnHardSwishEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnHardSwish hardSwish) + { + var input = context.GetOrtArgumentValue(hardSwish, NcnnHardSwish.Input); + return OrtKI.HardSwish(input).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnHardSwish target) + { + var input = context.CheckArgumentType(target, NcnnHardSwish.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnHardSwish target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnHardSwish target) + { + var inputType = context.GetArgumentType(target, NcnnHardSwish.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.PowFLOPs + MetricUtility.MulFLOPs + MetricUtility.AddFLOPs + (MetricUtility.CmpFLOPs * 2)), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnHardSwish target) => context.GetArgumentShape(target, NcnnHardSwish.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnInstanceNorm.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnInstanceNorm.cs new file mode 100644 index 0000000000..ee54186dba --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnInstanceNorm.cs @@ -0,0 +1,74 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnInstanceNormEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnInstanceNorm instanceNorm) + { + var input = context.GetOrtArgumentValue(instanceNorm, NcnnInstanceNorm.Input); + + return OrtKI.InstanceNormalization(input, instanceNorm.GammaData, instanceNorm.BetaData, instanceNorm.Eps).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnInstanceNorm target) + { + var input = context.CheckArgumentType(target, NcnnInstanceNorm.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnInstanceNorm target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnInstanceNorm target) + { + var inputType = context.GetArgumentType(target, NcnnInstanceNorm.Input); + var gamma = new TensorType(DataTypes.Float32, new[] { target.Channels }); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = (CostUtility.GetMemoryAccess(returnType) * 2) + (CostUtility.GetMemoryAccess(gamma) * 2), + + // x = (x - mean)/(standard_deviation) * gamma + beta; + // mean = sum(x)/N; + // standard-deviation = sqrt(sum(square(x-mean))/N + eps); + [MetricFactorNames.FLOPs] = + (MetricUtility.GetFLOPs(inputType) * (MetricUtility.AddFLOPs + MetricUtility.SubFLOPs)) + MetricUtility.DivFLOPs + // x = x-mean + (MetricUtility.GetFLOPs(inputType) * (MetricUtility.PowFLOPs + MetricUtility.AddFLOPs)) + MetricUtility.SqrtFLOPs + MetricUtility.DivFLOPs + MetricUtility.AddFLOPs + + (MetricUtility.GetFLOPs(inputType) * (MetricUtility.DivFLOPs + MetricUtility.MulFLOPs + MetricUtility.AddFLOPs)), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnInstanceNorm target) => context.GetArgumentShape(target, NcnnInstanceNorm.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLSTM.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLSTM.cs new file mode 100644 index 0000000000..7f2f7576b1 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLSTM.cs @@ -0,0 +1,135 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using Nncase.IR.RNN; +using OrtKISharp; +using static Nncase.LSTMHelper; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnLSTMEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnLSTM lstm) + { + var input = context.GetOrtArgumentValue(lstm, NcnnLSTM.Input); + + // TODO: reorder w,r to iofg, b to numDirection-8*hiddensize (Now, numDirection-4*hiddensize (bw+br)) + var w = lstm.W; + var r = lstm.R; + var b = lstm.B; + var direction_ = lstm.Direction > 1 ? 2 : 1; + var mode = direction_ switch + { + 0 => LSTMDirection.Forward, + 1 => LSTMDirection.Reverse, + 2 => LSTMDirection.Bidirectional, + _ => throw new NotImplementedException(), + }; + + var seqLens = (int)input.Shape[0]; + var initH = Tensor.Zeros(new[] { direction_, seqLens, lstm.HiddenSize }); + var initC = Tensor.Zeros(new[] { direction_, seqLens, lstm.HiddenSize }); + var p = Tensor.Zeros(new[] { 1, seqLens, lstm.HiddenSize }); + var actAlpha = new[] { 0.0f }; + var actBeta = new[] { 0.0f }; + var clip = float.NaN; + var hiddenSize = lstm.HiddenSize; + var inputForget = 1; + var outputSize = 1; + var result = OrtKI.LSTM(input, w, r, b, seqLens, initH.ToOrtTensor(), initC.ToOrtTensor(), p.ToOrtTensor(), actAlpha, actBeta, new string[] { "sigmoid", "sigmoid", "tanh" }, clip, LSTMDirectionToValue(mode), hiddenSize, inputForget, 0, !clip.Equals(float.NaN), outputSize); + return Value.FromTensors(result.Select(t => t.ToTensor()).ToArray()); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnLSTM target) + { + var input = context.CheckArgumentType(target, NcnnLSTM.Input); + + // [num_direction, batch_size:1, hidden_size] + var initH = new TensorType(DataTypes.Float32, new[] { target.Direction > 1 ? 2 : 1, 1, target.HiddenSize }); + var initC = new TensorType(DataTypes.Float32, new[] { target.Direction > 1 ? 2 : 1, 1, target.HiddenSize }); + if (input.Shape.Rank != 3) + { + // TODO: confirm ncnn input dims when direction not 1. + input.Shape.InsertAndClone(1, 1); + } + + return Visit(context, input, initH, initC, target.OutputSize); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnLSTM target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnLSTM target_) + { + var direction_ = target_.Direction > 1 ? 2 : 1; + + var xType = context.GetArgumentType(target_, NcnnLSTM.Input); + var wType = new TensorType(DataTypes.Float32, new[] { direction_, 4 * target_.HiddenSize, xType.Shape[^1] }); + var rType = new TensorType(DataTypes.Float32, new[] { direction_, 4 * target_.HiddenSize, target_.HiddenSize }); + var bType = new TensorType(DataTypes.Float32, new[] { direction_, 8 * target_.HiddenSize }); + var returnType = context.GetReturnType(); + var outputYType = (TensorType)returnType[0]; + var outputYShape = outputYType.Shape.ToValueArray().Select(s => (UInt128)s).ToArray(); + var (sequence_len, num_directions, batch_size, hidden_size) = (outputYShape[0], outputYShape[1], outputYShape[2], outputYShape[3]); + var embbeding_size = (UInt128)xType.Shape[^1].FixedValue; + + var flops = num_directions * batch_size * sequence_len * ( + MetricUtility.GetMatMulFLOPs(1, 4 * hidden_size, embbeding_size) + + // [1,embbeding_size] @ [embbeding_size, 4 * hidden_size] + + MetricUtility.GetMatMulFLOPs(1, 4 * hidden_size, hidden_size) // [1,hidden_size] @ [hidden_size, 4 * hidden_size] + + (4 * hidden_size) + + (MetricUtility.SigmoidFLOPs * hidden_size) // ft = sigmoid(g[2]) + + hidden_size // ct = init_c * ft + + (MetricUtility.SigmoidFLOPs * hidden_size) // it = sigmoid(g[0]) + + (MetricUtility.TanhFLOPs * hidden_size) // c_t = tanh(g[3]) + + hidden_size // c_t_it = it * c_t + + hidden_size // ct = ct + c_t_it + + (MetricUtility.SigmoidFLOPs * hidden_size) // ot = sigmoid(g[1]) + + (MetricUtility.TanhFLOPs * hidden_size) // tanh_ct = tanh(ct_o) + + hidden_size); // ht = tanh_ct * ot + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(xType) + CostUtility.GetMemoryAccess(wType) + CostUtility.GetMemoryAccess(rType) + CostUtility.GetMemoryAccess(bType) + returnType.Select(t => t switch + { + TensorType tensorType => CostUtility.GetMemoryAccess(tensorType), + _ => UInt128.One, + }).Sum(), + [MetricFactorNames.FLOPs] = flops, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnLSTM target) => context.GetArgumentShape(target, NcnnLSTM.Input); + + private IRType Visit(ITypeInferenceContext context, TensorType x, TensorType initH, TensorType initC, int outputSize) + { + // TODO: confirm ncnn output + _ = initH.Shape[0]; + var yType = new TensorType(DataTypes.Float32, new[] { x.Shape[0].FixedValue, initH.Shape[2].FixedValue }); + Console.WriteLine($"x.Shape[0].FixedValue: {x.Shape[0].FixedValue}, initH.Shape[2].FixedValue:{initH.Shape[2].FixedValue}"); + var result = new[] { yType, initH, initC }; + return new TupleType(result[..outputSize]); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLayernorm.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLayernorm.cs new file mode 100644 index 0000000000..219228c94a --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLayernorm.cs @@ -0,0 +1,74 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnLayerNormEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnLayerNorm layerNorm) + { + var input = context.GetOrtArgumentValue(layerNorm, NcnnLayerNorm.Input); + + return OrtKI.LayerNormalization(input, layerNorm.GammaData, layerNorm.BetaData, layerNorm.AffineSize, layerNorm.Eps, 1L)[0].ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnLayerNorm target) + { + var input = context.CheckArgumentType(target, NcnnLayerNorm.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnLayerNorm target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnLayerNorm target) + { + var inputType = context.GetArgumentType(target, NcnnLayerNorm.Input); + var gamma = new TensorType(DataTypes.Float32, new[] { target.AffineSize }); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = (CostUtility.GetMemoryAccess(returnType) * 2) + (CostUtility.GetMemoryAccess(gamma) * 2), + + // x = (x - mean)/(standard_deviation) * gamma + beta; + // mean = sum(x)/N; + // standard-deviation = sqrt(sum(square(x-mean))/N + eps); + [MetricFactorNames.FLOPs] = + (MetricUtility.GetFLOPs(inputType) * (MetricUtility.AddFLOPs + MetricUtility.SubFLOPs)) + MetricUtility.DivFLOPs + // x = x-mean + (MetricUtility.GetFLOPs(inputType) * (MetricUtility.PowFLOPs + MetricUtility.AddFLOPs)) + MetricUtility.SqrtFLOPs + MetricUtility.DivFLOPs + MetricUtility.AddFLOPs + + (MetricUtility.GetFLOPs(inputType) * (MetricUtility.DivFLOPs + MetricUtility.MulFLOPs + MetricUtility.AddFLOPs)), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnLayerNorm target) => context.GetArgumentShape(target, NcnnLayerNorm.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLrn.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLrn.cs new file mode 100644 index 0000000000..592a148dee --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLrn.cs @@ -0,0 +1,69 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnLRNEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnLRN lrn) + { + var input = context.GetOrtArgumentValue(lrn, NcnnLRN.Input); + var alpha = lrn.Alpha; + var beta = lrn.Beta; + var bias = lrn.Bias; + var size = lrn.Size; + return OrtKI.LRN(input, alpha, beta, bias, size).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnLRN target) + { + var input = context.CheckArgumentType(target, NcnnLRN.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnLRN target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnLRN target) + { + var inputType = context.GetArgumentType(target, NcnnLRN.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.ExpFLOPs + MetricUtility.MulFLOPs + MetricUtility.SubFLOPs + MetricUtility.CmpFLOPs), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnLRN target) => context.GetArgumentShape(target, NcnnLRN.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnMatMul.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnMatMul.cs new file mode 100644 index 0000000000..a312f8a60a --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnMatMul.cs @@ -0,0 +1,146 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.CostModel; +using Nncase.Evaluator; +using Nncase.IR; +using Nncase.IR.Ncnn; +using Nncase.Utilities; +using OrtKISharp; +using static Nncase.IR.F.Tensors; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnMatMulEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + public static IRType VisitTensorType(TensorType lhs, TensorType rhs) + { + if (lhs.Shape.IsUnranked || rhs.Shape.IsUnranked) + { + return new TensorType(lhs.DType, Shape.Unranked); + } + + // if (lhs.Shape[^1].IsUnknown || rhs.Shape[^2].IsUnknown) + // { + // return new TensorType(lhs.DType, Shape.Unranked); + // } + if (lhs.DType != rhs.DType) + { + return new InvalidType("MatMul lhs and rhs have different DType"); + } + + if (lhs.Shape[^1] != rhs.Shape[^2] && lhs.Shape[^1] != Dimension.Unknown && rhs.Shape[^2] != Dimension.Unknown) + { + return new InvalidType("MatMul lhs and rhs have not compatiable shape"); + } + + if (lhs.Shape.Count == 2 && rhs.Shape.Count == 2) + { + return new TensorType(lhs.DType, new[] { lhs.Shape[0], rhs.Shape[1] }); + } + + var lhsShape = lhs.Shape.Rank >= rhs.Shape.Rank ? lhs.Shape.ToArray() : Enumerable.Repeat((Dimension)1, rhs.Shape.Rank - lhs.Shape.Rank).Concat(lhs.Shape).ToArray(); + var rhsShape = lhs.Shape.Rank <= rhs.Shape.Rank ? rhs.Shape.ToArray() : Enumerable.Repeat((Dimension)1, lhs.Shape.Rank - rhs.Shape.Rank).Concat(rhs.Shape).ToArray(); + + var bigShape = Enumerable.Zip(lhsShape, rhsShape).SkipLast(2).Select(t => + t.First == Dimension.Unknown || t.Second == Dimension.Unknown + ? Dimension.Unknown + : System.Math.Max(t.First.FixedValue, t.Second.FixedValue)).ToArray(); + + // batch and channel + var front = bigShape; + var end = new[] { lhs.Shape[^2], rhs.Shape[^1] }; + return new TensorType(lhs.DType, front.Concat(end).ToArray()); + } + + /// + public IValue Visit(IEvaluateContext context, NcnnMatMul matMul) + { + var inputA = context.GetOrtArgumentValue(matMul, NcnnMatMul.InputA); + var inputB = context.GetOrtArgumentValue(matMul, NcnnMatMul.InputB); + + // return OrtKI. (input, dim).ToValue(); + return OrtKI.MatMul(inputA, inputB).ToValue(); + } + + public IRType Visit(NcnnMatMul target, TensorType lhs, TensorType rhs) + { + return TypeInference.BroadcastType(lhs, rhs); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnMatMul target) + { + TensorType inputA, inputB; + var dataType = target.LorR switch + { + 0 => context.CheckArgumentType(target, NcnnMatMul.InputA).DType, + 1 => context.CheckArgumentType(target, NcnnMatMul.InputB).DType, + 2 => context.CheckArgumentType(target, NcnnMatMul.InputA).DType, + _ => throw new NotSupportedException("never reach here"), + }; + + inputA = target.LorR switch + { + 1 => new TensorType(dataType, target.ConstShape), + _ => context.CheckArgumentType(target, NcnnMatMul.InputA), + }; + + inputB = target.LorR switch + { + 2 => new TensorType(inputA.DType, target.ConstShape), + _ => context.CheckArgumentType(target, NcnnMatMul.InputB), + }; + + return VisitTensorType(inputA, inputB); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnMatMul target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnMatMul target) + { + var lhs = context.GetArgumentType(target, NcnnMatMul.InputA); + var rhs = context.GetArgumentType(target, NcnnMatMul.InputB); + var outputType = context.GetReturnType(); + + var k = (UInt128)lhs.Shape[^1].FixedValue; + var m = MetricUtility.GetFLOPs(lhs) / k; + var n = MetricUtility.GetFLOPs(rhs) / k; + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(lhs) + CostUtility.GetMemoryAccess(rhs) + CostUtility.GetMemoryAccess(outputType), + [MetricFactorNames.FLOPs] = m * n * ((2 * k) - 1), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnMatMul target) + { + var lhs = context.GetArgumentShape(target, NcnnMatMul.InputA); + var rhs = context.GetArgumentShape(target, NcnnMatMul.InputB); + return ShapeExprUtility.BroadcastShape(lhs, rhs); + } + + private Expr Visit(TensorType inputA, TensorType inputB) + { + return Cast(IR.F.ShapeExpr.MatMulShape(inputA.Shape, inputB.Shape), DataTypes.Int32); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs new file mode 100644 index 0000000000..c31062a221 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -0,0 +1,53 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using DryIoc; +using Nncase.Hosting; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Ncnn module. +/// +internal class NcnnModule : IApplicationPart +{ + public void ConfigureServices(IRegistrator registrator) + { + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPRelu.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPRelu.cs new file mode 100644 index 0000000000..3e5fd294c7 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPRelu.cs @@ -0,0 +1,66 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnPReluEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnPReLU pReLU) + { + var input = context.GetOrtArgumentValue(pReLU, NcnnPReLU.Input); + var slope = pReLU.Slope; + return OrtKI.PRelu(input, slope).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnPReLU target) + { + var input = context.CheckArgumentType(target, NcnnPReLU.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnPReLU target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnPReLU target) + { + var inputType = context.GetArgumentType(target, NcnnPReLU.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.CmpFLOPs + MetricUtility.MulFLOPs), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnPReLU target) => context.GetArgumentShape(target, NcnnPReLU.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPadding.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPadding.cs new file mode 100644 index 0000000000..5f5bcfc131 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPadding.cs @@ -0,0 +1,84 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnPaddingEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnPadding padding) + { + var input = context.GetOrtArgumentValue(padding, NcnnPadding.Input); + + // ncnn not support N + var pads = new Tensor(new int[] { 0, 0, padding.Front, padding.Top, padding.Left, padding.Behind, padding.Bottom, padding.Right }); + return OrtKI.Pad( + input, + pads.ToOrtTensor(), + padding.Value, + padding.Type switch { 0 => "Constant", 1 => "Reflect", 2 => "Edge", _ => "Symmetric", }).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnPadding target) + { + var input = context.CheckArgumentType(target, NcnnPadding.Input); + var c = target.Front + target.Behind; + var h = target.Top + target.Bottom; + var w = target.Left + target.Right; + return Visit(input, c, h, w); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnPadding target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnPadding target) + { + var inputType = context.GetArgumentType(target, NcnnPadding.Input); + var outputType = context.GetReturnType(); + + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(inputType), + [CostFactorNames.MemoryStore] = outputType is TensorType outT ? CostUtility.GetMemoryAccess(outT) : CostUtility.GetMemoryAccess(inputType), + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnPadding target) => context.GetArgumentShape(target, NcnnPadding.Input); + + private IRType Visit(TensorType input, int c, int h, int w) + { + if (input.Shape.Count != 3) + { + throw new ArgumentException($"{nameof(NcnnPadding)} only supports 3-dims input."); + } + + var newShape = new int[] { 0, 0, 0 }; // Must be 3-dims without BatchSize. + newShape[0] = input.Shape[0].FixedValue + c; + newShape[1] = input.Shape[1].FixedValue + h; + newShape[2] = input.Shape[2].FixedValue + w; + return new TensorType(input.DType, newShape); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPermute.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPermute.cs new file mode 100644 index 0000000000..9dbd499bd7 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPermute.cs @@ -0,0 +1,75 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using DryIoc.ImTools; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnPermuteEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnPermute permute) + { + var input = context.GetOrtArgumentValue(permute, NcnnPermute.Input); + return OrtKI.Transpose(input, permute.Perm.ToLongs()).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnPermute target) + { + var input = context.CheckArgumentType(target, NcnnPermute.Input); + return Visit(input, target.Perm); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnPermute target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnPermute target) + { + var inputType = context.GetArgumentType(target, NcnnPermute.Input); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(inputType) * 2, + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnPermute target) => context.GetArgumentShape(target, NcnnPermute.Input); + + private IRType Visit(TensorType input, int[] perm) + { + var outputShape = input.Shape.ToValueList(); + if (perm.Length - input.Shape.Count == 1) + { + perm = perm.Remove(0); + var realPerm = perm.Select(x => x - 1).ToArray(); + for (int i = 0; i < realPerm.Length; i++) + { + outputShape[i] = input.Shape.ToValueList()[realPerm[i]]; + } + } + + return new TensorType(input.DType, outputShape.ToArray()); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPooling.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPooling.cs new file mode 100644 index 0000000000..bd806a09f2 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPooling.cs @@ -0,0 +1,98 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using Nncase.IR.Tensors; +using OrtKISharp; +using static Nncase.Evaluator.EvaluatorUtil; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnPoolingEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnPooling pooling) + { + var input = context.GetOrtArgumentValue(pooling, NcnnPooling.Input); + var kernelSize = new long[] { pooling.Args.KernelH, pooling.Args.KernelW }; + var stride = new long[] { pooling.Args.StrideH, pooling.Args.StrideW }; + var dilation = new long[] { 1, 1 }; + var padData = new long[] { pooling.Args.PadTop, pooling.Args.PadLeft, pooling.Args.PadBottom, pooling.Args.PadRight }; + + // var ceilMode = pooling.Args.PadMode == 0 ? 1 : 0; + long countIncludePad = pooling.Args.AvgPoolCountIncludePad ? 1 : 0; + + return (pooling.Args.PoolingType switch + { + 0 => OrtKI.MaxPool(input, "NOTSET", pooling.Args.CeilMode ? 1 : 0, dilation, kernelSize, padData, countIncludePad, stride)[0], + 1 => OrtKI.AveragePool(input, "NOTSET", pooling.Args.CeilMode ? 1 : 0, countIncludePad, kernelSize, padData, stride), + _ => throw + new NotImplementedException($"Pooling type {pooling.Args.PoolingType} is not supported."), + }).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnPooling target) + { + var input = context.CheckArgumentType(target, NcnnPooling.Input); + return Visit(target, input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnPooling target) + { + var inputType = context.GetArgumentType(target, NcnnPooling.Input); + var kernelSize = new int[] { target.Args.KernelH, target.Args.KernelW }; + var outputType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(inputType) + CostUtility.GetMemoryAccess(outputType), + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(outputType, kernelSize[0] * kernelSize[1]), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnPooling target) + { + var inputType = context.GetArgumentType(target, NcnnPooling.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.DivFLOPs + MetricUtility.ExpFLOPs + MetricUtility.MulFLOPs + MetricUtility.SubFLOPs + MetricUtility.AddFLOPs + (MetricUtility.CmpFLOPs * 2)), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnPooling target) => context.GetArgumentShape(target, NcnnPooling.Input); + + private IRType Visit(NcnnPooling pooling, TensorType input) + { + // var input = context.GetOrtArgumentValue(pooling, NcnnPooling.Input); + var kernelSize = new long[] { pooling.Args.KernelH, pooling.Args.KernelW }; + var stride = new long[] { pooling.Args.StrideH, pooling.Args.StrideW }; + var padData = new long[] { pooling.Args.PadTop, pooling.Args.PadBottom, pooling.Args.PadLeft, pooling.Args.PadRight }; + _ = pooling.Args.AvgPoolCountIncludePad ? 1 : 0; + var newInput = new TensorType(input.DType, input.Shape.InsertAndClone(0, 1)); + var output_ = TypeInference.ReduceWindow2DType(newInput, kernelSize, stride, Tensor.From(padData, new[] { 2, 2, }), pooling.Args.CeilMode); + if (output_ is TensorType t) + { + var newShape = t.Shape.ToArray(); + return new TensorType(t.DType, newShape[1..]); + } + + return output_; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnReduction.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnReduction.cs new file mode 100644 index 0000000000..505a403f35 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnReduction.cs @@ -0,0 +1,104 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using Nncase.IR.Tensors; +using OrtKISharp; +using static Nncase.Evaluator.EvaluatorUtil; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnReductionEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnReduction reduction) + { + var input = context.GetOrtArgumentValue(reduction, NcnnReduction.Input); + var axes = reduction.Args.Axes; + var keepDims = reduction.Args.Keepdims; + + return (reduction.Args.OpType switch + { + 0 => OrtKI.ReduceSum(input, axes, keepDims, 0), + 3 => OrtKI.ReduceMean(input, axes.Cast().ToArray(), keepDims), + 4 => OrtKI.ReduceMax(input, axes.Cast().ToArray(), keepDims), + 5 => OrtKI.ReduceMin(input, axes.Cast().ToArray(), keepDims), + 6 => OrtKI.ReduceProd(input, axes.Cast().ToArray(), keepDims), + _ => throw + new NotImplementedException($"Reduction opType {reduction.Args.OpType} is not supported."), + }).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnReduction target) + { + var input = context.CheckArgumentType(target, NcnnReduction.Input); + return Visit(target, input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnReduction target) + { + var inputType = context.GetArgumentType(target, NcnnReduction.Input); + var returnType = context.GetReturnType(); + var rF = MetricUtility.GetFLOPs(returnType); + var iF = MetricUtility.GetFLOPs(inputType); + var inner = iF / rF; + _ = iF / inner; + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(inputType) + CostUtility.GetMemoryAccess(returnType), + [MetricFactorNames.FLOPs] = iF, + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnReduction target) + { + var inputType = context.GetArgumentType(target, NcnnReduction.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.DivFLOPs + MetricUtility.ExpFLOPs + MetricUtility.MulFLOPs + MetricUtility.SubFLOPs + MetricUtility.AddFLOPs + (MetricUtility.CmpFLOPs * 2)), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnReduction target) => context.GetArgumentShape(target, NcnnReduction.Input); + + private IRType Visit(NcnnReduction reduction, TensorType input) + { + var inputShape = input.Shape.ToList(); + var axis = reduction.Args.Axes.ToList(); + var keepDims = reduction.Args.Keepdims; + + while (axis.Count > 0) + { + var (maxValue, maxIndex) = (axis.Max(), axis.IndexOf(axis.Max())); + if (keepDims == 1) + { + inputShape[(int)maxValue] = 1; + } + else + { + inputShape.RemoveAt((int)maxValue); + } + + axis.RemoveAt(maxIndex); + } + + return new TensorType(input.DType, inputShape.ToArray()); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnReshape.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnReshape.cs new file mode 100644 index 0000000000..77dec2298b --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnReshape.cs @@ -0,0 +1,83 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnReshapeEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnReshape reshape) + { + var input = context.GetOrtArgumentValue(reshape, NcnnReshape.Input); + var newShape = reshape.Shape; + return OrtKI.Reshape(input, newShape, 0).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnReshape target) + { + var input = context.CheckArgumentType(target, NcnnReshape.Input); + var newShape = target.Shape; + return Visit(input, newShape); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnReshape target) + { + return new() + { + [CostFactorNames.CPUCycles] = 1, + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnReshape target) + { + _ = context.GetArgumentType(target, NcnnReshape.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = 0, + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnReshape target) => context.GetArgumentShape(target, NcnnReshape.Input); + + private IRType Visit(TensorType input, int[] newShape) + { + var inputSize = input.Shape.Aggregate((a, b) => a * b).FixedValue; + var outputShape = newShape; + int negAxis = -1; + for (int i = 0; i < newShape.Length; i++) + { + if (newShape[i] == -1) + { + negAxis = i; + break; + } + } + + if (negAxis != -1) + { + int otherSize = newShape.Aggregate(1, (currentProduct, num) => num == 0 ? currentProduct : currentProduct * (num > 0 ? num : 1)); + outputShape[negAxis] = inputSize / otherSize; + } + + return new TensorType(input.DType, outputShape); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSELU.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSELU.cs new file mode 100644 index 0000000000..1b9e4f07e1 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSELU.cs @@ -0,0 +1,70 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnSELUEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnSELU selu) + { + var input = context.GetOrtArgumentValue(selu, NcnnSELU.Input); + var alpha = selu.Alpha; + var gamma = selu.Gamma; + return OrtKI.Selu(input, alpha, gamma).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnSELU target) + { + var input = context.CheckArgumentType(target, NcnnSELU.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnSELU target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnSELU target) + { + var inputType = context.GetArgumentType(target, NcnnSELU.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = + MetricUtility.GetFLOPs(inputType) * (MetricUtility.ExpFLOPs + (MetricUtility.MulFLOPs * 2) + + MetricUtility.SubFLOPs + MetricUtility.AddFLOPs + + (MetricUtility.CmpFLOPs * 2)), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnSELU target) => context.GetArgumentShape(target, NcnnSELU.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSigmoid.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSigmoid.cs new file mode 100644 index 0000000000..6c5184a934 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSigmoid.cs @@ -0,0 +1,65 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnSigmoidEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnSigmoid sigmoid) + { + var input = context.GetOrtArgumentValue(sigmoid, NcnnSigmoid.Input); + return OrtKI.Sigmoid(input).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnSigmoid target) + { + var input = context.CheckArgumentType(target, NcnnSigmoid.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnSigmoid target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnSigmoid target) + { + var inputType = context.GetArgumentType(target, NcnnSigmoid.Input); + + return new() + { + // y = 1 / (1 + exp(-x)). + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(inputType) * 2, + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.AddFLOPs + MetricUtility.ExpFLOPs + MetricUtility.DivFLOPs + MetricUtility.MulFLOPs), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnSigmoid target) => context.GetArgumentShape(target, NcnnSigmoid.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSlice.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSlice.cs new file mode 100644 index 0000000000..4a3fccffcb --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSlice.cs @@ -0,0 +1,74 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnSliceEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnSlice slice) + { + var input = context.GetOrtArgumentValue(slice, NcnnSlice.Input); + var split = slice.Slices; + var axis = slice.Axis; + var result = OrtKI.Split(input, split, axis); + return Value.FromTensors(result.Select(t => t.ToTensor()).ToArray()); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnSlice target) + { + var input = context.CheckArgumentType(target, NcnnSlice.Input); + return Visit(input, target.Slices, target.Axis); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnSlice target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnSlice target) + { + var inputType = context.GetArgumentType(target, NcnnSlice.Input); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(inputType) * 2, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnSlice target) => context.GetArgumentShape(target, NcnnSlice.Input); + + private IRType Visit(TensorType input, int[] slices, int axis) + { + var outputInfo = new List(); + var outputShape = new List>(); + for (int i = 0; i < slices.Length; i++) + { + outputShape.Add(input.Shape.ToValueList()); + outputShape[i][axis] = slices[i]; + outputInfo.Add(new TensorType(input.DType, outputShape[i].ToArray())); + } + + return new TupleType(outputInfo); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSoftmax.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSoftmax.cs new file mode 100644 index 0000000000..36a962d3d9 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSoftmax.cs @@ -0,0 +1,69 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnSoftmaxEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnSoftmax softmax) + { + var input = context.GetOrtArgumentValue(softmax, NcnnSoftmax.Input); + var dim = softmax.Axis; + return OrtKI.Softmax(input, dim).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnSoftmax target) + { + var input = context.CheckArgumentType(target, NcnnSoftmax.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnSoftmax target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnSoftmax target) + { + var inputType = context.GetArgumentType(target, NcnnSoftmax.Input); + var returnType = context.GetReturnType(); + var returnF = MetricUtility.GetFLOPs(returnType); + var inputF = MetricUtility.GetFLOPs(inputType); + var inner = inputF / returnF; + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = (inner * 2) + (inputF * (MetricUtility.SubFLOPs + MetricUtility.ExpFLOPs + MetricUtility.DivFLOPs)), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnSoftmax target) => context.GetArgumentShape(target, NcnnSoftmax.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSoftplus.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSoftplus.cs new file mode 100644 index 0000000000..ff936aa387 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSoftplus.cs @@ -0,0 +1,65 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnSoftplusEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnSoftplus softplus) + { + var input = context.GetOrtArgumentValue(softplus, NcnnSoftplus.Input); + return OrtKI.Softplus(input).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnSoftplus target) + { + var input = context.CheckArgumentType(target, NcnnSoftplus.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnSoftplus target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnSoftplus target) + { + var inputType = context.GetArgumentType(target, NcnnSoftplus.Input); + + return new() + { + // y = log(exp(x)+1). + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(inputType) * 2, + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.AddFLOPs + MetricUtility.ExpFLOPs + MetricUtility.LogFLOPs), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnSoftplus target) => context.GetArgumentShape(target, NcnnSoftplus.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSqueeze.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSqueeze.cs new file mode 100644 index 0000000000..90d9c4f953 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSqueeze.cs @@ -0,0 +1,78 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnSqueezeEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnSqueeze squeeze) + { + var input = context.GetOrtArgumentValue(squeeze, NcnnSqueeze.Input); + var dims = squeeze.Dims; + return OrtKI.Squeeze(input, dims).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnSqueeze target) + { + var input = context.CheckArgumentType(target, NcnnSqueeze.Input); + var dims = target.Dims; + return Visit(input, dims); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnSqueeze target) + { + return new() + { + [CostFactorNames.CPUCycles] = 1, + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnSqueeze target) + { + _ = context.GetArgumentType(target, NcnnSqueeze.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = 0, + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnSqueeze target) => context.GetArgumentShape(target, NcnnSqueeze.Input); + + private IRType Visit(TensorType input, int[] dims) + { + var outputShape = input.Shape.ToValueArray(); + if (dims.Length == 0) + { + outputShape = outputShape.Where(x => x != 1).ToArray(); + } + else + { + // outputShape = outputShape.Select((value, idx) => dims.Contains(idx) ? value : (value == 1 ? 0 : value)).Where(v => v != 0).ToArray(); + outputShape = outputShape.Select((value, idx) => dims.Contains(idx) ? (value == 1 ? 0 : value) : value).Where(v => v != 0) + .ToArray(); + } + + return new TensorType(input.DType, outputShape); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnTile.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnTile.cs new file mode 100644 index 0000000000..af25c17621 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnTile.cs @@ -0,0 +1,69 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnTileEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnTile tile) + { + var input = context.GetOrtArgumentValue(tile, NcnnTile.Input); + return OrtKI.Tile(input, tile.Repeats).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnTile target) + { + var input = context.CheckArgumentType(target, NcnnTile.Input); + return Visit(input, target.Repeats); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnTile target) + { + var inputType = context.GetArgumentType(target, NcnnTile.Input); + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(inputType), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnTile target) + { + var inputType = context.GetArgumentType(target, NcnnTile.Input); + var ret = context.GetReturnType(); + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(inputType) + CostUtility.GetMemoryAccess(ret), + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnTile target) => context.GetArgumentShape(target, NcnnTile.Input); + + private IRType Visit(TensorType input, int[] repeats) + { + var outputShape = input.Shape.ToValueArray(); + for (int i = 0; i < repeats.Length; i++) + { + outputShape[i] *= repeats[i]; + } + + return new TensorType(input.DType, outputShape); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnUnary.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnUnary.cs new file mode 100644 index 0000000000..dde4c66e7b --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnUnary.cs @@ -0,0 +1,117 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnUnaryEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnUnary unary) + { + var input = context.GetOrtArgumentValue(unary, NcnnUnary.Input); + var opType = unary.OpType; + + // return OrtKI. (input, dim).ToValue(); + switch (opType) + { + case UnaryOperationType.ABS: + return OrtKI.Abs(input).ToValue(); + case UnaryOperationType.NEG: + return OrtKI.Neg(input).ToValue(); + case UnaryOperationType.FLOOR: + return OrtKI.Floor(input).ToValue(); + case UnaryOperationType.CEIL: + return OrtKI.Ceil(input).ToValue(); + case UnaryOperationType.SQUARE: + return OrtKI.Square(input).ToValue(); + case UnaryOperationType.SQRT: + return OrtKI.Sqrt(input).ToValue(); + case UnaryOperationType.RSQRT: + return OrtKI.Rsqrt(input).ToValue(); + case UnaryOperationType.EXP: + return OrtKI.Exp(input).ToValue(); + case UnaryOperationType.LOG: + return OrtKI.Log(input).ToValue(); + case UnaryOperationType.SIN: + return OrtKI.Sin(input).ToValue(); + case UnaryOperationType.COS: + return OrtKI.Cos(input).ToValue(); + case UnaryOperationType.TAN: + return OrtKI.Tan(input).ToValue(); + case UnaryOperationType.ASIN: + return OrtKI.Asin(input).ToValue(); + case UnaryOperationType.ACOS: + return OrtKI.Acos(input).ToValue(); + case UnaryOperationType.ATAN: + return OrtKI.Atan(input).ToValue(); + case UnaryOperationType.RECIPROCAL: + return OrtKI.Reciprocal(input).ToValue(); + case UnaryOperationType.TANH: + return OrtKI.Tanh(input).ToValue(); + case UnaryOperationType.LOG10: + double ln10 = 2.3025850929940456840179914546844; + return OrtKI.Div(OrtKI.Log(input), OrtKISharp.Tensor.FromScalar(ln10)).ToValue(); + case UnaryOperationType.ROUND: + return OrtKI.Round(input).ToValue(); + + // TODO: trunc + default: + throw new NotSupportedException("Ncnn unary ops"); + } + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnUnary target) + { + var input = context.CheckArgumentType(target, NcnnUnary.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnUnary target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnUnary target) + { + var inputType = context.GetArgumentType(target, NcnnUnary.Input); + var returnType = context.GetReturnType(); + var returnF = MetricUtility.GetFLOPs(returnType); + var inputF = MetricUtility.GetFLOPs(inputType); + var inner = inputF / returnF; + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = (inner * 2) + (inputF * (MetricUtility.SubFLOPs + MetricUtility.ExpFLOPs + MetricUtility.DivFLOPs)), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnUnary target) => context.GetArgumentShape(target, NcnnUnary.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnUnsqueeze.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnUnsqueeze.cs new file mode 100644 index 0000000000..a9083f71a2 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnUnsqueeze.cs @@ -0,0 +1,72 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnUnsqueezeEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnUnsqueeze unsqueeze) + { + var input = context.GetOrtArgumentValue(unsqueeze, NcnnUnsqueeze.Input); + var dims = unsqueeze.Dims; + return OrtKI.Unsqueeze(input, dims).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnUnsqueeze target) + { + var input = context.CheckArgumentType(target, NcnnUnsqueeze.Input); + var dims = target.Dims; + return Visit(input, dims); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnUnsqueeze target) + { + return new() + { + [CostFactorNames.CPUCycles] = 1, + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnUnsqueeze target) + { + _ = context.GetArgumentType(target, NcnnUnsqueeze.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = 0, + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnUnsqueeze target) => context.GetArgumentShape(target, NcnnUnsqueeze.Input); + + private IRType Visit(TensorType input, int[] dims) + { + var outputShape = input.Shape.ToValueArray().ToList(); + for (int i = dims.Length - 1; i >= 0; i--) + { + outputShape.Insert(i, 1); + } + + return new TensorType(input.DType, outputShape.ToArray()); + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs new file mode 100644 index 0000000000..60d1e59d14 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -0,0 +1,113 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.ArgsStruct; +using Nncase.IR.Ncnn; + +namespace Nncase.IR.F; + +public sealed class Ncnn +{ + public static Call NcnnUnary(Expr expr, UnaryOperationType unaryOp) => + new Call(new NcnnUnary(unaryOp), expr); + + public static Call NcnnSoftmax(Expr expr, int axis) => + new Call(new NcnnSoftmax(axis), expr); + + public static Call NcnnBatchNorm(Expr expr, int channels, float eps, float[] slopeData, float[] meanData, float[] varData, float[] biasData) => + new Call(new NcnnBatchNorm(channels, eps, slopeData, meanData, varData, biasData), expr); + + public static Call NcnnBinary(Expr[] inExpr, BinaryOperationType binaryOp, int lorR, float[]? constInput, int[]? constShape) => + new Call(new NcnnBinary(binaryOp, lorR, constInput, constShape), inExpr); + + public static Call NcnnCelu(Expr expr, float alpha) => + new Call(new NcnnCelu(alpha), expr); + + public static Call NcnnClip(Expr expr, float min, float max) => + new Call(new NcnnClip(min, max), expr); + + public static Call NcnnConcat(Expr[] expr, int axis) => + new Call(new NcnnConcat(axis), new IR.Tuple(expr)); + + // In ncnn param file, lauout is [w, h] for kernel, dilation + public static Call NcnnConv(Expr expr, ConvArgs args) => + new Call(new NcnnConv(args), expr); + + public static Call NcnnCumsum(Expr expr, int axis) => new Call(new NcnnCumsum(axis), expr); + + public static Call NcnnElu(Expr expr, float alpha) => new Call(new NcnnElu(alpha), expr); + + public static Call NcnnErf(Expr expr) => new Call(new NcnnErf(), expr); + + public static Call NcnnHardSigmoid(Expr expr, float alpha, float beta) => new Call(new NcnnHardSigmoid(alpha, beta), expr); + + public static Call NcnnHardSwish(Expr expr, float alpha, float beta) => new Call(new NcnnHardSwish(alpha, beta), expr); + + public static Call NcnnInstanceNorm(Expr expr, int channels, float eps, int affine, float[] gammaData, float[] betaData) => + new Call(new NcnnInstanceNorm(channels, eps, affine, gammaData, betaData), expr); + + public static Call NcnnLayerNorm(Expr expr, int affineSize, float eps, int affine, float[] gammaData, float[] betaData) => + new Call(new NcnnLayerNorm(affineSize, eps, affine, gammaData, betaData), expr); + + public static Call NcnnLRN(Expr expr, float alpha, float beta, float bias, int size) => new Call(new NcnnLRN(alpha, beta, bias, size), expr); + + public static Call NcnnLSTM(Expr expr, int outputSize, int hiddenSize, int weightDataSize, int direction, float[] w, float[] b, float[] r) => + new Call(new NcnnLSTM(outputSize, hiddenSize, weightDataSize, direction, w, b, r), expr); + + public static Call NcnnPadding(Expr expr, int top, int bottom, int left, int right, int type, float value, int front, int behind) => + new Call(new NcnnPadding(top, bottom, left, right, type, value, front, behind), expr); + + public static Call NcnnPooling(Expr expr, PoolingArgs poolingArgs) => + new Call(new NcnnPooling(poolingArgs), expr); + + public static Call NcnnPReLU(Expr expr, float[] slope) => + new Call(new NcnnPReLU(slope), expr); + + public static Call NcnnReduction(Expr expr, ReductionArgs reductionArgs) => + new Call(new NcnnReduction(reductionArgs), expr); + + public static Call NcnnReshape(Expr expr, int[] shape) => + new Call(new NcnnReshape(shape), expr); + + public static Call NcnnSELU(Expr expr, float alpha, float gamma) => + new Call(new NcnnSELU(alpha, gamma), expr); + + public static Call NcnnSigmoid(Expr expr) => + new Call(new NcnnSigmoid(), expr); + + public static Call NcnnCrop(Expr expr, CropArgs args) => + new Call(new NcnnCrop(args), expr); + + public static Call NcnnSoftplus(Expr expr) => + new Call(new NcnnSoftplus(), expr); + + public static Call NcnnSlice(Expr expr, int[] slices, int axis) => + new Call(new NcnnSlice(slices, axis), expr); + + public static Call NcnnTile(Expr expr, int[] repeats) => + new Call(new NcnnTile(repeats), expr); + + public static Call NcnnPermute(Expr expr, int orderType, int[] perm) => + new Call(new NcnnPermute(orderType, perm), expr); + + public static Call NcnnMatMul(Expr[] inExpr, int lorR, float[]? constInput, int[]? constShape) => + new Call(new NcnnMatMul(lorR, constInput, constShape), inExpr); + + public static Call NcnnConvTranspose(Expr expr, ConvTransposeArgs args) => new Call(new NcnnConvTranspose(args), expr); + + public static Call NcnnCast(Expr expr, int fromType, int toType) => + new Call(new NcnnCast(fromType, toType), expr); + + public static Call NcnnGELU(Expr expr) => new Call(new NcnnGELU(), expr); + + public static Call NcnnDequantize(Expr expr, float[] scale, float[] bias) => new Call(new NcnnDequantize(scale, bias), expr); + + public static Call NcnnSqueeze(Expr expr, int[] dims) => new Call(new NcnnSqueeze(dims), expr); + + public static Call NcnnUnsqueeze(Expr expr, int[] dims) => new Call(new NcnnUnsqueeze(dims), expr); +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBatchnorm.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBatchnorm.cs new file mode 100644 index 0000000000..386caff1d7 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBatchnorm.cs @@ -0,0 +1,61 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// BatchNorm expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnBatchNorm : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnBatchNorm), 0, "input"); + + /// + /// Gets channels of Ncnn BatchNorm. + /// + public int Channels { get; } + + /// + /// Gets eps. + /// + public float Eps { get; } + + /// + /// Gets slopeData of Ncnn BatchNorm. + /// + public float[] SlopeData { get; } + + /// + /// Gets meanData of Ncnn BatchNorm. + /// + public float[] MeanData { get; } + + /// + /// Gets varData of Ncnn BatchNorm. + /// + public float[] VarData { get; } + + /// + /// Gets biasData of Ncnn BatchNorm. + /// + public float[] BiasData { get; } + + /// + public override string DisplayProperty() + { + return $"{Channels},{Eps}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBinary.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBinary.cs new file mode 100644 index 0000000000..1ada289725 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBinary.cs @@ -0,0 +1,80 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +public enum BinaryOperationType +{ + ADD = 0, + SUB = 1, + MUL = 2, + DIV = 3, + MAX = 4, + MIN = 5, + POW = 6, + + // Not support below + // RSUB = 7, + // RDIV = 8, + // RPOW = 9, + // ATAN2 = 10, + // RATAN2 = 11, +} + +/// +/// Binary expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnBinary : Op +{ + /// + /// Gets inputA. + /// + public static readonly ParameterInfo InputA = new(typeof(NcnnBinary), 0, "inputA"); + + /// + /// Gets inputB. + /// + public static readonly ParameterInfo InputB = new(typeof(NcnnBinary), 1, "inputB"); + + public BinaryOperationType OpType { get; } + + /// + /// Gets the flag of which input is const. + /// + public int LorR { get; } + + // These args will never used in nncase, scaler input was convert to single constant. + // So, it was no longer be scaler. + // public int WithScaler { get; } + // public float B { get; } + + /// + /// Gets constant data. + /// + public float[] ConstInput { get; } + + /// + /// Gets shape of constant data. + /// + public int[] ConstShape { get; } + + /// + public override string DisplayProperty() + { + if (ConstShape != null) + { + return $"BinaryOp.{OpType}, Const Shape: {string.Join(",", ConstShape!)}"; + } + + return $"BinaryOp.{OpType}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCast.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCast.cs new file mode 100644 index 0000000000..30ccd54df9 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCast.cs @@ -0,0 +1,41 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.ArgsStruct; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Cast expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnCast : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnCast), 0, "input"); + + /// + /// Gets FromType of Cast. + /// + public int FromType { get; } + + /// + /// Gets ToType of Cast. + /// + public int ToType { get; } + + /// + public override string DisplayProperty() + { + return $"FromType: {FromType}, ToType: {ToType}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCelu.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCelu.cs new file mode 100644 index 0000000000..2755fc7d35 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCelu.cs @@ -0,0 +1,35 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Celu expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnCelu : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnCelu), 0, "input"); + + /// + /// Gets Alpha of Ncnn Celu. + /// + public float Alpha { get; } + + /// + public override string DisplayProperty() + { + return $"{Alpha}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnClip.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnClip.cs new file mode 100644 index 0000000000..d64ceb265f --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnClip.cs @@ -0,0 +1,40 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Clip expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnClip : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnClip), 0, "input"); + + /// + /// Gets min of Ncnn Clip. + /// + public float Min { get; } + + /// + /// Gets max of Ncnn Clip. + /// + public float Max { get; } + + /// + public override string DisplayProperty() + { + return $"{Min}, {Max}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConcat.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConcat.cs new file mode 100644 index 0000000000..df8503f499 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConcat.cs @@ -0,0 +1,32 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Concat expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnConcat : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnConcat), 0, "inputs", ParameterKind.Input); + + /// + /// Gets axis. + /// + public int Axis { get; } + + /// + public override string DisplayProperty() => $"Axis: {Axis}"; +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConv.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConv.cs new file mode 100644 index 0000000000..f8faf9351e --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConv.cs @@ -0,0 +1,36 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.ArgsStruct; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Conv expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnConv : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnConv), 0, "input"); + + /// + /// Gets args of NcnnConv. + /// + public ConvArgs Args { get; } + + /// + public override string DisplayProperty() + { + return Args.ToString(); + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConvTranspose.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConvTranspose.cs new file mode 100644 index 0000000000..d579ec07a5 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConvTranspose.cs @@ -0,0 +1,33 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.ArgsStruct; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// ConvTranspose expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnConvTranspose : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnConvTranspose), 0, "input"); + + public ConvTransposeArgs Args { get; } + + /// + public override string DisplayProperty() + { + return Args.ToString(); + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCrop.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCrop.cs new file mode 100644 index 0000000000..423b605dc2 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCrop.cs @@ -0,0 +1,37 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.ArgsStruct; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Crop expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnCrop : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnCrop), 0, "input"); + + /// + /// Gets Args of Crop. + /// + public CropArgs Args { get; } + + /// + public override string DisplayProperty() + { + return + $"starts:{string.Join(",", (Args.Starts ?? Array.Empty()).Select(x => x.ToString()))}, ends: {string.Join(",", (Args.Ends ?? Array.Empty()).Select(x => x.ToString()))}, axes: {string.Join(",", (Args.Axes ?? Array.Empty()).Select(x => x.ToString()))}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCumsum.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCumsum.cs new file mode 100644 index 0000000000..01612188ea --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCumsum.cs @@ -0,0 +1,35 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Cumsum expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnCumsum : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnCumsum), 0, "input"); + + /// + /// Gets Axis of Ncnn Cumsum. + /// + public int Axis { get; } + + /// + public override string DisplayProperty() + { + return $"{Axis}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnDequantize.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnDequantize.cs new file mode 100644 index 0000000000..23e9a27514 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnDequantize.cs @@ -0,0 +1,40 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Dequantize expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnDequantize : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnDequantize), 0, "input"); + + /// + /// Gets scale of Ncnn Dequantize. + /// + public float[] Scale { get; } + + /// + /// Gets scale of Ncnn Dequantize. + /// + public float[] Bias { get; } + + /// + public override string DisplayProperty() + { + return $"scale_size: {Scale.Length}, bias_size: {Bias.Length}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnElu.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnElu.cs new file mode 100644 index 0000000000..48f285306e --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnElu.cs @@ -0,0 +1,35 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Gets expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnElu : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnElu), 0, "input"); + + /// + /// Gets Alpha of Ncnn Elu. + /// + public float Alpha { get; } + + /// + public override string DisplayProperty() + { + return $"{Alpha}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnErf.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnErf.cs new file mode 100644 index 0000000000..84fa7d9038 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnErf.cs @@ -0,0 +1,30 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Gets expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnErf : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnErf), 0, "input"); + + /// + public override string DisplayProperty() + { + return $""; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnGELU.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnGELU.cs new file mode 100644 index 0000000000..bf70a88ea3 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnGELU.cs @@ -0,0 +1,30 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// GELU expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnGELU : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnGELU), 0, "input"); + + /// + public override string DisplayProperty() + { + return $""; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnHardSigmoid.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnHardSigmoid.cs new file mode 100644 index 0000000000..4843fa26b6 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnHardSigmoid.cs @@ -0,0 +1,40 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Gets expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnHardSigmoid : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnHardSigmoid), 0, "input"); + + /// + /// Gets Alpha of Ncnn HardSigmoid. + /// + public float Alpha { get; } + + /// + /// Gets Beta of Ncnn HardSigmoid. + /// + public float Beta { get; } + + /// + public override string DisplayProperty() + { + return $"alpha = {Alpha}, beta = {Beta}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnHardSwish.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnHardSwish.cs new file mode 100644 index 0000000000..4ae10054cc --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnHardSwish.cs @@ -0,0 +1,40 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Gets expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnHardSwish : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnHardSwish), 0, "input"); + + /// + /// Gets Alpha of Ncnn HardSwish. + /// + public float Alpha { get; } + + /// + /// Gets Beta of Ncnn HardSwish. + /// + public float Beta { get; } + + /// + public override string DisplayProperty() + { + return $"alpha = {Alpha}, beta = {Beta}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnInstanceNorm.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnInstanceNorm.cs new file mode 100644 index 0000000000..4eaad36c3f --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnInstanceNorm.cs @@ -0,0 +1,55 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Gets expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnInstanceNorm : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnInstanceNorm), 0, "input"); + + /// + /// Gets Channels of Ncnn InstanceNorm. + /// + public int Channels { get; } + + /// + /// Gets Eps of Ncnn InstanceNorm. + /// + public float Eps { get; } + + /// + /// Gets Affine of Ncnn InstanceNorm. + /// + public int Affine { get; } + + /// + /// Gets GammaData of InstanceNorm. + /// + public float[] GammaData { get; } + + /// + /// Gets BetaData of InstanceNorm. + /// + public float[] BetaData { get; } + + /// + public override string DisplayProperty() + { + return $"channels={Channels}, eps={Eps}, affine={Affine}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnLSTM.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnLSTM.cs new file mode 100644 index 0000000000..d33df75b7e --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnLSTM.cs @@ -0,0 +1,65 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Gets expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnLSTM : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnLSTM), 0, "input"); + + /// + /// Gets OutputSize of Ncnn LSTM. + /// + public int OutputSize { get; } + + /// + /// Gets HiddenSize of Ncnn LSTM. + /// + public int HiddenSize { get; } + + /// + /// Gets WeightDataSize of Ncnn LSTM. + /// + public int WeightDataSize { get; } + + /// + /// Gets Direction of Ncnn LSTM. + /// + public int Direction { get; } + + /// + /// Gets W of Ncnn LSTM. + /// + public float[] W { get; } + + /// + /// Gets B of Ncnn LSTM. + /// + public float[] B { get; } + + /// + /// Gets R of Ncnn LSTM. + /// + public float[] R { get; } + + /// + public override string DisplayProperty() + { + return $"HiddenSize: {HiddenSize}, WeightDataSize: {WeightDataSize}, Direction: {Direction}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnLayerNorm.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnLayerNorm.cs new file mode 100644 index 0000000000..e85854c46a --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnLayerNorm.cs @@ -0,0 +1,55 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Gets expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnLayerNorm : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnLayerNorm), 0, "input"); + + /// + /// Gets AffineSize of Ncnn LayerNorm. + /// + public int AffineSize { get; } + + /// + /// Gets Eps of Ncnn LayerNorm. + /// + public float Eps { get; } + + /// + /// Gets Affine of Ncnn LayerNorm. + /// + public int Affine { get; } + + /// + /// Gets GammaData of LayerNorm. + /// + public float[] GammaData { get; } + + /// + /// Gets BetaData of LayerNorm. + /// + public float[] BetaData { get; } + + /// + public override string DisplayProperty() + { + return $"AffineSize={AffineSize}, eps={Eps}, affine={Affine}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnLrn.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnLrn.cs new file mode 100644 index 0000000000..baa57b8b63 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnLrn.cs @@ -0,0 +1,50 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Gets expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnLRN : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnLRN), 0, "input"); + + /// + /// Gets alpha of Ncnn LRN. + /// + public float Alpha { get; } + + /// + /// Gets Beta of Ncnn LRN. + /// + public float Beta { get; } + + /// + /// Gets Bias of Ncnn LRN. + /// + public float Bias { get; } + + /// + /// Gets Size of Ncnn LRN. + /// + public int Size { get; } + + /// + public override string DisplayProperty() + { + return $"alpha={Alpha}, beta={Beta}, bias={Bias}, size={Size}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnMatMul.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnMatMul.cs new file mode 100644 index 0000000000..b739696e63 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnMatMul.cs @@ -0,0 +1,50 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Matmul expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnMatMul : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo InputA = new(typeof(NcnnMatMul), 0, "inputA"); + + /// + /// Gets input. + /// + public static readonly ParameterInfo InputB = new(typeof(NcnnMatMul), 1, "inputB"); + + /// + /// Gets the flag of which input is const. + /// + public int LorR { get; } + + /// + /// Gets constant data. + /// + public float[] ConstInput { get; } + + /// + /// Gets shape of constant data. + /// + public int[] ConstShape { get; } + + /// + public override string DisplayProperty() + { + return $"constantShape:{string.Join(",", ConstShape ?? Array.Empty())}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPReLU.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPReLU.cs new file mode 100644 index 0000000000..4c06407f2c --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPReLU.cs @@ -0,0 +1,35 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// PReLU expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnPReLU : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnPReLU), 0, "input"); + + /// + /// Gets Slope of Ncnn PReLU. + /// + public float[] Slope { get; } + + /// + public override string DisplayProperty() + { + return $"{string.Join(", ", Slope)}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPadding.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPadding.cs new file mode 100644 index 0000000000..1f6d11ece5 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPadding.cs @@ -0,0 +1,75 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Gets expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnPadding : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnPadding), 0, "input"); + + /// + /// Gets Top of Ncnn Padding. + /// + public int Top { get; } + + /// + /// Gets Bottom of Ncnn Padding. + /// + public int Bottom { get; } + + /// + /// Gets Left of Ncnn Padding. + /// + public int Left { get; } + + /// + /// Gets Right of Ncnn Padding. + /// + public int Right { get; } + + /// + /// Gets Type of Ncnn Padding. + /// + public int Type { get; } + + /// + /// Gets Value of Ncnn Padding. + /// + public float Value { get; } + + // /// + // /// Gets PerChannelPadSize of Ncnn Padding. Do not need in onnx model. + // /// + // public int PerChannelPadSize { get; } + + /// + /// Gets Front of Ncnn Padding. + /// + public int Front { get; } + + /// + /// Gets Behind of Ncnn Padding. + /// + public int Behind { get; } + + /// + public override string DisplayProperty() + { + return $"Top: {Top}, Bottom: {Bottom}, Left: {Left}, Right: {Right}, Type: {Type}, Value: {Value}, Front: {Front}, Behind: {Behind}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPermute.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPermute.cs new file mode 100644 index 0000000000..3920f0b7fc --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPermute.cs @@ -0,0 +1,40 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Permute expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnPermute : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnPermute), 0, "input"); + + /// + /// Gets OrderType of NcnnPermute. + /// + public int OrderType { get; } + + /// + /// Gets perm of source transpose used to eval. + /// + public int[] Perm { get; } + + /// + public override string DisplayProperty() + { + return $"OrderType:{OrderType}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPooling.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPooling.cs new file mode 100644 index 0000000000..77522cc0cd --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPooling.cs @@ -0,0 +1,36 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.ArgsStruct; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Pooling expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnPooling : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnPooling), 0, "input"); + + /// + /// Gets PoolingArgs of Ncnn Pooling. + /// + public PoolingArgs Args { get; } + + /// + public override string DisplayProperty() + { + return $"{Args.PoolingType}, Kernel: {Args.KernelW}-{Args.KernelH}, Stride: {Args.StrideW}-{Args.StrideH}, Padding: {Args.PadLeft}-{Args.PadRight}-{Args.PadTop}-{Args.PadBottom}, CeilMode: {Args.CeilMode}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnReduction.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnReduction.cs new file mode 100644 index 0000000000..4cfd9ba943 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnReduction.cs @@ -0,0 +1,36 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.ArgsStruct; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Reduction expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnReduction : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnReduction), 0, "input"); + + /// + /// Gets ReductionArgs of Ncnn Reduction. + /// + public ReductionArgs Args { get; } + + /// + public override string DisplayProperty() + { + return $"opType:{Args.OpType}, reduceAll:{Args.ReduceAll}, coeff:{Args.Coeff}, axes:{string.Join(",", Args.Axes)}, keepdims:{Args.Keepdims}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnReshape.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnReshape.cs new file mode 100644 index 0000000000..6988d22673 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnReshape.cs @@ -0,0 +1,35 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Reshape expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnReshape : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnReshape), 0, "input"); + + /// + /// Gets Shape of Ncnn Reshape. + /// + public int[] Shape { get; } + + /// + public override string DisplayProperty() + { + return $"shape:{string.Join(",", Shape)}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSELU.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSELU.cs new file mode 100644 index 0000000000..8743b3488b --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSELU.cs @@ -0,0 +1,40 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// SELU expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnSELU : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnSELU), 0, "input"); + + /// + /// Gets Alpha of Ncnn SELU. + /// + public float Alpha { get; } + + /// + /// Gets Gamma of Ncnn SELU. + /// + public float Gamma { get; } + + /// + public override string DisplayProperty() + { + return $"Alpha:{Alpha}, Gamma:{Gamma}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSigmoid.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSigmoid.cs new file mode 100644 index 0000000000..df65b4b1b6 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSigmoid.cs @@ -0,0 +1,30 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Sigmoid expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnSigmoid : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnSigmoid), 0, "input"); + + /// + public override string DisplayProperty() + { + return $""; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSlice.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSlice.cs new file mode 100644 index 0000000000..856678187d --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSlice.cs @@ -0,0 +1,34 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Slice expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnSlice : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnSlice), 0, "input"); + + public int[] Slices { get; } + + public int Axis { get; } + + /// + public override string DisplayProperty() + { + return $"slices:{string.Join(",", Slices.Select(x => x.ToString()))}, axis:{Axis}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSoftmax.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSoftmax.cs new file mode 100644 index 0000000000..c028da8923 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSoftmax.cs @@ -0,0 +1,35 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Softmax expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnSoftmax : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnSoftmax), 0, "input"); + + /// + /// Gets axis of Ncnn Softmax. + /// + public int Axis { get; } + + /// + public override string DisplayProperty() + { + return $"{Axis}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSoftplus.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSoftplus.cs new file mode 100644 index 0000000000..1619e28e01 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSoftplus.cs @@ -0,0 +1,30 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Softplus expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnSoftplus : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnSoftplus), 0, "input"); + + /// + public override string DisplayProperty() + { + return $""; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSqueeze.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSqueeze.cs new file mode 100644 index 0000000000..3cb0fb8420 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSqueeze.cs @@ -0,0 +1,35 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Squeeze expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnSqueeze : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnSqueeze), 0, "input"); + + /// + /// Gets dims of Ncnn Squeeze. + /// + public int[] Dims { get; } + + /// + public override string DisplayProperty() + { + return $"dims:{string.Join(",", Dims)}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnTile.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnTile.cs new file mode 100644 index 0000000000..044366f006 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnTile.cs @@ -0,0 +1,37 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Tile expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnTile : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnTile), 0, "input"); + + // TODO: not support Tile with axis. + + /// + /// Gets repeats of NcnnTile. + /// + public int[] Repeats { get; } + + /// + public override string DisplayProperty() + { + return $"repeats:{string.Join(",", Repeats)}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnUnary.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnUnary.cs new file mode 100644 index 0000000000..ec2cb4065e --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnUnary.cs @@ -0,0 +1,56 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +public enum UnaryOperationType +{ + ABS = 0, + NEG = 1, + FLOOR = 2, + CEIL = 3, + SQUARE = 4, + SQRT = 5, + RSQRT = 6, + EXP = 7, + LOG = 8, + SIN = 9, + COS = 10, + TAN = 11, + ASIN = 12, + ACOS = 13, + ATAN = 14, + RECIPROCAL = 15, + TANH = 16, + LOG10 = 17, + ROUND = 18, + TRUNC = 19, +} + +/// +/// Unary expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnUnary : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnUnary), 0, "input"); + + public UnaryOperationType OpType { get; } + + /// + public override string DisplayProperty() + { + return $"UnaryOp.{OpType}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnUnsqueeze.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnUnsqueeze.cs new file mode 100644 index 0000000000..2eb1edb63c --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnUnsqueeze.cs @@ -0,0 +1,35 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Unsqueeze expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnUnsqueeze : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnUnsqueeze), 0, "input"); + + /// + /// Gets dims of Ncnn Unsqueeze. + /// + public int[] Dims { get; } + + /// + public override string DisplayProperty() + { + return $"dims:{string.Join(",", Dims)}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/NcnnApplicationPart.cs b/modules/Nncase.Modules.Ncnn/NcnnApplicationPart.cs new file mode 100644 index 0000000000..e492c65665 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/NcnnApplicationPart.cs @@ -0,0 +1,30 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Threading.Tasks; +using DryIoc; +using Nncase.Evaluator.Ncnn; +using Nncase.Hosting; + +namespace Nncase; + +/// +/// Ncnn application part extensions. +/// +public static class NcnnApplicationPart +{ + /// + /// Add ncnn assembly. + /// + /// Service registrator. + /// Configured service registrator. + public static IRegistrator AddNcnn(this IRegistrator registrator) + { + return registrator.RegisterModule(); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Nncase.Modules.Ncnn.csproj b/modules/Nncase.Modules.Ncnn/Nncase.Modules.Ncnn.csproj new file mode 100644 index 0000000000..3d14c19536 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Nncase.Modules.Ncnn.csproj @@ -0,0 +1,20 @@ + + + + Nncase + enable + true + true + True + + + + + + + + + + + + diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBatchnorm.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBatchnorm.cs new file mode 100644 index 0000000000..2358d5b2c0 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBatchnorm.cs @@ -0,0 +1,54 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; +using static Nncase.IR.F.Math; +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerBatchNorm : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsBatchNormalization( + IsWildcard("input") with { TypePattern = HasFixedShape() }, + IsTensorConst("scale"), + IsTensorConst("bias"), + IsTensorConst("mean"), + IsTensorConst("var"), + IsTensorConst("eps")); + + private Expr? GetReplace(Expr input, Tensor scale, Tensor bias, Tensor mean, Tensor var, Tensor eps) + { + if (input.CheckedShape.Rank <= 1) + { + return null; + } + + var shape = input.CheckedShape.ToValueArray().Reverse().Take(input.CheckedShape.Rank - 1).Reverse().ToArray(); + + var newVar = Add(var, eps).Evaluate().AsTensor().ToArray(); + + var inRes = Reshape(input, shape); + var inResO = new Var(inRes.CheckedType); + + var ncnnBatchNorm = new Call(new Fusion("ncnn", NcnnBatchNorm(inResO, shape[0], 0.0f, scale.ToArray(), mean.ToArray(), newVar.ToArray(), bias.ToArray()), new[] { inResO }), inRes); + + var outRes = Reshape(ncnnBatchNorm, input.CheckedShape); + return outRes; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs new file mode 100644 index 0000000000..adef2362b8 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs @@ -0,0 +1,217 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using DryIoc.ImTools; +using NetFabric.Hyperlinq; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.Math; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerBinary : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsBinary( + target_name: "binary", + _ => true, + IsWildcard("inputA") with { TypePattern = IsFloat() & HasFixedShape() }, + IsWildcard("inputB") with { TypePattern = IsFloat() & HasFixedShape() }); + + private static BinaryOperationType? MapBinaryOp(BinaryOp binaryOp) => + binaryOp switch + { + BinaryOp.Add => BinaryOperationType.ADD, + BinaryOp.Sub => BinaryOperationType.SUB, + BinaryOp.Mul => BinaryOperationType.MUL, + BinaryOp.Div => BinaryOperationType.DIV, + BinaryOp.Max => BinaryOperationType.MAX, + BinaryOp.Min => BinaryOperationType.MIN, + BinaryOp.Pow => BinaryOperationType.POW, + _ => null, + + // unsupported Binary ops + // BinaryOp.Mod => + // BitwiseAnd + // BitwiseOr + // BitwiseXor + // LogicalAnd + // LogicalOr + // LogicalXor + // LeftShift + // RightShift + // => BinaryOperationType.RSUB, + // => BinaryOperationType.RDIV, + // => BinaryOperationType.RPOW, + // => BinaryOperationType.ATAN2, + // => BinaryOperationType.RATAN2, + }; + + private int[] FixShape(int[] shape, int r) + { + var newShape = shape.ToList(); + for (int i = r - shape.Length; i > 0; i--) + { + newShape.Insert(0, 1); + } + + return newShape.ToArray(); + } + + private bool HasGap(List aShape, List bShape) + { + if ((aShape.Count == 1 && bShape.Count != 1 && aShape[0] != bShape[^1]) || (bShape.Count == 1 && aShape.Count != 1 && bShape[0] != aShape[^1])) + { + return true; + } + + return false; + } + + private Expr? GetReplace(Binary binary, Expr inputA, Expr inputB) + { + if (MapBinaryOp(binary.BinaryOp) is BinaryOperationType op) + { + bool needUnSqueeze = false; + int r = Math.Max(inputA.CheckedShape.Rank, inputB.CheckedShape.Rank); + if (r > 4) + { + return null; + } + + // ncnn [1, 3, 1, 1], [3] --> [1,3,1,3] + // onnx [1, 3, 1, 1], [3] --> [1,3,1,1] + if (inputA.CheckedShape.Rank != 0 && inputB.CheckedShape.Rank != 0 && HasGap(inputA.CheckedShape.ToValueArray().ToList(), inputB.CheckedShape.ToValueArray().ToList())) + { + return null; + } + + Call b; + + if (inputA is Const) + { + // A + var constA = ((TensorConst)inputA).Value; + var aShape = FixShape(inputA.CheckedShape.ToValueArray(), r).ToList(); + + // B + var newB = Reshape(inputB, FixShape(inputB.CheckedShape.ToValueArray(), r)); + var newInputB = new Var(newB.CheckedType); + + // Constant can not support 4D unless 0-D is 1. + if (r == 4) + { + if (newB.CheckedShape[0].FixedValue != 1 || aShape[0] != 1) + { + return null; + } + + newB = Squeeze(newB, new[] { 0 }); + newInputB = new Var(newB.CheckedType); + needUnSqueeze = true; + } + + while (aShape[0] == 1 && aShape.Count > 3 && aShape.Count > newB.CheckedShape.Count) + { + aShape.RemoveAt(0); + } + + b = new Call(new Fusion("ncnn", NcnnBinary(new Expr[] { newInputB }, op, 1, constA.ToArray(), aShape.ToArray()), new[] { newInputB }), newB); + } + else if (inputB is Const) + { + // A + var newA = Reshape(inputA, FixShape(inputA.CheckedShape.ToValueArray(), r)); + var newInputA = new Var(newA.CheckedType); + + // B + var constB = ((TensorConst)inputB).Value; + var bShape = FixShape(inputB.CheckedShape.ToValueArray(), r).ToList(); + + if (r == 4) + { + if (newA.CheckedShape[0].FixedValue != 1 || bShape[0] != 1) + { + return null; + } + + newA = Squeeze(newA, new[] { 0 }); + newInputA = new Var(newA.CheckedType); + needUnSqueeze = true; + } + + while (bShape[0] == 1 && bShape.Count > 3 && bShape.Count > newA.CheckedShape.Count) + { + bShape.RemoveAt(0); + } + + b = new Call(new Fusion("ncnn", NcnnBinary(new Expr[] { newInputA }, op, 2, constB.ToArray(), bShape.ToArray()), new[] { newInputA }), newA); + } + else + { + var inA = Reshape(inputA, FixShape(inputA.CheckedShape.ToValueArray(), r)); + var inB = Reshape(inputB, FixShape(inputB.CheckedShape.ToValueArray(), r)); + var newInputA = new Var(inA.CheckedType); + var newInputB = new Var(inB.CheckedType); + if (r < 4) + { + b = new Call(new Fusion("ncnn", NcnnBinary(new Expr[] { newInputA, newInputB }, op, 0, null, null), new[] { newInputA, newInputB }), inA, inB); + } + else + { + var newA = inA; + var newB = inB; + newInputA = new Var(newA.CheckedType); + newInputB = new Var(newB.CheckedType); + if (inA.CheckedShape[0].FixedValue != 1 && inA.CheckedShape.Rank == r) + { + return null; + } + else if (inA.CheckedShape[0].FixedValue == 1) + { + newA = Squeeze(inA, new[] { 0 }); + newInputA = new Var(newA.CheckedType); + needUnSqueeze = true; + } + + if (inB.CheckedShape[0].FixedValue != 1 && inB.CheckedShape.Rank == r) + { + return null; + } + else if (inB.CheckedShape[0].FixedValue == 1) + { + newB = Squeeze(inB, new[] { 0 }); + newInputB = new Var(newB.CheckedType); + needUnSqueeze = true; + } + + b = new Call(new Fusion("ncnn", NcnnBinary(new Expr[] { newInputA, newInputB }, op, 0, null, null), new[] { newInputA, newInputB }), newA, newB); + } + } + + if (needUnSqueeze) + { + return Unsqueeze(b, new[] { 0 }); + } + else + { + return b; + } + } + + return null; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCast.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCast.cs new file mode 100644 index 0000000000..55f9f2c903 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCast.cs @@ -0,0 +1,77 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using DryIoc.ImTools; +using Nncase.ArgsStruct; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.IR.Tensors; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.F.Tensors; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerCast : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsCast( + "cast", + _ => true, + IsWildcard("input") with { TypePattern = HasFixedShape() }); + + private int GetNcnnType(DataType inType) + { + if (inType == DataTypes.Float32) + { + return 1; + } + else if (inType == DataTypes.Float16) + { + return 2; + } + else if (inType == DataTypes.BFloat16) + { + return 4; + } + else + { + return -1; + } + } + + private Expr? GetReplace(Expr input, Cast cast) + { + if (input.CheckedShape.Count > 4 || input.CheckedShape[0].FixedValue != 1) + { + Console.WriteLine("ncnn not support more than 4D or batchSize > 1"); + return null; + } + + // ncnn only support f32->f16, f32->bf16, f16->f32, bf16->f32 + int fromType = GetNcnnType(input.CheckedDataType); + int toType = GetNcnnType(cast.NewType); + + if (fromType == -1 || toType == -1) + { + return null; + } + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + var c = new Call(new Fusion("ncnn", NcnnCast(inResO, fromType, toType), new[] { inResO }), inRes); + return Unsqueeze(c, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCelu.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCelu.cs new file mode 100644 index 0000000000..d9b94c1936 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCelu.cs @@ -0,0 +1,43 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerCelu : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsCelu( + IsWildcard("input") with { TypePattern = IsFloat() & HasFixedShape() }, + IsTensorConst("alpha") with { TypePattern = IsFloatScalar() }); + + private Expr? GetReplace(Expr input, float alpha) + { + if (input.CheckedShape.Count > 4 || input.CheckedShape[0].FixedValue != 1) + { + Console.WriteLine("ncnn not support more than 4D or batchSize > 1"); + return null; + } + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + var celu = new Call(new Fusion("ncnn", NcnnCelu(inResO, alpha), new[] { inResO }), inRes); + return Unsqueeze(celu, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerClamp.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerClamp.cs new file mode 100644 index 0000000000..5a7c72d413 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerClamp.cs @@ -0,0 +1,36 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.Math; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerClamp : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsClamp( + IsWildcard("input") with { TypePattern = HasFixedShape() }, + IsTensorConst("min", t => t.Value.ElementType == DataTypes.Float32), + IsTensorConst("max", t => t.Value.ElementType == DataTypes.Float32)); + + private Expr? GetReplace(Expr input, float min, float max) + { + var newInput = new Var(input.CheckedType); + + return new Call(new Fusion("ncnn", NcnnClip(newInput, min, max), new[] { newInput }), input); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConcat.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConcat.cs new file mode 100644 index 0000000000..69544b21f6 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConcat.cs @@ -0,0 +1,78 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.Tensors; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerConcat : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsConcat( + "concat", + _ => true, + IsTuple("tuple", IsVArgsRepeat("tupleInputs", () => IsWildcard() with { TypePattern = HasFixedShape() }))); + + // squeeze concat to 3D, get outputShape，set new axis as 1 + private static (List> NewShape, List OldOutputShape) GetFixedShapeAndOldOutputShape(IReadOnlyList tupleInputs, int oldAxis) + { + var newShapes = new List>(); + var outputShape = tupleInputs[0].CheckedShape.ToValueList(); + int positive_axis = oldAxis < 0 ? tupleInputs[0].CheckedShape.Count + oldAxis : oldAxis; + outputShape[positive_axis] = 0; + foreach (var input in tupleInputs) + { + var oldInputShape = input.CheckedShape.ToValueList(); + outputShape[positive_axis] += oldInputShape[positive_axis]; + var newShape = new List { 1, oldInputShape[positive_axis], 1 }; + for (int i = 0; i < positive_axis; i++) + { + newShape[0] *= oldInputShape[i]; + } + + for (int i = positive_axis + 1; i < oldInputShape.Count; i++) + { + newShape[2] *= oldInputShape[i]; + } + + newShapes.Add(newShape); + } + + return (newShapes, outputShape); + } + + private Expr? GetReplace(IReadOnlyList tupleInputs, IR.Tensors.Concat concat) + { + var (newInputShapes, oldOutputShape) = GetFixedShapeAndOldOutputShape(tupleInputs, concat.Axis); + + var varOfNewInputs = new List(); + var callOfNewInputs = new List(); + for (int i = 0; i < tupleInputs.Count; i++) + { + var inRes = Reshape(tupleInputs[i], newInputShapes[i].ToArray()); + var inResO = new Var(inRes.CheckedType); + varOfNewInputs.Add(inResO); + callOfNewInputs.Add(inRes); + } + + var ncnnConcatCall = new Call(new Fusion("ncnn", NcnnConcat(varOfNewInputs.ToArray(), 1), varOfNewInputs.ToArray()), callOfNewInputs.ToArray()); + + var outRes = Reshape(ncnnConcatCall, oldOutputShape.ToArray()); + + return outRes; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConv2d.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConv2d.cs new file mode 100644 index 0000000000..87a879bacd --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConv2d.cs @@ -0,0 +1,93 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.ArgsStruct; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.F.Tensors; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerConv : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsConv2D( + "conv", + conv => conv.PadMode == PadMode.Constant, + IsWildcard("input") with { TypePattern = HasFixedShape() }, + IsTensorConst("weights"), + IsTensorConst("bias"), + IsTensorConst("strides"), + IsTensorConst("paddings"), + IsTensorConst("dilation"), + IsTensorConst("groups"), + IsTensorConst("fusedClamp")); + + private Expr? GetReplace(Expr input, Tensor weights, Tensor bias, int[] strides, int[] paddings, int[] dilation, int groups, float[] fusedClamp) + { + if (input.CheckedShape.Rank != 4 || input.CheckedShape[0].FixedValue != 1) + { + return null; + } + + var (numOutput, kernelH, kernelW) = (weights.Shape[0], weights.Shape[2], weights.Shape[3]); + var (dilationH, dilationW) = (dilation[0], dilation.Length == 2 ? dilation[1] : dilation[0]); + var (strideH, strideW) = (strides[0], strides.Length == 2 ? strides[1] : strides[0]); + + // int[] paddingsValue = ((TensorConst)paddings).Value.ToArray(); + var (padLeft, padRight, padTop, padBottom) = (paddings[2], paddings[3], paddings[0], paddings[1]); + float padValue = 0; + int biasTerm = 1; + var weightsDataSize = weights.Shape[0] * weights.Shape[1] * weights.Shape[2] * weights.Shape[3]; + int int8ScaleTerm = 0; + + // 1:Relu 2: leaky relu 3:clip 4: sigmoid 5: mish 6: hardswish + int activationType = 3; + int dynamicWeight = 0; + + var args = new ConvArgs( + weights.ToArray(), + bias.ToArray(), + numOutput.FixedValue, + kernelW.FixedValue, + kernelH.FixedValue, + dilationW, + dilationH, + strideW, + strideH, + padLeft, + padRight, + padTop, + padBottom, + padValue, + biasTerm, + weightsDataSize.FixedValue, + int8ScaleTerm, + activationType, + fusedClamp, + dynamicWeight, + groups); + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + var conv = new Call(new Fusion("ncnn", NcnnConv(inResO, args), new[] { inResO }), inRes); + + var outRes = Unsqueeze(conv, new[] { 0 }); + + return outRes; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConvTranspose.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConvTranspose.cs new file mode 100644 index 0000000000..189dc404b8 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConvTranspose.cs @@ -0,0 +1,77 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.ArgsStruct; +using Nncase.Evaluator; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.F.Tensors; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerConvTranspose : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsConv2DTranspose( + "conv", + conv => conv.PadMode == PadMode.Constant, + IsWildcard("input") with { TypePattern = HasFixedShape() }, + IsTensorConst("weights"), + IsTensorConst("bias"), + IsTensorConst("outputShape"), + IsTensorConst("strides"), + IsTensorConst("paddings"), + IsTensorConst("outputPadding"), + IsTensorConst("dilation"), + IsTensorConst("group"), + IsTensorConst("fusedClamp")); + + private Expr? GetReplace(Expr input, Tensor weights, Tensor bias, int[] outputShape, int[] strides, int[] paddings, int[] outputPadding, int[] dilation, int group, float[] fusedClamp) + { + if (input.CheckedShape.Rank != 4 || input.CheckedShape[0].FixedValue != 1) + { + return null; + } + + int[] weightShape = weights.Shape.ToValueArray(); + var (numOutput, kernelH, kernelW) = (weights.Shape[0], weights.Shape[2], weights.Shape[3]); + var (dilationH, dilationW) = (dilation[0], dilation.Length == 2 ? dilation[1] : dilation[0]); + var (strideH, strideW) = (strides[0], strides.Length == 2 ? strides[1] : strides[0]); + + var (padLeft, padRight, padTop, padBottom) = (paddings[2], paddings[3], paddings[0], paddings[1]); + int biasTerm = 1; + var weightsDataSize = weightShape[0] * weightShape[1] * weightShape[2] * weightShape[3]; + + // 1:Relu 2: leaky relu 3:clip 4: sigmoid 5: mish 6: hardswish + int activationType = 3; + + // Note: Has reordered in importer. + // reorder weights(Cin Cout H W) to (Cout Cin H W) + // var newWeights = Transpose(weights, new int[] { 1, 0, 2, 3 }).Evaluate().AsTensor(); + + // actType and actParams not used in ncnn: onnx2ncnn. + var args = new ConvTransposeArgs(weights, bias.ToArray(), numOutput.FixedValue, kernelW.FixedValue, kernelH.FixedValue, dilationW, dilationH, strideW, strideH, padLeft, padRight, padTop, padBottom, biasTerm, weightsDataSize, activationType, fusedClamp, outputPadding[1], outputPadding[0], outputShape[3], outputShape[2]); + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + var conv = new Call(new Fusion("ncnn", NcnnConvTranspose(inResO, args), new[] { inResO }), inRes); + + var outRes = Unsqueeze(conv, new[] { 0 }); + + return outRes; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCrop.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCrop.cs new file mode 100644 index 0000000000..7d5844fa0b --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCrop.cs @@ -0,0 +1,77 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using DryIoc.ImTools; +using Nncase.ArgsStruct; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.F.Tensors; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerCrop : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsSlice( + IsWildcard("input") with { TypePattern = HasFixedShape() }, + IsTensorConst("starts"), + IsTensorConst("ends"), + IsTensorConst("axes"), + IsTensorConst("steps")); + + private Expr? GetReplace(Expr input, int[] starts, int[] ends, int[] axes, int[] steps) + { + if (steps.All(x => x != 1)) + { + Console.WriteLine("ncnn not support slice with step"); + return null; + } + + if (input.CheckedShape.Count > 4 || input.CheckedShape[0].FixedValue != 1) + { + Console.WriteLine("ncnn not support more than 4D or batchSize > 1"); + return null; + } + + var tStart = starts.ToList(); + var tEnds = ends.ToList(); + var tAxes = axes.ToList(); + + for (int i = 0; i < axes.Length; i++) + { + if (axes[i] == 0) + { + tStart.RemoveAt(i); + tEnds.RemoveAt(i); + tAxes.RemoveAt(i); + break; + } + } + + for (int i = 0; i < tAxes.Count; i++) + { + tAxes[i] -= 1; + } + + var args = new CropArgs(tStart.ToArray(), tEnds.ToArray(), tAxes.ToArray()); + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + var crop = new Call(new Fusion("ncnn", NcnnCrop(inResO, args), new[] { inResO }), inRes); + return Unsqueeze(crop, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCumsum.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCumsum.cs new file mode 100644 index 0000000000..5c45a9ed16 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCumsum.cs @@ -0,0 +1,51 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.Math; +using static Nncase.PatternMatch.Utility; +using static Nncase.Utilities.ShapeUtility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerCumsum : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsCumSum( + IsWildcard("input") with { TypePattern = HasFixedShape() }, + IsTensorConst("axis"), + IsTensorConst("exclusive"), + IsTensorConst("reverse")); + + private Expr? GetReplace(Expr input, Tensor axis, Tensor exclusive, Tensor reverse) + { + if (exclusive[0] == false && reverse[0] == false) + { + var inResShape = FitNcnnShape(input.CheckedShape.ToValueList(), axis[0]); + var inRes = Reshape(input, inResShape.ToArray()); + _ = new Var(inRes.CheckedType); + + var newInput = new Var(input.CheckedType); + var cumsum_ = new Call(new Fusion("ncnn", NcnnCumsum(newInput, 1), new[] { newInput }), inRes); + + var outRes = Reshape(cumsum_, input.CheckedShape.ToValueList().ToArray()); + + return outRes; + } + + return null; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerDequantize.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerDequantize.cs new file mode 100644 index 0000000000..4652892db7 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerDequantize.cs @@ -0,0 +1,57 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.Math; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerDequantize : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsDequantize( + "deq", + "output", + _ => true, + IsWildcard("input") with { TypePattern = HasFixedShape() }, + IsTensorConst("param")); + + private Expr? GetReplace(Expr input, Dequantize deq, QuantParam[] param) + { + if (input.CheckedShape.Count > 4 || input.CheckedShape[0].FixedValue != 1 || deq.TargetType != DataTypes.Float32) + { + Console.WriteLine("ncnn not support more than 4D or batchSize > 1"); + return null; + } + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + + float[] scale = new float[param.Length]; + float[] bias = new float[param.Length]; + for (int i = 0; i < param.Length; i++) + { + scale[i] = param[i].Scale; + bias[i] = param[i].ZeroPoint; + } + + var dequantize = new Call(new Fusion("ncnn", NcnnDequantize(inResO, scale, bias), new[] { inResO }), inRes); + + return Unsqueeze(dequantize, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerElu.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerElu.cs new file mode 100644 index 0000000000..329c4f34b2 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerElu.cs @@ -0,0 +1,36 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerElu : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsElu( + IsWildcard("input") with { TypePattern = IsFloat() & HasFixedShape() }, + IsTensorConst("alpha") with { TypePattern = IsFloatScalar() }); + + private Expr? GetReplace(Expr input, float alpha) + { + var newInput = new Var(input.CheckedType); + + return new Call(new Fusion("ncnn", NcnnElu(newInput, alpha), new[] { newInput }), input); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerErf.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerErf.cs new file mode 100644 index 0000000000..8c265d6433 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerErf.cs @@ -0,0 +1,44 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerErf : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsErf( + IsWildcard("input") with { TypePattern = IsFloat() & HasFixedShape() }); + + private Expr? GetReplace(Expr input) + { + // TODO: split input + if (input.CheckedShape.ToList()[0] != 1) + { + return null; + } + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + + var erf = new Call(new Fusion("ncnn", NcnnErf(inResO), new[] { inResO }), inRes); + + return Unsqueeze(erf, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerGELU.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerGELU.cs new file mode 100644 index 0000000000..509b231cde --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerGELU.cs @@ -0,0 +1,53 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using DryIoc.ImTools; +using Nncase.ArgsStruct; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.IR.Tensors; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.F.Tensors; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerGELU : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsGelu( + IsWildcard("input") with { TypePattern = HasFixedShape() }, + IsTensorConst("alpha")); + + private Expr? GetReplace(Expr input, float alpha) + { + if (input.CheckedShape.Count > 4 || input.CheckedShape[0].FixedValue != 1) + { + Console.WriteLine("ncnn not support more than 4D or batchSize > 1"); + return null; + } + + // TODO: support GELU with scale. + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + if (Math.Abs(alpha - 1.0) > 1e-06) + { + return null; + } + + var c = new Call(new Fusion("ncnn", NcnnGELU(inResO), new[] { inResO }), inRes); + return Unsqueeze(c, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerHardSigmoid.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerHardSigmoid.cs new file mode 100644 index 0000000000..97de87a795 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerHardSigmoid.cs @@ -0,0 +1,46 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerHardSigmoid : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsHardSigmoid( + IsWildcard("input") with { TypePattern = IsFloat() & HasFixedShape() }, + IsTensorConst("alpha") with { TypePattern = IsScalar() }, + IsTensorConst("beta") with { TypePattern = IsScalar() }); + + private Expr? GetReplace(Expr input, float alpha, float beta) + { + // TODO: split input + if (input.CheckedShape.ToList()[0] != 1) + { + return null; + } + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + + var hardSigmoid = new Call(new Fusion("ncnn", NcnnHardSigmoid(inResO, alpha, beta), new[] { inResO }), inRes); + + return Unsqueeze(hardSigmoid, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerHardSwish.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerHardSwish.cs new file mode 100644 index 0000000000..6b281b6359 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerHardSwish.cs @@ -0,0 +1,44 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerHardSwish : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsHardSwish( + IsWildcard("input") with { TypePattern = HasFixedShape() }); + + private Expr? GetReplace(Expr input) + { + // TODO: split input + if (input.CheckedShape.ToList()[0] != 1) + { + return null; + } + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + + var hardSwish = new Call(new Fusion("ncnn", NcnnHardSwish(inResO, 1.0f / 6, 0.5f), new[] { inResO }), inRes); + + return Unsqueeze(hardSwish, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerInstanceNorm.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerInstanceNorm.cs new file mode 100644 index 0000000000..a4e5a845d8 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerInstanceNorm.cs @@ -0,0 +1,49 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerInstanceNorm : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsInstanceNormalization( + IsWildcard("input") with { TypePattern = HasFixedShape() }, + IsTensorConst("gamma"), + IsTensorConst("beta"), + IsTensorConst("epsilon") with { TypePattern = IsFloatScalar() }); + + private Expr? GetReplace(Expr input, float[] gamma, float[] beta, float epsilon) + { + // TODO: split input + if (input.CheckedShape.ToList()[0] != 1) + { + return null; + } + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + + int affine = gamma.All(a => a != 1) && beta.All(a => a != 0) ? 1 : 0; + + var instanceNorm = new Call(new Fusion("ncnn", NcnnInstanceNorm(inResO, gamma.Length, epsilon, affine, gamma, beta), new[] { inResO }), inRes); + + return Unsqueeze(instanceNorm, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLSTM.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLSTM.cs new file mode 100644 index 0000000000..02125b4d97 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLSTM.cs @@ -0,0 +1,131 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using DryIoc.ImTools; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.IR.RNN; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.F.RNN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerLSTM : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsLSTM( + target_name: "lstm", + call_name: "call", + _ => true, + IsWildcard("input") with { TypePattern = HasFixedShape() }, + IsTensorConst("w"), + IsTensorConst("r"), + IsTensorConst("b"), + IsTensorConst(), + IsTensorConst("initialH"), + IsTensorConst("initialC"), + IsTensorConst(), + IsTensorConst(), + IsTensorConst(), + IsTensorConst(), + IsTensorConst("hiddenSize"), + IsTensorConst(), + IsTensorConst("outputSize")); + + private Expr? GetReplace(LSTM lstm, Call call, Expr input, Tensor w, Tensor r, Tensor b, Expr initialH, Expr initialC, int hiddenSize, int outputSize) + { + // x 先Squeeze掉batch size axis=1， + // ncnn : initial(H,C)默认为0，不起作用 + var inRes = Squeeze(input, new[] { 1 }); + var inResO = new Var(inRes.CheckedType); + + // args + // 0: hiddensize + // 1: weight_data_size : w.count/num_direction + var weightDataSize = w.Shape[1].FixedValue * w.Shape[2].FixedValue; + + // 2: direction [0,1,2] + int direction; + switch (lstm.Direction) + { + case LSTMDirection.Forward: + direction = 0; + break; + case LSTMDirection.Reverse: + direction = 1; + break; + default: // Bidirectional + direction = 2; + break; + } + + // bin + // reorder num_directions-IOFG - hidden_size to num_directions-IFOG - hidden_size + // weights: i f o g: + reverse i f o g + var (n_, c_, h_, w_) = (w.Shape[0].FixedValue, 4, hiddenSize, w.Shape[2].FixedValue); + var newWeights_ = Reshape(w, new[] { n_, c_, h_, w_ }); + var strides = new int[1] { 1 }; + var newWeights = Concat( + new IR.Tuple(new[] + { + Slice(newWeights_, new int[] { 0 }, new int[] { 1 }, new int[] { 1 }, strides), + Slice(newWeights_, new int[] { 2 }, new int[] { 3 }, new int[] { 1 }, strides), + Slice(newWeights_, new int[] { 1 }, new int[] { 2 }, new int[] { 1 }, strides), + Slice(newWeights_, new int[] { 3 }, new int[] { 4 }, new int[] { 1 }, strides), + }), + 1).Evaluate().AsTensor().ToArray(); + + // bias : x_b + w_b_re + var newBias_ = Reshape(b, new[] { w.Shape[0].FixedValue, 2, 4, hiddenSize }); + var biasW_ = Slice(newBias_, new[] { 0 }, new[] { 1 }, new int[] { 1 }, strides); + var biasR_ = Slice(newBias_, new[] { 1 }, new[] { 2 }, new int[] { 1 }, strides); + var biasW = Concat( + new IR.Tuple(new[] + { + Slice(biasW_, new int[] { 0 }, new int[] { 1 }, new int[] { 2 }, strides), + Slice(biasW_, new int[] { 2 }, new int[] { 3 }, new int[] { 2 }, strides), + Slice(biasW_, new int[] { 1 }, new int[] { 2 }, new int[] { 2 }, strides), + Slice(biasW_, new int[] { 3 }, new int[] { 4 }, new int[] { 2 }, strides), + }), + 2); + var biasR = Concat( + new IR.Tuple(new[] + { + Slice(biasR_, new int[] { 0 }, new int[] { 1 }, new int[] { 2 }, strides), + Slice(biasR_, new int[] { 2 }, new int[] { 3 }, new int[] { 2 }, strides), + Slice(biasR_, new int[] { 1 }, new int[] { 2 }, new int[] { 2 }, strides), + Slice(biasR_, new int[] { 3 }, new int[] { 4 }, new int[] { 2 }, strides), + }), + 2); + float[] newBias = (biasW + biasR).Evaluate().AsTensor().ToArray(); + + // R: 处理同weights + var newR_ = Reshape(r, new[] { n_, c_, h_, hiddenSize }); + float[] newR = Concat( + new IR.Tuple(new[] + { + Slice(newR_, new int[] { 0 }, new int[] { 1 }, new int[] { 1 }, strides), + Slice(newR_, new int[] { 2 }, new int[] { 3 }, new int[] { 1 }, strides), + Slice(newR_, new int[] { 1 }, new int[] { 2 }, new int[] { 1 }, strides), + Slice(newR_, new int[] { 3 }, new int[] { 4 }, new int[] { 1 }, strides), + }), + 1).Evaluate().AsTensor().ToArray(); + + var lstm_ = new Call(new Fusion("ncnn", NcnnLSTM(inResO, outputSize, hiddenSize, weightDataSize, direction, newWeights, newBias, newR), new[] { inResO }), inRes); + return new IR.Tuple(new[] { Unsqueeze(lstm_[0], new[] { 1, 1 }), lstm_[1], lstm_[2] }); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLayernorm.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLayernorm.cs new file mode 100644 index 0000000000..de53fd37e7 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLayernorm.cs @@ -0,0 +1,60 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.IR.NN; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; +using static Nncase.Utilities.ShapeUtility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerLayerNorm : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsLayerNorm( + "op", + _ => true, + IsWildcard("input") with { TypePattern = HasFixedShape() }, + IsTensorConst("gamma"), + IsTensorConst("beta")); + + private Expr? GetReplace(LayerNorm op, Expr input, float[] gamma, float[] beta) + { + // TODO: split input + if (input.CheckedShape.ToList()[0] != 1) + { + return null; + } + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + + int affine = gamma.All(a => a != 1) && beta.All(a => a != 0) ? 1 : 0; + int affineSize = 1; + + int newAxis = op.Axis > 0 ? op.Axis - 1 : 0; // BatchSize has been squeezed. + + for (int i = newAxis; i < inRes.CheckedShape.Rank; i++) + { + affineSize *= inRes.CheckedShape.ToValueArray()[i]; + } + + var layerNorm = new Call(new Fusion("ncnn", NcnnLayerNorm(inResO, affineSize, op.Epsilon, affine, gamma, beta), new[] { inResO }), inRes); + + return Unsqueeze(layerNorm, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLrn.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLrn.cs new file mode 100644 index 0000000000..1981f6f94b --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLrn.cs @@ -0,0 +1,48 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerLRN : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsLRN( + IsWildcard("input") with { TypePattern = HasFixedShape() }, + IsTensorConst("alpha") with { TypePattern = IsScalar() }, + IsTensorConst("beta") with { TypePattern = IsScalar() }, + IsTensorConst("bias") with { TypePattern = IsScalar() }, + IsTensorConst("size") with { TypePattern = IsIntegralScalar() }); + + private Expr? GetReplace(Expr input, float alpha, float beta, float bias, int size) + { + // TODO: split input + if (input.CheckedShape.ToList()[0] != 1) + { + return null; + } + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + + var lrn = new Call(new Fusion("ncnn", NcnnLRN(inResO, alpha, beta, bias, size), new[] { inResO }), inRes); + + return Unsqueeze(lrn, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerMatmul.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerMatmul.cs new file mode 100644 index 0000000000..5235a913a5 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerMatmul.cs @@ -0,0 +1,60 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using DryIoc.ImTools; +using NetFabric.Hyperlinq; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.Math; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerMatmul : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsMatMul( + IsWildcard("inputA") with { TypePattern = HasFixedShape() }, + IsWildcard("inputB") with { TypePattern = HasFixedShape() }); + + private Expr? GetReplace(Expr inputA, Expr inputB) + { + if (inputA is Const) + { + var constA = ((TensorConst)inputA).Value; + var constShape = inputA.CheckedShape.ToValueArray(); + + // var newB = Reshape(inputB, FixShape(inputB.CheckedShape.ToValueArray(), r)); + var newInputB = new Var(inputB.CheckedType); + return new Call(new Fusion("ncnn", NcnnMatMul(new Expr[] { newInputB }, 1, constA.ToArray(), constShape), new[] { newInputB }), inputB); + } + + if (inputB is Const) + { + // var newA = Reshape(inputA, FixShape(inputA.CheckedShape.ToValueArray(), r)); + var newInputA = new Var(inputA.CheckedType); + var constB = ((TensorConst)inputB).Value; + var constShape = inputB.CheckedShape.ToValueArray(); + return new Call(new Fusion("ncnn", NcnnMatMul(new Expr[] { newInputA }, 2, constB.ToArray(), constShape), new[] { newInputA }), inputA); + } + + { + var newInputA = new Var(inputA.CheckedType); + var newInputB = new Var(inputB.CheckedType); + return new Call( + new Fusion("ncnn", NcnnMatMul(new Expr[] { newInputA, newInputB }, 0, null, null), new[] { newInputA, newInputB }), inputA, inputB); + } + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPReLU.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPReLU.cs new file mode 100644 index 0000000000..5d68910827 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPReLU.cs @@ -0,0 +1,45 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.IR.NN; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerPReLU : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsPRelu( + IsWildcard("input") with { TypePattern = HasFixedShape() }, + IsTensorConst("slope")); + + private Expr? GetReplace(Expr input, Tensor slope) + { + // TODO: split input + if (input.CheckedShape.ToList()[0] != 1 || input.CheckedShape.Rank > 4) + { + return null; + } + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + + var pReLU = new Call(new Fusion("ncnn", NcnnPReLU(inResO, slope.ToArray()), new[] { inResO }), inRes); + return Unsqueeze(pReLU, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPadding.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPadding.cs new file mode 100644 index 0000000000..360812ee64 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPadding.cs @@ -0,0 +1,61 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerPadding : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsPad( + "pad", + "padCall", + x => true, + IsWildcard("input") with { TypePattern = HasFixedShape() }, + IsTensorConst("pads"), + IsTensorConst("padValue")); + + private Expr? GetReplace(IR.NN.Pad pad, Expr input, Expr pads, Expr padValue) + { + // TODO: split input + if (input.CheckedShape.ToList()[0] != 1) + { + return null; + } + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + var paddings = pads.Evaluate().AsTensor().ToArray(); + var (front, behind, top, bottom, left, right) = + (paddings[2], paddings[3], paddings[4], paddings[5], paddings[6], paddings[7]); + var padV = padValue.Evaluate().AsTensor().ToArray()[0]; // TODO: IsScaler or not. + int type = pad.PadMode switch + { + PadMode.Constant => 0, + PadMode.Edge => 1, + PadMode.Reflect => 2, + _ => throw new NotImplementedException("Ncnn not support other pad mode"), + }; + + var erf = new Call(new Fusion("ncnn", NcnnPadding(inResO, top, bottom, left, right, type, padV, front, behind), new[] { inResO }), inRes); + + return Unsqueeze(erf, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPermute.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPermute.cs new file mode 100644 index 0000000000..f42d0de4c9 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPermute.cs @@ -0,0 +1,100 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.F.Tensors; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerPermute : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsTranspose( + IsWildcard("input") with { TypePattern = HasFixedShape() }, + IsTensorConst("perm")); + + private Expr? GetReplace(Expr input, int[] perm) + { + if (input.CheckedShape[0] != 1 || input.CheckedShape.Rank > 5 || perm.Length == 2) + { + return null; + } + + int orderType = 0; + if (perm.Length == 3) + { + if (perm.Skip(1).SequenceEqual(new[] { 1, 2 })) + { + orderType = 0; + } + else if (perm.Skip(1).SequenceEqual(new[] { 2, 1 })) + { + orderType = 1; + } + else if (perm.SequenceEqual(new[] { 1, 0, 2 })) + { + orderType = 0; + } + else if (perm.SequenceEqual(new[] { 2, 0, 1 })) + { + orderType = 1; + } + else + { + return null; + } + } + else if (perm.Length == 4) + { + if (perm.Skip(1).SequenceEqual(new[] { 1, 2, 3 })) + { + orderType = 0; + } + else if (perm.Skip(1).SequenceEqual(new[] { 1, 3, 2 })) + { + orderType = 1; + } + else if (perm.Skip(1).SequenceEqual(new[] { 2, 1, 3 })) + { + orderType = 2; + } + else if (perm.Skip(1).SequenceEqual(new[] { 2, 3, 1 })) + { + orderType = 3; + } + else if (perm.Skip(1).SequenceEqual(new[] { 3, 1, 2 })) + { + orderType = 4; + } + else if (perm.Skip(1).SequenceEqual(new[] { 3, 2, 1 })) + { + orderType = 5; + } + else + { + return null; + } + } + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + + var permute = new Call(new Fusion("ncnn", NcnnPermute(inResO, orderType, perm), inResO), inRes); + return Unsqueeze(permute, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPooling.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPooling.cs new file mode 100644 index 0000000000..8418318697 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPooling.cs @@ -0,0 +1,72 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.ArgsStruct; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.IR.NN; +using Nncase.PatternMatch; +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerPooling : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsReduceWindow2D( + "pdp", + _ => true, + IsWildcard("input") with { TypePattern = HasFixedShape() }, + IsWildcard("initValue"), + IsWildcard("filter"), + IsWildcard("stride"), + IsTensorConst("padding"), + IsWildcard("dilation"), + IsWildcard("ceilMode"), + IsWildcard("countIncludePad")); + + private Expr? GetReplace(ReduceWindow2D pdp, Expr input, float initValue, Expr filter, Expr stride, Expr padding, Expr dilation, bool ceilMode, bool countIncludePad) + { + // TODO: split input + if (input.CheckedShape.ToList()[0] != 1) + { + return null; + } + + var poolingType = pdp.ReduceOp switch + { + ReduceOp.Max => 0, + ReduceOp.Mean => 1, + _ => throw new NotImplementedException($"{pdp.ReduceOp} not support in ncnn!"), + }; + + var kernel_ = filter.Evaluate().AsTensor().ToArray(); + var (kernelW, kernelH) = (kernel_[1], kernel_[0]); + + var stride_ = stride.Evaluate().AsTensor().ToArray(); + var (strideW, strideH) = (stride_[1], stride_[0]); + + var padding_ = padding.Evaluate().AsTensor().ToArray(); + var (padLeft, padRight, padTop, padBottom) = (padding_[2], padding_[3], padding_[0], padding_[1]); + + // Globalpool has been converted to pool, reflected in the size of the kernel. + var args = new PoolingArgs(poolingType, kernelW, kernelH, strideW, strideH, padLeft, padRight, padTop, padBottom, false, 1, countIncludePad, false, 0, 0, ceilMode); + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + + var pool = new Call(new Fusion("ncnn", NcnnPooling(inResO, args), new[] { inResO }), inRes); + return Unsqueeze(pool, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReduction.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReduction.cs new file mode 100644 index 0000000000..42f088b17c --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReduction.cs @@ -0,0 +1,229 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using DryIoc.ImTools; +using Nncase.ArgsStruct; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.IR.NN; +using Nncase.PatternMatch; +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.Math; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerReduction : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsReduce( + "reduce", + "reduceCall", + _ => true, + IsWildcard("input") with { TypePattern = IsFloat() & HasFixedShape() }, + IsTensorConst("axis"), + IsTensorConst("initValue"), + IsTensorConst("keepDims")); + + public virtual int GetOpT() + { + return -1; + } + + private Expr? GetReplace(Reduce reduce, Expr input, long[] axis, Expr initValue, bool keepDims) + { + // TODO: split input + if ((input.CheckedShape.ToList()[0] != 1 && input.CheckedShape.Rank == 4) || input.CheckedShape.Rank > 4 || input.CheckedDataType == DataTypes.Float16) + { + return null; + } + + // Support other reduction ops combined with several ops. + var otherOpType = GetOpT(); + var reductionType = otherOpType == -1 + ? reduce.ReduceOp switch + { + ReduceOp.Sum => 0, + ReduceOp.Mean => 3, + ReduceOp.Max => 4, + ReduceOp.Min => 5, + ReduceOp.Prod => 6, + _ => -1, + } + : otherOpType; + if (reductionType == -1) + { + return null; + } + + var newAxis = axis; + + var newInput = input; + var newInputVar = new Var(newInput.CheckedType); + if (input.CheckedShape.Rank == 4) + { + if (axis.Length == input.CheckedShape.Rank) + { + // 排除batch维度 + newAxis = newAxis.Remove(0); + newAxis = newAxis.Remove(-input.CheckedShape.Rank); + } + + for (int i = 0; i < newAxis.Length; i++) + { + if (newAxis[i] == 0 || newAxis[i] > 4 || newAxis[i] < -3) + { + return null; + } + + newAxis[i] = newAxis[i] > 0 ? newAxis[i] - 1 : newAxis[i]; + } + + newInput = Squeeze(input, new[] { 0 }); + newInputVar = new Var(newInput.CheckedType); + } + + var args = new ReductionArgs(reductionType, newAxis.Length == newInput.CheckedShape.Rank ? 1 : 0, 0, newAxis, keepDims ? 1 : 0); + + var pool = new Call(new Fusion("ncnn", NcnnReduction(newInputVar, args), new[] { newInputVar }), newInput); + + if (input.CheckedShape.Rank == 4) + { + return Unsqueeze(pool, new[] { 0 }); + } + else + { + return pool; + } + } +} + +[RuleGenerator] +public partial class LowerReductionSumSquare : LowerReduction +{ + /// + public override Pattern Pattern { get; } = + IsReduce( + "reduce", + "reduceCall", + ReduceOp.Sum, + IsUnary( + "square", + UnaryOp.Square, + IsWildcard("input") with { TypePattern = IsFloat() & HasFixedShape() }), + IsTensorConst("axis"), + IsTensorConst("initValue"), + IsTensorConst("keepDims")); + + public override int GetOpT() + { + return 2; + } +} + +[RuleGenerator] +public partial class LowerReductionL1 : LowerReduction +{ + /// + public override Pattern Pattern { get; } = + IsReduce( + "reduce", + "reduceCall", + ReduceOp.Sum, + IsUnary( + "abs", + UnaryOp.Abs, + IsWildcard("input") with { TypePattern = IsFloat() & HasFixedShape() }), + IsTensorConst("axis"), + IsTensorConst("initValue"), + IsTensorConst("keepDims")); + + public override int GetOpT() + { + return 7; + } +} + +[RuleGenerator] +public class LowerReductionL2 : LowerReduction +{ + /// + public override Pattern Pattern { get; } = + IsUnary( + "sqrt", + UnaryOp.Sqrt, + IsReduce( + "reduce", + "reduceCall", + ReduceOp.Sum, + IsUnary( + "square", + UnaryOp.Square, + IsWildcard("input") with { TypePattern = IsFloat() & HasFixedShape() }), + IsTensorConst("axis"), + IsTensorConst("initValue"), + IsTensorConst("keepDims"))); + + public override int GetOpT() + { + return 8; + } +} + +[RuleGenerator] +public partial class LowerReductionLogSum : LowerReduction +{ + /// + public override Pattern Pattern { get; } = + IsUnary( + "log", + UnaryOp.Log, + IsReduce( + "reduce", + "reduceCall", + ReduceOp.Sum, + IsWildcard("input") with { TypePattern = IsFloat() & HasFixedShape() }, + IsTensorConst("axis"), + IsTensorConst("initValue"), + IsTensorConst("keepDims"))); + + public override int GetOpT() + { + return 9; + } +} + +[RuleGenerator] +public partial class LowerReductionLogSumExp : LowerReduction +{ + /// + public override Pattern Pattern { get; } = + IsUnary( + "log", + UnaryOp.Log, + IsReduce( + "reduce", + "reduceCall", + ReduceOp.Sum, + IsUnary( + "exp", + UnaryOp.Exp, + IsWildcard("input") with { TypePattern = IsFloat() & HasFixedShape() }), + IsTensorConst("axis"), + IsTensorConst("initValue"), + IsTensorConst("keepDims"))); + + public override int GetOpT() + { + return 10; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReshape.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReshape.cs new file mode 100644 index 0000000000..eb13c91cb1 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReshape.cs @@ -0,0 +1,73 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using DryIoc.ImTools; +using NetFabric.Hyperlinq; +using Nncase.IR; +using Nncase.PatternMatch; +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.Tensors; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerReshape : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsReshape( + IsWildcard("input") with { TypePattern = HasFixedShape() & !IsScalar() }, + IsTensorConst("shape")); + + private Expr? GetReplace(Expr input, Expr shape) + { + if (input.CheckedShape.Rank < 5) + { + Call r; + var newInput = input; + var newInputVar = new Var(input.CheckedType); + + var outputShape = shape.Evaluate().AsTensor().ToArray().ToList(); + + int needSqueeze = 0; + if (newInput.CheckedShape[0].FixedValue != 1 && newInput.CheckedShape.Rank == 4) + { + return null; + } + else if (newInput.CheckedShape[0].FixedValue == 1 && newInput.CheckedShape.Rank == 4) + { + newInput = Squeeze(newInput, new[] { 0 }); + newInputVar = new Var(newInput.CheckedType); + needSqueeze += 1; + } + + if (outputShape.Count > 4 || (outputShape.Count == 4 && outputShape[0] != 1)) + { + return null; + } + else if (outputShape.Count == 4 && outputShape[0] == 1 && needSqueeze == 1) + { + outputShape.RemoveAt(0); // Avoid reshape input to 4D with multi batchsize. + needSqueeze += 2; + } // left 3D shape. + + if (outputShape.Count == 0) + { + return null; + } + + r = new Call(new Fusion("ncnn", NcnnReshape(newInputVar, outputShape.ToArray()), new[] { newInputVar }), newInput); + + if (needSqueeze == 3) + { + return Unsqueeze(r, new[] { 0 }); + } + + return r; + } + + return null; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSELU.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSELU.cs new file mode 100644 index 0000000000..7e1148fef8 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSELU.cs @@ -0,0 +1,40 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerSELU : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsSelu( + IsWildcard("input") with { TypePattern = HasFixedShape() }, + IsTensorConst("alpha"), + IsTensorConst("gamma")); + + private Expr? GetReplace(Expr input, float alpha, float gamma) + { + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + + var selu = new Call(new Fusion("ncnn", NcnnSELU(inResO, alpha, gamma), new[] { inResO }), inRes); + + return Unsqueeze(selu, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSigmoid.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSigmoid.cs new file mode 100644 index 0000000000..b7ae02fc2f --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSigmoid.cs @@ -0,0 +1,56 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerSigmoid : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsSigmoid( + IsWildcard("input") with { TypePattern = HasFixedShape() }); + + private Expr? GetReplace(Expr input) + { + if (input.CheckedShape.Rank > 4 || (input.CheckedShape.Rank == 4 && input.CheckedShape[0].FixedValue != 1)) + { + return null; + } + + var newInput = input; + var newInputVar = new Var(newInput.CheckedType); + + if (input.CheckedShape.Count == 4 && input.CheckedShape[0].FixedValue == 1) + { + newInput = Squeeze(input, new[] { 0 }); + newInputVar = new Var(newInput.CheckedType); + } + + var sigmoid = new Call(new Fusion("ncnn", NcnnSigmoid(newInputVar), new[] { newInputVar }), newInput); + + if (input.CheckedShape.Count == 4 && input.CheckedShape[0].FixedValue == 1) + { + return Unsqueeze(sigmoid, new int[] { 0 }); + } + else + { + return sigmoid; + } + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSlice.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSlice.cs new file mode 100644 index 0000000000..21eea2e09a --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSlice.cs @@ -0,0 +1,60 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.IR.Tensors; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.F.Tensors; +using static Nncase.PatternMatch.Utility; +using Unsqueeze = Nncase.IR.Tensors.Unsqueeze; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerSlice : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsSplit( + IsWildcard("input") with { TypePattern = HasFixedShape() }, + IsTensorConst("axis") with { TypePattern = IsScalar() }, + IsTensorConst("slices")); + + private Expr? GetReplace(Expr input, int[] slices, int axis) + { + // TODO: split input + if (input.CheckedShape.ToList()[0] != 1) + { + return null; + } + + if (input.CheckedShape.Count <= 5 && axis != 0) + { + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + var newAxis = axis < 0 ? axis + input.CheckedShape.Count - 1 : axis - 1; + var slice = new Call(new Fusion("ncnn", NcnnSlice(inResO, slices, newAxis), new[] { inResO }), inRes); + var result = new List(); + for (int i = 0; i < slices.Length; i++) + { + result.Add(Unsqueeze(slice[i], new[] { 0 })); + } + + return new IR.Tuple(result.ToArray()); + } + + return null; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSoftmax.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSoftmax.cs new file mode 100644 index 0000000000..2732e0cb23 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSoftmax.cs @@ -0,0 +1,68 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerSoftmax : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsSoftmax( + IsWildcard("input") with { TypePattern = IsFloat() & HasFixedShape() }, + IsTensorConst("axis")); + + // squeeze softmax to 3D，set axis to 1 + private (List NewShape, int NewAxis) GetFixedShapeAndAxis(List oldShape, int oldAxis) + { + int positive_axis = oldAxis < 0 ? oldShape.Count + oldAxis : oldAxis; + var newShape = new List { 1, oldShape[positive_axis], 1 }; + for (int i = 0; i < positive_axis; i++) + { + newShape[0] *= oldShape[i]; + } + + for (int i = positive_axis + 1; i < oldShape.Count; i++) + { + newShape[2] *= oldShape[i]; + } + + return (newShape, 1); + } + + private Expr? GetReplace(Expr input, int axis) + { + var newInput = new Var(input.CheckedType); + if (input.CheckedShape.Rank > 3) + { + var (newShape, newAxis) = GetFixedShapeAndAxis(input.CheckedShape.ToValueList(), axis); + + var inRes = Reshape(input, newShape.ToArray()); + var inResO = new Var(inRes.CheckedType); + + var ncnnSoftmaxCall = new Call(new Fusion("ncnn", NcnnSoftmax(inResO, newAxis), new[] { inResO }), inRes); + + var outRes = Reshape(ncnnSoftmaxCall, input.CheckedShape.ToValueList().ToArray()); + return outRes; + } + else + { + return new Call(new Fusion("ncnn", NcnnSoftmax(newInput, axis), new[] { newInput }), input); + } + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSoftplus.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSoftplus.cs new file mode 100644 index 0000000000..db25ebd1b7 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSoftplus.cs @@ -0,0 +1,42 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerSoftplus : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsSoftplus( + IsWildcard("input") with { TypePattern = HasFixedShape() }); + + private Expr? GetReplace(Expr input) + { + if (input.CheckedShape.ToList()[0] != 1) + { + return null; + } + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + + var softplus = new Call(new Fusion("ncnn", NcnnSoftplus(inResO), new[] { inResO }), inRes); + return Unsqueeze(softplus, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSqueeze.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSqueeze.cs new file mode 100644 index 0000000000..f8990d52e1 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSqueeze.cs @@ -0,0 +1,52 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using NetFabric.Hyperlinq; +using Nncase.IR; +using Nncase.PatternMatch; +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.Tensors; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerSqueeze : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsSqueeze( + IsWildcard("input") with { TypePattern = HasFixedShape() & !IsScalar() }, + IsTensorConst("dims")); + + private Expr? GetReplace(Expr input, int[] dims) + { + if (input.CheckedShape.Count < 5) + { + var inResO = new Var(input.CheckedType); + + var newDims = new List(); + foreach (int item in dims) + { + if (item > 4 || item < -3) + { + return null; + } + else + { + newDims.Add(item); + } + + if (input.CheckedShape[item] != 1) + { + return null; + } + } + + return new Call(new Fusion("ncnn", NcnnSqueeze(inResO, newDims.ToArray()), new[] { inResO }), input); + } + + return null; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerTile.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerTile.cs new file mode 100644 index 0000000000..4074972494 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerTile.cs @@ -0,0 +1,53 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.F.Tensors; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerTile : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsTile( + IsWildcard("input") with { TypePattern = HasFixedShape() }, + IsTensorConst("repeats")); + + private Expr? GetReplace(Expr input, int[] repeats) + { + // TODO: split input + if (input.CheckedShape.ToList()[0] != 1) + { + return null; + } + + // TODO: confirm d dim. + if (input.CheckedShape.Count < 5 && repeats[0] == 1) + { + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + var newRepeats = repeats[1..]; + + var tile = new Call(new Fusion("ncnn", NcnnTile(inResO, newRepeats), new[] { inResO }), inRes); + return Unsqueeze(tile, new[] { 0 }); + } + + // if repeats[0] != 1, means that input can't squeeze batchSize dim. + return null; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnary.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnary.cs new file mode 100644 index 0000000000..9b91e1d9d7 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnary.cs @@ -0,0 +1,99 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.Math; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerUnary : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsUnary( + target_name: "unary", + _ => true, + IsWildcard("input") with { TypePattern = IsFloat() & HasFixedShape() }); + + private static UnaryOperationType? MapUnaryOp(UnaryOp unaryOp) => + unaryOp switch + { + UnaryOp.Abs => UnaryOperationType.ABS, + UnaryOp.Neg => UnaryOperationType.NEG, + UnaryOp.Floor => UnaryOperationType.FLOOR, + UnaryOp.Ceil => UnaryOperationType.CEIL, + UnaryOp.Square => UnaryOperationType.SQUARE, + UnaryOp.Sqrt => UnaryOperationType.SQRT, + UnaryOp.Rsqrt => UnaryOperationType.RSQRT, + UnaryOp.Exp => UnaryOperationType.EXP, + UnaryOp.Log => UnaryOperationType.LOG, + UnaryOp.Sin => UnaryOperationType.SIN, + UnaryOp.Cos => UnaryOperationType.COS, + UnaryOp.Asin => UnaryOperationType.ASIN, + UnaryOp.Acos => UnaryOperationType.ACOS, + UnaryOp.Tanh => UnaryOperationType.TANH, + UnaryOp.Round => UnaryOperationType.ROUND, + _ => null, + + // unsupported unary ops + // UnaryOp.TAN => UnaryOperationType.ABS, + // UnaryOp.Atan => UnaryOperationType.ATAN, + // UnaryOp.Reciprocal => UnaryOperationType.RECIPROCAL, + // UnaryOp.Log10 => UnaryOperationType.LOG10, + // UnaryOp.Trunc => UnaryOperationType.TRUNC, + }; + + // squeeze unary shape to 3D + private List GetFixedShape(List oldShape) + { + var newShape = new List(); + + newShape.AddRange(oldShape.GetRange(oldShape.Count - 3, 3)); + + for (int i = 0; i < oldShape.Count - 3; i++) + { + newShape[0] *= oldShape[i]; + } + + return newShape; + } + + private Expr? GetReplace(Unary unary, Expr input) + { + if (MapUnaryOp(unary.UnaryOp) is UnaryOperationType op) + { + var newInput = new Var(input.CheckedType); + if (input.CheckedShape.Rank > 3) + { + var newShape = GetFixedShape(input.CheckedShape.ToValueList()); + + var inRes = Reshape(input, newShape.ToArray()); + var inResO = new Var(inRes.CheckedType); + + var ncnnUnaryCall = new Call(new Fusion("ncnn", NcnnUnary(inResO, op), new[] { inResO }), inRes); + + var outRes = Reshape(ncnnUnaryCall, input.CheckedShape.ToValueList().ToArray()); + return outRes; + } + else + { + return new Call(new Fusion("ncnn", NcnnUnary(newInput, op), new[] { newInput }), input); + } + } + + return null; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnsqueeze.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnsqueeze.cs new file mode 100644 index 0000000000..c494de8fb2 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnsqueeze.cs @@ -0,0 +1,33 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using NetFabric.Hyperlinq; +using Nncase.IR; +using Nncase.PatternMatch; +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.Tensors; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerUnsqueeze : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsUnsqueeze( + IsWildcard("input") with { TypePattern = HasFixedShape() & !IsScalar() }, + IsTensorConst("dims")); + + private Expr? GetReplace(Expr input, int[] dims) + { + if (input.CheckedShape.Count + dims.Length < 5) + { + var inResO = new Var(input.CheckedType); + return new Call(new Fusion("ncnn", NcnnUnsqueeze(inResO, dims), new[] { inResO }), input); + } + + return null; + } +} diff --git a/modules/Nncase.Modules.Ncnn/packages.lock.json b/modules/Nncase.Modules.Ncnn/packages.lock.json new file mode 100644 index 0000000000..e5ee906aff --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/packages.lock.json @@ -0,0 +1,306 @@ +{ + "version": 2, + "dependencies": { + "net7.0": { + "StyleCop.Analyzers": { + "type": "Direct", + "requested": "[1.2.0-beta.435, )", + "resolved": "1.2.0-beta.435", + "contentHash": "TADk7vdGXtfTnYCV7GyleaaRTQjfoSfZXprQrVMm7cSJtJbFc1QIbWPyLvrgrfGdfHbGmUPvaN4ODKNxg2jgPQ==", + "dependencies": { + "StyleCop.Analyzers.Unstable": "1.2.0.435" + } + }, + "Google.OrTools.runtime.linux-arm64": { + "type": "Transitive", + "resolved": "9.4.1874", + "contentHash": "Z46ndZcZa2Lt5b76xU9kxVYbPLg/LfuMufhUVsu3Qo3L7Bibf7WXd9j7RRldjnuv8RIHWTqb0b+2FwwMxs0c5A==" + }, + "Google.OrTools.runtime.linux-x64": { + "type": "Transitive", + "resolved": "9.4.1874", + "contentHash": "zGeDb8FuvP9HXjrsU7krVXtSDFpR+DUGNEsH51k94jL9tzf2vWYI8+WUBRHZ/cGe50dpLr+vIjfcNo3gFyOpkQ==" + }, + "Google.OrTools.runtime.osx-arm64": { + "type": "Transitive", + "resolved": "9.4.1874", + "contentHash": "Wo0ZfDaH6DhiQw0jZm4HWJm/oPGPpWNwOLUz+EYaoH3MLtocSxItHGQj/Ta3HyhXnYNOv+TliAH8L+8RCXu/2w==" + }, + "Google.OrTools.runtime.osx-x64": { + "type": "Transitive", + "resolved": "9.4.1874", + "contentHash": "IAfGgKR1og6vU87axK1d37Ak/4jy8B4NMoElovG/KZc/2UY+cJEAQDA709UMegtI4lBhuxTWFNUiHQYmRIB9yQ==" + }, + "Google.OrTools.runtime.win-x64": { + "type": "Transitive", + "resolved": "9.4.1874", + "contentHash": "fUs5qDnZA6itygolcX6nPuachQkY9CVvQbakIzIiRAWKcaj8umQAbFdGwbkyzp3qp34BKW5mtPVsmMyfQBBjOQ==" + }, + "libortki": { + "type": "Transitive", + "resolved": "0.0.2", + "contentHash": "svfuG5mxGY/QC/5DVheHOCELmdSP90RtxQ73j23KarPXZ9ZXW+7v1l5J77hGDyQbEh1BGrnGgKBlyn76RauGHg==", + "dependencies": { + "libortki-linux": "0.0.2", + "libortki-osx": "0.0.2", + "libortki-osx-arm64": "0.0.2", + "libortki-win": "0.0.2" + } + }, + "libortki-linux": { + "type": "Transitive", + "resolved": "0.0.2", + "contentHash": "b04LWD4lgGy60tys3hPFhnUpgWDM6dN5r1PI7GOcPj8VupXCaI70LKNQ5/5twbDE6rkowOGanVTw0S2wBGBqBQ==" + }, + "libortki-osx": { + "type": "Transitive", + "resolved": "0.0.2", + "contentHash": "O6Q9GLULkDkZEPAZJVKLPH0ROXGVOE7BxuddgOcHNK2oiTEM7wIRnzp2OIlYgLpaOLyxJMisbGOhtWgdzt2Wng==" + }, + "libortki-osx-arm64": { + "type": "Transitive", + "resolved": "0.0.2", + "contentHash": "4Qn2dirJmRicnUG945oWpq7HVGwgqCKKxYPMISv/MRvmpZBbXrZ1cVvRaF8WwTu4XXgfKTa1sLv+i8zLifUMeQ==" + }, + "libortki-win": { + "type": "Transitive", + "resolved": "0.0.2", + "contentHash": "HAoROgAKn8XBun11X43HZuspKlo5JGy8/OYw5IUPo7FVh5TCaPrLjGmyGYYZ2dqLlv31yv/b6s254PIRGn95cA==" + }, + "Microsoft.Extensions.Configuration.Abstractions": { + "type": "Transitive", + "resolved": "8.0.0", + "contentHash": "3lE/iLSutpgX1CC0NOW70FJoGARRHbyKmG7dc0klnUZ9Dd9hS6N/POPWhKhMLCEuNN5nXEY5agmlFtH562vqhQ==", + "dependencies": { + "Microsoft.Extensions.Primitives": "8.0.0" + } + }, + "Microsoft.Extensions.DependencyInjection.Abstractions": { + "type": "Transitive", + "resolved": "8.0.1", + "contentHash": "fGLiCRLMYd00JYpClraLjJTNKLmMJPnqxMaiRzEBIIvevlzxz33mXy39Lkd48hu1G+N21S7QpaO5ZzKsI6FRuA==" + }, + "Microsoft.Extensions.Diagnostics.Abstractions": { + "type": "Transitive", + "resolved": "8.0.0", + "contentHash": "JHYCQG7HmugNYUhOl368g+NMxYE/N/AiclCYRNlgCY9eVyiBkOHMwK4x60RYMxv9EL3+rmj1mqHvdCiPpC+D4Q==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "8.0.0", + "Microsoft.Extensions.Options": "8.0.0", + "System.Diagnostics.DiagnosticSource": "8.0.0" + } + }, + "Microsoft.Extensions.FileProviders.Abstractions": { + "type": "Transitive", + "resolved": "8.0.0", + "contentHash": "ZbaMlhJlpisjuWbvXr4LdAst/1XxH3vZ6A0BsgTphZ2L4PGuxRLz7Jr/S7mkAAnOn78Vu0fKhEgNF5JO3zfjqQ==", + "dependencies": { + "Microsoft.Extensions.Primitives": "8.0.0" + } + }, + "Microsoft.Extensions.Primitives": { + "type": "Transitive", + "resolved": "8.0.0", + "contentHash": "bXJEZrW9ny8vjMF1JV253WeLhpEVzFo1lyaZu1vQ4ZxWUlVvknZ/+ftFgVheLubb4eZPSwwxBeqS1JkCOjxd8g==" + }, + "NetFabric.Hyperlinq.Abstractions": { + "type": "Transitive", + "resolved": "1.3.0", + "contentHash": "WXnEcGwmXfa8gW9N2MlcaPNUzM3NLMwnAhacbtH554F8YcoXbIkTB+uGa1Aa+9gyb/9JZgYVHnmADgJUKP52nA==" + }, + "StyleCop.Analyzers.Unstable": { + "type": "Transitive", + "resolved": "1.2.0.435", + "contentHash": "ouwPWZxbOV3SmCZxIRqHvljkSzkCyi1tDoMzQtDb/bRP8ctASV/iRJr+A2Gdj0QLaLmWnqTWDrH82/iP+X80Lg==" + }, + "System.Buffers": { + "type": "Transitive", + "resolved": "4.5.1", + "contentHash": "Rw7ijyl1qqRS0YQD/WycNst8hUUMgrMH4FCn1nNm27M4VxchZ1js3fVjQaANHO5f3sN4isvP4a+Met9Y4YomAg==" + }, + "System.Diagnostics.DiagnosticSource": { + "type": "Transitive", + "resolved": "8.0.0", + "contentHash": "c9xLpVz6PL9lp/djOWtk5KPDZq3cSYpmXoJQY524EOtuFl5z9ZtsotpsyrDW40U1DRnQSYvcPKEUV0X//u6gkQ==" + }, + "System.Runtime.CompilerServices.Unsafe": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "ZD9TMpsmYJLrxbbmdvhwt9YEgG5WntEnZ/d1eH8JBX9LBp+Ju8BSBhUGbZMNVHHomWo2KVImJhTDl2hIgw/6MA==" + }, + "nncase.codegen": { + "type": "Project", + "dependencies": { + "Extension.Mathematics": "[1.2.12, )", + "Nncase.Core": "[1.0.0, )", + "Nncase.IO": "[1.0.0, )" + } + }, + "nncase.core": { + "type": "Project", + "dependencies": { + "CommunityToolkit.HighPerformance": "[8.2.2, )", + "DryIoc.dll": "[5.3.1, )", + "GiGraph.Dot": "[2.0.0, )", + "Microsoft.Extensions.Hosting.Abstractions": "[8.0.0, )", + "Microsoft.Extensions.Logging.Abstractions": "[8.0.1, )", + "Microsoft.Extensions.Options": "[8.0.2, )", + "NetFabric.Hyperlinq": "[3.0.0-beta48, )", + "System.CommandLine": "[2.0.0-beta4.22272.1, )", + "System.Reactive": "[6.0.0, )" + } + }, + "nncase.egraph": { + "type": "Project", + "dependencies": { + "GiGraph.Dot": "[2.0.0, )", + "Google.OrTools": "[9.4.1874, )", + "NetFabric.Hyperlinq": "[3.0.0-beta48, )", + "Nncase.Core": "[1.0.0, )", + "Nncase.Evaluator": "[1.0.0, )", + "Singulink.Collections.Weak": "[1.0.2, )" + } + }, + "nncase.evaluator": { + "type": "Project", + "dependencies": { + "Nncase.Core": "[1.0.0, )", + "OrtKISharp": "[0.0.2, )" + } + }, + "nncase.graph": { + "type": "Project", + "dependencies": { + "Nncase.Core": "[1.0.0, )", + "Nncase.Evaluator": "[1.0.0, )" + } + }, + "nncase.io": { + "type": "Project" + }, + "nncase.passes": { + "type": "Project", + "dependencies": { + "Nncase.Core": "[1.0.0, )", + "Nncase.EGraph": "[1.0.0, )", + "Nncase.Evaluator": "[1.0.0, )", + "Nncase.Graph": "[1.0.0, )" + } + }, + "CommunityToolkit.HighPerformance": { + "type": "CentralTransitive", + "requested": "[8.2.2, )", + "resolved": "8.2.2", + "contentHash": "+zIp8d3sbtYaRbM6hqDs4Ui/z34j7DcUmleruZlYLE4CVxXq+MO8XJyIs42vzeTYFX+k0Iq1dEbBUnQ4z/Gnrw==" + }, + "DryIoc.dll": { + "type": "CentralTransitive", + "requested": "[5.3.1, )", + "resolved": "5.3.1", + "contentHash": "E3zclUh2CIBks1t2uBD1k18pyGFJ1YSKCrbCDbB7qCdl2RAB+k68AyDpjeplhF1ot2XPV82AgyCWBXMf0ggL1g==" + }, + "Extension.Mathematics": { + "type": "CentralTransitive", + "requested": "[1.2.12, )", + "resolved": "1.2.12", + "contentHash": "D4mn5Cab4ztPLJ0V8uMErDrO/Y61098nwrvyIOLZymVAYOQcwP1vomVWKbTagf1aPU3cX5Q7adZtQEQwOy6XEg==" + }, + "GiGraph.Dot": { + "type": "CentralTransitive", + "requested": "[2.0.0, )", + "resolved": "2.0.0", + "contentHash": "ThvS2mQVveSkTMUm04tMbRYzu1XFPV8xBHISrUMp02APjhv9IRbLu3v3upTPCywORx2Ds/c6AqEUL1WU6kPfuQ==" + }, + "Google.OrTools": { + "type": "CentralTransitive", + "requested": "[9.4.1874, )", + "resolved": "9.4.1874", + "contentHash": "jqRoI+pYlym+fhoU25u+13oti5h+772bllQ9zDitTVMclDXVTiG6pxzvmYO74wnADBMdpb2SQlgiNQxoNk5dlA==", + "dependencies": { + "Google.OrTools.runtime.linux-arm64": "9.4.1874", + "Google.OrTools.runtime.linux-x64": "9.4.1874", + "Google.OrTools.runtime.osx-arm64": "9.4.1874", + "Google.OrTools.runtime.osx-x64": "9.4.1874", + "Google.OrTools.runtime.win-x64": "9.4.1874", + "Google.Protobuf": "3.19.4" + } + }, + "Google.Protobuf": { + "type": "CentralTransitive", + "requested": "[3.19.4, )", + "resolved": "3.19.4", + "contentHash": "fd07/ykL4O4FhqrZIELm5lmiyOHfdPg9+o+hWr6tcfRdS7tHXnImg/2wtogLzlW2eEmr0J7j6ZrZvaWOLiJbxQ==" + }, + "Microsoft.Extensions.Hosting.Abstractions": { + "type": "CentralTransitive", + "requested": "[8.0.0, )", + "resolved": "8.0.0", + "contentHash": "AG7HWwVRdCHlaA++1oKDxLsXIBxmDpMPb3VoyOoAghEWnkUvEAdYQUwnV4jJbAaa/nMYNiEh5ByoLauZBEiovg==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "8.0.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "8.0.0", + "Microsoft.Extensions.Diagnostics.Abstractions": "8.0.0", + "Microsoft.Extensions.FileProviders.Abstractions": "8.0.0", + "Microsoft.Extensions.Logging.Abstractions": "8.0.0" + } + }, + "Microsoft.Extensions.Logging.Abstractions": { + "type": "CentralTransitive", + "requested": "[8.0.1, )", + "resolved": "8.0.1", + "contentHash": "RIFgaqoaINxkM2KTOw72dmilDmTrYA0ns2KW4lDz4gZ2+o6IQ894CzmdL3StM2oh7QQq44nCWiqKqc4qUI9Jmg==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "8.0.1" + } + }, + "Microsoft.Extensions.Options": { + "type": "CentralTransitive", + "requested": "[8.0.2, )", + "resolved": "8.0.2", + "contentHash": "dWGKvhFybsaZpGmzkGCbNNwBD1rVlWzrZKANLW/CcbFJpCEceMCGzT7zZwHOGBCbwM0SzBuceMj5HN1LKV1QqA==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "8.0.0", + "Microsoft.Extensions.Primitives": "8.0.0" + } + }, + "NetFabric.Hyperlinq": { + "type": "CentralTransitive", + "requested": "[3.0.0-beta48, )", + "resolved": "3.0.0-beta48", + "contentHash": "oYUhXvxNS8bBJWqNkvx5g8y0P/0LtyqS2pN0w4OWjVDNWEpLbdbvPy9w/9z1n2PrqIjX3jxUsEnoCmxxGnI3gw==", + "dependencies": { + "NetFabric.Hyperlinq.Abstractions": "1.3.0", + "System.Buffers": "4.5.1", + "System.Runtime.CompilerServices.Unsafe": "5.0.0" + } + }, + "OrtKISharp": { + "type": "CentralTransitive", + "requested": "[0.0.2, )", + "resolved": "0.0.2", + "contentHash": "q8j0yR5836Zhv9WB9BFkQt1UaEFyibq8bqJcTiULlILF6/sz8z7Wy2N8sgYdDKsdW25zncIz7j6IDbKM5ynePg==", + "dependencies": { + "libortki": "0.0.2" + } + }, + "Singulink.Collections.Weak": { + "type": "CentralTransitive", + "requested": "[1.0.2, )", + "resolved": "1.0.2", + "contentHash": "giLAHrjJe0Bh7yhNexR6pmcv02+Fi+lEPxQVdB8zvkuJCmy6rnqu8CZLIpxrUfLcWDuTCSiK0IfGmMhig3UDhA==" + }, + "System.CommandLine": { + "type": "CentralTransitive", + "requested": "[2.0.0-beta4.22272.1, )", + "resolved": "2.0.0-beta4.22272.1", + "contentHash": "1uqED/q2H0kKoLJ4+hI2iPSBSEdTuhfCYADeJrAqERmiGQ2NNacYKRNEQ+gFbU4glgVyK8rxI+ZOe1onEtr/Pg==" + }, + "System.Reactive": { + "type": "CentralTransitive", + "requested": "[6.0.0, )", + "resolved": "6.0.0", + "contentHash": "31kfaW4ZupZzPsI5PVe77VhnvFF55qgma7KZr/E0iFTs6fmdhhG8j0mgEx620iLTey1EynOkEfnyTjtNEpJzGw==" + } + } + } +} \ No newline at end of file diff --git a/modules/Nncase.Modules.StackVM/Nncase.Modules.StackVM.csproj b/modules/Nncase.Modules.StackVM/Nncase.Modules.StackVM.csproj index 3d14c19536..45983364c7 100644 --- a/modules/Nncase.Modules.StackVM/Nncase.Modules.StackVM.csproj +++ b/modules/Nncase.Modules.StackVM/Nncase.Modules.StackVM.csproj @@ -15,6 +15,7 @@ + diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs new file mode 100644 index 0000000000..3a326a04c9 --- /dev/null +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -0,0 +1,170 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.CommandLine.Invocation; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Options; +using Nncase.CodeGen; +using Nncase.CodeGen.Ncnn; +using Nncase.CodeGen.StackVM; +using Nncase.IR; +using Nncase.Passes; +using Nncase.Passes.Rules.Neutral; +using Nncase.Passes.Transforms; +using Nncase.Quantization; + +namespace Nncase.Targets; + +/// +/// Target for CPU. +/// +public class CPUTarget : ITarget +{ + public const string Kind = "cpu"; + + string ITarget.Kind => Kind; + + public (System.CommandLine.Command Command, Func Parser) RegisterCommandAndParser() + { + return (new System.CommandLine.Command(Kind), (_, _) => DefaultTargetCompileOptions.Instance); + } + + /// + public void ParseTargetDependentOptions(IConfigurationSection configure) + { + } + + /// + public void RegisterTargetInDependentPass(IPassManager passManager, CompileOptions options) + { + } + + /// + public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions options) + { + passManager.AddWithName("LowerNcnnIR").Configure(p => + { + p.Add(); + + // Fold reduce + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + + // single op + p.Add(); + p.Add(); + p.Add(); + p.Add(); + + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + + p.Add(); + p.Add(); + p.Add(); + + // p.Add(); // ncnn dequantize int to float. + }); + + passManager.AddWithName("RemoveGlueOp").Configure(p => + { + p.Add(); + p.Add(); + p.Add(); + }); + + passManager.AddWithName("RemoveSingleSqueezeAndUnsqueeze").Configure(p => + { + p.Add(); + p.Add(); + }); + } + + /// + public Task, List>, float>>>> BindQuantMethodCosine(ICalibrationDatasetProvider calibrationDataset, List rangeOfs, List childrenOfRangeOfs, QuantizeOptions quantizeOptions) + { + var enodeQuantCosineDict = new Dictionary, List>, float>>>(); + return Task.FromResult(enodeQuantCosineDict); + } + + /// + public Task AdaRoundWeights(ICalibrationDatasetProvider calibrationDataset, List rangeOfs, List childrenOfRangeOfs, QuantizeOptions quantizeOptions) + { + return Task.CompletedTask; + } + + /// + public void RegisterQuantizePass(IPassManager passManager, CompileOptions options) + { + } + + /// + public void RegisterTargetDependentAfterQuantPass(IPassManager passManager, CompileOptions options) + { + if (options.QuantizeOptions.ModelQuantMode == ModelQuantMode.UsePTQ) + { + passManager.AddWithName("RemoveMarker").Configure(p => + { + p.Add(); + }); + } + } + + public void RegisterTargetDependentBeforeCodeGen(IPassManager passManager, CompileOptions options) + { + passManager.Add(); + } + + /// + public IModuleBuilder CreateModuleBuilder(string moduleKind, CompileOptions options) + { + if (moduleKind == Callable.StackVMModuleKind) + { + return new StackVMModuleBuilder(); + } + else if (moduleKind == "ncnn") + { + return new NcnnModuleBuilder(); + } + else + { + throw new NotSupportedException($"{moduleKind} module is not supported."); + } + } +} diff --git a/modules/Nncase.Modules.StackVM/packages.lock.json b/modules/Nncase.Modules.StackVM/packages.lock.json index e5ee906aff..119af5fdfd 100644 --- a/modules/Nncase.Modules.StackVM/packages.lock.json +++ b/modules/Nncase.Modules.StackVM/packages.lock.json @@ -178,6 +178,13 @@ "nncase.io": { "type": "Project" }, + "nncase.modules.ncnn": { + "type": "Project", + "dependencies": { + "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Passes": "[1.0.0, )" + } + }, "nncase.passes": { "type": "Project", "dependencies": { diff --git a/modules/ncnn/CMakeLists.txt b/modules/ncnn/CMakeLists.txt new file mode 100644 index 0000000000..b2d953d3b4 --- /dev/null +++ b/modules/ncnn/CMakeLists.txt @@ -0,0 +1,59 @@ +cmake_minimum_required (VERSION 3.8) + +if (BUILDING_RUNTIME) + add_definitions(-DBUILDING_RUNTIME) +endif() + +include_directories(include) +add_subdirectory(src/runtime) + +if (BUILDING_RUNTIME) + add_library(nncase_rt_modules_ncnn STATIC ${SRCS}) + target_include_directories(nncase_rt_modules_ncnn PRIVATE include) + target_link_libraries(nncase_rt_modules_ncnn PRIVATE runtime_ncnn) + set_target_properties(nncase_rt_modules_ncnn PROPERTIES + OUTPUT_NAME "nncase.rt_modules.ncnn") + + install(DIRECTORY include/nncase/kernels + DESTINATION include/nncase + COMPONENT nncase-headers + FILES_MATCHING + PATTERN "*.def" + PATTERN "*.h" + PATTERN "*.hpp" + PATTERN "*.td" + PATTERN "*.inc" + PATTERN "LICENSE.TXT" + ) + + install(DIRECTORY include/nncase/runtime + DESTINATION include/nncase + COMPONENT nncase-headers + FILES_MATCHING + PATTERN "*.def" + PATTERN "*.h" + PATTERN "*.hpp" + PATTERN "*.td" + PATTERN "*.inc" + PATTERN "LICENSE.TXT" + ) + + install(TARGETS nncase_rt_modules_ncnn EXPORT nncase_rt_modules_ncnnTargets + ARCHIVE DESTINATION lib + LIBRARY DESTINATION lib + RUNTIME DESTINATION bin + INCLUDES DESTINATION include + ) + + install(EXPORT nncase_rt_modules_ncnnTargets + DESTINATION lib/cmake/nncase_rt_modules_ncnn) +else() + add_library(nncase_modules_ncnn SHARED ${SRCS}) + target_include_directories(nncase_modules_ncnn PUBLIC include) + target_link_libraries(nncase_modules_ncnn PRIVATE simulator_ncnn) + set_target_properties(nncase_modules_ncnn PROPERTIES + OUTPUT_NAME "nncase.simulator.ncnn") + install(TARGETS nncase_modules_ncnn + COMPONENT nncase-runtime) + +endif() diff --git a/modules/ncnn/include/nncase/runtime/ncnn/compiler_defs.h b/modules/ncnn/include/nncase/runtime/ncnn/compiler_defs.h new file mode 100644 index 0000000000..2285e8abae --- /dev/null +++ b/modules/ncnn/include/nncase/runtime/ncnn/compiler_defs.h @@ -0,0 +1,28 @@ +/* Copyright 2019-2021 Canaan Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once +#include + +#if defined(_MSC_VER) +#ifdef NNCASE_MODULES_NCNN_DLL +#define NNCASE_MODULES_NCNN_API __declspec(dllexport) +#elif NNCASE_SHARED_LIBS +#define NNCASE_MODULES_NCNN_API __declspec(dllimport) +#else +#define NNCASE_MODULES_NCNN_API +#endif +#else +#define NNCASE_MODULES_NCNN_API __attribute__((visibility("default"))) +#endif diff --git a/modules/ncnn/include/nncase/runtime/ncnn/runtime_module.h b/modules/ncnn/include/nncase/runtime/ncnn/runtime_module.h new file mode 100644 index 0000000000..689b4ae62d --- /dev/null +++ b/modules/ncnn/include/nncase/runtime/ncnn/runtime_module.h @@ -0,0 +1,28 @@ +/* Copyright 2019-2021 Canaan Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once +#include "compiler_defs.h" +#include + +BEGIN_NS_NNCASE_RT_MODULE(ncnn) + +NNCASE_INLINE_VAR constexpr module_kind_t ncnn_module_kind = + to_module_kind("ncnn"); +NNCASE_INLINE_VAR constexpr uint32_t ncnn_module_version = 1; + +NNCASE_MODULES_NCNN_API result> +create_ncnn_runtime_module(); + +END_NS_NNCASE_RT_MODULE diff --git a/modules/ncnn/src/runtime/CMakeLists.txt b/modules/ncnn/src/runtime/CMakeLists.txt new file mode 100644 index 0000000000..f5354f48c3 --- /dev/null +++ b/modules/ncnn/src/runtime/CMakeLists.txt @@ -0,0 +1,23 @@ +cmake_minimum_required (VERSION 3.13) + +set(SRCS runtime_module.cpp + runtime_function.cpp) + +if (BUILDING_RUNTIME) + add_library(runtime_ncnn OBJECT ${SRCS}) + if (NOT MSVC) + target_compile_options(runtime_ncnn PRIVATE -Wno-unused-parameter) + endif() + + target_link_libraries(runtime_ncnn PUBLIC nncaseruntime ncnn::ncnn OpenMP::OpenMP_CXX) + set_target_properties(runtime_ncnn PROPERTIES POSITION_INDEPENDENT_CODE ON) + install(TARGETS runtime_ncnn EXPORT nncase_rt_modules_ncnnTargets) +else() + add_library(simulator_ncnn OBJECT ${SRCS}) + if (NOT MSVC) + target_compile_options(simulator_ncnn PRIVATE -Wno-unused-parameter) + endif() + target_link_libraries(simulator_ncnn PRIVATE nncaseruntime ncnn::ncnn OpenMP::OpenMP_CXX) + target_compile_definitions(simulator_ncnn PUBLIC -DNNCASE_MODULES_NCNN_DLL -DNNCASE_SIMULATOR) + set_target_properties(simulator_ncnn PROPERTIES POSITION_INDEPENDENT_CODE ON) +endif() diff --git a/modules/ncnn/src/runtime/runtime_function.cpp b/modules/ncnn/src/runtime/runtime_function.cpp new file mode 100644 index 0000000000..60eb323d48 --- /dev/null +++ b/modules/ncnn/src/runtime/runtime_function.cpp @@ -0,0 +1,225 @@ +/* Copyright 2019-2021 Canaan Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "runtime_function.h" +#include +#include +#include +#include +#include +#include + +#ifdef ENABLE_OPENMP +#include +#define OMP_MAX_THREAD omp_get_max_threads() +#else +#define OMP_MAX_THREAD 1 +#endif // ENABLE_OPENMP + +size_t rdata_offset = 0; + +using namespace nncase; +using namespace nncase::runtime; +using namespace nncase::runtime::ncnn; + +namespace { +class DataReaderFromEmpty : public ::ncnn::DataReader { + public: + virtual int scan(const char *format, void *p) const { return 0; } + virtual size_t read(void *buf, size_t size) const { + memset(buf, 0, size); + return size; + } +}; +} // namespace + +ncnn_runtime_function::ncnn_runtime_function(runtime_module &rt_module) + : runtime_function(rt_module) {} + +ncnn_runtime_module &ncnn_runtime_function::module() const noexcept { + return static_cast(runtime_function::module()); +} + +result ncnn_runtime_function::initialize_core( + runtime_function_init_context &context) noexcept { + try_(context.read_section(".inputs", + [this](auto sr, size_t) -> result { + input_names_ = sr.read_string_array(); + return ok(); + })); + try_(context.read_section(".outputs", + [this](auto sr, size_t) -> result { + output_names_ = sr.read_string_array(); + return ok(); + })); + + NNCASE_UNUSED stream_reader *sr = nullptr; + section_header h; + try_set(sr, context.seek_section(".rdata", h)); + auto param_mem = reinterpret_cast( + module().text().data() + context.header().entrypoint); + if (context.header().entrypoint == 0) + rdata_offset = 0; + auto bin_mem = reinterpret_cast(module().rdata().data() + + rdata_offset); + + // auto bin_mem = reinterpret_cast(rdata_.data()); + ::ncnn::DataReaderFromMemory paramdr(param_mem); + ::ncnn::DataReaderFromMemory bindr(bin_mem); + + CHECK_WITH_ERR(!net_.load_param(paramdr), std::errc::invalid_argument); + CHECK_WITH_ERR(!net_.load_model(bindr), std::errc::invalid_argument); + + net_.opt.num_threads = OMP_MAX_THREAD; + rdata_offset += h.memory_size; + + return ok(); +} + +result ncnn_runtime_function::invoke_core( + gsl::span parameters, + [[maybe_unused]] value_t return_value) noexcept { + + auto ex = net_.create_extractor(); + + // 1. Set input + for (size_t i = 0; i < parameters.size(); i++) { + try_var(t, parameters[i].as()); + ::ncnn::Mat mat; + mat.elempack = 1; + mat.elemsize = runtime::get_bytes(t->dtype()); + auto shape = t->shape(); + switch (shape.size()) { + case 1: + mat.dims = 1; + mat.w = shape[0]; + mat.h = 1; + mat.d = 1; + mat.c = 1; + mat.cstep = mat.w; + break; + case 2: + mat.dims = 2; + mat.w = shape[1]; + mat.h = shape[0]; + mat.d = 1; + mat.c = 1; + mat.cstep = (size_t)mat.w * mat.h; + break; + case 3: + mat.dims = 3; + mat.w = shape[2]; + mat.h = shape[1]; + mat.d = 1; + mat.c = shape[0]; + mat.cstep = (size_t)mat.w * mat.h; + break; + case 4: + mat.dims = 4; + mat.w = shape[3]; + mat.h = shape[2]; + mat.d = shape[1]; + mat.c = shape[0]; + mat.cstep = (size_t)mat.w * mat.h * mat.d; + break; + default: + return err(std::errc::invalid_argument); + } + + try_var(hb, t->buffer().as_host()); + try_var(map, hb.map(map_read)); + mat.data = map.buffer().data(); + + // Must clone to full fill ncnn implicit contracts: + // 1. Input is not user mananged data + // 2. Data must be aligned + auto cloned_input = mat.clone(); + CHECK_WITH_ERR(!ex.input(input_names_[i].c_str(), cloned_input), + std::errc::invalid_argument); + } + + // 2. Extract outputs + std::vector outputs; + for (size_t i = 0; i < output_names_.size(); i++) { + ::ncnn::Mat mat; + CHECK_WITH_ERR(!ex.extract(output_names_[i].c_str(), mat), + std::errc::invalid_argument); + auto mat_size = (size_t)mat.c * mat.w * mat.h * mat.d * mat.elemsize; + gsl::span data{reinterpret_cast(mat.data), + mat_size}; + + dims_t shape; + switch (mat.dims) { + case 1: + shape = {(size_t)mat.w}; + break; + case 2: + shape = {(size_t)mat.h, (size_t)mat.w}; + break; + case 3: + shape = {(size_t)mat.c, (size_t)mat.h, (size_t)mat.w}; + break; + case 4: + shape = {(size_t)mat.c, (size_t)mat.d, (size_t)mat.h, + (size_t)mat.w}; + break; + default: + return err(std::errc::invalid_argument); + } + + datatype_t dt; + switch (mat.elemsize) { + case 1: + dt = datatype_t::uint8; + break; + case 2: + dt = datatype_t::float16; + break; + case 4: + dt = datatype_t::float32; + break; + default: + return err(std::errc::invalid_argument); + } + + buffer_allocate_options options{}; + options.flags = HOST_BUFFER_ALLOCATE_SHARED; + try_var(buf, buffer_allocator::host().allocate(mat_size, options)); + try_var(hb, buf.as()); + + { + try_var(map, hb->map(map_write)); + auto csize = (size_t)mat.w * mat.h * mat.d * mat.elemsize; + for (size_t i = 0; i < (size_t)mat.c; i++) { + void *dest = + (unsigned char *)map.buffer().data() + (size_t)i * csize; + const void *src = (unsigned char *)mat.channel(i); + memcpy(dest, src, csize); + } + } + + tensor t(std::in_place, dt, shape, get_default_strides(shape), buf); + outputs.emplace_back(t); + } + auto ret_val = output_names_.size() == 1 + ? outputs[0] + : tuple(std::in_place, std::move(outputs)); + + if (!return_value.empty()) { + try_(ret_val->copy_to(return_value)); + return ok(return_value); + } + + return ok(ret_val); +} diff --git a/modules/ncnn/src/runtime/runtime_function.h b/modules/ncnn/src/runtime/runtime_function.h new file mode 100644 index 0000000000..f5d4e97f47 --- /dev/null +++ b/modules/ncnn/src/runtime/runtime_function.h @@ -0,0 +1,45 @@ +/* Copyright 2019-2021 Canaan Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once +#include "runtime_module.h" +#include +#include +#include +#include + +BEGIN_NS_NNCASE_RT_MODULE(ncnn) + +class ncnn_runtime_function final : public runtime_function { + public: + ncnn_runtime_function(runtime_module &rt_module); + + ncnn_runtime_module &module() const noexcept; + + protected: + result + initialize_core(runtime_function_init_context &context) noexcept override; + result invoke_core(gsl::span parameters, + value_t return_value) noexcept override; + + private: + result run(gsl::span text) noexcept; + + private: + ::ncnn::Net net_; + std::vector input_names_; + std::vector output_names_; +}; + +END_NS_NNCASE_RT_MODULE diff --git a/modules/ncnn/src/runtime/runtime_module.cpp b/modules/ncnn/src/runtime/runtime_module.cpp new file mode 100644 index 0000000000..88d71305d5 --- /dev/null +++ b/modules/ncnn/src/runtime/runtime_module.cpp @@ -0,0 +1,65 @@ +/* Copyright 2019-2021 Canaan Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "runtime_module.h" +#include "runtime_function.h" +#include +#include +#include +#include + +using namespace nncase; +using namespace nncase::runtime; +using namespace nncase::runtime::ncnn; + +result ncnn_runtime_module::initialize_before_functions( + runtime_module_init_context &context) noexcept { + try_set(text_, context.get_or_read_section(".text", text_storage_, false)); + try_set(rdata_, + context.get_or_read_section(".rdata", rdata_storage_, false)); + return ok(); +} + +kernels::kernel_context &ncnn_runtime_module::kernel_context() noexcept { + auto &context = kernels::default_kernel_context(); +#ifdef NNCASE_DUMP_MANAGER + context.dump_manager = interp().dump_manager(); +#endif + return context; +} + +result> +ncnn_runtime_module::create_function() noexcept { + std::unique_ptr mod(new (std::nothrow) + ncnn_runtime_function(*this)); + if (mod) + return ok(std::move(mod)); + return err(std::errc::not_enough_memory); +} + +result> +nncase::runtime::ncnn::create_ncnn_runtime_module() { + std::unique_ptr mod(new (std::nothrow) + ncnn_runtime_module()); + if (mod) + return ok(std::move(mod)); + return err(std::errc::not_enough_memory); +} + +extern "C" { +NNCASE_MODULES_NCNN_API void +RUNTIME_MODULE_ACTIVATOR_NAME(result> &result) { + result = create_ncnn_runtime_module(); +} +} diff --git a/modules/ncnn/src/runtime/runtime_module.h b/modules/ncnn/src/runtime/runtime_module.h new file mode 100644 index 0000000000..e0f286d417 --- /dev/null +++ b/modules/ncnn/src/runtime/runtime_module.h @@ -0,0 +1,42 @@ +/* Copyright 2019-2021 Canaan Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once +#include +#include +#include + +BEGIN_NS_NNCASE_RT_MODULE(ncnn) + +class ncnn_runtime_module : public runtime_module { + public: + kernels::kernel_context &kernel_context() noexcept; + + gsl::span text() const noexcept { return text_; } + gsl::span rdata() const noexcept { return rdata_; } + + protected: + result initialize_before_functions( + runtime_module_init_context &context) noexcept override; + result> + create_function() noexcept override; + + private: + gsl::span text_; + gsl::span rdata_; + host_buffer_t text_storage_; + host_buffer_t rdata_storage_; +}; + +END_NS_NNCASE_RT_MODULE diff --git a/modules/ncnn/src/runtime/tensor_utils.h b/modules/ncnn/src/runtime/tensor_utils.h new file mode 100644 index 0000000000..400787097a --- /dev/null +++ b/modules/ncnn/src/runtime/tensor_utils.h @@ -0,0 +1,55 @@ +/* Copyright 2019-2021 Canaan Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once +#include +#include +#include +#include + +BEGIN_NS_NNCASE_RT_MODULE(ncnn) + +inline result<::ncnn::Mat> to_ncnn_mat(value_t value) { + try_var(t, value.as()); + ::ncnn::Mat mat; + auto shape = t->shape(); + auto elemsize = runtime::get_bytes(t->dtype()); + switch (shape.size()) { + case 1: + mat.create((int)shape[0], elemsize); + break; + case 2: + mat.create((int)shape[1], (int)shape[0], elemsize); + break; + case 3: + mat.create((int)shape[2], (int)shape[1], (int)shape[0], elemsize); + break; + case 4: + mat.create((int)shape[3], (int)shape[2], (int)shape[1], (int)shape[0], + elemsize); + break; + default: + return err(std::errc::invalid_argument); + } + + try_var(hb, t->buffer().as_host()); + try_var(map, hb.map(map_read)); + return kernels::stackvm::optimized::slice( + datatype, src_map.buffer().data() + src_start * datatype->size_bytes(), + dest_map.buffer().data() + dest_start * datatype->size_bytes(), shape, + src_strides, dest_strides, begins, ends, strides, + kernels::default_kernel_context()); +} + +END_NS_NNCASE_RT_MODULE diff --git a/nncase.sln b/nncase.sln index e1d5a9b5f1..ad20eb7ca9 100644 --- a/nncase.sln +++ b/nncase.sln @@ -77,6 +77,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Nncase.Tests.TestFixture", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Nncase.Passes", "src\Nncase.Passes\Nncase.Passes.csproj", "{E6462E82-B48F-4AFA-AE34-725EF0A9CB42}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nncase.Modules.Ncnn", "modules\Nncase.Modules.Ncnn\Nncase.Modules.Ncnn.csproj", "{FA535CB0-C8B9-4CA5-B3E3-8B7912E5F061}" +EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Nncase.Studio", "src\Nncase.Studio\Nncase.Studio.csproj", "{0E5BF964-B878-4BD6-8C84-FFE85E23994B}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Nncase.Modules.CPU", "modules\Nncase.Modules.CPU\Nncase.Modules.CPU.csproj", "{6AEE2334-CCF4-464E-8C90-C6BC0D930327}" diff --git a/requirements.test.txt b/requirements.test.txt index 00d2282ef7..c65210af8b 100644 --- a/requirements.test.txt +++ b/requirements.test.txt @@ -17,3 +17,4 @@ pytest-xdist pyyaml pandas tabulate +ncnn==1.0.20240102 \ No newline at end of file diff --git a/src/Native/include/nncase/runtime/stream_reader.h b/src/Native/include/nncase/runtime/stream_reader.h index fbe5d6ae01..b3d11a70e8 100644 --- a/src/Native/include/nncase/runtime/stream_reader.h +++ b/src/Native/include/nncase/runtime/stream_reader.h @@ -58,6 +58,32 @@ class stream_reader { stream_.read(reinterpret_cast(span.data()), span.size_bytes()); } + std::string read_string() { + std::string str; + while (true) { + auto c = read(); + if (c) { + str.push_back(c); + } else { + break; + } + } + + return str; + } + + std::vector read_string_array() { + std::vector array; + while (true) { + if (peek() == '\0') { + skip(1); + break; + } + array.emplace_back(read_string()); + } + return array; + } + void skip(size_t count) { stream_.seekg(count, std::ios::cur); } private: diff --git a/src/Native/src/runtime/CMakeLists.txt b/src/Native/src/runtime/CMakeLists.txt index 4b51ac899e..a3bc2691b7 100644 --- a/src/Native/src/runtime/CMakeLists.txt +++ b/src/Native/src/runtime/CMakeLists.txt @@ -53,7 +53,7 @@ else() target_include_directories(simulator PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) target_link_libraries(simulator PUBLIC fmt::fmt) target_link_libraries(simulator PRIVATE kernels) - target_compile_definitions(simulator PUBLIC -DNNCASE_DLL -DNNCASE_SIMULATOR) + target_compile_definitions(simulator PUBLIC -DNNCASE_DLL -DNNCASE_SIMULATOR -DNNCASE_SHARED_LIBS) if (DEFAULT_BUILTIN_RUNTIMES) target_compile_definitions(simulator PRIVATE -DNNCASE_DEFAULT_BUILTIN_RUNTIMES) endif() @@ -69,6 +69,7 @@ else() endif() set_target_properties(nncaseruntime PROPERTIES OUTPUT_NAME "Nncase.Runtime.Native") + target_compile_definitions(nncaseruntime PUBLIC -DNNCASE_SIMULATOR -DNNCASE_SHARED_LIBS) install(TARGETS nncaseruntime EXPORT nncaseTargets COMPONENT nncase-runtime diff --git a/src/Native/src/runtime/dump_manager_impl.h b/src/Native/src/runtime/dump_manager_impl.h index 35364638a8..e5199b25d1 100644 --- a/src/Native/src/runtime/dump_manager_impl.h +++ b/src/Native/src/runtime/dump_manager_impl.h @@ -87,6 +87,7 @@ void dump_data(std::ostream &stream, const T *data, stream << std::to_string(data[i]) << "\n"; } } + stream << "------------------\n" << std::endl; } END_NS_NNCASE_RUNTIME \ No newline at end of file diff --git a/src/Nncase.Cli/packages.lock.json b/src/Nncase.Cli/packages.lock.json index 8b147bbc48..968cf8bfe4 100644 --- a/src/Nncase.Cli/packages.lock.json +++ b/src/Nncase.Cli/packages.lock.json @@ -765,10 +765,18 @@ "Razor.Templating.Core": "[1.9.0, )" } }, + "nncase.modules.ncnn": { + "type": "Project", + "dependencies": { + "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Passes": "[1.0.0, )" + } + }, "nncase.modules.stackvm": { "type": "Project", "dependencies": { "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Modules.Ncnn": "[1.0.0, )", "Nncase.Passes": "[1.0.0, )" } }, diff --git a/src/Nncase.CodeGen/CodeGen/LinkedSection.cs b/src/Nncase.CodeGen/CodeGen/LinkedSection.cs index 2bbddefd90..aeaa2c9739 100644 --- a/src/Nncase.CodeGen/CodeGen/LinkedSection.cs +++ b/src/Nncase.CodeGen/CodeGen/LinkedSection.cs @@ -43,6 +43,37 @@ public LinkedSection(Stream? content, string name, uint flags, uint alignment, u public ulong SizeInMemory { get; } + public static LinkedSection FromStrings(IReadOnlyCollection strings, string name) + { + var ms = new MemoryStream(); + using (var bw = new BinaryWriter(ms, Encoding.UTF8, true)) + { + foreach (string s in strings) + { + bw.Write(Encoding.UTF8.GetBytes(s)); + bw.Write((byte)0); + } + + bw.Write((byte)0); + } + + return new LinkedSection(ms, name, 0, 1, (ulong)ms.Length); + } + + public static LinkedSection FromData(IReadOnlyCollection datas, string name) + { + var ms = new MemoryStream(); + using (var bw = new BinaryWriter(ms, Encoding.UTF8, true)) + { + foreach (float s in datas) + { + bw.Write(s); + } + } + + return new LinkedSection(ms, name, 0, 1, (ulong)ms.Length); + } + public void Serialize(Stream output) { if (_content != null) diff --git a/src/Nncase.Compiler/Hosting/CompilerHostBuilderExtensions.cs b/src/Nncase.Compiler/Hosting/CompilerHostBuilderExtensions.cs index 628dc907de..e8b4712237 100644 --- a/src/Nncase.Compiler/Hosting/CompilerHostBuilderExtensions.cs +++ b/src/Nncase.Compiler/Hosting/CompilerHostBuilderExtensions.cs @@ -53,6 +53,8 @@ private static void ConfigureBuiltinModules(Container builder) .AddEGraph() .AddCodeGen() .AddPasses() + .AddStackVM() + .AddNcnn() .AddSchedule() .AddCPU() .AddStackVM(); diff --git a/src/Nncase.Compiler/Hosting/PluginLoader.cs b/src/Nncase.Compiler/Hosting/PluginLoader.cs index 014ab29fb1..af90a533ed 100644 --- a/src/Nncase.Compiler/Hosting/PluginLoader.cs +++ b/src/Nncase.Compiler/Hosting/PluginLoader.cs @@ -26,6 +26,7 @@ public sealed class PluginLoader private static readonly string[] _builtinModules = new[] { "Nncase.Modules.StackVM.dll", + "Nncase.Modules.Ncnn.dll", "Nncase.Modules.CPU.dll", "Nncase.Modules.K210.dll", }; diff --git a/src/Nncase.Compiler/packages.lock.json b/src/Nncase.Compiler/packages.lock.json index f63ed9a24a..bf1efa248f 100644 --- a/src/Nncase.Compiler/packages.lock.json +++ b/src/Nncase.Compiler/packages.lock.json @@ -746,10 +746,18 @@ "Razor.Templating.Core": "[1.9.0, )" } }, + "nncase.modules.ncnn": { + "type": "Project", + "dependencies": { + "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Passes": "[1.0.0, )" + } + }, "nncase.modules.stackvm": { "type": "Project", "dependencies": { "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Modules.Ncnn": "[1.0.0, )", "Nncase.Passes": "[1.0.0, )" } }, diff --git a/src/Nncase.Core/IR/ExprCollector.cs b/src/Nncase.Core/IR/ExprCollector.cs index 5eff7385a6..e76084fa25 100644 --- a/src/Nncase.Core/IR/ExprCollector.cs +++ b/src/Nncase.Core/IR/ExprCollector.cs @@ -13,20 +13,27 @@ namespace Nncase.IR; public sealed class ExprCollector : ExprWalker> { - private ExprCollector() + private readonly Func? _condition; + + private ExprCollector(Func? condition = null) { + _condition = condition; } - public static IReadOnlyList Collect(Expr expr) + public static IReadOnlyList Collect(Expr expr, Func? condition = null) { var exprs = new List(); - new ExprCollector().Visit(expr, exprs); + new ExprCollector(condition).Visit(expr, exprs); return exprs; } protected override Unit DefaultVisitLeaf(Expr expr, List context) { - context.Add(expr); + if (_condition?.Invoke(expr) ?? true) + { + context.Add(expr); + } + return default; } } diff --git a/src/Nncase.Core/IR/Function.cs b/src/Nncase.Core/IR/Function.cs index 742d696b81..cb698a31c9 100644 --- a/src/Nncase.Core/IR/Function.cs +++ b/src/Nncase.Core/IR/Function.cs @@ -27,8 +27,8 @@ public sealed class Function : BaseFunction /// Initializes a new instance of the class. /// build function. /// - public Function(string name, Expr body, ReadOnlySpan parameters) - : this(name, body, parameters, new Dictionary()) + public Function(string name, Expr body, ReadOnlySpan parameters, string? moduleKind = null) + : this(name, body, parameters, new Dictionary(), moduleKind) { } @@ -49,8 +49,8 @@ public Function(string name, Expr body, ReadOnlySpan parameters, Dictionary /// Initializes a new instance of the class. /// build function. /// - public Function(Expr body, ReadOnlySpan parameters) - : this($"func_{_globalFuncIndex++}", body, parameters, new Dictionary()) + public Function(Expr body, ReadOnlySpan parameters, string? moduleKind = null) + : this($"func_{_globalFuncIndex++}", body, parameters, new Dictionary(), moduleKind) { } diff --git a/src/Nncase.Core/IR/Tensors/Functional.cs b/src/Nncase.Core/IR/Tensors/Functional.cs index d20f69cbf5..d8df282669 100644 --- a/src/Nncase.Core/IR/Tensors/Functional.cs +++ b/src/Nncase.Core/IR/Tensors/Functional.cs @@ -35,26 +35,31 @@ public static Expr NHWCToNCHW(Expr input) } else { - throw new InvalidOperationException(); + return input; } return Transpose(input, perm); } - public static Expr NCHWToNHWC(Expr input) + public static Expr NCHWToNHWC(Expr input, int rank = 0) { + if (rank == 0) + { + rank = input.CheckedShape.Rank; + } + int[] perm; - if (input.CheckedShape.Rank == 4) + if (rank == 4) { perm = new[] { 0, 2, 3, 1 }; } - else if (input.CheckedShape.Rank == 3) + else if (rank == 3) { perm = new[] { 0, 2, 1 }; } else { - throw new InvalidOperationException(); + return input; } return Transpose(input, perm); @@ -187,4 +192,35 @@ public static Call TopK(Expr x, Expr k, Expr axis, Expr largest, Expr sorted) => public static Call IndexOf(Expr input, Expr value) => new Call(new IndexOf(), input, value); public static Call Trilu(Expr input, Expr k, Expr upper) => new Call(new Trilu(), input, k, upper); + + public static Expr PaddingNHWCToNCHW(Expr paddings, int rank) + { + switch (rank) + { + case 1: + case 2: + return paddings; + case 3: + return Concat( + new IR.Tuple( + new[] { + Slice(paddings, new int[] { 0 }, new int[] { 1 }, new int[] { 0 }, new int[] { 1 }), + Slice(paddings, new int[] { 2 }, new int[] { 3 }, new int[] { 0 }, new int[] { 1 }), + Slice(paddings, new int[] { 1 }, new int[] { 2 }, new int[] { 0 }, new int[] { 1 }), + }), + 0); + case 4: + return Concat( + new IR.Tuple( + new[] { + Slice(paddings, new int[] { 0 }, new int[] { 1 }, new int[] { 0 }, new int[] { 1 }), + Slice(paddings, new int[] { 3 }, new int[] { 4 }, new int[] { 0 }, new int[] { 1 }), + Slice(paddings, new int[] { 1 }, new int[] { 2 }, new int[] { 0 }, new int[] { 1 }), + Slice(paddings, new int[] { 2 }, new int[] { 3 }, new int[] { 0 }, new int[] { 1 }), + }), + 0); + default: + return paddings; + } + } } diff --git a/src/Nncase.Core/IR/TypePattern.cs b/src/Nncase.Core/IR/TypePattern.cs index a779261747..c863cf0d8e 100644 --- a/src/Nncase.Core/IR/TypePattern.cs +++ b/src/Nncase.Core/IR/TypePattern.cs @@ -134,8 +134,8 @@ public static TypePattern HasShape(Shape target_shape) => HasShape( /// /// is custom rank. /// - public static TypePattern HasRank(Func cond, string reason) => HasShape( - inshape => cond(inshape.Rank), reason); + public static TypePattern HasRank(Func cond, [CallerArgumentExpression(nameof(cond))] string? reason = null) => HasShape( + inshape => cond(inshape.Rank), reason!); /// /// is target rank. diff --git a/src/Nncase.Core/Runtime/Ncnn/NcnnLayer.cs b/src/Nncase.Core/Runtime/Ncnn/NcnnLayer.cs new file mode 100644 index 0000000000..0fe6734820 --- /dev/null +++ b/src/Nncase.Core/Runtime/Ncnn/NcnnLayer.cs @@ -0,0 +1,62 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; + +namespace Nncase.Runtime.Ncnn; + +public class NcnnTensor +{ + public string Name { get; set; } = string.Empty; + + public Shape ShapeHint { get; set; } = Shape.Unranked; + + public override string ToString() => $"{Name}: {ShapeHint}"; +} + +public class NcnnLayer +{ + public NcnnLayer(string type, string name, int bottomCount, int topCount) + { + Type = type; + Name = name; + Bottoms = new NcnnTensor[bottomCount]; + Tops = new NcnnTensor[topCount]; + } + + public string Type { get; } + + public string Name { get; } + + public NcnnTensor[] Bottoms { get; } + + public NcnnTensor[] Tops { get; } + + public ParamDict ParamDict { get; set; } = new(); + + public override string ToString() => $"[{Type}] {Name}"; + + public void Serialize(TextWriter writer) + { + writer.Write($"{Type,-16} {Name,-24} {Bottoms.Length} {Tops.Length} "); + + foreach (var bottom in Bottoms) + { + writer.Write($"{bottom.Name} "); + } + + foreach (var top in Tops) + { + writer.Write($"{top.Name} "); + } + + ParamDict.Serialize(writer); + writer.WriteLine(); + } +} diff --git a/src/Nncase.Importer/Ncnn/NcnnModel.cs b/src/Nncase.Core/Runtime/Ncnn/NcnnModel.cs similarity index 62% rename from src/Nncase.Importer/Ncnn/NcnnModel.cs rename to src/Nncase.Core/Runtime/Ncnn/NcnnModel.cs index ec67287e97..7e1362f2ef 100644 --- a/src/Nncase.Importer/Ncnn/NcnnModel.cs +++ b/src/Nncase.Core/Runtime/Ncnn/NcnnModel.cs @@ -7,55 +7,36 @@ using System.Linq; using System.Text; using System.Threading.Tasks; -using Nncase.IR; -namespace Nncase.Importer.Ncnn; +namespace Nncase.Runtime.Ncnn; -internal class NcnnTensor +public class NcnnModel { - public string Name { get; set; } = string.Empty; - - public Shape ShapeHint { get; set; } = Shape.Unranked; - - public override string ToString() => $"{Name}: {ShapeHint}"; -} + public static readonly int ExpectedMagic = 7767517; -internal class NcnnLayer -{ - public NcnnLayer(string type, string name, int bottomCount, int topCount) + public NcnnModel() { - Type = type; - Name = name; - Bottoms = new NcnnTensor[bottomCount]; - Tops = new NcnnTensor[topCount]; + Magic = ExpectedMagic; + ModelInputs = new List(); + Layers = new List(); + MemoryDatas = new List(); } - public string Type { get; } - - public string Name { get; } - - public NcnnTensor[] Bottoms { get; } - - public NcnnTensor[] Tops { get; } - - public ParamDict ParamDict { get; } = new(); - - public override string ToString() => $"[{Type}] {Name}"; -} - -internal class NcnnModel -{ - public static readonly int ExpectedMagic = 7767517; - - public NcnnModel(int magic, IReadOnlyList layers) + public NcnnModel(int magic, IList layers, IList? modelInputs = null, IList? memoryDatas = null) { Magic = magic; + ModelInputs = modelInputs ?? new List(); Layers = layers; + MemoryDatas = memoryDatas ?? new List(); } public int Magic { get; } - public IReadOnlyList Layers { get; } + public IList ModelInputs { get; } + + public IList MemoryDatas { get; } + + public IList Layers { get; } public static NcnnModel ParseFromStream(Stream stream) { @@ -105,4 +86,31 @@ public static NcnnModel ParseFromStream(Stream stream) return new NcnnModel(magic, layers); } + + public void Serialize(TextWriter writer) + { + // 1. Magic + writer.WriteLine(Magic); + + // 2. layer_count & blob_count + writer.WriteLine($"{Layers.Count + MemoryDatas.Count + ModelInputs.Count} {Layers.Select(x => x.Tops.Length).Sum() + MemoryDatas.Select(x => x.Tops.Length).Sum() + ModelInputs.Select(x => x.Tops.Length).Sum()}"); + + // 3. inputs + foreach (var modelInput in ModelInputs) + { + modelInput.Serialize(writer); + } + + // 4. memorydatas + foreach (var memoryData in MemoryDatas) + { + memoryData.Serialize(writer); + } + + // 5. layers + foreach (var layer in Layers) + { + layer.Serialize(writer); + } + } } diff --git a/src/Nncase.Importer/Ncnn/NcnnModelBin.cs b/src/Nncase.Core/Runtime/Ncnn/NcnnModelBin.cs similarity index 96% rename from src/Nncase.Importer/Ncnn/NcnnModelBin.cs rename to src/Nncase.Core/Runtime/Ncnn/NcnnModelBin.cs index 3957e1fbba..e9cbc98ba8 100644 --- a/src/Nncase.Importer/Ncnn/NcnnModelBin.cs +++ b/src/Nncase.Core/Runtime/Ncnn/NcnnModelBin.cs @@ -11,9 +11,9 @@ using System.Threading.Tasks; using CommunityToolkit.HighPerformance; -namespace Nncase.Importer.Ncnn; +namespace Nncase.Runtime.Ncnn; -internal class NcnnModelBin +public class NcnnModelBin { private readonly Stream _stream; diff --git a/src/Nncase.Importer/Ncnn/ParamDict.cs b/src/Nncase.Core/Runtime/Ncnn/ParamDict.cs similarity index 58% rename from src/Nncase.Importer/Ncnn/ParamDict.cs rename to src/Nncase.Core/Runtime/Ncnn/ParamDict.cs index bc5c77e6d7..57efc2ad46 100644 --- a/src/Nncase.Importer/Ncnn/ParamDict.cs +++ b/src/Nncase.Core/Runtime/Ncnn/ParamDict.cs @@ -7,12 +7,11 @@ using System.Linq; using System.Text; using System.Threading.Tasks; -using Google.Protobuf.WellKnownTypes; using Nncase.IR; -namespace Nncase.Importer.Ncnn; +namespace Nncase.Runtime.Ncnn; -internal enum ParamKind +public enum ParamKind { Null, IntOrFloat, @@ -23,7 +22,7 @@ internal enum ParamKind ArrayOfFloat, } -internal struct ParamValue +public struct ParamValue { public ParamKind Kind; @@ -34,12 +33,23 @@ internal struct ParamValue public Tensor? TensorValue; } -internal class ParamDict +public class ParamDict { public static readonly int NcnnMaxParamCount = 32; private readonly Dictionary _values = new(); + public ParamValue this[int index] + { + get => _values[index]; + set => _values[index] = value; + } + + public void Add(int index, ParamValue paramValue) + { + _values[index] = paramValue; + } + public void LoadFrom(ReadOnlySpan fields) { foreach (var field in fields) @@ -116,4 +126,71 @@ public void LoadFrom(ReadOnlySpan fields) public Tensor Get(int id, Tensor defaultValue) where T : unmanaged, IEquatable => _values.TryGetValue(id, out var value) ? value.TensorValue!.Cast() : defaultValue; + + public void Serialize(TextWriter writer) + { + int index = 0; + foreach (var field in _values) + { + var id = field.Key; + var paramValue = field.Value; + var isArray = paramValue.Kind is ParamKind.ArrayOfFloat or ParamKind.ArrayOfIntOrFloat or ParamKind.ArrayOfInt; + var isFloat = paramValue.Kind is ParamKind.ArrayOfFloat or ParamKind.ArrayOfIntOrFloat or ParamKind.Float; + var mixIntAndFloat = paramValue.Kind is ParamKind.ArrayOfIntOrFloat; + + if (isArray) + { + id = id - 23300; + } + + writer.Write($"{id}="); + + if (isArray) + { + if (isFloat) + { + var sourceData = paramValue.TensorValue!.ToArray(); + + if (mixIntAndFloat) + { + int size = (int)sourceData[0]; + writer.Write($"{size},"); + } + + // Console.WriteLine($"{String.Join(",", sourceData[1..])}"); + writer.Write(string.Join(',', sourceData[1..].Select(x => + { + return x switch + { + _ when float.IsPositiveInfinity(x) => "3.402823e+38", + _ when float.IsNegativeInfinity(x) => "-3.402823e+38", + _ => x.ToString("e"), + }; + }))); + } + else + { + writer.Write(string.Join(',', paramValue.TensorValue!.Cast())); + } + } + else + { + if (isFloat) + { + writer.Write(paramValue.FloatValue.ToString("e")); + } + else + { + writer.Write(paramValue.IntValue); + } + } + + if (++index != _values.Count) + { + writer.Write(' '); + } + } + + writer.Write(' '); + } } diff --git a/src/Nncase.Core/Utilities/ShapeUtility.cs b/src/Nncase.Core/Utilities/ShapeUtility.cs new file mode 100644 index 0000000000..bc39790ec9 --- /dev/null +++ b/src/Nncase.Core/Utilities/ShapeUtility.cs @@ -0,0 +1,34 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.TIR; +using static NetFabric.Hyperlinq.ArrayExtensions; + +namespace Nncase.Utilities; + +public static class ShapeUtility +{ + public static List FitNcnnShape(List shape, int axis) + { + int positive_axis = axis < 0 ? shape.Count + axis : axis; + var newShape = new List { 1, shape[positive_axis], 1 }; + for (int i = 0; i < positive_axis; i++) + { + newShape[0] *= shape[i]; + } + + for (int i = positive_axis + 1; i < shape.Count; i++) + { + newShape[2] *= shape[i]; + } + + return newShape; + } +} diff --git a/src/Nncase.Importer/Ncnn/Concat.cs b/src/Nncase.Importer/Ncnn/Concat.cs index 2363144d4c..c565dbfc31 100644 --- a/src/Nncase.Importer/Ncnn/Concat.cs +++ b/src/Nncase.Importer/Ncnn/Concat.cs @@ -11,6 +11,7 @@ using Nncase.IR.Buffers; using Nncase.IR.F; using Nncase.IR.NN; +using Nncase.Runtime.Ncnn; namespace Nncase.Importer.Ncnn; diff --git a/src/Nncase.Importer/Ncnn/Convolution.cs b/src/Nncase.Importer/Ncnn/Convolution.cs index 5885feef46..be28b4fd63 100644 --- a/src/Nncase.Importer/Ncnn/Convolution.cs +++ b/src/Nncase.Importer/Ncnn/Convolution.cs @@ -12,6 +12,7 @@ using Nncase.IR.Buffers; using Nncase.IR.F; using Nncase.IR.NN; +using Nncase.Runtime.Ncnn; namespace Nncase.Importer.Ncnn; diff --git a/src/Nncase.Importer/Ncnn/InnerProduct.cs b/src/Nncase.Importer/Ncnn/InnerProduct.cs index 80aeb98bb2..e051e82b85 100644 --- a/src/Nncase.Importer/Ncnn/InnerProduct.cs +++ b/src/Nncase.Importer/Ncnn/InnerProduct.cs @@ -12,6 +12,7 @@ using Nncase.IR.Buffers; using Nncase.IR.F; using Nncase.IR.NN; +using Nncase.Runtime.Ncnn; namespace Nncase.Importer.Ncnn; diff --git a/src/Nncase.Importer/Ncnn/NcnnImporter.cs b/src/Nncase.Importer/Ncnn/NcnnImporter.cs index a67542aeab..7e66018743 100644 --- a/src/Nncase.Importer/Ncnn/NcnnImporter.cs +++ b/src/Nncase.Importer/Ncnn/NcnnImporter.cs @@ -17,6 +17,7 @@ using Nncase.IR.Math; using Nncase.IR.NN; using Nncase.IR.Tensors; +using Nncase.Runtime.Ncnn; using Math = System.Math; using Tuple = Nncase.IR.Tuple; diff --git a/src/Nncase.Importer/Ncnn/Pooling.cs b/src/Nncase.Importer/Ncnn/Pooling.cs index edc486d71d..dc48d9d04b 100644 --- a/src/Nncase.Importer/Ncnn/Pooling.cs +++ b/src/Nncase.Importer/Ncnn/Pooling.cs @@ -11,6 +11,7 @@ using Nncase.IR.Buffers; using Nncase.IR.F; using Nncase.IR.NN; +using Nncase.Runtime.Ncnn; namespace Nncase.Importer.Ncnn; diff --git a/src/Nncase.Importer/Ncnn/ShuffleChannel.cs b/src/Nncase.Importer/Ncnn/ShuffleChannel.cs index dd4f760951..0d88ff7d29 100644 --- a/src/Nncase.Importer/Ncnn/ShuffleChannel.cs +++ b/src/Nncase.Importer/Ncnn/ShuffleChannel.cs @@ -11,6 +11,7 @@ using Nncase.IR.Buffers; using Nncase.IR.F; using Nncase.IR.NN; +using Nncase.Runtime.Ncnn; namespace Nncase.Importer.Ncnn; diff --git a/src/Nncase.Importer/Ncnn/Softmax.cs b/src/Nncase.Importer/Ncnn/Softmax.cs index 0e1744992b..8c6c3620f7 100644 --- a/src/Nncase.Importer/Ncnn/Softmax.cs +++ b/src/Nncase.Importer/Ncnn/Softmax.cs @@ -11,6 +11,7 @@ using Nncase.IR.Buffers; using Nncase.IR.F; using Nncase.IR.NN; +using Nncase.Runtime.Ncnn; namespace Nncase.Importer.Ncnn; diff --git a/src/Nncase.Importer/Ncnn/Split.cs b/src/Nncase.Importer/Ncnn/Split.cs index a6327ee653..6ab464c106 100644 --- a/src/Nncase.Importer/Ncnn/Split.cs +++ b/src/Nncase.Importer/Ncnn/Split.cs @@ -11,6 +11,7 @@ using Nncase.IR.Buffers; using Nncase.IR.F; using Nncase.IR.NN; +using Nncase.Runtime.Ncnn; namespace Nncase.Importer.Ncnn; diff --git a/src/Nncase.Importer/TFLite/Pad.cs b/src/Nncase.Importer/TFLite/Pad.cs index 7cf10f8998..6ed91debe2 100644 --- a/src/Nncase.Importer/TFLite/Pad.cs +++ b/src/Nncase.Importer/TFLite/Pad.cs @@ -4,6 +4,7 @@ using System; using System.Diagnostics; using System.Linq; +using NetFabric.Hyperlinq; using Nncase.IR; using Nncase.IR.Tensors; using tflite; @@ -18,6 +19,8 @@ public partial class TFLiteImporter private Expr VisitPad(in tflite.Operator op) { var (input, paddings) = GetInputExprs(op, 0, 1); + input = F.Tensors.NHWCToNCHW(input); + paddings = F.Tensors.PaddingNHWCToNCHW(paddings, input.CheckedShape.Rank); var padValue = GetInputTensor(op, 0).Type switch { TensorType.FLOAT32 => 0.0f, @@ -26,20 +29,23 @@ private Expr VisitPad(in tflite.Operator op) _ => throw new NotSupportedException("Unsupported Constant Pad Value"), }; - return F.NN.Pad(input, paddings, PadMode.Constant, padValue); + return F.Tensors.NCHWToNHWC(F.NN.Pad(input, paddings, PadMode.Constant, padValue), input.CheckedShape.Rank); } private Expr VisitPadV2(in tflite.Operator op) { var (input, paddings) = GetInputExprs(op, 0, 1); + input = F.Tensors.NHWCToNCHW(input); + paddings = F.Tensors.PaddingNHWCToNCHW(paddings, input.CheckedShape.Rank); var padValue = GetInputExprs(op, 2); - return F.NN.Pad(input, paddings, PadMode.Constant, padValue); + return F.Tensors.NCHWToNHWC(F.NN.Pad(input, paddings, PadMode.Constant, padValue), input.CheckedShape.Rank); } private Expr VisitMirrorPad(in tflite.Operator op) { var (input, paddings) = GetInputExprs(op, 0, 1); - + input = F.Tensors.NHWCToNCHW(input); + paddings = F.Tensors.PaddingNHWCToNCHW(paddings, input.CheckedShape.Rank); var padMode = op.BuiltinOptionsAsMirrorPadOptions().Mode switch { tflite.MirrorPadMode.REFLECT => PadMode.Reflect, @@ -47,7 +53,7 @@ private Expr VisitMirrorPad(in tflite.Operator op) _ => throw new NotSupportedException("Unsupported Mirror Pad Mode"), }; - return F.NN.Pad(input, paddings, padMode, 0.0f); + return F.Tensors.NCHWToNHWC(F.NN.Pad(input, paddings, padMode, 0.0f), input.CheckedShape.Rank); } } } diff --git a/src/Nncase.Importer/TFLite/ReduceWindow2D.cs b/src/Nncase.Importer/TFLite/ReduceWindow2D.cs index 1fabda70ed..75f099169c 100644 --- a/src/Nncase.Importer/TFLite/ReduceWindow2D.cs +++ b/src/Nncase.Importer/TFLite/ReduceWindow2D.cs @@ -25,9 +25,10 @@ private Expr VisitReduceWindow2D(in tflite.Operator op, ReduceOp reduceOp, float var filter = Tensor.From(new[] { filterH, filterW }, new[] { 2 }); var stride = Tensor.From(new[] { strideH, strideW }, new[] { 2 }); var padding = Util.ConcatPadding(padH, padW); + var countIncludePad = option.Padding == tflite.Padding.SAME ? false : true; return F.Tensors.NCHWToNHWC( F.NN.ReduceWindow2D( - reduceOp, input, initValue, filter, stride, padding, Tensor.From(new long[] { 1, 1 }), false, false)); + reduceOp, input, initValue, filter, stride, padding, Tensor.From(new long[] { 1, 1 }), false, countIncludePad)); } } } diff --git a/src/Nncase.Passes/Rules/Neutral/FoldPad.cs b/src/Nncase.Passes/Rules/Neutral/FoldPad.cs index 2ea62feff0..ef0a5506af 100644 --- a/src/Nncase.Passes/Rules/Neutral/FoldPad.cs +++ b/src/Nncase.Passes/Rules/Neutral/FoldPad.cs @@ -132,7 +132,7 @@ public sealed partial class FoldReduceWindow2DPads : IRewriteRule IsWildcard("ceilMode"), IsWildcard("countIncludePad")); - private Expr? GetReplace(ReduceWindow2D pdp, Expr input, Expr initValue, Expr filter, Expr stride, Tensor padding, Expr dilation, Expr ceilMode, Expr countIncludePad, Tensor ext_pad, float ext_pad_init) + private Expr? GetReplace(ReduceWindow2D pdp, Expr input, Expr initValue, Expr filter, Expr stride, Tensor padding, Expr dilation, Expr ceilMode, bool countIncludePad, Tensor ext_pad, float ext_pad_init) { if (!(ext_pad[0, 0] == 0 && ext_pad[0, 1] == 0 && ext_pad[1, 0] == 0 && ext_pad[1, 1] == 0)) @@ -140,6 +140,11 @@ public sealed partial class FoldReduceWindow2DPads : IRewriteRule return null; } + if (countIncludePad || pdp.ReduceOp != ReduceOp.Max) + { + return null; + } + var new_pad = padding.Clone(); new_pad[0, 0] += ext_pad[2, 0]; new_pad[0, 1] += ext_pad[2, 1]; diff --git a/src/Nncase.Passes/Transforms/FusionToFunctionPass.cs b/src/Nncase.Passes/Transforms/FusionToFunctionPass.cs new file mode 100644 index 0000000000..6464f21efd --- /dev/null +++ b/src/Nncase.Passes/Transforms/FusionToFunctionPass.cs @@ -0,0 +1,45 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reactive; +using System.Text; +using System.Threading.Tasks; +using DryIoc.ImTools; +using Nncase.IR; + +namespace Nncase.Passes.Transforms; + +public sealed class FusionToFunctionPass : ModulePass +{ + protected override Task RunCoreAsync(IRModule input, RunPassContext context) + { + var count = input.Functions.Count; + for (int i = 0; i < count; i++) + { + var replacer = new FusionReplacer(input); + input.Replace(i, (BaseFunction)replacer.Rewrite(input.Functions[i])); + } + + return Task.FromResult(input); + } + + private sealed class FusionReplacer : ExprRewriter + { + private readonly IRModule _module; + + public FusionReplacer(IRModule module) + { + _module = module; + } + + protected override Expr RewriteLeafFusion(Fusion expr) + { + var func = new Function(expr.Body, expr.Parameters, expr.ModuleKind); + _module.Add(func); + return func; + } + } +} diff --git a/src/Nncase.Studio/packages.lock.json b/src/Nncase.Studio/packages.lock.json index 8d4952e0c0..df84b3b59f 100644 --- a/src/Nncase.Studio/packages.lock.json +++ b/src/Nncase.Studio/packages.lock.json @@ -1016,10 +1016,18 @@ "Razor.Templating.Core": "[1.9.0, )" } }, + "nncase.modules.ncnn": { + "type": "Project", + "dependencies": { + "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Passes": "[1.0.0, )" + } + }, "nncase.modules.stackvm": { "type": "Project", "dependencies": { "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Modules.Ncnn": "[1.0.0, )", "Nncase.Passes": "[1.0.0, )" } }, diff --git a/src/Nncase.Tests.TestFixture/packages.lock.json b/src/Nncase.Tests.TestFixture/packages.lock.json index 8ab7507a8f..e12016f4a6 100644 --- a/src/Nncase.Tests.TestFixture/packages.lock.json +++ b/src/Nncase.Tests.TestFixture/packages.lock.json @@ -818,10 +818,18 @@ "Razor.Templating.Core": "[1.9.0, )" } }, + "nncase.modules.ncnn": { + "type": "Project", + "dependencies": { + "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Passes": "[1.0.0, )" + } + }, "nncase.modules.stackvm": { "type": "Project", "dependencies": { "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Modules.Ncnn": "[1.0.0, )", "Nncase.Passes": "[1.0.0, )" } }, diff --git a/src/Nncase.Tests/packages.lock.json b/src/Nncase.Tests/packages.lock.json index 2baef5a63b..dad914a726 100644 --- a/src/Nncase.Tests/packages.lock.json +++ b/src/Nncase.Tests/packages.lock.json @@ -934,10 +934,18 @@ "Razor.Templating.Core": "[1.9.0, )" } }, + "nncase.modules.ncnn": { + "type": "Project", + "dependencies": { + "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Passes": "[1.0.0, )" + } + }, "nncase.modules.stackvm": { "type": "Project", "dependencies": { "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Modules.Ncnn": "[1.0.0, )", "Nncase.Passes": "[1.0.0, )" } }, diff --git a/tests/importer/onnx_/basic/test_binary.py b/tests/importer/onnx_/basic/test_binary.py index 6a4cecf9c0..16dca0f08b 100644 --- a/tests/importer/onnx_/basic/test_binary.py +++ b/tests/importer/onnx_/basic/test_binary.py @@ -23,7 +23,7 @@ def _make_module(v_shape): class BinaryModule(torch.nn.Module): def __init__(self): super(BinaryModule, self).__init__() - self.v = torch.from_numpy(np.ones(v_shape).astype(np.float32)) + self.v = torch.from_numpy(np.random.random(v_shape).astype(np.float32)) def forward(self, x): outs = [] @@ -44,6 +44,7 @@ def forward(self, x): [64, 3], [3, 64, 3], [8, 3, 64, 3], + [1, 1, 64, 3], ] rhs_shapes = [ diff --git a/tests/importer/onnx_/basic/test_cast.py b/tests/importer/onnx_/basic/test_cast.py index 3fc91c3303..5d15111823 100644 --- a/tests/importer/onnx_/basic/test_cast.py +++ b/tests/importer/onnx_/basic/test_cast.py @@ -18,6 +18,7 @@ from onnx import helper from onnx import AttributeProto, TensorProto, GraphProto from onnx_test_runner import OnnxTestRunner +import numpy as np def _make_module(in_shape, in_type, out_type, op_version): @@ -52,6 +53,9 @@ def _make_module(in_shape, in_type, out_type, op_version): ([8, 3, 12, 3], TensorProto.FLOAT, TensorProto.FLOAT16), ([8, 3, 12, 3], TensorProto.FLOAT, TensorProto.UINT8), ([8, 3, 12, 3], TensorProto.FLOAT, TensorProto.INT32), + ([1, 3, 12, 3], TensorProto.FLOAT16, TensorProto.FLOAT), + ([1, 3, 12, 3], TensorProto.FLOAT, TensorProto.FLOAT16), + ] op_versions = [ diff --git a/tests/importer/onnx_/basic/test_gelu.py b/tests/importer/onnx_/basic/test_gelu.py new file mode 100644 index 0000000000..54003d7347 --- /dev/null +++ b/tests/importer/onnx_/basic/test_gelu.py @@ -0,0 +1,51 @@ +# Copyright 2019-2021 Canaan Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# pylint: disable=invalid-name, unused-argument, import-outside-toplevel + +import pytest +import torch +import torch.nn as nn +from onnx_test_runner import OnnxTestRunner + + +def _make_module(v_shape): + class GELUModule(torch.nn.Module): + def __init__(self,): + super(GELUModule, self).__init__() + self.gelu = nn.GELU() + + def forward(self, x): + x = self.gelu(x) + return x + + return GELUModule() + + +lhs_shapes = [ + [1, 3, 16, 32], + [1, 3, 16] +] + + +@pytest.mark.parametrize('lhs_shape', lhs_shapes) +def test_gelu(lhs_shape, request): + module = _make_module(lhs_shape) + + runner = OnnxTestRunner(request.node.name) + model_file = runner.from_torch(module, lhs_shape) + runner.run(model_file) + + +if __name__ == "__main__": + pytest.main(['-vv', 'test_gelu.py']) diff --git a/tests/importer/onnx_/basic/test_layernorm.py b/tests/importer/onnx_/basic/test_layernorm.py new file mode 100644 index 0000000000..d07555f724 --- /dev/null +++ b/tests/importer/onnx_/basic/test_layernorm.py @@ -0,0 +1,68 @@ +# Copyright 2019-2021 Canaan Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# pylint: disable=invalid-name, unused-argument, import-outside-toplevel + +import pytest +import torch +import torch.nn as nn +import numpy as np +from onnx_test_runner import OnnxTestRunner + + +def _make_module(v_shape, axis): + class LayerNormModule(torch.nn.Module): + def __init__(self) -> None: + super(LayerNormModule, self).__init__() + reduce_dim = [v_shape[i] for i in range(len(v_shape)) if i > axis] + self.scale = torch.from_numpy(np.random.rand(*reduce_dim).astype(np.float32)) + self.bias = torch.from_numpy(np.random.rand(*reduce_dim).astype(np.float32)) + self.axis = [i for i in range(len(v_shape)) if i > axis] + + def forward(self, x): + reduce_mean = torch.mean(x, self.axis, keepdim=True) + x_sub = x - reduce_mean + x_pow = torch.pow(x_sub, 2) + x_pow_mean = torch.mean(x_pow, self.axis, keepdim=True) + x_add = x_pow_mean + 1e-06 + x_sqrt = torch.sqrt(x_add) + x_div = x_sub / x_sqrt + x_mul = x_div * self.scale + x_add_bias = x_mul + self.bias + return x_add_bias + + return LayerNormModule() + + +lhs_shapes = [ + [1, 3, 16, 32], + [1, 3, 16] +] + +axises = [ + 1 +] + + +@pytest.mark.parametrize('lhs_shape', lhs_shapes) +@pytest.mark.parametrize('axis', axises) +def test_LayerNorm(lhs_shape, axis, request): + module = _make_module(lhs_shape, axis) + + runner = OnnxTestRunner(request.node.name) + model_file = runner.from_torch(module, lhs_shape) + runner.run(model_file) + + +if __name__ == "__main__": + pytest.main(['-vv', 'test_LayerNorm.py']) diff --git a/tests/importer/onnx_/basic/test_reshape.py b/tests/importer/onnx_/basic/test_reshape.py index b263cb3fa6..4939e6da99 100644 --- a/tests/importer/onnx_/basic/test_reshape.py +++ b/tests/importer/onnx_/basic/test_reshape.py @@ -18,54 +18,40 @@ from onnx_test_runner import OnnxTestRunner -def _make_module(in_shape, out_channel, kernel_size): +def _make_module(in_shape): class ReshapeModule(torch.nn.Module): def __init__(self): super(ReshapeModule, self).__init__() - self.conv2d = torch.nn.Conv2d(in_shape[1], out_channel, kernel_size) self.n = in_shape[0] - self.c = out_channel - self.h = in_shape[2] - kernel_size + 1 - self.w = in_shape[3] - kernel_size + 1 + self.c = in_shape[1] + self.h = in_shape[2] + self.w = in_shape[3] def forward(self, x): - x = self.conv2d(x) - x = torch.reshape(x, (self.n, self.c * self.h * self.w)) - x = torch.reshape(x, (self.n, self.c, self.h * self.w)) - x = torch.reshape(x, (self.n, self.c * self.h, self.w)) - x = torch.reshape(x, (self.n * self.c, self.h * self.w)) - x = torch.reshape(x, (self.n * self.c, self.h, self.w)) - x = torch.reshape(x, (self.n * self.c * self.h, self.w)) - x = torch.reshape(x, (-1, self.w)) - return x + out = [] + out.append(torch.reshape(x, (self.n, self.c * self.h * self.w))) + out.append(torch.reshape(x, (self.n, self.c, self.h * self.w))) + out.append(torch.reshape(x, (self.n, self.c * self.h, self.w))) + out.append(torch.reshape(x, (self.n * self.c, self.h * self.w))) + out.append(torch.reshape(x, (self.n * self.c, self.h, self.w))) + out.append(torch.reshape(x, (self.n * self.c * self.h, self.w))) + out.append(torch.reshape(x, (-1, self.w))) + return out return ReshapeModule() in_shapes = [ [1, 4, 60, 72], - [1, 3, 224, 224] -] - -out_channels = [ - 1, - 3, - 16 -] - -kernel_sizes = [ - 1, - 3, - 5 + [1, 3, 224, 224], + [3, 4, 5, 6] ] @pytest.mark.parametrize('in_shape', in_shapes) -@pytest.mark.parametrize('out_channel', out_channels) -@pytest.mark.parametrize('kernel_size', kernel_sizes) -def test_reshape(in_shape, out_channel, kernel_size, request): - module = _make_module(in_shape, out_channel, kernel_size) +def test_reshape(in_shape, request): + module = _make_module(in_shape) runner = OnnxTestRunner(request.node.name) model_file = runner.from_torch(module, in_shape) diff --git a/tests/importer/onnx_/basic/test_slice.py b/tests/importer/onnx_/basic/test_slice.py index 0d10f1f204..65f345c051 100644 --- a/tests/importer/onnx_/basic/test_slice.py +++ b/tests/importer/onnx_/basic/test_slice.py @@ -158,19 +158,15 @@ def _make_module(in_shape, start, end, axes, step, outshape, op_version, value_f return model_def -in_shapes = [ - [20, 10, 5] -] - -starts_ends_axes_steps_outshapes = [ - [[0, 0], [3, 10], [0, 1], [1, 1], [3, 10, 5]], - [[0, 0, 3], [20, 10, 4], None, None, [20, 10, 1]], - [[0, 0, 3], [20, 10, 4], [0, 1, 2], None, [20, 10, 1]], - [[1], [1000], [1], [1], [20, 9, 5]], - [[0], [-1], [1], [1], [20, 9, 5]], - [[0, 0, 3], [20, 10, 4], [0, -2, -1], None, [20, 10, 1]], - [[20, 10, 4], [0, 0, 1], [0, 1, 2], [-1, -3, -2], [19, 3, 2]], - [[-1], [-9223372036854775807], [0], [-1], [20, 10, 5]] +in_shapes_starts_ends_axes_steps_outshapes = [ + [[1, 20, 10, 5], [0, 0], [3, 10], [1, 2], [1, 1], [1, 3, 10, 5]], + [[1, 20, 10, 5], [0, 0, 0, 3], [1, 20, 10, 4], None, None, [1, 20, 10, 1]], + [[1, 20, 10, 5], [0, 0, 0, 3], [1, 20, 10, 4], [0, 1, 2, 3], None, [1, 20, 10, 1]], + [[1, 20, 10, 5], [1], [1000], [2], [1], [1, 20, 9, 5]], + [[1, 20, 10, 5], [0], [-1], [1], [1], [1, 19, 10, 5]], + [[1, 20, 10, 5], [0, 0, 3], [20, 10, 4], [-3, -2, -1], None, [1, 20, 10, 1]], + [[1, 20, 10, 5], [20, 10, 4], [0, 0, 1], [1, 2, 3], [-1, -3, -2], [1, 20, 4, 2]], + [[1, 20, 10, 5], [-1], [-9223372036854775807], [1], [-1], [1, 20, 10, 5]] ] @@ -185,11 +181,10 @@ def _make_module(in_shape, start, end, axes, step, outshape, op_version, value_f ] -@pytest.mark.parametrize('in_shape', in_shapes) -@pytest.mark.parametrize('start_end_axes_step_outshape', starts_ends_axes_steps_outshapes) +@pytest.mark.parametrize('in_shape_start_end_axis_step_outshape', in_shapes_starts_ends_axes_steps_outshapes) @pytest.mark.parametrize('op_versions_and_value_format', op_versions_and_value_formats) -def test_slice(in_shape, start_end_axes_step_outshape, op_versions_and_value_format, request): - start, end, axes, step, outshape = start_end_axes_step_outshape +def test_slice(in_shape_start_end_axis_step_outshape, op_versions_and_value_format, request): + in_shape, start, end, axes, step, outshape = in_shape_start_end_axis_step_outshape op_version, value_format = op_versions_and_value_format if op_version != 1 or (op_version == 1 and step is not None and all([x == 1 for x in step])): model_def = _make_module(in_shape, start, end, axes, step, diff --git a/tests/importer/onnx_/basic/test_softmax.py b/tests/importer/onnx_/basic/test_softmax.py index 31f98158b2..3f16c20bb4 100644 --- a/tests/importer/onnx_/basic/test_softmax.py +++ b/tests/importer/onnx_/basic/test_softmax.py @@ -63,8 +63,8 @@ def _make_module(in_shape, axis, op_version): in_shapes = [ - [2, 3, 8, 1], - [1, 3, 8, 5], + [2, 32, 16, 4], + [5, 13, 28, 64], ] axes = [ diff --git a/tests/importer/onnx_/basic/test_split.py b/tests/importer/onnx_/basic/test_split.py index d1a125a1cf..abe8e44442 100644 --- a/tests/importer/onnx_/basic/test_split.py +++ b/tests/importer/onnx_/basic/test_split.py @@ -100,12 +100,13 @@ def _make_module(in_shape, axis, split, output_size, op_version, value_format): in_shapes = [ - [4, 8, 8] + [1, 4, 8, 8] ] axes = [ None, - -3 + -2, + 1 ] splits = [ @@ -113,14 +114,15 @@ def _make_module(in_shape, axis, split, output_size, op_version, value_format): [2, 2], [1, 1, 2], [1, 2, 1], - [2, 1, 1], - [1, 1, 1, 1] + [3, 2, 2, 1], + [2, 1, 1, 1, 3], ] output_sizes = [ 2, 3, - 4 + 4, + 5 ] op_versions_and_value_formats = [ @@ -141,7 +143,7 @@ def _make_module(in_shape, axis, split, output_size, op_version, value_format): def test_split(in_shape, axis, split, output_size, op_versions_and_value_format, request): op_version, value_format = op_versions_and_value_format dim = axis if axis is not None else 0 - if (split is None and in_shape[dim] % output_size == 0) or (split is not None and len(split) == output_size): + if (split is None and in_shape[dim] % output_size == 0) or (split is not None and len(split) == output_size and sum(split) == in_shape[dim]): model_def = _make_module(in_shape, axis, split, output_size, op_version, value_format) runner = OnnxTestRunner(request.node.name) diff --git a/tests/importer/onnx_/basic/test_transpose.py b/tests/importer/onnx_/basic/test_transpose.py index e22d231185..20b6fa576d 100644 --- a/tests/importer/onnx_/basic/test_transpose.py +++ b/tests/importer/onnx_/basic/test_transpose.py @@ -63,6 +63,7 @@ def _make_module(in_shape, axis, op_version): in_shapes = [ [16, 16], [3, 16, 16], + [1, 8, 16], [1, 3, 16, 16] ] diff --git a/tests/importer/tflite_/basic/disabled_test_conv2d_transpose.py b/tests/importer/tflite_/basic/disabled_test_conv2d_transpose.py index 0b2a8b8c9a..a8eb0cf2eb 100644 --- a/tests/importer/tflite_/basic/disabled_test_conv2d_transpose.py +++ b/tests/importer/tflite_/basic/disabled_test_conv2d_transpose.py @@ -91,13 +91,14 @@ def __call__(self, x): @pytest.mark.parametrize('dilations', dilations) @pytest.mark.parametrize('bias', biases) def test_conv2d_transpose(n, i_channels, i_size, k_size, o_channels, strides, padding, dilations, bias, request): - module = _make_module(n, i_channels, i_size, k_size, o_channels, - strides, padding, dilations, bias) - - #runner = TfliteTestRunner(request.node.name, ['k510']) - runner = TfliteTestRunner(request.node.name) - model_file = runner.from_tensorflow(module) - runner.run(model_file) + if k_size[0] >= strides[0] and k_size[1] >= strides[1]: + module = _make_module(n, i_channels, i_size, k_size, o_channels, + strides, padding, dilations, bias) + + #runner = TfliteTestRunner(request.node.name, ['k510']) + runner = TfliteTestRunner(request.node.name) + model_file = runner.from_tensorflow(module) + runner.run(model_file) if __name__ == "__main__": diff --git a/toolchains/riscv64-unknown-linux.profile.jinja b/toolchains/riscv64-unknown-linux.profile.jinja index 93ee41b644..a067bc6d73 100644 --- a/toolchains/riscv64-unknown-linux.profile.jinja +++ b/toolchains/riscv64-unknown-linux.profile.jinja @@ -22,4 +22,4 @@ compiler=gcc build_type=Release compiler.cppstd=20 compiler.libcxx=libstdc++11 -compiler.version=12 +compiler.version=13